fix(compose): serve TLS ingest via dfe-proxy and mount file input

.env.example

@@ -141,6 +141,7 @@
 ## =============================================================
 ## DFE Fetcher
 ## =============================================================
+# DFE_FETCHER_INPUT_DIR=
 # DFE_FETCHER_VERSION=
 
 ## =============================================================
@@ -157,6 +158,7 @@
 # DFE_RECEIVER_GRPC_PORT=
 # DFE_RECEIVER_HEC_PORT=
 # DFE_RECEIVER_HTTP_PORT=8080
+# DFE_RECEIVER_HTTPS_PORT=
 # DFE_RECEIVER_IPFIX_PORT=
 # DFE_RECEIVER_MAX_BODY_SIZE=8000000
 # DFE_RECEIVER_NETFLOW_PORT=

Makefile

@@ -281,6 +281,12 @@ ifneq ($(RECEIVER_PORTS_SET),)
     $(eval $(call chain_fragment,.tmp/docker-compose.receiver-ports.yml))
 endif
 
+# A host directory mounts read only into dfe-fetcher once DFE_FETCHER_INPUT_DIR is set, for a `sources.file` source to read. `create_host_path: false` makes a mistyped path fail the start instead of mounting an empty directory Docker made
+ifneq ($(strip $(DFE_FETCHER_INPUT_DIR)),)
+    $(file > .tmp/docker-compose.fetcher-input.yml,# Generated by make$(newline)services:$(newline)  dfe-fetcher:$(newline)    volumes:$(newline)      - type: bind$(newline)        source: $${DFE_FETCHER_INPUT_DIR}$(newline)        target: /mnt/input$(newline)        read_only: true$(newline)        bind:$(newline)          create_host_path: false)
+    $(eval $(call chain_fragment,.tmp/docker-compose.fetcher-input.yml))
+endif
+
 # One service per app instance dfe-engine has rendered a config for, generated
 # with the profile (scripts/instances.py). Empty for the bootstrap goals, which
 # reads as off -- the safe direction for a target that only stops a stack.
@@ -356,6 +362,16 @@ ifneq ($(PROXY_TLS_START),)
     endif
 endif
 
+# dfe-proxy's TLS ingest listener (config/proxy/envoy.tls.yaml) publishes once its key is set. The plain config has no such listener, so the key without the dial is refused on the goals that start a stack
+ifneq ($(strip $(DFE_RECEIVER_HTTPS_PORT)),)
+    ifeq ($(strip $(DFE_PROXY_TLS)),true)
+        $(file > .tmp/docker-compose.receiver-https.yml,# Generated by make$(newline)services:$(newline)  dfe-proxy:$(newline)    ports:$(newline)      - $${DFE_INGRESS_BIND_HOST:-0.0.0.0}:$${DFE_RECEIVER_HTTPS_PORT}:8443)
+        $(eval $(call chain_fragment,.tmp/docker-compose.receiver-https.yml))
+    else ifneq (,$(filter $(ORIGIN_GOALS),$(MAKECMDGOALS)))
+        $(error DFE_RECEIVER_HTTPS_PORT='$(strip $(DFE_RECEIVER_HTTPS_PORT))' publishes dfe-proxy's TLS ingest listener, which only exists under DFE_PROXY_TLS=true. Set DFE_PROXY_TLS=true and run `make certs`, or unset DFE_RECEIVER_HTTPS_PORT. Plain http ingest stays on DFE_RECEIVER_HTTP_PORT)
+    endif
+endif
+
 # These goals recreate services on the network that already exists with no `make down` first; the e2e suites keep ClickHouse, and so its network, between tests. Docker cannot re-address a live network, so a stack whose network predates a DFE_PROXY_TLS flip would leave every recreated service Exited.
 NETWORK_REUSE_GOALS := apply apply-services infra $(E2E_GOALS)
 PROXY_NETWORK_REUSE := $(filter $(NETWORK_REUSE_GOALS),$(or $(MAKECMDGOALS),help))

config/proxy/envoy.tls.yaml

@@ -8,6 +8,7 @@
 #
 # Mounted instead of envoy.yaml when DFE_PROXY_TLS=true
 # Must equal envoy.yaml plus the marked TLS blocks (test_proxy_tls_config.py)
+# The receiver-ingest listener exists only here: TLS is its whole purpose
 
 admin:
   address:
@@ -45,6 +46,18 @@ static_resources:
                   address:
                     socket_address: { address: dfe-ui, port_value: 3000 }
       type: STRICT_DNS
+    # BEGIN TLS
+    - name: dfe-receiver
+      dns_lookup_family: V4_ONLY
+      load_assignment:
+        cluster_name: dfe-receiver
+        endpoints:
+          - lb_endpoints:
+              - endpoint:
+                  address:
+                    socket_address: { address: dfe-receiver, port_value: 8080 }
+      type: STRICT_DNS
+    # END TLS
   listeners:
     - name: ingress
       address:
@@ -149,3 +162,30 @@ static_resources:
           # An http embed inside the https console is blocked as mixed content
           transport_socket: *console_tls
           # END TLS
+    # BEGIN TLS
+    # External pushers send to the receiver here, over the console's certificate
+    - name: receiver-ingest
+      address:
+        socket_address: { address: 0.0.0.0, port_value: 8443 }
+      filter_chains:
+        - filters:
+            - name: envoy.filters.network.http_connection_manager
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                http_filters:
+                  - name: envoy.filters.http.router
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+                route_config:
+                  name: receiver_ingest
+                  virtual_hosts:
+                    - name: receiver
+                      domains: ["*"]
+                      routes:
+                        # The timeout bounds the wait for the receiver's reply. The receiver enforces its own body size and request timeout
+                        - match: { prefix: / }
+                          route: { cluster: dfe-receiver, timeout: 120s }
+                stat_prefix: receiver_ingest
+                use_remote_address: true
+          transport_socket: *console_tls
+    # END TLS

docs/architecture.md

@@ -147,7 +147,7 @@ flowchart LR
 
 | Container port | Host port | Owner | Protocol | Audience | Binds |
 |---|---|---|---|---|---|
-| 6000 | none, until `DFE_RECEIVER_GRPC_PORT` is set | dfe-receiver | Vector protocol, an ingest source that listens only with `DFE_RECEIVER_GRPC__ENABLED=true` | external clients | `DFE_INGRESS_BIND_HOST` (`0.0.0.0`) |
+| 6000 | none, until `DFE_RECEIVER_GRPC_PORT` is set | dfe-receiver | Vector protocol, an ingest source that listens only once `config.grpc.enabled` is set through the engine | external clients | `DFE_INGRESS_BIND_HOST` (`0.0.0.0`) |
 | 6000 | none | dfe-loader | `DfeTransport/Push`, internal transport | receiver, fetcher, the transforms | the compose network |
 
 Every Push listener in the suite answers on 6000 and dfe-engine compiles each sender's endpoint against that port. The receiver's listener is meant to be reachable from outside the stack. The loader's is not, so it publishes no host port.

docs/configuration.md

@@ -241,13 +241,16 @@ Only `single-kafka` runs dfe-archiver, so archiving needs a Kafka profile ([prof
 | Variable                                         | Use                                                                                  | Default                                                             |
 |--------------------------------------------------|--------------------------------------------------------------------------------------|---------------------------------------------------------------------|
 | `DFE_FETCHER_VERSION`                            | Version of dfe-fetcher to use                                                        | none -- `make stack` pins it; unset is a hard-fail                                                            |
+| `DFE_FETCHER_INPUT_DIR`                          | Absolute host directory mounted read only into dfe-fetcher at `/mnt/input`           | unset, nothing mounted                                                |
 | `AWS_ACCESS_KEY_ID`                              | AWS access key. Goes in `env/fetcher.env`, NOT `.env` -- the fetcher config reads it via `env:`  | -                                    |
 | `AWS_SECRET_ACCESS_KEY`                          | AWS secret key. Same file as above                                                   | -                                                                   |
 | `AZURE_CLIENT_ID`                                | In `env/fetcher.env`. Client ID an Azure source names with `env:AZURE_CLIENT_ID`     | -                                                                   |
 | `AZURE_CLIENT_SECRET`                            | In `env/fetcher.env`. That client's secret, named the same way                       | -                                                                   |
 | `AZURE_SUBSCRIPTION_ID`                          | In `env/fetcher.env`. Subscription an Azure source reads                             | -                                                                   |
 | `AZURE_TENANT_ID`                                | In `env/fetcher.env`. Tenant the client belongs to                                   | -                                                                   |
 
+A `sources.file` source reads files from the fetcher's container, so its globs name the container path, such as `/mnt/input/*.log`. A dump source lists them under `paths` and a tail source under `include`. Set `DFE_FETCHER_INPUT_DIR` to the host directory holding them. `make` mounts it only when the key is set, and a path that does not exist stops the start. The fetcher image runs as uid 1000, so that user must be able to read the directory. A `0700` directory owned by root passes the start and fails when the source runs.
+
 The AWS **region** and poll interval are not environment variables: they are literals in each source's fetcher config (`region: us-west-2` and `interval_secs` in `resources/seed/sources/aws-*.yaml`, 30 seconds and 300 for GuardDuty, whose every poll stores all its findings again), because only the two credential fields are `env:`-interpolated.
 
 ### DFE Loader
@@ -267,6 +270,7 @@ The AWS **region** and poll interval are not environment variables: they are lit
 | `DFE_RECEIVER_GRPC_PORT`                         | Host port for the receiver's Vector gRPC listener (container port 6000)             | unset, unpublished                                                  |
 | `DFE_RECEIVER_HEC_PORT`                          | Host port for the receiver's HEC listener (container port 8088)                     | unset, unpublished                                                  |
 | `DFE_RECEIVER_HTTP_PORT`                         | Receiver HTTP port                                                                   | `8080`                                                              |
+| `DFE_RECEIVER_HTTPS_PORT`                        | Host port for TLS ingest to the receiver, which dfe-proxy serves on container port 8443. Needs `DFE_PROXY_TLS=true` | unset, unpublished                                                  |
 | `DFE_RECEIVER_IPFIX_PORT`                        | Host UDP port for the receiver's flow listener on container port 4739 (IPFIX)        | unset, unpublished                                                  |
 | `DFE_RECEIVER_MAX_BODY_SIZE`                     | Largest record the receiver accepts in bytes, under every broker's 8 MiB message limit | `8000000`                                                           |
 | `DFE_RECEIVER_NETFLOW_PORT`                      | Host UDP port for the receiver's flow listener on container port 2055 (NetFlow)      | unset, unpublished                                                  |
@@ -278,7 +282,13 @@ The AWS **region** and poll interval are not environment variables: they are lit
 | `DFE_RECEIVER_SYSLOG_TLS_PORT`                   | Host TCP port for the receiver's syslog over TLS listener (container port 6514)      | unset, unpublished                                                  |
 | `DFE_RECEIVER_SYSLOG_UDP_PORT`                   | Host UDP port for the receiver's syslog listener (container port 514)                | unset, unpublished                                                  |
 
-Setting a listener's port key publishes it through a fragment `make` writes, so a raw `docker compose` outside `make` does not publish it. The listener still has to be enabled in the receiver's config, such as `DFE_RECEIVER_GRPC__ENABLED=true` in `env/receiver.env`. Syslog takes `DFE_RECEIVER_SYSLOG__ENABLED=true` and `DFE_RECEIVER_SYSLOG__ACCEPT_UNAUTHENTICATED=true`. Its UDP and plain TCP listeners bind whenever syslog is on and have no handshake to authenticate at, so the receiver refuses to start without the second key, even when only the TLS port is published. Flow takes `DFE_RECEIVER_FLOW__ENABLED=true`.
+Setting a listener's port key publishes it through a fragment `make` writes, so a raw `docker compose` outside `make` does not publish it. The listener still has to be enabled in the receiver's config. Flow takes `DFE_RECEIVER_FLOW_ENABLED=true` in `env/receiver.env`.
+
+Syslog and the Vector gRPC listener have no environment key on Docker, because the receiver reads only a fixed set of flat keys when it runs from a config file. Turn them on through the engine, as `make demo` does for syslog: `PUT /api/v1/apps/dfe-receiver/<instance>/config` with `{"changes": {"config.syslog.enabled": true, "config.syslog.accept_unauthenticated": true}}`, then `make apply`. The gRPC listener takes `config.grpc.enabled` the same way. The UDP and plain TCP syslog listeners bind whenever syslog is on and have no handshake to authenticate at, so the receiver refuses to start without the second key, even when only the TLS port is published.
+
+`DFE_RECEIVER_HTTPS_PORT` publishes TLS ingest to the receiver's HTTP listener, bound to `DFE_INGRESS_BIND_HOST`. dfe-proxy terminates it with the same `console.crt` and `console.key` as the console and forwards to `dfe-receiver:8080`, so callers on the compose network keep posting plain http. It exists only under `DFE_PROXY_TLS=true` and `make` refuses the key without it. A pusher outside the stack connects to the hostname the certificate names and trusts the `ca.crt` beside the pair.
+
+The receiver sees dfe-proxy as the TCP peer, so its `ip_filter` cannot tell pushers apart on this port. Its rate limit and auth-failure events key on dfe-proxy too, until `server.trusted_proxies` in the receiver's config file lists dfe-proxy's address, the last host of `DFE_NETWORK_SUBNET`. That key has no environment spelling.
 
 A host port under 1024, such as 514, needs a Docker daemon allowed to bind it. Rootless Docker is not, so pick a high port such as 1514 there.
 

scripts/check_compose.py

@@ -256,13 +256,15 @@
     "DFE_ENGINE_API_EXTERNAL": "false",
     "DFE_PROXY_TLS": "true",
     "DFE_RECEIVER_BEATS_PORT": "5044",
+    "DFE_RECEIVER_HTTPS_PORT": "8443",
     "DFE_RECEIVER_SYSLOG_UDP_PORT": "1514",
+    "DFE_FETCHER_INPUT_DIR": "/srv/compose-check-input",
 }
 # Console TLS refuses to start beside the auth profile, so `make dev` runs once per side, each held to the chain its own dials produce. The TLS run also needs what that refusal and the TLS precheck look for: an https origin and a certificate pair openssl accepts.
 _DEV_PATH_CERT_DIR = REPO_ROOT / ".tmp" / "compose-check-certs"
 _DEV_PATH_BIN_DIR = REPO_ROOT / ".tmp" / "compose-check-bin"
 _DEV_PATH_RUNS = {
-    "auth": {"DFE_PROXY_TLS": "false"},
+    "auth": {"DFE_PROXY_TLS": "false", "DFE_RECEIVER_HTTPS_PORT": ""},
     "console TLS": {
         "DFE_AUTH_ENABLED": "false",
         "DFE_AUTH_RESOLVED": "false",

scripts/tests/test_check_compose.py

@@ -567,7 +567,7 @@ def _proxy_text(name: str) -> str:
         ("hyperdx.yaml", 2),
         ("hyperdx.tls.yaml", 4),
         ("envoy.yaml", 2),
-        ("envoy.tls.yaml", 2),
+        ("envoy.tls.yaml", 3),
     ],
 )
 def test_no_proxy_adds_an_identity_header(hosts: int, name: str) -> None:

scripts/tests/test_hyperdx_embed_exposure.py

@@ -31,6 +31,12 @@
 # Container ports dfe-receiver publishes whatever its port keys say.
 _RECEIVER_BASE = {"8080", "9090"}
 _RECEIVER_FRAGMENT = ".tmp/docker-compose.receiver-ports.yml"
+# The TLS ingest listener dfe-proxy publishes for the receiver, and where dfe-fetcher mounts a host directory.
+_INGEST_FRAGMENT = ".tmp/docker-compose.receiver-https.yml"
+_INGEST_TARGET = "8443"
+_FETCHER_FRAGMENT = ".tmp/docker-compose.fetcher-input.yml"
+_FETCHER_MOUNT = "/mnt/input"
+_TLS = {"DFE_PROXY_TLS": "true", "DFE_EXTERNAL_ORIGIN": "https://dfe.example.test"}
 
 _RESOLVER = '''\
 import os
@@ -222,3 +228,113 @@ def test_a_receiver_port_key_publishes_its_listener(
         for port in ports
         if str(port.get("target")) not in _RECEIVER_BASE
     } == published
+
+
+@pytest.mark.parametrize(
+    ("settings", "published"),
+    [
+        pytest.param({}, None, id="none-set"),
+        pytest.param({"DFE_RECEIVER_HTTPS_PORT": "8443"}, None, id="key-without-tls"),
+        pytest.param(dict(_TLS), None, id="tls-without-key"),
+        pytest.param(
+            {**_TLS, "DFE_RECEIVER_HTTPS_PORT": "8443"},
+            ("8443", "0.0.0.0"),
+            id="key-with-tls",
+        ),
+        pytest.param(
+            {
+                **_TLS,
+                "DFE_RECEIVER_HTTPS_PORT": "443",
+                "DFE_INGRESS_BIND_HOST": "192.0.2.10",
+            },
+            ("443", "192.0.2.10"),
+            id="binds-the-ingest-host",
+        ),
+        pytest.param(
+            {
+                **_TLS,
+                "DFE_RECEIVER_HTTPS_PORT": "8443",
+                "DFE_UI_EXTERNAL": "false",
+                "DFE_INFRA_UIS_EXTERNAL": "false",
+            },
+            ("8443", "0.0.0.0"),
+            id="survives-the-console-opt-outs",
+        ),
+    ],
+)
+def test_the_https_ingest_key_publishes_the_proxy_listener_under_tls(
+    checkout: Path,
+    settings: dict[str, str],
+    published: tuple[str, str] | None,
+    tmp_path: Path,
+) -> None:
+    if shutil.which("docker") is None:
+        pytest.skip("`docker compose config` renders the model this asserts on")
+    fragment = _chained_fragment(
+        checkout=checkout, name=_INGEST_FRAGMENT, settings=settings
+    )
+    unpublish = _chained_fragment(
+        checkout=checkout, name=check_compose._UNPUBLISH_FRAGMENT, settings=settings
+    )
+    environ, _ = check_compose._check_env()
+
+    assert (fragment is None) == (published is None)
+    overlays = []
+    for number, text in enumerate(part for part in (unpublish, fragment) if part):
+        overlay = tmp_path / f"overlay{number}.yml"
+        overlay.write_text(data=text, encoding="utf-8", newline="\n")
+        overlays.append(str(overlay))
+    model = check_compose._config_json(
+        env={**environ, **settings},
+        files=[check_compose.COMPOSE_FILE.name, *overlays],
+    )
+
+    assert model is not None
+    ports = model["services"]["dfe-proxy"].get("ports") or []
+    assert {
+        (str(port.get("published")), str(port.get("host_ip")))
+        for port in ports
+        if str(port.get("target")) == _INGEST_TARGET
+    } == ({published} if published else set())
+    assert (_INGEST_TARGET in {str(port.get("target")) for port in ports}) == bool(
+        published
+    )
+
+
+@pytest.mark.parametrize(
+    ("settings", "mounted"),
+    [
+        pytest.param({}, False, id="none-set"),
+        pytest.param({"DFE_FETCHER_INPUT_DIR": " "}, False, id="blank"),
+        pytest.param({"DFE_FETCHER_INPUT_DIR": "/srv/dfe-input"}, True, id="set"),
+    ],
+)
+def test_the_fetcher_input_key_mounts_a_host_directory_read_only(
+    checkout: Path, settings: dict[str, str], mounted: bool
+) -> None:
+    if shutil.which("docker") is None:
+        pytest.skip("`docker compose config` renders the model this asserts on")
+    fragment = _chained_fragment(
+        checkout=checkout, name=_FETCHER_FRAGMENT, settings=settings
+    )
+    environ, _ = check_compose._check_env()
+
+    assert (fragment is None) == (not (mounted))
+    model = check_compose._config_json(
+        env={**environ, **settings},
+        files=[check_compose.COMPOSE_FILE.name],
+        fragment=fragment,
+    )
+
+    assert model is not None
+    volumes = model["services"]["dfe-fetcher"].get("volumes") or []
+    inputs = [volume for volume in volumes if volume.get("target") == _FETCHER_MOUNT]
+    assert [
+        (
+            volume["type"],
+            volume["source"],
+            volume.get("read_only"),
+            volume["bind"]["create_host_path"],
+        )
+        for volume in inputs
+    ] == ([("bind", "/srv/dfe-input", True, False)] if mounted else [])