test: producer-to-collector routing on pg_stat_ch.block_format

docker/docker-compose.arrow-route.yml

@@ -0,0 +1,26 @@
+# Standalone collector profile for t/037_arrow_routing.pl. Kept separate
+# from docker-compose.otel.yml (which serves t/024_otel_export.pl) so the
+# two tests can run concurrently — different ports + container names.
+#
+# Output JSONL files land in ./otel-routing/output/, bind-mounted into the
+# container at /output. The test cleans this directory before each run.
+
+services:
+  routing-otelcol:
+    image: otel/opentelemetry-collector-contrib:0.120.0
+    container_name: psch-routing-otelcol
+    command: ["--config=/etc/otelcol-contrib/config.yaml"]
+    volumes:
+      - ./otel-routing/collector-config.yaml:/etc/otelcol-contrib/config.yaml
+      - ./otel-routing/output:/output
+    ports:
+      - "14317:4317"   # gRPC OTLP receiver (host:14317 so it doesn't clash with otel-test's 4317)
+      - "23133:13133"  # Health check HTTP endpoint
+    # No Docker-level healthcheck: otel/opentelemetry-collector-contrib is a
+    # distroless image with no wget/curl/shell, so a CMD-based healthcheck
+    # can never succeed and would always report "unhealthy" regardless of
+    # actual readiness — misleading in `docker compose ps`, and a landmine
+    # for anyone who later adds `--wait` to the start command (would hang
+    # forever). Readiness is checked from the host via a real HTTP request
+    # in psch_start_routing_collector (t/psch.pm), independent of Docker's
+    # own healthcheck machinery.

docker/otel-routing/collector-config.yaml

@@ -0,0 +1,69 @@
+# Minimum-faithful OTel collector config for the producer-to-collector
+# routing test (t/037_arrow_routing.pl).
+#
+# Mirrors the SHAPE of the prod datagres-otelcol pipeline (routing connector
+# fanning Arrow batches between the legacy query_logs_arrow target and the new
+# events_raw target based on the pg_stat_ch.block_format resource attribute)
+# but with stdout/file sinks instead of real receivers. The test asserts that
+# routing decisions match the producer-side marker emission contract — it does
+# NOT validate any downstream wire format. That's covered by:
+#   - t/036_unified_arrow_e2e.pl (producer Arrow IPC -> direct curl into CH)
+#   - clickgres-platform's receiver-faithful e2e test (producer -> real
+#     datagres-arrow-exporter -> CH events_raw)
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+
+connectors:
+  routing:
+    # Default pipeline catches anything that doesn't match a routing rule.
+    # In the test we treat hits on the default pipeline as routing failures
+    # (the producer should always emit one of the two known marker values).
+    default_pipelines: [logs/default]
+    error_mode: ignore
+    table:
+      - context: resource
+        statement: route() where attributes["pg_stat_ch.block_format"] == "arrow_events_raw"
+        pipelines: [logs/events_raw]
+      - context: resource
+        statement: route() where attributes["pg_stat_ch.block_format"] == "arrow_ipc"
+        pipelines: [logs/legacy]
+
+exporters:
+  # JSONL sinks — one file per route. Tests parse these directly to assert
+  # that batches landed on the expected pipeline.
+  file/legacy:
+    path: /output/legacy.jsonl
+  file/events_raw:
+    path: /output/events_raw.jsonl
+  file/default:
+    path: /output/default.jsonl
+  # Keep one stdout sink for easier debugging when a test fails locally.
+  debug:
+    verbosity: basic
+
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
+
+service:
+  extensions: [health_check]
+  pipelines:
+    # Ingress: receive OTLP gRPC and hand off to the routing connector.
+    logs/in:
+      receivers: [otlp]
+      exporters: [routing]
+    # Routed legs. Each pipeline takes its input from the connector and
+    # writes to a dedicated file plus the shared debug stdout.
+    logs/legacy:
+      receivers: [routing]
+      exporters: [file/legacy, debug]
+    logs/events_raw:
+      receivers: [routing]
+      exporters: [file/events_raw, debug]
+    logs/default:
+      receivers: [routing]
+      exporters: [file/default, debug]

docker/otel-routing/output/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

t/013_clickhouse_tls.pl

@@ -88,8 +88,18 @@ sub ch_tls_ready {
 
     $node->safe_psql('postgres', 'SELECT 200');
     $node->safe_psql('postgres', 'SELECT pg_stat_ch_flush()');
+    # 90s, not 30s: the exporter's reconnect backoff (stats_exporter.cc,
+    # PschGetRetryDelayMs) is exponential — base 1s, doubling per
+    # consecutive failure, capped at 60s. Failed flush attempts start
+    # accumulating the moment CH goes down (well before docker restart's
+    # container comes back), and the cumulative wait before the Nth retry
+    # is the sum of all prior delays: 1, 3, 7, 15, 31, 63, 123s... A CH
+    # restart slow enough to rack up 5-6 failures (very plausible under a
+    # loaded CI runner) pushes the bgworker's next attempt past the 30s
+    # mark even though CH itself came back much sooner. 90s comfortably
+    # covers the 63s (6-failure) cumulative-backoff case plus round-trip.
     my $after = psch_wait_for_clickhouse_query(
-        'SELECT count() FROM pg_stat_ch.events_raw', sub { $_[0] >= 1 }, 30);
+        'SELECT count() FROM pg_stat_ch.events_raw', sub { $_[0] >= 1 }, 90);
     cmp_ok($after, '>=', 1, "export resumed over TLS after reconnect (got $after)");
 };