From dcffa2d992575108da732f91a5da5649a9d6fe39 Mon Sep 17 00:00:00 2001 From: Matt Johnston Date: Fri, 11 Sep 2026 11:25:27 -0300 Subject: [PATCH 1/3] refactor(api): filter on subservice column instead of log_attributes['source'] --- .../experimental/compute/logs/SIDE_EFFECTS.md | 6 ++--- .../compute/logs/logs.integration.test.ts | 5 ++-- .../src/shared/compute/compute-logs.sql.ts | 26 +++++++++++-------- .../compute/compute-logs.sql.unit.test.ts | 6 +++-- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/apps/cli/src/commands/experimental/compute/logs/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/compute/logs/SIDE_EFFECTS.md index 12eb2fcc8c..2c12f24001 100644 --- a/apps/cli/src/commands/experimental/compute/logs/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/compute/logs/SIDE_EFFECTS.md @@ -49,9 +49,9 @@ Compute private-alpha allow-list — an unenrolled project answers 404. ### The query SQL in **ClickHouse dialect** against the project's unified `logs` table, filtered -on `log_attributes['worker']` and `log_attributes['source']`. It does **not** filter -the top-level `source` column: compute rows carry an empty string there, because the -Compute Logflare source is not enrolled as a category in the generic logs path. +on `log_attributes['worker']` and the top-level `subservice` column. It does **not** +filter the top-level `source` column: compute rows carry an empty string there, because +the Compute Logflare source is not enrolled as a category in the generic logs path. ### The window diff --git a/apps/cli/src/commands/experimental/compute/logs/logs.integration.test.ts b/apps/cli/src/commands/experimental/compute/logs/logs.integration.test.ts index 757116a0ae..1bc9228100 100644 --- a/apps/cli/src/commands/experimental/compute/logs/logs.integration.test.ts +++ b/apps/cli/src/commands/experimental/compute/logs/logs.integration.test.ts @@ -258,7 +258,7 @@ describe("compute logs", () => { }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), ); - it.live("filters on log_attributes, never the empty source column", () => + it.live("filters on the subservice column, never the empty source column", () => Effect.gen(function* () { const repo = yield* project(); const { layer, http } = setupCompute({ @@ -271,7 +271,8 @@ describe("compute logs", () => { const sql = sentQuery(http.requests[0]!).sql ?? ""; expect(sql).toContain("log_attributes['worker'] = 'api'"); - expect(sql).toContain("log_attributes['source'] in ("); + expect(sql).toContain("and subservice in ("); + expect(sql).not.toContain("log_attributes['source']"); expect(sql).not.toMatch(/where source =/); }).pipe(Effect.provide(layer)); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), diff --git a/apps/cli/src/shared/compute/compute-logs.sql.ts b/apps/cli/src/shared/compute/compute-logs.sql.ts index 2f79a94ff2..2b402e86cd 100644 --- a/apps/cli/src/shared/compute/compute-logs.sql.ts +++ b/apps/cli/src/shared/compute/compute-logs.sql.ts @@ -44,8 +44,14 @@ export const ALL_COMPUTE_LOG_STREAMS: ReadonlyArray = Object.values(COMP */ const COMPUTE_LOG_NAME_ATTRIBUTE = "worker"; -/** Which key carries the stream name. See {@link computeLogsQuery} for why. */ -const COMPUTE_LOG_STREAM_ATTRIBUTE = "source"; +/** + * Which top-level column carries the stream name. + * + * The writer publishes it beside `project` rather than inside `metadata`, so it is a column + * here rather than a `log_attributes` key. `metadata.source` carries the same value for + * readers that have not moved across; this is the one to filter on. + */ +const COMPUTE_LOG_STREAM_COLUMN = "subservice"; /** * The server clamps a span of more than 24 hours, so the default window sits just under the @@ -127,13 +133,11 @@ function quote(value: string): string { } /** - * The logs query for one compute. Two things about the projection are load-bearing: the filter - * is `log_attributes`, not the `source` column, since compute rows carry an empty top-level - * `source` (the stream survives only in `log_attributes['source']`); and the `in (...)` list - * is a tenancy guard, not a convenience — with `source` empty, it's the only thing keeping a - * non-compute row with a `worker` attribute out of the result. `toUnixTimestamp64Milli` rather - * than a formatter, since ClickHouse's `%M` is the month name and bare `toString(timestamp)` - * has no zone. + * The logs query for one compute. Two things about the projection are load-bearing: the + * `in (...)` list is a tenancy guard, not a convenience — compute rows carry an empty + * top-level `source`, so the stream list is the only thing keeping a non-compute row with a + * `worker` attribute out of the result; and `toUnixTimestamp64Milli` rather than a formatter, + * since ClickHouse's `%M` is the month name and bare `toString(timestamp)` has no zone. */ export function computeLogsQuery(options: { readonly name: string; @@ -144,12 +148,12 @@ export function computeLogsQuery(options: { return ( `select id, ` + `toUnixTimestamp64Milli(timestamp) as ts_ms, ` + - `log_attributes['${COMPUTE_LOG_STREAM_ATTRIBUTE}'] as stream, ` + + `${COMPUTE_LOG_STREAM_COLUMN} as stream, ` + `event_message, ` + `log_attributes ` + `from logs ` + `where log_attributes['${COMPUTE_LOG_NAME_ATTRIBUTE}'] = ${quote(options.name)} ` + - `and log_attributes['${COMPUTE_LOG_STREAM_ATTRIBUTE}'] in (${streams}) ` + + `and ${COMPUTE_LOG_STREAM_COLUMN} in (${streams}) ` + `order by timestamp desc ` + `limit ${options.tail}` ); diff --git a/apps/cli/src/shared/compute/compute-logs.sql.unit.test.ts b/apps/cli/src/shared/compute/compute-logs.sql.unit.test.ts index 07acfce967..1dbc9c42df 100644 --- a/apps/cli/src/shared/compute/compute-logs.sql.unit.test.ts +++ b/apps/cli/src/shared/compute/compute-logs.sql.unit.test.ts @@ -12,11 +12,13 @@ import { } from "./compute-logs.sql.ts"; describe("computeLogsQuery", () => { - it("filters on log_attributes, never the source column", () => { + it("filters on the subservice column, never the source column", () => { const sql = computeLogsQuery({ name: "api", streams: ALL_COMPUTE_LOG_STREAMS, tail: 100 }); expect(sql).toContain("log_attributes['worker'] = 'api'"); - expect(sql).toContain("log_attributes['source'] in ("); + expect(sql).toContain("subservice as stream"); + expect(sql).toContain("and subservice in ("); + expect(sql).not.toContain("log_attributes['source']"); expect(sql).not.toMatch(/(?:^|\s)where source =/); expect(sql).not.toMatch(/(?:^|\s)and source =/); }); From 095b5aafc595f1c0b44bd03016215af5dfa07237 Mon Sep 17 00:00:00 2001 From: Matt Johnston Date: Thu, 17 Sep 2026 07:11:31 -0300 Subject: [PATCH 2/3] fix(cli): filter on log_attributes[subservice] --- .../experimental/compute/logs/SIDE_EFFECTS.md | 8 +++++--- .../compute/logs/logs.integration.test.ts | 5 +++-- apps/cli/src/shared/compute/compute-logs.sql.ts | 17 ++++++++++------- .../compute/compute-logs.sql.unit.test.ts | 8 +++++--- apps/cli/tests/helpers/compute.ts | 2 +- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/apps/cli/src/commands/experimental/compute/logs/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/compute/logs/SIDE_EFFECTS.md index 2c12f24001..babc1b631c 100644 --- a/apps/cli/src/commands/experimental/compute/logs/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/compute/logs/SIDE_EFFECTS.md @@ -49,9 +49,11 @@ Compute private-alpha allow-list — an unenrolled project answers 404. ### The query SQL in **ClickHouse dialect** against the project's unified `logs` table, filtered -on `log_attributes['worker']` and the top-level `subservice` column. It does **not** -filter the top-level `source` column: compute rows carry an empty string there, because -the Compute Logflare source is not enrolled as a category in the generic logs path. +on `log_attributes['worker']` and `log_attributes['subservice']`. It does **not** filter +the top-level `source` column, nor a top-level `subservice` one: the Compute Logflare +source is not enrolled as a category in the generic logs path, so `source` is an empty +string on a compute row and every key the writer publishes — the top-level `subservice` +included — arrives flattened into `log_attributes`. ### The window diff --git a/apps/cli/src/commands/experimental/compute/logs/logs.integration.test.ts b/apps/cli/src/commands/experimental/compute/logs/logs.integration.test.ts index 1bc9228100..8825802215 100644 --- a/apps/cli/src/commands/experimental/compute/logs/logs.integration.test.ts +++ b/apps/cli/src/commands/experimental/compute/logs/logs.integration.test.ts @@ -258,7 +258,7 @@ describe("compute logs", () => { }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), ); - it.live("filters on the subservice column, never the empty source column", () => + it.live("filters on the subservice attribute, never the empty source column", () => Effect.gen(function* () { const repo = yield* project(); const { layer, http } = setupCompute({ @@ -271,8 +271,9 @@ describe("compute logs", () => { const sql = sentQuery(http.requests[0]!).sql ?? ""; expect(sql).toContain("log_attributes['worker'] = 'api'"); - expect(sql).toContain("and subservice in ("); + expect(sql).toContain("and log_attributes['subservice'] in ("); expect(sql).not.toContain("log_attributes['source']"); + expect(sql).not.toMatch(/(? = Object.values(COMP const COMPUTE_LOG_NAME_ATTRIBUTE = "worker"; /** - * Which top-level column carries the stream name. + * Which `log_attributes` key carries the stream name. * - * The writer publishes it beside `project` rather than inside `metadata`, so it is a column - * here rather than a `log_attributes` key. `metadata.source` carries the same value for - * readers that have not moved across; this is the one to filter on. + * The writer publishes `subservice` at the top level, beside `project`, but that does not + * make it a column here: the Compute Logflare source is not enrolled as a category in the + * generic logs path, so nothing it sends is promoted, and every key — top-level ones + * included — arrives flattened into `log_attributes`. Filtering on a bare `subservice` + * fails the whole query with `Field "subservice" does not exist`. Once the source is + * enrolled it becomes a real column and this moves with it. */ -const COMPUTE_LOG_STREAM_COLUMN = "subservice"; +const COMPUTE_LOG_STREAM_ATTRIBUTE = "subservice"; /** * The server clamps a span of more than 24 hours, so the default window sits just under the @@ -148,12 +151,12 @@ export function computeLogsQuery(options: { return ( `select id, ` + `toUnixTimestamp64Milli(timestamp) as ts_ms, ` + - `${COMPUTE_LOG_STREAM_COLUMN} as stream, ` + + `log_attributes['${COMPUTE_LOG_STREAM_ATTRIBUTE}'] as stream, ` + `event_message, ` + `log_attributes ` + `from logs ` + `where log_attributes['${COMPUTE_LOG_NAME_ATTRIBUTE}'] = ${quote(options.name)} ` + - `and ${COMPUTE_LOG_STREAM_COLUMN} in (${streams}) ` + + `and log_attributes['${COMPUTE_LOG_STREAM_ATTRIBUTE}'] in (${streams}) ` + `order by timestamp desc ` + `limit ${options.tail}` ); diff --git a/apps/cli/src/shared/compute/compute-logs.sql.unit.test.ts b/apps/cli/src/shared/compute/compute-logs.sql.unit.test.ts index 1dbc9c42df..b8eb2f9a7d 100644 --- a/apps/cli/src/shared/compute/compute-logs.sql.unit.test.ts +++ b/apps/cli/src/shared/compute/compute-logs.sql.unit.test.ts @@ -12,13 +12,15 @@ import { } from "./compute-logs.sql.ts"; describe("computeLogsQuery", () => { - it("filters on the subservice column, never the source column", () => { + it("filters on the subservice attribute, never a bare column or the legacy source key", () => { const sql = computeLogsQuery({ name: "api", streams: ALL_COMPUTE_LOG_STREAMS, tail: 100 }); expect(sql).toContain("log_attributes['worker'] = 'api'"); - expect(sql).toContain("subservice as stream"); - expect(sql).toContain("and subservice in ("); + expect(sql).toContain("log_attributes['subservice'] as stream"); + expect(sql).toContain("and log_attributes['subservice'] in ("); expect(sql).not.toContain("log_attributes['source']"); + // A bare `subservice` is not a column on an unenrolled source; it fails the query. + expect(sql).not.toMatch(/(? Date: Fri, 18 Sep 2026 01:42:18 -0300 Subject: [PATCH 3/3] docs(cli): drop the unplanned subservice-column note --- apps/cli/src/shared/compute/compute-logs.sql.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/cli/src/shared/compute/compute-logs.sql.ts b/apps/cli/src/shared/compute/compute-logs.sql.ts index 062c31963d..507423c216 100644 --- a/apps/cli/src/shared/compute/compute-logs.sql.ts +++ b/apps/cli/src/shared/compute/compute-logs.sql.ts @@ -51,8 +51,7 @@ const COMPUTE_LOG_NAME_ATTRIBUTE = "worker"; * make it a column here: the Compute Logflare source is not enrolled as a category in the * generic logs path, so nothing it sends is promoted, and every key — top-level ones * included — arrives flattened into `log_attributes`. Filtering on a bare `subservice` - * fails the whole query with `Field "subservice" does not exist`. Once the source is - * enrolled it becomes a real column and this moves with it. + * fails the whole query with `Field "subservice" does not exist`. */ const COMPUTE_LOG_STREAM_ATTRIBUTE = "subservice";