Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,11 @@ Compute private-alpha allow-list — an unenrolled project answers 404.
### The query

SQL in **ClickHouse dialect** against the project's unified `logs` table, filtered
on `log_attributes['worker']` and `log_attributes['source']`. It does **not** filter
the top-level `source` column: compute rows carry an empty string there, because the
Compute Logflare source is not enrolled as a category in the generic logs path.
on `log_attributes['worker']` and `log_attributes['subservice']`. It does **not** filter
the top-level `source` column, nor a top-level `subservice` one: the Compute Logflare
source is not enrolled as a category in the generic logs path, so `source` is an empty
string on a compute row and every key the writer publishes — the top-level `subservice`
included — arrives flattened into `log_attributes`.

### The window

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ describe("compute logs", () => {
}).pipe(Effect.scoped, Effect.provide(BunServices.layer)),
);

it.live("filters on log_attributes, never the empty source column", () =>
it.live("filters on the subservice attribute, never the empty source column", () =>
Effect.gen(function* () {
const repo = yield* project();
const { layer, http } = setupCompute({
Expand All @@ -271,7 +271,9 @@ describe("compute logs", () => {

const sql = sentQuery(http.requests[0]!).sql ?? "";
expect(sql).toContain("log_attributes['worker'] = 'api'");
expect(sql).toContain("log_attributes['source'] in (");
expect(sql).toContain("and log_attributes['subservice'] in (");
expect(sql).not.toContain("log_attributes['source']");
expect(sql).not.toMatch(/(?<!\[')subservice/);
expect(sql).not.toMatch(/where source =/);
}).pipe(Effect.provide(layer));
}).pipe(Effect.scoped, Effect.provide(BunServices.layer)),
Expand Down
24 changes: 15 additions & 9 deletions apps/cli/src/shared/compute/compute-logs.sql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,16 @@ export const ALL_COMPUTE_LOG_STREAMS: ReadonlyArray<string> = Object.values(COMP
*/
const COMPUTE_LOG_NAME_ATTRIBUTE = "worker";

/** Which key carries the stream name. See {@link computeLogsQuery} for why. */
const COMPUTE_LOG_STREAM_ATTRIBUTE = "source";
/**
* Which `log_attributes` key carries the stream name.
*
* The writer publishes `subservice` at the top level, beside `project`, but that does not
* make it a column here: the Compute Logflare source is not enrolled as a category in the
* generic logs path, so nothing it sends is promoted, and every key — top-level ones
* included — arrives flattened into `log_attributes`. Filtering on a bare `subservice`
* fails the whole query with `Field "subservice" does not exist`.
*/
const COMPUTE_LOG_STREAM_ATTRIBUTE = "subservice";

/**
* The server clamps a span of more than 24 hours, so the default window sits just under the
Expand Down Expand Up @@ -127,13 +135,11 @@ function quote(value: string): string {
}

/**
* The logs query for one compute. Two things about the projection are load-bearing: the filter
* is `log_attributes`, not the `source` column, since compute rows carry an empty top-level
* `source` (the stream survives only in `log_attributes['source']`); and the `in (...)` list
* is a tenancy guard, not a convenience — with `source` empty, it's the only thing keeping a
* non-compute row with a `worker` attribute out of the result. `toUnixTimestamp64Milli` rather
* than a formatter, since ClickHouse's `%M` is the month name and bare `toString(timestamp)`
* has no zone.
* The logs query for one compute. Two things about the projection are load-bearing: the
* `in (...)` list is a tenancy guard, not a convenience — compute rows carry an empty
* top-level `source`, so the stream list is the only thing keeping a non-compute row with a
* `worker` attribute out of the result; and `toUnixTimestamp64Milli` rather than a formatter,
* since ClickHouse's `%M` is the month name and bare `toString(timestamp)` has no zone.
*/
export function computeLogsQuery(options: {
readonly name: string;
Expand Down
8 changes: 6 additions & 2 deletions apps/cli/src/shared/compute/compute-logs.sql.unit.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,15 @@ import {
} from "./compute-logs.sql.ts";

describe("computeLogsQuery", () => {
it("filters on log_attributes, never the source column", () => {
it("filters on the subservice attribute, never a bare column or the legacy source key", () => {
const sql = computeLogsQuery({ name: "api", streams: ALL_COMPUTE_LOG_STREAMS, tail: 100 });

expect(sql).toContain("log_attributes['worker'] = 'api'");
expect(sql).toContain("log_attributes['source'] in (");
expect(sql).toContain("log_attributes['subservice'] as stream");
expect(sql).toContain("and log_attributes['subservice'] in (");
expect(sql).not.toContain("log_attributes['source']");
// A bare `subservice` is not a column on an unenrolled source; it fails the query.
expect(sql).not.toMatch(/(?<!\[')subservice/);
expect(sql).not.toMatch(/(?:^|\s)where source =/);
expect(sql).not.toMatch(/(?:^|\s)and source =/);
});
Expand Down
2 changes: 1 addition & 1 deletion apps/cli/tests/helpers/compute.ts
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ export function computeLogRow(options: {
stream,
event_message: options.message ?? "compute shim: listening on :8080 (serving)",
log_attributes: {
source: stream,
subservice: stream,
worker: options.compute ?? "api",
project: COMPUTE_PROJECT_REF,
...options.attributes,
Expand Down
Loading