diff --git a/graphile/graphile-llm/src/__tests__/vector-model.test.ts b/graphile/graphile-llm/src/__tests__/vector-model.test.ts new file mode 100644 index 0000000000..34048d8605 --- /dev/null +++ b/graphile/graphile-llm/src/__tests__/vector-model.test.ts @@ -0,0 +1,175 @@ +/** + * A vector column that records the model of each row's vector is searched with + * that model, and only against rows the same model embedded. + * + * `@embeddingModel` on a vector column names the model its definition asks for; + * `@embeddingModelOf ` marks the sibling that holds, per row, the model + * that produced the vector. Two models' vectors are not comparable even at the + * same length, so one search text is embedded once per model its columns use. + */ + +import type { GraphileConfig } from 'graphile-config'; +import { ConnectionFilterPreset } from 'graphile-connection-filter'; +import { createPgvectorAdapter } from 'graphile-search/adapters/pgvector'; +import { createTsvectorAdapter } from 'graphile-search/adapters/tsvector'; +import { TsvectorCodecPlugin } from 'graphile-search/codecs/tsvector-codec'; +import { VectorCodecPlugin } from 'graphile-search/codecs/vector-codec'; +import { createUnifiedSearchPlugin } from 'graphile-search/plugin'; +import type { GraphQLResponse } from 'graphile-test'; +import { getConnections, seed } from 'graphile-test'; +import type { PgTestClient } from 'pgsql-test'; + +import { createLlmModulePlugin } from '../../src/plugins/llm-module-plugin'; +import { createLlmTextMutationPlugin } from '../../src/plugins/text-mutation-plugin'; +import { createLlmTextSearchPlugin } from '../../src/plugins/text-search-plugin'; + +type QueryFn = (query: string, variables?: Record) => Promise>; +type Notes = { allNotes: { nodes: Array<{ name: string }> } }; + +const REQUEST_MODEL = 'request-model'; +const calls: Array<{ text: string; model: string | undefined }> = []; + +// Every model answers with the same vector, so only the model stamp can tell rows apart. +const FakeEmbedderPlugin: GraphileConfig.Plugin = { + name: 'FakeEmbedderPlugin', + version: '0.0.0', + after: ['LlmModulePlugin'], + schema: { + hooks: { + build(build) { + (build as any).llmEmbedder = async (text: string, model?: string) => { + calls.push({ text, model }); + return { embedding: [1, 0, 0], promptTokens: 1, model: model ?? REQUEST_MODEL }; + }; + return build; + } + } + } +}; + +const SETUP_SQL = ` + CREATE EXTENSION IF NOT EXISTS vector; + CREATE SCHEMA llm_vector_model; + CREATE TABLE llm_vector_model.notes ( + id serial PRIMARY KEY, + name text NOT NULL, + tsv tsvector GENERATED ALWAYS AS (to_tsvector('english', name || ' note')) STORED, + embedding vector(3), + embedding_model text, + summary vector(3), + summary_model text, + legacy vector(3) + ); + COMMENT ON COLUMN llm_vector_model.notes.embedding IS E'@embeddingModel model-a'; + COMMENT ON COLUMN llm_vector_model.notes.embedding_model IS E'@embeddingModelOf embedding'; + COMMENT ON COLUMN llm_vector_model.notes.summary_model IS E'@embeddingModelOf summary'; + INSERT INTO llm_vector_model.notes (name, embedding, embedding_model, summary, summary_model, legacy) VALUES + ('current', '[1,0,0]', 'model-a', '[1,0,0]', '${REQUEST_MODEL}', '[1,0,0]'), + ('older', '[1,0,0]', 'model-b', '[1,0,0]', 'model-b', '[1,0,0]'), + ('pending', NULL, NULL, NULL, NULL, NULL); +`; + +describe('searching a vector column that records its model', () => { + let db: PgTestClient; + let pg: PgTestClient; + let query: QueryFn; + let teardown: () => Promise; + + const names = async (where: string) => { + const res = await query(`{ allNotes(where: ${where}) { nodes { name } } }`); + expect(res.errors).toBeUndefined(); + return res.data!.allNotes.nodes.map((n) => n.name).sort(); + }; + + beforeAll(async () => { + const connections = await getConnections( + { + schemas: ['llm_vector_model'], + preset: { + extends: [ConnectionFilterPreset()], + plugins: [ + TsvectorCodecPlugin, + VectorCodecPlugin, + createUnifiedSearchPlugin({ + adapters: [createTsvectorAdapter(), createPgvectorAdapter()], + enableSearchScore: true, + enableUnifiedSearch: true + }), + createLlmModulePlugin(), + FakeEmbedderPlugin, + createLlmTextSearchPlugin(), + createLlmTextMutationPlugin() + ] + }, + useRoot: true, + authRole: 'postgres' + }, + [seed.fn(async (ctx) => { await ctx.pg.query(SETUP_SQL); })] + ); + ({ db, pg, teardown } = connections); + query = connections.query as QueryFn; + }); + + afterAll(async () => { await teardown(); }); + + beforeEach(async () => { + calls.length = 0; + await pg.beforeEach(); + await db.beforeEach(); + }); + + afterEach(async () => { + await db.afterEach(); + await pg.afterEach(); + }); + + it('embeds the text with the model the column names and skips rows of another model', async () => { + expect(await names('{ vectorEmbedding: { text: "q" } }')).toEqual(['current']); + expect(calls).toEqual([{ text: 'q', model: 'model-a' }]); + }); + + it('uses the request model for a column that names none, and still skips other models', async () => { + expect(await names('{ vectorSummary: { text: "q" } }')).toEqual(['current']); + expect(calls).toEqual([{ text: 'q', model: undefined }]); + }); + + it('compares every row of a column that records no model', async () => { + expect(await names('{ vectorLegacy: { text: "q" } }')).toEqual(['current', 'older']); + }); + + it('filters a client-sent vector by the model the client says produced it', async () => { + expect(await names('{ vectorEmbedding: { vector: [1, 0, 0], model: "model-b" } }')).toEqual(['older']); + expect(await names('{ vectorEmbedding: { vector: [1, 0, 0] } }')).toEqual(['current', 'older']); + expect(calls).toEqual([]); + }); + + it('labels an embedded text with the model that embedded it, whatever label the client sent', async () => { + expect(await names('{ vectorEmbedding: { text: "q", model: "model-b" } }')).toEqual(['current']); + }); + + it('stamps the model beside a vector it embeds on a write, so the row is found by a search', async () => { + const res = await query<{ createNote: { note: { name: string; embeddingModel: string; summaryModel: string } } }>(`mutation { + createNote(input: { note: { name: "written", embeddingText: "q", summaryText: "q" } }) { + note { name embeddingModel summaryModel } + } + }`); + expect(res.errors).toBeUndefined(); + expect(res.data!.createNote.note).toEqual({ name: 'written', embeddingModel: 'model-a', summaryModel: REQUEST_MODEL }); + expect(calls.map((c) => c.model).sort()).toEqual(['model-a', undefined]); + expect(await names('{ vectorEmbedding: { text: "q" } }')).toEqual(['current', 'written']); + }); + + it('embeds unifiedSearch once per model and scores each column with its own', async () => { + const res = await query<{ allNotes: { nodes: Array> } }>(`{ + allNotes(where: { unifiedSearch: "note" }) { + nodes { name embeddingVectorDistance summaryVectorDistance legacyVectorDistance } + } + }`); + expect(res.errors).toBeUndefined(); + const byName = Object.fromEntries(res.data!.allNotes.nodes.map((n) => [n.name, n])); + expect(calls.map((c) => c.model).sort()).toEqual(['model-a', undefined]); + expect(byName.current).toMatchObject({ embeddingVectorDistance: 0, summaryVectorDistance: 0, legacyVectorDistance: 0 }); + // Another model produced these two vectors: no distance, not a wrong one. + expect(byName.older).toMatchObject({ embeddingVectorDistance: null, summaryVectorDistance: null, legacyVectorDistance: 0 }); + }); +}); diff --git a/graphile/graphile-llm/src/embed-inputs.ts b/graphile/graphile-llm/src/embed-inputs.ts index 1496d57e05..213d65c7cf 100644 --- a/graphile/graphile-llm/src/embed-inputs.ts +++ b/graphile/graphile-llm/src/embed-inputs.ts @@ -27,11 +27,47 @@ export type EmbedRule = /** `unifiedSearch: String`: becomes `{ __text, __vector }` for graphile-search. */ | { kind: 'unified'; onQuotaExceeded: QuotaPolicy } /** `{col}Text` on a create input or patch. `required` means the column is NOT NULL. */ - | { kind: 'companion'; into: string; required: boolean }; + | { + kind: 'companion'; + into: string; + required: boolean; + /** The model the column's vectors are declared to come from. */ + model?: string; + /** The input field of the column that records each vector's model. */ + modelField?: string; + }; export const EMBED_EXT = 'graphileLlmEmbed'; -export type VectorEmbedder = (text: string) => Promise; +/** + * Set by graphile-search on a filter field that searches vector columns: which + * columns, and the model each one's own definition names (`@embeddingModel`). + * Vectors of two models are not comparable, so a search text is embedded once + * per model its columns use, never once for all of them. + */ +export const VECTOR_COLUMNS_EXT = 'graphileSearchVectorColumns'; +export interface VectorColumnRef { + attributeName: string; + model?: string; +} + +/** A query vector and the model that produced it. */ +export interface Embedded { + vector: number[]; + model?: string; +} + +export interface VectorEmbedder { + (text: string): Promise; + /** Embed with `model` when given, else the request's; reports the model used. */ + withModel?: (text: string, model?: string) => Promise; +} + +const embedWith = async (embed: VectorEmbedder, text: string, model?: string): Promise => { + if (embed.withModel) return embed.withModel(text, model); + const vector = await embed(text); + return vector ? { vector } : null; +}; export function embedField(rule: EmbedRule, spec: T): T { return { ...spec, extensions: { ...(spec as { extensions?: object }).extensions, [EMBED_EXT]: rule } }; @@ -40,11 +76,12 @@ export function embedField(rule: EmbedRule, spec: T): T { /** `build.llmEmbedder` resolves to `{ embedding }`; an empty or missing vector is a failed embed. */ export function toVectorEmbedder(raw: unknown): VectorEmbedder | null { if (typeof raw !== 'function') return null; - return async (text) => { - const result = await raw(text); + const withModel = async (text: string, model?: string): Promise => { + const result = await raw(text, model); const vector = result?.embedding; - return isNonEmptyVector(vector) ? vector : null; + return isNonEmptyVector(vector) ? { vector, model: result.model ?? model } : null; }; + return Object.assign(async (text: string) => (await withModel(text))?.vector ?? null, { withModel }); } function isNonEmptyVector(value: unknown): boolean { @@ -58,16 +95,29 @@ function isNonEmptyVector(value: unknown): boolean { function forRequest(embedder: VectorEmbedder | null, graphqlContext: unknown): VectorEmbedder | null { if (!embedder) return null; let scope: Promise | undefined; - return async (text) => { - scope ??= resolveRequestScope(graphqlContext); - const resolved = await scope; - const startTime = Date.now(); - const vector = await llmConfigStore.run(resolved, () => embedder(text)); - if (vector) { - console.log(`[graphile-llm] embed: dims=${vector.length}, latency=${Date.now() - startTime}ms`); + // One search may name the same text for several columns of one model. + const memo = new Map>(); + const withModel = (text: string, model?: string): Promise => { + const memoKey = JSON.stringify([model ?? null, text]); + let pending = memo.get(memoKey); + if (!pending) { + pending = (async () => { + scope ??= resolveRequestScope(graphqlContext); + const resolved = await scope; + const startTime = Date.now(); + const embedded = await llmConfigStore.run(resolved, () => embedWith(embedder, text, model)); + if (embedded) { + console.log( + `[graphile-llm] embed: model=${embedded.model ?? 'default'}, dims=${embedded.vector.length}, latency=${Date.now() - startTime}ms` + ); + } + return embedded; + })(); + memo.set(memoKey, pending); } - return vector; + return pending; }; + return Object.assign(async (text: string) => (await withModel(text))?.vector ?? null, { withModel }); } export function createEmbedIndex(graphql: any) { @@ -132,7 +182,8 @@ export function createEmbedIndex(graphql: any) { typeName: string, value: Record, next: Record, - embed: VectorEmbedder | null + embed: VectorEmbedder | null, + columns: VectorColumnRef[] | undefined ): Promise { const text = value[fieldName]; @@ -159,9 +210,13 @@ export function createEmbedIndex(graphql: any) { `send \`${rule.into}\` instead.` ); } - const vector = await embed(text); - if (vector) { - next[rule.into] = vector; + // The filter field this input sits under searches one column. + const embedded = await embedWith(embed, text, columns?.[0]?.model); + if (embedded) { + next[rule.into] = embedded.vector; + // The label is the model that made this vector, whatever the client sent. + if (embedded.model) next.model = embedded.model; + else delete next.model; } else if (rule.onQuotaExceeded === 'throw') { throw new Error( `${typeName}: embedding failed (provider unavailable). ` + @@ -173,9 +228,31 @@ export function createEmbedIndex(graphql: any) { case 'unified': { if (typeof text !== 'string' || text.trim().length === 0 || !embed) return; - const vector = await embed(text); - if (vector) { - next[fieldName] = { __text: text, __vector: vector }; + // One vector per model the table's vector columns use; `__vector` stays + // the request model's, for a column nothing is known about. + const models = [...new Set(columns?.length ? columns.map((c) => c.model) : [undefined])]; + const byModel = new Map( + await Promise.all(models.map(async (m) => [m, await embedWith(embed, text, m)] as const)) + ); + if (rule.onQuotaExceeded === 'throw' && models.some((m) => !byModel.get(m))) { + throw new Error( + `${fieldName}: embedding failed (provider unavailable). ` + + 'onQuotaExceeded is set to \'throw\'; set it to \'degrade\' for text-only fallback.' + ); + } + const base = byModel.get(undefined); + const perColumn: Record = {}; + for (const column of columns ?? []) { + const embedded = byModel.get(column.model); + if (embedded) perColumn[column.attributeName] = embedded; + } + if (base || Object.keys(perColumn).length > 0) { + next[fieldName] = { + __text: text, + __vector: base?.vector ?? null, + __model: base?.model, + __vectors: perColumn + }; } else if (rule.onQuotaExceeded === 'throw') { throw new Error( `${fieldName}: embedding failed (provider unavailable). ` + @@ -209,25 +286,33 @@ export function createEmbedIndex(graphql: any) { 'Set defaultEmbedder in GraphileLlmPreset options or EMBEDDER_PROVIDER env var.' ); } - const vector = await embed(text); - if (!vector) { + const embedded = await embedWith(embed, text, rule.model); + if (!embedded) { throw new Error( `EMBED_FAILED: Cannot embed ${fieldName} — the embedding provider returned no vector.` ); } - next[rule.into] = vector; + next[rule.into] = embedded.vector; + // The vector and the model that made it are written together: a search + // compares only rows whose stamp is the model of its query vector. + if (rule.modelField && embedded.model) next[rule.modelField] = embedded.model; return; } } } /** Returns `value` itself when its type cannot reach a tagged field. */ - async function embedValue(type: any, value: any, embed: VectorEmbedder | null): Promise { + async function embedValue( + type: any, + value: any, + embed: VectorEmbedder | null, + columns?: VectorColumnRef[] + ): Promise { if (value == null) return value; const inner = isNonNullType(type) ? type.ofType : type; if (isListType(inner)) { if (!Array.isArray(value) || !reaches(inner.ofType)) return value; - return Promise.all(value.map((item: any) => embedValue(inner.ofType, item, embed))); + return Promise.all(value.map((item: any) => embedValue(inner.ofType, item, embed, columns))); } if (!isInputObjectType(inner) || !reaches(inner)) return value; @@ -235,10 +320,11 @@ export function createEmbedIndex(graphql: any) { const jobs: Promise[] = []; for (const [name, field] of Object.entries(inner.getFields())) { const rule = field.extensions?.[EMBED_EXT] as EmbedRule | undefined; + const own = field.extensions?.[VECTOR_COLUMNS_EXT] as VectorColumnRef[] | undefined; if (rule) { - jobs.push(applyRule(rule, name, inner.name, value, next, embed)); + jobs.push(applyRule(rule, name, inner.name, value, next, embed, own ?? columns)); } else if (value[name] != null && reaches(field.type)) { - jobs.push(embedValue(field.type, value[name], embed).then((embedded) => { next[name] = embedded; })); + jobs.push(embedValue(field.type, value[name], embed, own).then((embedded) => { next[name] = embedded; })); } } await Promise.all(jobs); diff --git a/graphile/graphile-llm/src/embedder.ts b/graphile/graphile-llm/src/embedder.ts index 097ecf272f..403dba9f21 100644 --- a/graphile/graphile-llm/src/embedder.ts +++ b/graphile/graphile-llm/src/embedder.ts @@ -123,8 +123,10 @@ export function createGatewayEmbedder( ): EmbedderFunction { const root = normalizeGatewayUrl(gatewayUrl); - return async (text: string): Promise => { - const model = llmConfigStore.getStore()?.embeddingModel ?? defaultModel; + // A caller that names a model (a vector column that records its own) gets + // that one; otherwise the database's, then the deployment's. + return async (text: string, named?: string): Promise => { + const model = named ?? llmConfigStore.getStore()?.embeddingModel ?? defaultModel; const response = await fetch(`${root}/v1/embeddings`, { method: 'POST', headers: gatewayHeaders(provider), @@ -137,6 +139,7 @@ export function createGatewayEmbedder( const payload = (await response.json()) as GatewayEmbeddingsResponse; return { embedding: payload.data?.[0]?.embedding ?? [], + model, promptTokens: payload.usage?.prompt_tokens ?? 0 }; }; diff --git a/graphile/graphile-llm/src/plugins/text-mutation-plugin.ts b/graphile/graphile-llm/src/plugins/text-mutation-plugin.ts index d5847645a6..35193eff23 100644 --- a/graphile/graphile-llm/src/plugins/text-mutation-plugin.ts +++ b/graphile/graphile-llm/src/plugins/text-mutation-plugin.ts @@ -152,11 +152,26 @@ export function createLlmTextMutationPlugin(): GraphileConfig.Plugin { }; } + const attributes = pgCodec.attributes as Record; + const model = attributes[columnName].extensions?.tags?.embeddingModel; + const modelColumn = Object.keys(attributes).find( + (name) => attributes[name].extensions?.tags?.embeddingModelOf === columnName + ); + const modelField = modelColumn + ? build.inflection.attribute({ codec: pgCodec, attributeName: modelColumn }) + : undefined; + newFields = build.extend( newFields, { [textFieldName]: embedField( - { kind: 'companion', into: fieldName, required }, + { + kind: 'companion', + into: fieldName, + required, + ...(typeof model === 'string' && model && { model }), + ...(modelField && modelField in newFields && { modelField }) + }, { type: GraphQLString, description: diff --git a/graphile/graphile-llm/src/types.ts b/graphile/graphile-llm/src/types.ts index 795c9256de..463773ff5a 100644 --- a/graphile/graphile-llm/src/types.ts +++ b/graphile/graphile-llm/src/types.ts @@ -14,12 +14,14 @@ export interface EmbeddingResult { embedding: number[]; /** Number of prompt tokens consumed (from provider; 0 if unavailable) */ promptTokens: number; + /** The model the gateway was asked for */ + model?: string; } /** * A function that converts text into a vector embedding with token usage. */ -export type EmbedderFunction = (text: string) => Promise; +export type EmbedderFunction = (text: string, model?: string) => Promise; /** * Configuration for an embedding provider. diff --git a/graphile/graphile-presigned-url-plugin/__tests__/custom-key.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/custom-key.test.ts index 0747733568..f1bbbc56ed 100644 --- a/graphile/graphile-presigned-url-plugin/__tests__/custom-key.test.ts +++ b/graphile/graphile-presigned-url-plugin/__tests__/custom-key.test.ts @@ -69,4 +69,14 @@ describe('validateCustomKey', () => { ])('rejects %s', (key) => { expect(validateCustomKey(key)).toMatch(/^INVALID_KEY:/); }); + + // A file's rendered versions live under `.versions/` and are served to + // whoever can read that file, so no file of its own may be stored there. + it.each(['photo.png.versions/secret.webp', 'a/b.versions/c/d.txt'])('rejects %s: the versions prefix is reserved', (key) => { + expect(validateCustomKey(key)).toMatch(/reserved/); + }); + + it('accepts a key that only mentions versions', () => { + expect(validateCustomKey('docs/versions/v1.versions.txt')).toBeNull(); + }); }); diff --git a/graphile/graphile-presigned-url-plugin/src/custom-key.ts b/graphile/graphile-presigned-url-plugin/src/custom-key.ts index 795af69dc9..ec3bbfe1f7 100644 --- a/graphile/graphile-presigned-url-plugin/src/custom-key.ts +++ b/graphile/graphile-presigned-url-plugin/src/custom-key.ts @@ -38,6 +38,11 @@ export function validateCustomKey(key: string): string | null { if (key.includes('\0')) { return 'INVALID_KEY: null bytes not allowed'; } + // `.versions/` is where a file's rendered versions are stored, and they + // are served to whoever can read that file: no file of its own may live there. + if (key.includes('.versions/')) { + return 'INVALID_KEY: ".versions/" is reserved for a file\'s rendered versions'; + } if (!CUSTOM_KEY_REGEX.test(key)) { return 'INVALID_KEY: must start with alphanumeric or underscore and contain only alphanumeric, dots, hyphens, underscores, slashes, and ! $ @'; } diff --git a/graphile/graphile-presigned-url-plugin/src/file-ref-urls.ts b/graphile/graphile-presigned-url-plugin/src/file-ref-urls.ts new file mode 100644 index 0000000000..6b2dbd4c05 --- /dev/null +++ b/graphile/graphile-presigned-url-plugin/src/file-ref-urls.ts @@ -0,0 +1,364 @@ +/** + * URLs for a file-reference column on a business table. + * + * A file-reference column is a column of one of the declared file domains + * (`FILE_REF_DOMAINS`: `image`, `upload`), or an array of one. It holds a + * document that projects a files row — `id`, `key`, `bucket_id` — and, when a + * lane has rendered versions beside the source, lists them under `versions` + * (`.versions/-.`). The column reaches GraphQL as JSON, + * so without these fields a client has no URL to fetch the file with. + * + * For each such column `photo` on a table type this adds + * photoUrl(width: Int): String the narrowest version covering `width`, else the source + * photoVersions: [FileVersion!] each rendered version, with its url + * and, for an array column, the same as lists with one entry per element. A + * document with no versions (an `upload` column's, say) gives its source URL + * and an empty list. + * + * The document is written by the client, `versions` included, so nothing in it + * is authority for a URL: + * 1. only a document with an `id` resolves, and the files row is read by that + * id on the reader's own request lane — the files table's RLS decides, + * exactly as it does for `downloadUrl`; + * 2. the source key, bucket, filename and publicness come from that row, + * never from the document; + * 3. a version is signed only when its key is a direct child of that row's + * own `.versions/` prefix. A key the document merely names is never + * signed. + * + * Files rows are read once per request, not once per document: every document + * a request resolves in one tick shares one lookup per storage module, and a + * row or bucket coordinate read once is kept for the rest of the request. + */ + +import 'graphile-build'; + +import { Logger } from '@pgpmjs/logger'; +import { context as grafastContext, lambda, object } from 'grafast'; +import type { GraphileConfig } from 'graphile-config'; +import { withRequestPgClient } from 'graphile-plugin-utils'; +import { FILE_REF_DOMAINS } from 'graphile-storage-registry'; + +import { isLiveFileRow } from './file-lifecycle'; +import { resolveS3ForDatabase } from './physical-bucket'; +import { generatePresignedGetUrl } from './s3-signer'; +import { loadAllStorageModules, readBucketPhysicalName } from './storage-module-cache'; +import type { PresignedUrlPluginOptions, StorageModuleConfig } from './types'; + +const log = new Logger('graphile-presigned-url:file-ref-urls'); + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +export interface FileVersion { + name: string | null; + key: string; + width: number | null; + height: number | null; + format: string | null; + size: number | null; + url: string; +} + +export interface ResolvedFile { + /** URL of the source object. */ + url: string; + /** The versions that belong to the source, narrowest first. */ + versions: FileVersion[]; +} + +interface FilesRow { + id: string; + key: string; + bucket_id: string; + is_public: boolean | null; + status: string | null; + filename: string | null; +} + +/** A files row the reader can fetch, with where its bytes live. */ +interface FetchableFile { + config: StorageModuleConfig; + row: FilesRow; + physicalName: string; +} + +type PgClientLike = { + query(opts: { text: string; values?: unknown[] }): Promise<{ rows: any[] }>; +}; + +/** + * Whether `variantKey` is an object a lane wrote beside the file at + * `sourceKey`: a direct child of `.versions/`. + */ +export function isVersionKeyOf(sourceKey: string, variantKey: unknown): variantKey is string { + if (typeof variantKey !== 'string' || !sourceKey) return false; + const prefix = `${sourceKey}.versions/`; + if (!variantKey.startsWith(prefix)) return false; + const name = variantKey.slice(prefix.length); + return name.length > 0 && !name.includes('/') && name !== '.' && name !== '..'; +} + +const numberOrNull = (v: unknown): number | null => (typeof v === 'number' && Number.isFinite(v) ? v : null); +const stringOrNull = (v: unknown): string | null => (typeof v === 'string' ? v : null); + +/** + * The version entries of a document that belong to the file at `sourceKey`, + * narrowest first. Entries naming any other key are dropped. + */ +export function ownVersions(sourceKey: string, versions: unknown): Array> { + if (!Array.isArray(versions)) return []; + return versions + .filter((v): v is Record => !!v && typeof v === 'object' && isVersionKeyOf(sourceKey, (v as any).key)) + .map((v) => ({ + name: stringOrNull(v.name), + key: v.key as string, + width: numberOrNull(v.width), + height: numberOrNull(v.height), + format: stringOrNull(v.format), + size: numberOrNull(v.size), + })) + .sort((a, b) => (a.width ?? Infinity) - (b.width ?? Infinity)); +} + +/** The URL for a requested width: the narrowest version at least that wide, else the source. */ +export function pickUrl(file: ResolvedFile, width: number | null | undefined): string { + if (width == null) return file.url; + return file.versions.find((v) => v.width !== null && v.width >= width)?.url ?? file.url; +} + +/** + * The files rows one request reads. Ids asked for in the same tick are read + * together: one lookup per storage module for all of them, one coordinate read + * per bucket. What was read stays for the request. + */ +class RequestFiles { + private readonly files = new Map>(); + private waiting: Array<{ id: string; settle: (hit: FetchableFile | null) => void; fail: (e: unknown) => void }> = []; + private readonly buckets = new Map(); + + constructor(private readonly withPgClient: any, private readonly pgSettings: any) {} + + get(id: string): Promise { + const key = id.toLowerCase(); + let hit = this.files.get(key); + if (!hit) { + hit = new Promise((settle, fail) => { + if (this.waiting.length === 0) process.nextTick(() => this.flush()); + this.waiting.push({ id: key, settle, fail }); + }); + this.files.set(key, hit); + } + return hit; + } + + private async flush(): Promise { + const batch = this.waiting; + this.waiting = []; + try { + const found = await withRequestPgClient(this.withPgClient, this.pgSettings, (pgClient: PgClientLike) => + this.read(pgClient, batch.map((w) => w.id)), + ); + for (const w of batch) w.settle(found.get(w.id) ?? null); + } catch (e) { + for (const w of batch) w.fail(e); + } + } + + private async read(pgClient: PgClientLike, ids: string[]): Promise> { + const found = new Map(); + const db = await pgClient.query({ text: `SELECT jwt_private.current_database_id() AS id` }); + const databaseId = db.rows[0]?.id as string | undefined; + if (!databaseId) return found; + + let missing = ids; + for (const config of await loadAllStorageModules(pgClient, databaseId)) { + if (missing.length === 0) break; + const { rows } = await pgClient.query({ + text: `SELECT id, key, bucket_id, is_public, status::text AS status, filename FROM ${config.filesQualifiedName} WHERE id = ANY($1::uuid[])`, + values: [missing], + }); + for (const row of rows as FilesRow[]) { + if (!row.key || !isLiveFileRow(row)) continue; + const bucketKey = `${config.id}:${row.bucket_id}`; + if (!this.buckets.has(bucketKey)) { + this.buckets.set(bucketKey, await readBucketPhysicalName(pgClient, config, row.bucket_id)); + } + const physicalName = this.buckets.get(bucketKey); + // No physical bucket was ever provisioned: no object can exist. + if (physicalName != null) found.set(String(row.id).toLowerCase(), { config, row, physicalName }); + } + const seen = new Set((rows as FilesRow[]).map((row) => String(row.id).toLowerCase())); + missing = missing.filter((id) => !seen.has(id)); + } + return found; + } +} + +const requests = new WeakMap(); + +function requestFiles(withPgClient: any, pgSettings: object): RequestFiles { + let files = requests.get(pgSettings); + if (!files) { + files = new RequestFiles(withPgClient, pgSettings); + requests.set(pgSettings, files); + } + return files; +} + +/** + * Resolve one file document to URLs, or null when it names no file the reader + * can fetch. + */ +export async function resolveFileDocument( + options: PresignedUrlPluginOptions, + withPgClient: any, + pgSettings: any, + doc: unknown, +): Promise { + if (!doc || typeof doc !== 'object' || Array.isArray(doc)) return null; + const id = (doc as Record).id; + if (typeof id !== 'string' || !UUID.test(id)) return null; + if (!withPgClient || !pgSettings) return null; + + const found = await requestFiles(withPgClient, pgSettings).get(id); + if (!found) return null; + + const { config, row, physicalName } = found; + const s3 = resolveS3ForDatabase(options, config, physicalName); + const urlFor = (key: string, filename?: string): Promise | string => + row.is_public && s3.publicUrlPrefix + ? `${s3.publicUrlPrefix}/${s3.bucket}/${key}` + : generatePresignedGetUrl(s3, key, config.downloadUrlExpirySeconds, filename); + + const versions = await Promise.all( + ownVersions(row.key, (doc as Record).versions).map(async (v) => ({ ...v, url: await urlFor(v.key) })), + ); + // As `downloadUrl` does, the source is served under the name it was uploaded with. + return { url: await urlFor(row.key, row.filename || undefined), versions }; +} + +/** `'scalar'` for a column of a file domain, `'list'` for an array of one, else null. */ +export function fileRefColumnKind(codec: any, domains: readonly string[]): 'scalar' | 'list' | null { + const isFileDomain = (c: any): boolean => + c?.domainOfCodec?.name === 'jsonb' && domains.includes(c.extensions?.pg?.name); + if (isFileDomain(codec)) return 'scalar'; + if (codec?.arrayOfCodec && isFileDomain(codec.arrayOfCodec)) return 'list'; + return null; +} + +const PLUGIN = 'PresignedUrlFileRefUrlsPlugin'; + +export function createFileRefUrlsPlugin(options: PresignedUrlPluginOptions): GraphileConfig.Plugin { + const domains = options.fileRefDomains ?? FILE_REF_DOMAINS; + + return { + name: PLUGIN, + version: '0.2.0', + description: 'Adds Url(width) and Versions beside each file-reference column', + + schema: { + hooks: { + init(_, build) { + const { + graphql: { GraphQLInt, GraphQLNonNull, GraphQLString }, + } = build; + build.registerObjectType( + 'FileVersion', + {}, + () => ({ + description: 'One version rendered beside a file, with a URL to fetch it.', + fields: { + name: { type: GraphQLString }, + key: { type: new GraphQLNonNull(GraphQLString) }, + width: { type: GraphQLInt }, + height: { type: GraphQLInt }, + format: { type: GraphQLString }, + size: { type: GraphQLInt }, + url: { type: new GraphQLNonNull(GraphQLString) }, + }, + }), + `${PLUGIN} registering FileVersion`, + ); + return _; + }, + + GraphQLObjectType_fields(fields, build, context) { + const { + scope: { pgCodec, isPgClassType }, + } = context as any; + if (!isPgClassType || !pgCodec?.attributes) return fields; + + const { + inflection, + graphql: { GraphQLInt, GraphQLList, GraphQLNonNull, GraphQLString }, + } = build; + const FileVersionType = build.getTypeByName('FileVersion') as any; + if (!FileVersionType) return fields; + + const added: Record = {}; + for (const [attributeName, attribute] of Object.entries(pgCodec.attributes as Record)) { + const kind = fileRefColumnKind(attribute.codec, domains); + if (!kind) continue; + // A column the API does not select stays unread here too. + if (!(build as any).behavior.pgCodecAttributeMatches([pgCodec, attributeName], 'attribute:select')) continue; + const base = (inflection as any).attribute({ attributeName, codec: pgCodec }); + const urlField = `${base}Url`; + const versionsField = `${base}Versions`; + if (fields[urlField] || fields[versionsField]) { + log.warn(`${pgCodec.name}.${attributeName}: ${urlField}/${versionsField} already exist; not adding file URL fields`); + continue; + } + + const resolved = ($parent: any) => { + const $combined = object({ + value: $parent.get(attributeName), + withPgClient: (grafastContext() as any).get('withPgClient'), + pgSettings: (grafastContext() as any).get('pgSettings'), + }); + return lambda($combined, async ({ value, withPgClient, pgSettings }: any) => { + if (value == null) return null; + if (kind === 'scalar') return resolveFileDocument(options, withPgClient, pgSettings, value); + return Promise.all( + (value as unknown[]).map((doc) => resolveFileDocument(options, withPgClient, pgSettings, doc)), + ); + }); + }; + const each = (file: any, fn: (one: ResolvedFile) => T) => + file == null ? null : kind === 'scalar' ? fn(file) : (file as Array).map((one) => (one ? fn(one) : null)); + + const versionList = new GraphQLList(new GraphQLNonNull(FileVersionType)); + added[urlField] = context.fieldWithHooks({ fieldName: urlField } as any, { + description: + `URL to fetch \`${base}\`: the narrowest rendered version at least \`width\` pixels wide, ` + + 'else the source file. Null when the column names no file the reader can fetch.', + type: kind === 'scalar' ? GraphQLString : new GraphQLList(GraphQLString), + args: { + width: { + type: GraphQLInt, + description: 'The width the file will be shown at, in device pixels. Without a version that wide, the source.', + }, + }, + plan($parent: any, fieldArgs: any) { + const $both = object({ file: resolved($parent), width: fieldArgs.getRaw('width') }); + return lambda($both, ({ file, width }: any) => each(file, (one) => pickUrl(one, width))); + }, + }); + added[versionsField] = context.fieldWithHooks({ fieldName: versionsField } as any, { + description: `The versions rendered beside \`${base}\`, narrowest first, each with a URL. Empty when there are none.`, + type: kind === 'scalar' ? versionList : new GraphQLList(versionList), + plan($parent: any) { + return lambda(resolved($parent), (file: any) => each(file, (one) => one.versions)); + }, + }); + } + + return Object.keys(added).length + ? build.extend(fields, added as any, `${PLUGIN} adding file URL fields`) + : fields; + }, + }, + }, + }; +} + +export default createFileRefUrlsPlugin; diff --git a/graphile/graphile-presigned-url-plugin/src/index.ts b/graphile/graphile-presigned-url-plugin/src/index.ts index 1dcb5c4890..1fd5d13b7b 100644 --- a/graphile/graphile-presigned-url-plugin/src/index.ts +++ b/graphile/graphile-presigned-url-plugin/src/index.ts @@ -27,6 +27,14 @@ export type { ResolvedBucketCoordinate } from './default-bucket'; export { resolveDefaultBucket } from './default-bucket'; export { createDownloadUrlPlugin } from './download-url-field'; export { buildFileProjection, type FileProjection } from './file-projection'; +export { + createFileRefUrlsPlugin, + type FileVersion, + isVersionKeyOf, + ownVersions, + pickUrl, + type ResolvedFile, +} from './file-ref-urls'; export { assertBucketReconciled, resolveS3ForDatabase, diff --git a/graphile/graphile-presigned-url-plugin/src/preset.ts b/graphile/graphile-presigned-url-plugin/src/preset.ts index d1ff7103b7..f37497f3b8 100644 --- a/graphile/graphile-presigned-url-plugin/src/preset.ts +++ b/graphile/graphile-presigned-url-plugin/src/preset.ts @@ -10,6 +10,7 @@ import type { GraphileConfig } from 'graphile-config'; import { createBulkLimitPlugin } from './bulk-limit-field'; import { createDownloadUrlPlugin } from './download-url-field'; +import { createFileRefUrlsPlugin } from './file-ref-urls'; import { createPresignedUrlPlugin } from './plugin'; import type { PresignedUrlPluginOptions } from './types'; @@ -38,6 +39,7 @@ export function PresignedUrlPreset( plugins: [ createPresignedUrlPlugin(options), createDownloadUrlPlugin(options), + createFileRefUrlsPlugin(options), createBulkLimitPlugin(), ], }; diff --git a/graphile/graphile-presigned-url-plugin/src/types.ts b/graphile/graphile-presigned-url-plugin/src/types.ts index 0cf5b19b87..8375d3d289 100644 --- a/graphile/graphile-presigned-url-plugin/src/types.ts +++ b/graphile/graphile-presigned-url-plugin/src/types.ts @@ -216,4 +216,10 @@ export interface StorageCredentials { */ export interface PresignedUrlPluginOptions { credentials: StorageCredentials | (() => StorageCredentials); + /** + * The jsonb domains whose columns hold a file document and get + * `Url` / `Versions`. Defaults to `FILE_REF_DOMAINS` of + * graphile-storage-registry (`image`, `upload`). + */ + fileRefDomains?: readonly string[]; } diff --git a/graphile/graphile-search/src/adapters/pgvector.ts b/graphile/graphile-search/src/adapters/pgvector.ts index 733cdb7851..44e0b4696e 100644 --- a/graphile/graphile-search/src/adapters/pgvector.ts +++ b/graphile/graphile-search/src/adapters/pgvector.ts @@ -93,11 +93,24 @@ export function createPgvectorAdapter( )) { if (isVectorCodec(attribute.codec)) { const dimensions: number | undefined = attribute.extensions?.vectorDimensions; + // `@embeddingModel` on the vector column: the model its definition + // names. `@embeddingModelOf ` on a sibling: the column that + // records, per row, the model that produced the vector. + const named = attribute.extensions?.tags?.embeddingModel; + const model = typeof named === 'string' && named ? named : undefined; + const modelColumn = Object.entries(codec.attributes as Record).find( + ([, other]) => other.extensions?.tags?.embeddingModelOf === attributeName + )?.[0]; columns.push({ attributeName, adapterData: - chunksInfo || dimensions - ? { ...(chunksInfo && { chunksInfo }), ...(dimensions && { dimensions }) } + chunksInfo || dimensions || model || modelColumn + ? { + ...(chunksInfo && { chunksInfo }), + ...(dimensions && { dimensions }), + ...(model && { model }), + ...(modelColumn && { modelColumn }), + } : undefined, }); } @@ -158,6 +171,13 @@ export function createPgvectorAdapter( ), description: 'Query vector for similarity search.', }, + model: { + type: build.graphql.GraphQLString, + description: + 'The model that produced `vector`. On a column that records the model of each ' + + 'row\'s vector, only rows embedded with this model are compared; vectors of ' + + 'two models are not comparable. Filled in server-side when `text` is embedded.', + }, metric: { type: VectorMetricEnum, description: `Similarity metric to use (default: ${defaultMetric}).`, @@ -196,7 +216,7 @@ export function createPgvectorAdapter( ): FilterApplyResult | null { if (filterValue == null) return null; - const { vector, metric, distance, includeChunks } = filterValue; + const { vector, metric, distance, includeChunks, model } = filterValue; if (!vector || !Array.isArray(vector) || vector.length === 0) return null; const resolvedMetric = metric || defaultMetric; @@ -204,9 +224,22 @@ export function createPgvectorAdapter( const vectorExpr = sql`${sql.value(vectorString)}::vector`; // Check if this column has chunks info and chunk querying is requested - const adapterData = column.adapterData as { chunksInfo?: ChunksInfo } | undefined; + const adapterData = column.adapterData as { chunksInfo?: ChunksInfo; modelColumn?: string } | undefined; const chunksInfo = adapterData?.chunksInfo; + // A row whose vector another model produced is not a worse match, it is + // not a match at all: its distance to this query vector means nothing. + const sameModel: SQL | null = + adapterData?.modelColumn && typeof model === 'string' && model + ? sql`${alias}.${sql.identifier(adapterData.modelColumn)} = ${sql.value(model)}` + : null; + // Narrows a clause that is there; where there is none, `scorableClause` + // carries the model so a search that only ranks (unifiedSearch) is not + // turned into one that filters. + const both = (a: SQL | null, b: SQL | null): SQL | null => (a && b ? sql`(${a}) AND (${b})` : b); + const scored = (distance: SQL): SQL => + sameModel ? sql`CASE WHEN ${sameModel} THEN ${distance} END` : distance; + if (chunksInfo && (includeChunks !== false)) { // Chunk-aware query: find the closest chunk for each parent row // Uses a lateral subquery to get the minimum distance across all chunks @@ -230,7 +263,11 @@ export function createPgvectorAdapter( // Direct parent distance (NULL when the parent has no embedding) const parentColumnExpr = sql`${alias}.${sql.identifier(column.attributeName)}`; - const parentDistanceExpr = buildDistanceExpr(sql, parentColumnExpr, vectorExpr, resolvedMetric); + // The model stamp is the parent vector's; a chunk's vector is not the parent's. + const parentDistanceExpr = scored(buildDistanceExpr(sql, parentColumnExpr, vectorExpr, resolvedMetric)); + const parentScorable = sameModel + ? sql`(${sameModel} AND ${parentColumnExpr} IS NOT NULL)` + : sql`${parentColumnExpr} IS NOT NULL`; // LEAST skips NULL operands, so this is the closest of parent + chunks // and NULL only when neither the parent nor any chunk is embedded. @@ -238,7 +275,7 @@ export function createPgvectorAdapter( // Rows with no embedding anywhere have no distance to rank: leave them out. // MIN(distance) is NULL exactly when no chunk has an embedding. - const scorableClause = sql`(${parentColumnExpr} IS NOT NULL OR ${chunkDistanceSubquery} IS NOT NULL)`; + const scorableClause = sql`(${parentScorable} OR ${chunkDistanceSubquery} IS NOT NULL)`; let whereClause: SQL | null = null; if (distance !== undefined && distance !== null) { @@ -262,10 +299,10 @@ export function createPgvectorAdapter( } return { - whereClause, - scoreExpression: distanceExpr, + whereClause: both(sameModel, whereClause), + scoreExpression: scored(distanceExpr), // A NULL embedding has no distance to rank: leave the row out. - scorableClause: sql`${columnExpr} IS NOT NULL`, + scorableClause: both(sameModel, sql`${columnExpr} IS NOT NULL`), }; }, }; diff --git a/graphile/graphile-search/src/plugin.ts b/graphile/graphile-search/src/plugin.ts index 0dd3aa4f6a..8d3b679fda 100644 --- a/graphile/graphile-search/src/plugin.ts +++ b/graphile/graphile-search/src/plugin.ts @@ -29,6 +29,19 @@ import type { SQL } from 'pg-sql2'; import type { SearchableColumn, SearchAdapter, UnifiedSearchOptions } from './types'; +/** + * Carried on a filter field that searches vector columns, for whatever embeds + * the search text (graphile-llm): which columns, and the model each column's + * own definition names. Vectors of two models are not comparable, so the text + * is embedded per model. + */ +export const VECTOR_COLUMNS_EXT = 'graphileSearchVectorColumns'; + +function vectorColumnRef(column: SearchableColumn): { attributeName: string; model?: string } { + const model = (column.adapterData as { model?: string } | undefined)?.model; + return { attributeName: column.attributeName, ...(model && { model }) }; +} + // ─── TypeScript Namespace Augmentations ────────────────────────────────────── declare global { @@ -1237,6 +1250,9 @@ export function createUnifiedSearchPlugin( 'field' ), type: build.getTypeByName(adapter.getFilterTypeName(build)) as any, + ...(adapter.name === 'vector' && { + extensions: { [VECTOR_COLUMNS_EXT]: [vectorColumnRef(column)] }, + }), apply: function plan($condition: any, val: any) { if (val == null) return; @@ -1302,15 +1318,24 @@ export function createUnifiedSearchPlugin( 'field' ), type: build.graphql.GraphQLString as any, + extensions: { + [VECTOR_COLUMNS_EXT]: vectorAdapterColumns.flatMap((ac) => ac.columns.map(vectorColumnRef)), + }, apply: function plan($condition: any, val: any) { if (val == null) return; // Support both plain string and { __text, __vector } from LLM plugin let text: string; let vector: number[] | null = null; + let model: string | undefined; + // Absent: one vector for every column. Present: a column it does + // not name has no vector of its own model and sits this search out. + let perColumn: Record | undefined; if (typeof val === 'object' && val.__text) { text = val.__text; vector = val.__vector ?? null; + model = val.__model ?? undefined; + perColumn = val.__vectors ?? undefined; } else { text = typeof val === 'string' ? val : String(val); if (text.trim().length === 0) return; @@ -1340,17 +1365,24 @@ export function createUnifiedSearchPlugin( } // Vector adapters (pgvector) — only when __vector is injected - if (vector && vectorAdapterColumns.length > 0) { + if ((vector || (perColumn && Object.keys(perColumn).length > 0)) && vectorAdapterColumns.length > 0) { for (const { adapter, columns } of vectorAdapterColumns) { for (const column of columns) { // One query vector, embedded by the text model: a column of // another declared length (an image-embedding column beside // the text one) holds vectors it cannot be compared with — // pgvector raises on the operator — so it sits this search out. + // Each column is searched with the vector its own model produced. + const query = perColumn ? perColumn[column.attributeName] : (vector ? { vector, model } : null); + if (!query) continue; const declared = (column.adapterData as { dimensions?: number } | undefined)?.dimensions; - if (declared && declared !== vector.length) continue; + if (declared && declared !== query.vector.length) continue; const applyFor = (alias: SQL) => - adapter.buildFilterApply(sql, alias, column, { vector, metric: 'COSINE' }, build); + adapter.buildFilterApply( + sql, alias, column, + { vector: query.vector, model: query.model, metric: 'COSINE' }, + build + ); const result = applyFor($condition.alias); if (!result) continue; diff --git a/graphile/graphile-storage-registry/src/index.ts b/graphile/graphile-storage-registry/src/index.ts index a2844b342c..81358e3129 100644 --- a/graphile/graphile-storage-registry/src/index.ts +++ b/graphile/graphile-storage-registry/src/index.ts @@ -1,5 +1,5 @@ export type { StorageInflection, UploadSurfaceNames } from './naming'; -export { DOWNLOAD_URL_FIELD, uploadSurfaceNames } from './naming'; +export { DOWNLOAD_URL_FIELD, FILE_REF_DOMAINS, uploadSurfaceNames } from './naming'; export type { StorageCodec, StorageCodecAttributes, diff --git a/graphile/graphile-storage-registry/src/naming.ts b/graphile/graphile-storage-registry/src/naming.ts index 380f8cb32f..b7461c15f1 100644 --- a/graphile/graphile-storage-registry/src/naming.ts +++ b/graphile/graphile-storage-registry/src/naming.ts @@ -64,3 +64,12 @@ export function uploadSurfaceNames( bulkUploadFilePayloadType: `Upload${filesTypeName}BulkFilePayload`, }; } + +/** + * The Postgres domains whose value is a file document: a jsonb object that + * projects a files row (`id`, `key`, `bucket_id`, …) and may list the + * `versions` rendered beside it. A column of one of these domains, or an array + * of one, is a file-reference column. One list, so that a new file type is a + * name added here and nothing else. + */ +export const FILE_REF_DOMAINS: readonly string[] = ['image', 'upload']; diff --git a/graphql/server-test/__fixtures__/seed/second-storage-module/schema.sql b/graphql/server-test/__fixtures__/seed/second-storage-module/schema.sql new file mode 100644 index 0000000000..95bb9c63d8 --- /dev/null +++ b/graphql/server-test/__fixtures__/seed/second-storage-module/schema.sql @@ -0,0 +1,60 @@ +-- A second storage module in Alice's database, with its own files and buckets +-- tables, so a file can live in a module other than the first one read. + +CREATE SCHEMA "second-storage-public"; +CREATE SCHEMA "second-storage-public-private"; +GRANT USAGE ON SCHEMA "second-storage-public", "second-storage-public-private" TO administrator, authenticated, anonymous; + +CREATE TABLE "second-storage-public".team_buckets ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + key text NOT NULL UNIQUE, + type text NOT NULL DEFAULT 'private', + is_public boolean NOT NULL DEFAULT false, + physical_name text NULL +); +COMMENT ON TABLE "second-storage-public".team_buckets IS E'@storageBuckets\nStorage buckets table'; + +CREATE TABLE "second-storage-public".team_files ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + bucket_id uuid NOT NULL REFERENCES "second-storage-public".team_buckets(id), + key text NOT NULL, + content_hash text NOT NULL, + mime_type text NOT NULL, + size bigint, + filename text, + is_public boolean NOT NULL DEFAULT false, + status text NOT NULL DEFAULT 'requested', + UNIQUE (bucket_id, key) +); +COMMENT ON TABLE "second-storage-public".team_files IS E'@storageFiles\nStorage files table'; + +GRANT SELECT, INSERT, UPDATE, DELETE ON "second-storage-public".team_buckets, "second-storage-public".team_files + TO administrator, authenticated, anonymous; + +CREATE FUNCTION "second-storage-public-private".team_buckets_physical_name(bucket_id uuid) RETURNS text +LANGUAGE sql SECURITY DEFINER STABLE AS $$ + SELECT b.physical_name FROM "second-storage-public".team_buckets b WHERE b.id = team_buckets_physical_name.bucket_id +$$; +GRANT EXECUTE ON FUNCTION "second-storage-public-private".team_buckets_physical_name(uuid) TO administrator, authenticated, anonymous; + +INSERT INTO metaschema_public.schema (id, database_id, name, schema_name, description, is_public) +VALUES + ('5ec0d000-0000-4000-8000-000000000001', '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9', 'second', 'second-storage-public', NULL, true), + ('5ec0d000-0000-4000-8000-000000000002', '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9', 'second_private', 'second-storage-public-private', NULL, false); + +INSERT INTO metaschema_public.table (id, database_id, schema_id, name, description) +VALUES + ('5ec0d000-0000-4000-8000-000000000003', '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9', '5ec0d000-0000-4000-8000-000000000001', 'team_buckets', NULL), + ('5ec0d000-0000-4000-8000-000000000004', '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9', '5ec0d000-0000-4000-8000-000000000001', 'team_files', NULL); + +INSERT INTO metaschema_modules_public.storage_module ( + id, database_id, schema_id, buckets_table_id, files_table_id, scope, private_schema_id +) +VALUES ( + '5ec0d000-0000-4000-8000-000000000005', '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9', + '5ec0d000-0000-4000-8000-000000000001', '5ec0d000-0000-4000-8000-000000000003', '5ec0d000-0000-4000-8000-000000000004', + 'team', '5ec0d000-0000-4000-8000-000000000002' +); + +INSERT INTO "second-storage-public".team_buckets (id, key, type, is_public, physical_name) +VALUES ('5ec0d000-0000-4000-8000-000000000006', 'private', 'private', false, 'team-80a2eaaf-private-0123456789ab'); diff --git a/graphql/server-test/__fixtures__/seed/upload-columns/schema.sql b/graphql/server-test/__fixtures__/seed/upload-columns/schema.sql index a276a2f91a..97dd01eb9f 100644 --- a/graphql/server-test/__fixtures__/seed/upload-columns/schema.sql +++ b/graphql/server-test/__fixtures__/seed/upload-columns/schema.sql @@ -24,3 +24,15 @@ GRANT SELECT, INSERT, UPDATE, DELETE ON upload_columns.products TO administrator GRANT USAGE, SELECT ON SEQUENCE upload_columns.products_id_seq TO administrator, authenticated, anonymous; INSERT INTO upload_columns.products (title) VALUES ('kettle'); + +-- A column holding a list of images: each element is its own document. +CREATE TABLE upload_columns.albums ( + id serial PRIMARY KEY, + shots public.image[] +); + +GRANT SELECT, INSERT, UPDATE, DELETE ON upload_columns.albums TO administrator, authenticated, anonymous; +GRANT USAGE, SELECT ON SEQUENCE upload_columns.albums_id_seq TO administrator, authenticated, anonymous; + +-- A column holding a list of uploads. +ALTER TABLE upload_columns.albums ADD COLUMN papers public.upload[]; diff --git a/graphql/server-test/__tests__/file-ref-urls.integration.test.ts b/graphql/server-test/__tests__/file-ref-urls.integration.test.ts new file mode 100644 index 0000000000..235bf8a5fe --- /dev/null +++ b/graphql/server-test/__tests__/file-ref-urls.integration.test.ts @@ -0,0 +1,436 @@ +/** + * Integration tests — URLs for a file-reference column (`image`, `upload`, or + * an array of one): `Url(width:)` and `Versions`. + * + * The document in the column is written by the client, its `versions` list + * included, so it is never the authority for a URL. The files row is read by + * the document's `id` as the reader; the source key comes from that row; and a + * version is signed only when its key is a direct child of that row's own + * `.versions/`. + * + * pnpm test -- --testPathPattern=file-ref-urls + */ + +import path from 'path'; +import pgModule from 'pg'; +import type supertest from 'supertest'; + +import { getConnections, seed } from '../src'; + +jest.setTimeout(120000); + +const sharedSeedRoot = path.join(__dirname, '..', '..', '..', '__fixtures__', 'seed'); +const localSeedRoot = path.join(__dirname, '..', '__fixtures__', 'seed'); +const pgpmWorkspace = path.join(sharedSeedRoot, '..', '..'); + +// Alice: no RLS on her files. Bob: anonymous reads only files in a public bucket. +const alice = { databaseId: '80a2eaaf-f77e-4bfe-8506-df929ef1b8d9', storageSchema: 'simple-storage-public' }; +const bob = { databaseId: 'a1a1a1a1-b2b2-4c3c-d4d4-e5e5e5e5e5e5', storageSchema: 'bob-storage-public' }; +const alicePublicBucketId = 'd0000001-0000-0000-0000-000000000001'; +const alicePrivateBucketId = 'd0000001-0000-0000-0000-000000000002'; +const alicePrivateBucket = 'app-80a2eaaf-private-cc079c527e7d'; +const secondBucketId = '5ec0d000-0000-4000-8000-000000000006'; +const cdn = 'https://cdn.files.test'; +const bobPrivateFileId = 'd3d3d3d3-0000-0000-0000-000000000001'; +const bobPublicFileId = 'd3d3d3d3-0000-0000-0000-000000000002'; +const schema = 'upload_columns'; +const metaSchemas = [ + 'catalog_private', + 'routing_public', + 'apps_public', + 'metaschema_public', + 'metaschema_modules_public' +]; + +const PRODUCT = ` + query Product($id: Int!, $width: Int) { + products(where: { id: { equalTo: $id } }) { + nodes { + photoUrl(width: $width) + source: photoUrl + photoVersions { name key width height format size url } + } + } + } +`; + +const pathOf = (url: string) => decodeURIComponent(new URL(url).pathname); + +describe('file-reference column URLs', () => { + let request: supertest.Agent; + let pg: { query: (text: string, values?: unknown[]) => Promise<{ rows: any[] }> }; + let teardown: () => Promise; + + const product = async (who: typeof alice, id: number, width?: number) => { + const res = await request + .post('/graphql') + .set('X-Database-Id', who.databaseId) + .set('X-Schemata', [who.storageSchema, schema].join(',')) + .send({ query: PRODUCT, variables: { id, width } }); + expect(res.body.errors).toBeUndefined(); + expect(res.status).toBe(200); + return res.body.data.products.nodes[0] as { + photoUrl: string | null; + source: string | null; + photoVersions: Array<{ name: string; key: string; width: number; url: string }> | null; + }; + }; + + const addFile = async (key: string, status = 'uploaded', bucketId = alicePrivateBucketId, isPublic = false) => + ( + await pg.query( + `INSERT INTO "${alice.storageSchema}".app_files (bucket_id, key, content_hash, mime_type, filename, status, is_public) + VALUES ($1, $2, $2, 'image/png', $4, $3, $5) RETURNING id`, + [bucketId, key, status, `${key.split('/').pop()}`, isPublic] + ) + ).rows[0].id as string; + + const addProduct = async (photo: unknown, manual: unknown = null) => + ( + await pg.query(`INSERT INTO ${schema}.products (title, photo, manual) VALUES ('p', $1::jsonb, $2::jsonb) RETURNING id`, [ + photo === null ? null : JSON.stringify(photo), + manual === null ? null : JSON.stringify(manual) + ]) + ).rows[0].id as number; + + const ask = async (who: typeof alice, query: string, variables: Record = {}) => { + const res = await request + .post('/graphql') + .set('X-Database-Id', who.databaseId) + .set('X-Schemata', [who.storageSchema, schema].join(',')) + .send({ query, variables }); + expect(res.body.errors).toBeUndefined(); + expect(res.status).toBe(200); + return res.body.data; + }; + + const version = (key: string, name: string, width: number) => ({ name, key, width, height: width, format: 'webp', size: 10 }); + + beforeAll(async () => { + ({ request, pg, teardown } = await getConnections( + { + schemas: [alice.storageSchema, schema], + authRole: 'anonymous', + server: { useRouting: true, api: { isPublic: false, metaSchemas } } + }, + [ + seed.pgpm(pgpmWorkspace), + seed.sqlfile([ + path.join(localSeedRoot, 'simple-seed-storage', 'schema.sql'), + path.join(localSeedRoot, 'simple-seed-storage', 'test-data.sql'), + path.join(localSeedRoot, 'upload-columns', 'schema.sql'), + path.join(localSeedRoot, 'second-storage-module', 'schema.sql') + ]) + ] + )); + // Before the first request reads the module: a public file is served from this prefix. + await pg.query(`UPDATE metaschema_modules_public.storage_module SET public_url_prefix = $1 WHERE database_id = $2`, [ + cdn, + alice.databaseId + ]); + }); + + afterAll(async () => { + if (teardown) await teardown(); + }); + + it('gives the narrowest version covering the width, else the source', async () => { + const id = await addFile('pick/photo.png'); + const small = version('pick/photo.png.versions/small-0a1b2c3d.webp', 'small', 320); + const large = version('pick/photo.png.versions/large-4e5f6a7b.webp', 'large', 1280); + const row = await addProduct({ id, key: 'pick/photo.png', versions: [large, small] }); + + const at300 = await product(alice, row, 300); + expect(pathOf(at300.photoUrl)).toMatch(new RegExp(`/${small.key}$`)); + expect(pathOf(at300.source)).toMatch(/\/pick\/photo\.png$/); + expect(at300.photoVersions.map((v) => v.name)).toEqual(['small', 'large']); + for (const v of at300.photoVersions) expect(pathOf(v.url)).toMatch(new RegExp(`/${v.key}$`)); + + expect(pathOf((await product(alice, row, 640)).photoUrl)).toMatch(new RegExp(`/${large.key}$`)); + expect(pathOf((await product(alice, row, 2000)).photoUrl)).toMatch(/\/pick\/photo\.png$/); + }); + + it('never signs a key the document names outside its own file', async () => { + const id = await addFile('forge/mine.png'); + await addFile('forge/victim.png'); + const own = version('forge/mine.png.versions/small-0a1b2c3d.webp', 'small', 320); + const forged = [ + version('forge/victim.png', 'other-file', 400), + version('forge/victim.png.versions/small-0a1b2c3d.webp', 'other-files-version', 400), + version('forge/mine.png.versions/../victim.png', 'traversal', 400), + version('forge/mine.png.versions/nested/deep.webp', 'nested', 400), + version('forge/mine.png.versions/', 'prefix-only', 400), + version('forge/mine.png', 'source-as-version', 400) + ]; + // The document also lies about its own key: the row's key is what is signed. + const row = await addProduct({ id, key: 'forge/victim.png', versions: [...forged, own] }); + + const got = await product(alice, row, 400); + expect(got.photoVersions.map((v) => v.key)).toEqual([own.key]); + expect(pathOf(got.photoUrl)).toMatch(/\/forge\/mine\.png$/); + expect(pathOf(got.source)).toMatch(/\/forge\/mine\.png$/); + expect(JSON.stringify(got)).not.toContain('victim'); + }); + + it('resolves nothing for a document without a files row the reader can fetch', async () => { + const requested = await addFile('dead/requested.png', 'requested'); + const noId = await addProduct({ key: 'pick/photo.png', versions: [version('pick/photo.png.versions/small-0a1b2c3d.webp', 'small', 320)] }); + const notLive = await addProduct({ id: requested, key: 'dead/requested.png' }); + const unknown = await addProduct({ id: '00000000-0000-4000-8000-000000000000', key: 'pick/photo.png' }); + const empty = await addProduct(null); + + for (const row of [noId, notLive, unknown, empty]) { + expect(await product(alice, row, 320)).toEqual({ photoUrl: null, source: null, photoVersions: null }); + } + }); + + it('reads the files row as the reader: a file RLS hides gives no URL', async () => { + // The seed leaves Bob's rows `requested`; both must be live so that only RLS tells them apart. + await pg.query(`UPDATE "${bob.storageSchema}".app_files SET status = 'uploaded' WHERE id = ANY($1::uuid[])`, [ + [bobPrivateFileId, bobPublicFileId] + ]); + const hidden = await addProduct({ + id: bobPrivateFileId, + key: 'seeded-private-hash', + versions: [version('seeded-private-hash.versions/small-0a1b2c3d.webp', 'small', 320)] + }); + const visible = await addProduct({ id: bobPublicFileId, key: 'seeded-public-hash' }); + + expect(await product(bob, hidden, 320)).toEqual({ photoUrl: null, source: null, photoVersions: null }); + expect((await product(bob, visible, 320)).source).toContain('seeded-public-hash'); + }); + + it('resolves each element of an image[] column on its own', async () => { + const one = await addFile('list/one.png'); + await addFile('list/two.png'); + const mine = version('list/one.png.versions/small-0a1b2c3d.webp', 'small', 320); + const theirs = version('list/two.png.versions/small-0a1b2c3d.webp', 'small', 320); + const { rows } = await pg.query( + `INSERT INTO ${schema}.albums (shots) + VALUES (ARRAY(SELECT jsonb_array_elements($1::jsonb))::public.image[]) RETURNING id`, + [JSON.stringify([{ id: one, key: 'list/one.png', versions: [mine, theirs] }, { key: 'list/two.png' }])] + ); + + const res = await request + .post('/graphql') + .set('X-Database-Id', alice.databaseId) + .set('X-Schemata', [alice.storageSchema, schema].join(',')) + .send({ + query: `query Album($id: Int!) { + albums(where: { id: { equalTo: $id } }) { nodes { shotsUrl(width: 300) shotsVersions { key url } } } + }`, + variables: { id: rows[0].id } + }); + expect(res.body.errors).toBeUndefined(); + const album = res.body.data.albums.nodes[0]; + + expect(album.shotsUrl).toHaveLength(2); + expect(pathOf(album.shotsUrl[0])).toMatch(new RegExp(`/${mine.key}$`)); + expect(album.shotsUrl[1]).toBeNull(); + expect(album.shotsVersions[0].map((v: { key: string }) => v.key)).toEqual([mine.key]); + expect(album.shotsVersions[1]).toBeNull(); + }); + + describe('an upload column', () => { + const MANUAL = `query Manual($id: Int!, $width: Int) { + products(where: { id: { equalTo: $id } }) { + nodes { manualUrl(width: $width) source: manualUrl manualVersions { name key width height format size url } } + } + }`; + const manual = async (id: number, width?: number) => (await ask(alice, MANUAL, { id, width })).products.nodes[0]; + + it('gives the source URL for a document without versions, whatever the width', async () => { + const id = await addFile('docs/resume.pdf'); + const row = await addProduct(null, { id, key: 'docs/resume.pdf' }); + + const got = await manual(row, 640); + expect(pathOf(got.manualUrl)).toMatch(/\/docs\/resume\.pdf$/); + expect(pathOf(got.source)).toMatch(/\/docs\/resume\.pdf$/); + expect(got.manualVersions).toEqual([]); + }); + + it('falls back to the source when no version states a width', async () => { + const id = await addFile('docs/deck.pdf'); + const preview = { name: 'preview', key: 'docs/deck.pdf.versions/preview-0a1b2c3d.png' }; + const row = await addProduct(null, { id, key: 'docs/deck.pdf', versions: [preview] }); + + const got = await manual(row, 320); + expect(pathOf(got.manualUrl)).toMatch(/\/docs\/deck\.pdf$/); + expect(got.manualVersions).toEqual([ + { name: 'preview', key: preview.key, width: null, height: null, format: null, size: null, url: expect.stringContaining('preview-0a1b2c3d.png') } + ]); + }); + + it('never signs a key the document names outside its own file', async () => { + const id = await addFile('docs/mine.pdf'); + await addFile('docs/victim.pdf'); + const own = version('docs/mine.pdf.versions/page-0a1b2c3d.png', 'page', 320); + const row = await addProduct(null, { + id, + key: 'docs/victim.pdf', + versions: [ + version('docs/victim.pdf', 'other-file', 400), + version('docs/victim.pdf.versions/page-0a1b2c3d.png', 'other-files-version', 400), + version('docs/mine.pdf.versions/../victim.pdf', 'traversal', 400), + own + ] + }); + + const got = await manual(row, 400); + expect(got.manualVersions.map((v: { key: string }) => v.key)).toEqual([own.key]); + expect(pathOf(got.source)).toMatch(/\/docs\/mine\.pdf$/); + expect(JSON.stringify(got)).not.toContain('victim'); + }); + + it('reads the files row as the reader: a file RLS hides gives no URL', async () => { + await pg.query(`UPDATE "${bob.storageSchema}".app_files SET status = 'uploaded' WHERE id = ANY($1::uuid[])`, [ + [bobPrivateFileId, bobPublicFileId] + ]); + const hidden = await addProduct(null, { id: bobPrivateFileId, key: 'seeded-private-hash' }); + const visible = await addProduct(null, { id: bobPublicFileId, key: 'seeded-public-hash' }); + const asBob = async (id: number) => (await ask(bob, MANUAL, { id })).products.nodes[0]; + + expect(await asBob(hidden)).toEqual({ manualUrl: null, source: null, manualVersions: null }); + expect((await asBob(visible)).source).toContain('seeded-public-hash'); + }); + + it('resolves each element of an upload[] column on its own', async () => { + const one = await addFile('papers/one.pdf'); + await addFile('papers/two.pdf'); + const theirs = version('papers/two.pdf.versions/page-0a1b2c3d.png', 'page', 320); + const { rows } = await pg.query( + `INSERT INTO ${schema}.albums (papers) + VALUES (ARRAY(SELECT jsonb_array_elements($1::jsonb))::public.upload[]) RETURNING id`, + [JSON.stringify([{ id: one, key: 'papers/two.pdf', versions: [theirs] }, { key: 'papers/two.pdf' }])] + ); + const album = ( + await ask(alice, `query Album($id: Int!) { + albums(where: { id: { equalTo: $id } }) { nodes { papersUrl papersVersions { key url } } } + }`, { id: rows[0].id }) + ).albums.nodes[0]; + + expect(album.papersUrl).toHaveLength(2); + expect(pathOf(album.papersUrl[0])).toMatch(/\/papers\/one\.pdf$/); + expect(album.papersUrl[1]).toBeNull(); + expect(album.papersVersions).toEqual([[], null]); + }); + }); + + it('names no type or field after one file kind', async () => { + const res = await request + .post('/graphql') + .set('X-Database-Id', alice.databaseId) + .set('X-Schemata', [alice.storageSchema, schema].join(',')) + .send({ query: `{ __schema { types { name fields { name type { name ofType { name ofType { name ofType { name } } } } } } } }` }); + const types = res.body.data.__schema.types as Array<{ name: string; fields: Array<{ name: string }> | null }>; + const fileVersion = types.find((t) => t.name === 'FileVersion'); + expect(fileVersion.fields.map((f) => f.name)).toEqual(['name', 'key', 'width', 'height', 'format', 'size', 'url']); + expect(types.find((t) => t.name === 'ImageVersion')).toBeUndefined(); + const product = types.find((t) => t.name === 'Product').fields.map((f) => f.name); + expect(product).toEqual(expect.arrayContaining(['photoUrl', 'photoVersions', 'manualUrl', 'manualVersions'])); + // `attachment` holds a link, not a file document. + expect(product).not.toContain('brochureUrl'); + }); + + describe('where the bytes live, as downloadUrl has it', () => { + const FILE = `query File($id: UUID!) { appFiles(where: { id: { equalTo: $id } }) { nodes { downloadUrl } } }`; + const urls = async (fileId: string, productId: number) => ({ + downloadUrl: (await ask(alice, FILE, { id: fileId })).appFiles.nodes[0].downloadUrl as string, + ...(await product(alice, productId)) + }); + + it('serves a public file from the public URL prefix, its versions too', async () => { + const id = await addFile('open/logo.png', 'uploaded', alicePublicBucketId, true); + const small = version('open/logo.png.versions/small-0a1b2c3d.webp', 'small', 320); + const got = await urls(id, await addProduct({ id, key: 'open/logo.png', versions: [small] })); + + expect(got.source).toBe(`${cdn}/app-80a2eaaf-public-c7156cb19fe3/open/logo.png`); + expect(got.source).toBe(got.downloadUrl); + expect(got.photoVersions[0].url).toBe(`${cdn}/app-80a2eaaf-public-c7156cb19fe3/${small.key}`); + }); + + it('presigns a private file in its database\'s own bucket, under the name it was uploaded with', async () => { + const id = await addFile('closed/photo.png'); + const got = await urls(id, await addProduct({ id, key: 'closed/photo.png' })); + + const source = new URL(got.source); + const download = new URL(got.downloadUrl); + expect(source.searchParams.get('X-Amz-Signature')).toEqual(expect.any(String)); + expect(source.host).toBe(download.host); + expect(source.pathname).toBe(download.pathname); + expect(decodeURIComponent(source.pathname)).toContain(`${alicePrivateBucket}/closed/photo.png`); + expect(source.searchParams.get('response-content-disposition')).toContain('photo.png'); + expect(source.searchParams.get('response-content-disposition')).toBe(download.searchParams.get('response-content-disposition')); + }); + + it('signs in another database\'s bucket for that database', async () => { + await pg.query(`UPDATE "${bob.storageSchema}".app_files SET status = 'uploaded' WHERE id = $1`, [bobPublicFileId]); + const got = await product(bob, await addProduct({ id: bobPublicFileId, key: 'seeded-public-hash' })); + expect(decodeURIComponent(got.source)).toContain('app-a1a1a1a1-public-4db6afbff1a2/seeded-public-hash'); + expect(got.source).not.toContain('app-80a2eaaf'); + }); + + it('finds a file that lives in the database\'s second storage module', async () => { + const { rows } = await pg.query( + `INSERT INTO "second-storage-public".team_files (bucket_id, key, content_hash, mime_type, filename, status) + VALUES ($1, 'team/plan.png', 'team/plan.png', 'image/png', 'plan.png', 'uploaded') RETURNING id`, + [secondBucketId] + ); + const small = version('team/plan.png.versions/small-0a1b2c3d.webp', 'small', 320); + const got = await product(alice, await addProduct({ id: rows[0].id, key: 'team/plan.png', versions: [small] }), 300); + + expect(pathOf(got.source)).toContain('team-80a2eaaf-private-0123456789ab/team/plan.png'); + expect(pathOf(got.photoUrl)).toContain(`team-80a2eaaf-private-0123456789ab/${small.key}`); + }); + + it('gives no URL for a file whose bucket was never provisioned', async () => { + const bucket = ( + await pg.query(`INSERT INTO "${alice.storageSchema}".app_buckets (key, type, physical_name) VALUES ('pending', 'private', NULL) RETURNING id`) + ).rows[0].id as string; + const id = await addFile('pending/photo.png', 'uploaded', bucket); + const got = await urls(id, await addProduct({ id, key: 'pending/photo.png' })); + + expect(got.downloadUrl).toBeNull(); + expect(got).toMatchObject({ photoUrl: null, source: null, photoVersions: null }); + }); + }); + + it('reads the files of a list once for the request, not once per row', async () => { + const title = 'fifty'; + for (let i = 0; i < 50; i++) { + const key = `fifty/${i}.png`; + const id = await addFile(key); + await pg.query(`INSERT INTO ${schema}.products (title, photo) VALUES ($1, $2::jsonb)`, [ + title, + JSON.stringify({ id, key, versions: [version(`${key}.versions/small-0a1b2c3d.webp`, 'small', 320)] }) + ]); + } + + const storage = /app_files|team_files|current_database_id|_physical_name|storage_modules/; + const seen: string[] = []; + const query = pgModule.Client.prototype.query; + const spy = jest.spyOn(pgModule.Client.prototype, 'query').mockImplementation(function (this: any, ...args: any[]) { + const text = typeof args[0] === 'string' ? args[0] : args[0]?.text; + if (typeof text === 'string' && storage.test(text)) seen.push(text); + return (query as any).apply(this, args); + } as any); + let nodes: Array<{ photoUrl: string; photoVersions: unknown[] }>; + try { + nodes = ( + await ask(alice, `{ products(where: { title: { equalTo: "${title}" } }) { nodes { photoUrl(width: 300) photoVersions { url } } } }`) + ).products.nodes; + } finally { + spy.mockRestore(); + } + + expect(nodes).toHaveLength(50); + for (const node of nodes) { + expect(node.photoUrl).toContain('small-0a1b2c3d.webp'); + expect(node.photoVersions).toHaveLength(1); + } + // eslint-disable-next-line no-console + console.log(`storage queries for a 50-row list: ${seen.length}`); + // The claimed database, the files rows, the bucket coordinate; the module list may be read once more. + expect(seen.length).toBeLessThanOrEqual(4); + }); +});