Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
175 changes: 175 additions & 0 deletions graphile/graphile-llm/src/__tests__/vector-model.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
/**
* A vector column that records the model of each row's vector is searched with
* that model, and only against rows the same model embedded.
*
* `@embeddingModel` on a vector column names the model its definition asks for;
* `@embeddingModelOf <column>` marks the sibling that holds, per row, the model
* that produced the vector. Two models' vectors are not comparable even at the
* same length, so one search text is embedded once per model its columns use.
*/

import type { GraphileConfig } from 'graphile-config';
import { ConnectionFilterPreset } from 'graphile-connection-filter';
import { createPgvectorAdapter } from 'graphile-search/adapters/pgvector';
import { createTsvectorAdapter } from 'graphile-search/adapters/tsvector';
import { TsvectorCodecPlugin } from 'graphile-search/codecs/tsvector-codec';
import { VectorCodecPlugin } from 'graphile-search/codecs/vector-codec';
import { createUnifiedSearchPlugin } from 'graphile-search/plugin';
import type { GraphQLResponse } from 'graphile-test';
import { getConnections, seed } from 'graphile-test';
import type { PgTestClient } from 'pgsql-test';

import { createLlmModulePlugin } from '../../src/plugins/llm-module-plugin';
import { createLlmTextMutationPlugin } from '../../src/plugins/text-mutation-plugin';
import { createLlmTextSearchPlugin } from '../../src/plugins/text-search-plugin';

type QueryFn = <TResult = unknown>(query: string, variables?: Record<string, unknown>) => Promise<GraphQLResponse<TResult>>;
type Notes = { allNotes: { nodes: Array<{ name: string }> } };

const REQUEST_MODEL = 'request-model';
const calls: Array<{ text: string; model: string | undefined }> = [];

// Every model answers with the same vector, so only the model stamp can tell rows apart.
const FakeEmbedderPlugin: GraphileConfig.Plugin = {
name: 'FakeEmbedderPlugin',
version: '0.0.0',
after: ['LlmModulePlugin'],
schema: {
hooks: {
build(build) {
(build as any).llmEmbedder = async (text: string, model?: string) => {
calls.push({ text, model });
return { embedding: [1, 0, 0], promptTokens: 1, model: model ?? REQUEST_MODEL };
};
return build;
}
}
}
};

const SETUP_SQL = `
CREATE EXTENSION IF NOT EXISTS vector;
CREATE SCHEMA llm_vector_model;
CREATE TABLE llm_vector_model.notes (
id serial PRIMARY KEY,
name text NOT NULL,
tsv tsvector GENERATED ALWAYS AS (to_tsvector('english', name || ' note')) STORED,
embedding vector(3),
embedding_model text,
summary vector(3),
summary_model text,
legacy vector(3)
);
COMMENT ON COLUMN llm_vector_model.notes.embedding IS E'@embeddingModel model-a';
COMMENT ON COLUMN llm_vector_model.notes.embedding_model IS E'@embeddingModelOf embedding';
COMMENT ON COLUMN llm_vector_model.notes.summary_model IS E'@embeddingModelOf summary';
INSERT INTO llm_vector_model.notes (name, embedding, embedding_model, summary, summary_model, legacy) VALUES
('current', '[1,0,0]', 'model-a', '[1,0,0]', '${REQUEST_MODEL}', '[1,0,0]'),
('older', '[1,0,0]', 'model-b', '[1,0,0]', 'model-b', '[1,0,0]'),
('pending', NULL, NULL, NULL, NULL, NULL);
`;

describe('searching a vector column that records its model', () => {
let db: PgTestClient;
let pg: PgTestClient;
let query: QueryFn;
let teardown: () => Promise<void>;

const names = async (where: string) => {
const res = await query<Notes>(`{ allNotes(where: ${where}) { nodes { name } } }`);
expect(res.errors).toBeUndefined();
return res.data!.allNotes.nodes.map((n) => n.name).sort();
};

beforeAll(async () => {
const connections = await getConnections(
{
schemas: ['llm_vector_model'],
preset: {
extends: [ConnectionFilterPreset()],
plugins: [
TsvectorCodecPlugin,
VectorCodecPlugin,
createUnifiedSearchPlugin({
adapters: [createTsvectorAdapter(), createPgvectorAdapter()],
enableSearchScore: true,
enableUnifiedSearch: true
}),
createLlmModulePlugin(),
FakeEmbedderPlugin,
createLlmTextSearchPlugin(),
createLlmTextMutationPlugin()
]
},
useRoot: true,
authRole: 'postgres'
},
[seed.fn(async (ctx) => { await ctx.pg.query(SETUP_SQL); })]
);
({ db, pg, teardown } = connections);
query = connections.query as QueryFn;
});

afterAll(async () => { await teardown(); });

beforeEach(async () => {
calls.length = 0;
await pg.beforeEach();
await db.beforeEach();
});

afterEach(async () => {
await db.afterEach();
await pg.afterEach();
});

it('embeds the text with the model the column names and skips rows of another model', async () => {
expect(await names('{ vectorEmbedding: { text: "q" } }')).toEqual(['current']);
expect(calls).toEqual([{ text: 'q', model: 'model-a' }]);
});

it('uses the request model for a column that names none, and still skips other models', async () => {
expect(await names('{ vectorSummary: { text: "q" } }')).toEqual(['current']);
expect(calls).toEqual([{ text: 'q', model: undefined }]);
});

it('compares every row of a column that records no model', async () => {
expect(await names('{ vectorLegacy: { text: "q" } }')).toEqual(['current', 'older']);
});

it('filters a client-sent vector by the model the client says produced it', async () => {
expect(await names('{ vectorEmbedding: { vector: [1, 0, 0], model: "model-b" } }')).toEqual(['older']);
expect(await names('{ vectorEmbedding: { vector: [1, 0, 0] } }')).toEqual(['current', 'older']);
expect(calls).toEqual([]);
});

it('labels an embedded text with the model that embedded it, whatever label the client sent', async () => {
expect(await names('{ vectorEmbedding: { text: "q", model: "model-b" } }')).toEqual(['current']);
});

it('stamps the model beside a vector it embeds on a write, so the row is found by a search', async () => {
const res = await query<{ createNote: { note: { name: string; embeddingModel: string; summaryModel: string } } }>(`mutation {
createNote(input: { note: { name: "written", embeddingText: "q", summaryText: "q" } }) {
note { name embeddingModel summaryModel }
}
}`);
expect(res.errors).toBeUndefined();
expect(res.data!.createNote.note).toEqual({ name: 'written', embeddingModel: 'model-a', summaryModel: REQUEST_MODEL });
expect(calls.map((c) => c.model).sort()).toEqual(['model-a', undefined]);
expect(await names('{ vectorEmbedding: { text: "q" } }')).toEqual(['current', 'written']);
});

it('embeds unifiedSearch once per model and scores each column with its own', async () => {
const res = await query<{ allNotes: { nodes: Array<Record<string, unknown>> } }>(`{
allNotes(where: { unifiedSearch: "note" }) {
nodes { name embeddingVectorDistance summaryVectorDistance legacyVectorDistance }
}
}`);
expect(res.errors).toBeUndefined();
const byName = Object.fromEntries(res.data!.allNotes.nodes.map((n) => [n.name, n]));
expect(calls.map((c) => c.model).sort()).toEqual(['model-a', undefined]);
expect(byName.current).toMatchObject({ embeddingVectorDistance: 0, summaryVectorDistance: 0, legacyVectorDistance: 0 });
// Another model produced these two vectors: no distance, not a wrong one.
expect(byName.older).toMatchObject({ embeddingVectorDistance: null, summaryVectorDistance: null, legacyVectorDistance: 0 });
});
});
140 changes: 113 additions & 27 deletions graphile/graphile-llm/src/embed-inputs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,47 @@ export type EmbedRule =
/** `unifiedSearch: String`: becomes `{ __text, __vector }` for graphile-search. */
| { kind: 'unified'; onQuotaExceeded: QuotaPolicy }
/** `{col}Text` on a create input or patch. `required` means the column is NOT NULL. */
| { kind: 'companion'; into: string; required: boolean };
| {
kind: 'companion';
into: string;
required: boolean;
/** The model the column's vectors are declared to come from. */
model?: string;
/** The input field of the column that records each vector's model. */
modelField?: string;
};

export const EMBED_EXT = 'graphileLlmEmbed';

export type VectorEmbedder = (text: string) => Promise<number[] | null>;
/**
* Set by graphile-search on a filter field that searches vector columns: which
* columns, and the model each one's own definition names (`@embeddingModel`).
* Vectors of two models are not comparable, so a search text is embedded once
* per model its columns use, never once for all of them.
*/
export const VECTOR_COLUMNS_EXT = 'graphileSearchVectorColumns';
export interface VectorColumnRef {
attributeName: string;
model?: string;
}

/** A query vector and the model that produced it. */
export interface Embedded {
vector: number[];
model?: string;
}

export interface VectorEmbedder {
(text: string): Promise<number[] | null>;
/** Embed with `model` when given, else the request's; reports the model used. */
withModel?: (text: string, model?: string) => Promise<Embedded | null>;
}

const embedWith = async (embed: VectorEmbedder, text: string, model?: string): Promise<Embedded | null> => {
if (embed.withModel) return embed.withModel(text, model);
const vector = await embed(text);
return vector ? { vector } : null;
};

export function embedField<T extends object>(rule: EmbedRule, spec: T): T {
return { ...spec, extensions: { ...(spec as { extensions?: object }).extensions, [EMBED_EXT]: rule } };
Expand All @@ -40,11 +76,12 @@ export function embedField<T extends object>(rule: EmbedRule, spec: T): T {
/** `build.llmEmbedder` resolves to `{ embedding }`; an empty or missing vector is a failed embed. */
export function toVectorEmbedder(raw: unknown): VectorEmbedder | null {
if (typeof raw !== 'function') return null;
return async (text) => {
const result = await raw(text);
const withModel = async (text: string, model?: string): Promise<Embedded | null> => {
const result = await raw(text, model);
const vector = result?.embedding;
return isNonEmptyVector(vector) ? vector : null;
return isNonEmptyVector(vector) ? { vector, model: result.model ?? model } : null;
};
return Object.assign(async (text: string) => (await withModel(text))?.vector ?? null, { withModel });
}

function isNonEmptyVector(value: unknown): boolean {
Expand All @@ -58,16 +95,29 @@ function isNonEmptyVector(value: unknown): boolean {
function forRequest(embedder: VectorEmbedder | null, graphqlContext: unknown): VectorEmbedder | null {
if (!embedder) return null;
let scope: Promise<LlmRequestScope> | undefined;
return async (text) => {
scope ??= resolveRequestScope(graphqlContext);
const resolved = await scope;
const startTime = Date.now();
const vector = await llmConfigStore.run(resolved, () => embedder(text));
if (vector) {
console.log(`[graphile-llm] embed: dims=${vector.length}, latency=${Date.now() - startTime}ms`);
// One search may name the same text for several columns of one model.
const memo = new Map<string, Promise<Embedded | null>>();
const withModel = (text: string, model?: string): Promise<Embedded | null> => {
const memoKey = JSON.stringify([model ?? null, text]);
let pending = memo.get(memoKey);
if (!pending) {
pending = (async () => {
scope ??= resolveRequestScope(graphqlContext);
const resolved = await scope;
const startTime = Date.now();
const embedded = await llmConfigStore.run(resolved, () => embedWith(embedder, text, model));
if (embedded) {
console.log(
`[graphile-llm] embed: model=${embedded.model ?? 'default'}, dims=${embedded.vector.length}, latency=${Date.now() - startTime}ms`
);
}
return embedded;
})();
memo.set(memoKey, pending);
}
return vector;
return pending;
};
return Object.assign(async (text: string) => (await withModel(text))?.vector ?? null, { withModel });
}

export function createEmbedIndex(graphql: any) {
Expand Down Expand Up @@ -132,7 +182,8 @@ export function createEmbedIndex(graphql: any) {
typeName: string,
value: Record<string, any>,
next: Record<string, any>,
embed: VectorEmbedder | null
embed: VectorEmbedder | null,
columns: VectorColumnRef[] | undefined
): Promise<void> {
const text = value[fieldName];

Expand All @@ -159,9 +210,13 @@ export function createEmbedIndex(graphql: any) {
`send \`${rule.into}\` instead.`
);
}
const vector = await embed(text);
if (vector) {
next[rule.into] = vector;
// The filter field this input sits under searches one column.
const embedded = await embedWith(embed, text, columns?.[0]?.model);
if (embedded) {
next[rule.into] = embedded.vector;
// The label is the model that made this vector, whatever the client sent.
if (embedded.model) next.model = embedded.model;
else delete next.model;
} else if (rule.onQuotaExceeded === 'throw') {
throw new Error(
`${typeName}: embedding failed (provider unavailable). ` +
Expand All @@ -173,9 +228,31 @@ export function createEmbedIndex(graphql: any) {

case 'unified': {
if (typeof text !== 'string' || text.trim().length === 0 || !embed) return;
const vector = await embed(text);
if (vector) {
next[fieldName] = { __text: text, __vector: vector };
// One vector per model the table's vector columns use; `__vector` stays
// the request model's, for a column nothing is known about.
const models = [...new Set<string | undefined>(columns?.length ? columns.map((c) => c.model) : [undefined])];
const byModel = new Map<string | undefined, Embedded | null>(
await Promise.all(models.map(async (m) => [m, await embedWith(embed, text, m)] as const))
);
if (rule.onQuotaExceeded === 'throw' && models.some((m) => !byModel.get(m))) {
throw new Error(
`${fieldName}: embedding failed (provider unavailable). ` +
'onQuotaExceeded is set to \'throw\'; set it to \'degrade\' for text-only fallback.'
);
}
const base = byModel.get(undefined);
const perColumn: Record<string, Embedded> = {};
for (const column of columns ?? []) {
const embedded = byModel.get(column.model);
if (embedded) perColumn[column.attributeName] = embedded;
}
if (base || Object.keys(perColumn).length > 0) {
next[fieldName] = {
__text: text,
__vector: base?.vector ?? null,
__model: base?.model,
__vectors: perColumn
};
} else if (rule.onQuotaExceeded === 'throw') {
throw new Error(
`${fieldName}: embedding failed (provider unavailable). ` +
Expand Down Expand Up @@ -209,36 +286,45 @@ export function createEmbedIndex(graphql: any) {
'Set defaultEmbedder in GraphileLlmPreset options or EMBEDDER_PROVIDER env var.'
);
}
const vector = await embed(text);
if (!vector) {
const embedded = await embedWith(embed, text, rule.model);
if (!embedded) {
throw new Error(
`EMBED_FAILED: Cannot embed ${fieldName} — the embedding provider returned no vector.`
);
}
next[rule.into] = vector;
next[rule.into] = embedded.vector;
// The vector and the model that made it are written together: a search
// compares only rows whose stamp is the model of its query vector.
if (rule.modelField && embedded.model) next[rule.modelField] = embedded.model;
return;
}
}
}

/** Returns `value` itself when its type cannot reach a tagged field. */
async function embedValue(type: any, value: any, embed: VectorEmbedder | null): Promise<any> {
async function embedValue(
type: any,
value: any,
embed: VectorEmbedder | null,
columns?: VectorColumnRef[]
): Promise<any> {
if (value == null) return value;
const inner = isNonNullType(type) ? type.ofType : type;
if (isListType(inner)) {
if (!Array.isArray(value) || !reaches(inner.ofType)) return value;
return Promise.all(value.map((item: any) => embedValue(inner.ofType, item, embed)));
return Promise.all(value.map((item: any) => embedValue(inner.ofType, item, embed, columns)));
}
if (!isInputObjectType(inner) || !reaches(inner)) return value;

const next: Record<string, any> = { ...value };
const jobs: Promise<void>[] = [];
for (const [name, field] of Object.entries<any>(inner.getFields())) {
const rule = field.extensions?.[EMBED_EXT] as EmbedRule | undefined;
const own = field.extensions?.[VECTOR_COLUMNS_EXT] as VectorColumnRef[] | undefined;
if (rule) {
jobs.push(applyRule(rule, name, inner.name, value, next, embed));
jobs.push(applyRule(rule, name, inner.name, value, next, embed, own ?? columns));
} else if (value[name] != null && reaches(field.type)) {
jobs.push(embedValue(field.type, value[name], embed).then((embedded) => { next[name] = embedded; }));
jobs.push(embedValue(field.type, value[name], embed, own).then((embedded) => { next[name] = embedded; }));
}
}
await Promise.all(jobs);
Expand Down
Loading
Loading