diff --git a/services/llmCache.ts b/services/llmCache.ts new file mode 100644 index 0000000..896a801 --- /dev/null +++ b/services/llmCache.ts @@ -0,0 +1,196 @@ +/** + * llmCache.ts + * + * IndexedDB-backed response cache for Gemini calls. + * + * Cache key is a deterministic hash of (model, compacted prompt, schema + * fingerprint, temperature). Entries expire after `TTL_MS` (default 24 h) + * so stale data is never served indefinitely. + * + * All operations are async and safe to call concurrently. + */ + +const DB_NAME = 'LlmCacheDB'; +const STORE_NAME = 'responses'; +const DB_VERSION = 1; +const TTL_MS = 24 * 60 * 60 * 1000; // 24 hours + +// ── DB helpers ──────────────────────────────────────────────────────────────── + +let _db: IDBDatabase | null = null; + +function openDb(): Promise { + if (_db) return Promise.resolve(_db); + return new Promise((resolve, reject) => { + const req = indexedDB.open(DB_NAME, DB_VERSION); + req.onerror = () => reject(req.error); + req.onupgradeneeded = (e) => { + const db = (e.target as IDBOpenDBRequest).result; + if (!db.objectStoreNames.contains(STORE_NAME)) { + const store = db.createObjectStore(STORE_NAME, { keyPath: 'key' }); + store.createIndex('expiresAt', 'expiresAt', { unique: false }); + } + }; + req.onsuccess = (e) => { + _db = (e.target as IDBOpenDBRequest).result; + resolve(_db); + }; + }); +} + +// ── Key generation ──────────────────────────────────────────────────────────── + +/** + * Simple, deterministic string hash (djb2). + * Not cryptographic – only needs to be collision-resistant within a cache. + */ +function hashString(s: string): string { + let h = 5381; + for (let i = 0; i < s.length; i++) { + h = ((h << 5) + h) ^ s.charCodeAt(i); + h = h >>> 0; // keep 32-bit unsigned + } + return h.toString(16); +} + +export interface CacheKeyParams { + model: string; + prompt: string; + /** Pass the schema object; it will be JSON-stringified for fingerprinting. */ + schema?: unknown; + temperature?: number; +} + +export function buildCacheKey(params: CacheKeyParams): string { + const raw = [ + params.model, + params.prompt, + params.schema ? JSON.stringify(params.schema) : '', + String(params.temperature ?? 1), + ].join('\x00'); + return hashString(raw); +} + +// ── Cache entry ─────────────────────────────────────────────────────────────── + +interface CacheEntry { + key: string; + response: string; + createdAt: number; + expiresAt: number; +} + +// ── Public API ──────────────────────────────────────────────────────────────── + +export async function cacheGet(key: string): Promise { + try { + const db = await openDb(); + return new Promise((resolve, reject) => { + const tx = db.transaction(STORE_NAME, 'readonly'); + const store = tx.objectStore(STORE_NAME); + const req = store.get(key); + req.onerror = () => reject(req.error); + req.onsuccess = () => { + const entry = req.result as CacheEntry | undefined; + if (!entry) { resolve(null); return; } + if (Date.now() > entry.expiresAt) { + // Expired – delete async, return null + cacheDelete(key).catch(() => {}); + resolve(null); + return; + } + resolve(entry.response); + }; + }); + } catch { + return null; // Never let cache errors break the main flow + } +} + +export async function cacheSet(key: string, response: string): Promise { + try { + const db = await openDb(); + const entry: CacheEntry = { + key, + response, + createdAt: Date.now(), + expiresAt: Date.now() + TTL_MS, + }; + return new Promise((resolve, reject) => { + const tx = db.transaction(STORE_NAME, 'readwrite'); + const store = tx.objectStore(STORE_NAME); + const req = store.put(entry); + req.onerror = () => reject(req.error); + req.onsuccess = () => resolve(); + }); + } catch { + // Cache write failures are non-fatal + } +} + +export async function cacheDelete(key: string): Promise { + try { + const db = await openDb(); + return new Promise((resolve, reject) => { + const tx = db.transaction(STORE_NAME, 'readwrite'); + const store = tx.objectStore(STORE_NAME); + const req = store.delete(key); + req.onerror = () => reject(req.error); + req.onsuccess = () => resolve(); + }); + } catch {} +} + +/** Remove all expired entries. Call this on app startup to prevent DB bloat. */ +export async function evictExpired(): Promise { + try { + const db = await openDb(); + return new Promise((resolve, reject) => { + const tx = db.transaction(STORE_NAME, 'readwrite'); + const store = tx.objectStore(STORE_NAME); + const idx = store.index('expiresAt'); + const range = IDBKeyRange.upperBound(Date.now()); + const req = idx.openCursor(range); + req.onerror = () => reject(req.error); + req.onsuccess = (e) => { + const cursor = (e.target as IDBRequest).result; + if (cursor) { + cursor.delete(); + cursor.continue(); + } else { + resolve(); + } + }; + }); + } catch {} +} + +/** Return approximate number of entries in the cache (for UI stats). */ +export async function cacheCount(): Promise { + try { + const db = await openDb(); + return new Promise((resolve, reject) => { + const tx = db.transaction(STORE_NAME, 'readonly'); + const store = tx.objectStore(STORE_NAME); + const req = store.count(); + req.onerror = () => reject(req.error); + req.onsuccess = () => resolve(req.result); + }); + } catch { + return 0; + } +} + +/** Clear all cache entries (e.g., user action). */ +export async function cacheClear(): Promise { + try { + const db = await openDb(); + return new Promise((resolve, reject) => { + const tx = db.transaction(STORE_NAME, 'readwrite'); + const store = tx.objectStore(STORE_NAME); + const req = store.clear(); + req.onerror = () => reject(req.error); + req.onsuccess = () => resolve(); + }); + } catch {} +} diff --git a/services/llmOrchestrator.ts b/services/llmOrchestrator.ts new file mode 100644 index 0000000..5333071 --- /dev/null +++ b/services/llmOrchestrator.ts @@ -0,0 +1,334 @@ +/** + * llmOrchestrator.ts + * + * Central orchestration layer for all Gemini LLM calls. + * + * Features + * ───────── + * • Prompt compaction — removes duplicate blank lines, normalises whitespace, + * deduplicates repeated instruction phrases, compresses bullet lists. + * • Context-window budgeting — hard token cap per request; long inputs are + * automatically chunked. + * • Jittered exponential back-off retry (replaces the simple retry in + * geminiService.ts). + * • Response caching — keyed by (model, compacted prompt, schema, temperature). + * Identical calls return instantly from IndexedDB with zero API spend. + * • Token-usage tracking — every call records estimated input/output tokens and + * whether it was a cache hit. The caller receives a `CallTokenUsage` record. + * + * Usage + * ───── + * import { orchestrate, OrchestrateParams } from './llmOrchestrator'; + * + * const { text, usage } = await orchestrate({ + * model: 'gemini-2.5-flash', + * prompt: longPromptString, + * schema: myResponseSchema, + * responseMimeType: 'application/json', + * }); + */ + +/// + +import { GoogleGenAI, GenerateContentResponse, Type } from '@google/genai'; +import { buildCacheKey, cacheGet, cacheSet } from './llmCache'; +import { + estimateTokens, + buildCallUsage, + CallTokenUsage, +} from './tokenEstimator'; + +// ── AI singleton (shared with geminiService) ───────────────────────────────── + +let _ai: GoogleGenAI | null = null; + +export function getAiInstance(): GoogleGenAI { + if (!_ai) { + const key = + (typeof import.meta !== 'undefined' && + (import.meta as any).env?.VITE_GOOGLE_API_KEY) || + (typeof process !== 'undefined' && process.env?.API_KEY); + if (!key) { + throw new Error( + 'API Key is missing. Please set VITE_GOOGLE_API_KEY in your .env file.' + ); + } + _ai = new GoogleGenAI({ apiKey: key as string }); + } + return _ai; +} + +// ── Constants ───────────────────────────────────────────────────────────────── + +/** Hard upper bound on estimated input tokens per single request. */ +export const MAX_INPUT_TOKENS = 32_000; + +/** Target chunk size when splitting long content for multi-pass processing. */ +export const CHUNK_TOKEN_TARGET = 8_000; + +// ── Prompt compaction ───────────────────────────────────────────────────────── + +/** + * Compact a prompt string to reduce token usage without losing information: + * + * 1. Normalise line endings to \n. + * 2. Collapse runs of blank lines to a single blank line. + * 3. Trim trailing whitespace from each line. + * 4. Remove duplicate sentences / instructions (exact match after trim). + * 5. Compress consecutive single-word bullet points into a comma list. + */ +export function compactPrompt(text: string): string { + if (!text) return text; + + // 1 & 3: normalise & trim each line + let lines = text + .replace(/\r\n?/g, '\n') + .split('\n') + .map((l) => l.trimEnd()); + + // 2: collapse consecutive blank lines + const collapsed: string[] = []; + let prevBlank = false; + for (const line of lines) { + const isBlank = line.trim() === ''; + if (isBlank && prevBlank) continue; + collapsed.push(line); + prevBlank = isBlank; + } + lines = collapsed; + + // 4: deduplicate lines that are identical instructions + const seen = new Set(); + const deduped: string[] = []; + for (const line of lines) { + const key = line.trim().toLowerCase(); + // Only deduplicate non-trivial lines (>20 chars, ends with period/colon) + if (key.length > 20 && (key.endsWith('.') || key.endsWith(':')) && seen.has(key)) { + continue; + } + seen.add(key); + deduped.push(line); + } + lines = deduped; + + // 5: compress adjacent short bullet-point lines into comma lists + const compressed: string[] = []; + let bulletBuffer: string[] = []; + + const flushBullets = () => { + if (bulletBuffer.length >= 3) { + // Emit as a single comma-separated line + const prefix = bulletBuffer[0].match(/^(\s*[-*•]\s*)/)?.[1] ?? '- '; + compressed.push( + prefix + bulletBuffer.map((b) => b.replace(/^\s*[-*•]\s*/, '')).join(', ') + ); + } else { + compressed.push(...bulletBuffer); + } + bulletBuffer = []; + }; + + for (const line of lines) { + const isBullet = /^\s*[-*•]\s+\S/.test(line); + const wordCount = line.trim().split(/\s+/).length; + if (isBullet && wordCount <= 4) { + bulletBuffer.push(line); + } else { + if (bulletBuffer.length > 0) flushBullets(); + compressed.push(line); + } + } + if (bulletBuffer.length > 0) flushBullets(); + + return compressed.join('\n').trim(); +} + +// ── Chunking ────────────────────────────────────────────────────────────────── + +/** + * Split a long text into chunks where each chunk is at most `maxTokens` + * estimated tokens. Splits prefer paragraph boundaries (double newline). + */ +export function chunkText( + text: string, + maxTokens: number = CHUNK_TOKEN_TARGET +): string[] { + if (estimateTokens(text) <= maxTokens) return [text]; + + const paragraphs = text.split(/\n{2,}/); + const chunks: string[] = []; + let current = ''; + + for (const para of paragraphs) { + const candidate = current ? `${current}\n\n${para}` : para; + if (estimateTokens(candidate) > maxTokens && current) { + chunks.push(current.trim()); + current = para; + } else { + current = candidate; + } + } + if (current.trim()) chunks.push(current.trim()); + return chunks; +} + +// ── Retry with jitter ───────────────────────────────────────────────────────── + +const BASE_DELAY_MS = 1000; +const MAX_DELAY_MS = 30_000; + +function jitter(ms: number): number { + // ±25% random jitter + return ms * (0.75 + Math.random() * 0.5); +} + +async function withRetry( + fn: () => Promise, + retries = 4 +): Promise { + for (let attempt = 0; attempt < retries; attempt++) { + try { + return await fn(); + } catch (err) { + if (attempt === retries - 1) throw err; + const delay = Math.min(BASE_DELAY_MS * Math.pow(2, attempt), MAX_DELAY_MS); + await new Promise((r) => setTimeout(r, jitter(delay))); + } + } + throw new Error('Max retries reached'); +} + +// ── Main orchestration entry-point ─────────────────────────────────────────── + +export interface OrchestrateParams { + model: string; + prompt: string; + /** JSON schema object (Gemini Type format). */ + schema?: object; + responseMimeType?: string; + temperature?: number; + /** Skip cache for this call (e.g., image generation). */ + noCache?: boolean; + /** System instruction to prepend (shared across agents). */ + systemInstruction?: string; +} + +export interface OrchestrateResult { + text: string; + usage: CallTokenUsage; +} + +export async function orchestrate( + params: OrchestrateParams +): Promise { + const { + model, + schema, + responseMimeType, + temperature, + noCache = false, + systemInstruction, + } = params; + + // 1. Compact the prompt + const compacted = compactPrompt(params.prompt); + + // 2. Budget check — warn if we're near the limit (but don't hard-block) + const inputEst = estimateTokens(compacted) + (systemInstruction ? estimateTokens(systemInstruction) : 0); + if (inputEst > MAX_INPUT_TOKENS) { + console.warn( + `[llmOrchestrator] Prompt estimated at ${inputEst} tokens, exceeds budget of ${MAX_INPUT_TOKENS}. Consider chunking.` + ); + } + + // 3. Check cache (skip for image calls and noCache) + if (!noCache) { + const cacheKey = buildCacheKey({ + model, + prompt: compacted, + schema, + temperature, + }); + + const cached = await cacheGet(cacheKey); + if (cached !== null) { + const usage = buildCallUsage(compacted, cached, true); + return { text: cached, usage }; + } + + // 4. Make the API call with retry + const rawResult = await withRetry(() => + callApi({ model, prompt: compacted, schema, responseMimeType, temperature, systemInstruction }) + ); + + // 5. Store in cache + await cacheSet(cacheKey, rawResult); + + const usage = buildCallUsage(compacted, rawResult, false); + return { text: rawResult, usage }; + } + + // noCache path (e.g., image generation) + const rawResult = await withRetry(() => + callApi({ model, prompt: compacted, schema, responseMimeType, temperature, systemInstruction }) + ); + const usage = buildCallUsage(compacted, rawResult, false); + return { text: rawResult, usage }; +} + +// ── Internal API call ───────────────────────────────────────────────────────── + +interface CallApiParams { + model: string; + prompt: string; + schema?: object; + responseMimeType?: string; + temperature?: number; + systemInstruction?: string; +} + +async function callApi(params: CallApiParams): Promise { + const ai = getAiInstance(); + + const config: Record = {}; + if (params.responseMimeType) config.responseMimeType = params.responseMimeType; + if (params.schema) config.responseSchema = params.schema; + if (params.temperature !== undefined) config.temperature = params.temperature; + if (params.systemInstruction) config.systemInstruction = params.systemInstruction; + + const response: GenerateContentResponse = await ai.models.generateContent({ + model: params.model, + contents: params.prompt, + config: Object.keys(config).length > 0 ? config : undefined, + }); + + return response.text || ''; +} + +// ── Multi-chunk orchestration ──────────────────────────────────────────────── + +/** + * Orchestrate a call where `contextText` might be very large. + * The context is split into chunks; each chunk is processed separately + * with `chunkPromptFn` and results are concatenated. + * + * Useful for full-chapter rewrites where we don't want to send 30k tokens. + */ +export async function orchestrateChunked( + contextText: string, + chunkPromptFn: (chunk: string, index: number, total: number) => string, + baseParams: Omit +): Promise<{ text: string; usages: CallTokenUsage[] }> { + const chunks = chunkText(contextText); + const results: string[] = []; + const usages: CallTokenUsage[] = []; + + for (let i = 0; i < chunks.length; i++) { + const prompt = chunkPromptFn(chunks[i], i, chunks.length); + const result = await orchestrate({ ...baseParams, prompt }); + results.push(result.text); + usages.push(result.usage); + } + + return { text: results.join('\n\n'), usages }; +} diff --git a/services/tokenEstimator.ts b/services/tokenEstimator.ts new file mode 100644 index 0000000..fa44a22 --- /dev/null +++ b/services/tokenEstimator.ts @@ -0,0 +1,99 @@ +/** + * tokenEstimator.ts + * + * Lightweight token-counting utilities. + * Uses the common "chars / 4" heuristic which is accurate to ~±10% for + * English text and avoids pulling in a full tokeniser library. + * + * All public functions are pure / synchronous so they can be used both in + * the UI thread and inside Web Workers without any async overhead. + */ + +/** Estimate the number of tokens in a text string. */ +export function estimateTokens(text: string): number { + if (!text) return 0; + return Math.ceil(text.length / 4); +} + +/** Estimate tokens for a JSON-serialisable value (schema objects, responses). */ +export function estimateObjectTokens(obj: unknown): number { + try { + return estimateTokens(JSON.stringify(obj)); + } catch { + return 0; + } +} + +/** + * Given a list of prompt parts, return the total estimated token count. + * Handy for budgeting before making a call. + */ +export function estimatePromptTokens(parts: string[]): number { + return parts.reduce((sum, p) => sum + estimateTokens(p), 0); +} + +/** Summarise token consumption for a single LLM call. */ +export interface CallTokenUsage { + inputTokens: number; + outputTokens: number; + totalTokens: number; + cacheHit: boolean; +} + +export function buildCallUsage( + inputText: string, + outputText: string, + cacheHit = false +): CallTokenUsage { + const inputTokens = estimateTokens(inputText); + const outputTokens = estimateTokens(outputText); + return { + inputTokens, + outputTokens, + totalTokens: inputTokens + outputTokens, + cacheHit, + }; +} + +/** Running aggregate for a full project session. */ +export interface SessionTokenStats { + totalInputTokens: number; + totalOutputTokens: number; + totalTokens: number; + cacheHits: number; + cacheMisses: number; + savedByCache: number; // input tokens avoided due to cache hits +} + +export function createEmptyStats(): SessionTokenStats { + return { + totalInputTokens: 0, + totalOutputTokens: 0, + totalTokens: 0, + cacheHits: 0, + cacheMisses: 0, + savedByCache: 0, + }; +} + +export function addCallToStats( + stats: SessionTokenStats, + usage: CallTokenUsage +): SessionTokenStats { + return { + totalInputTokens: stats.totalInputTokens + usage.inputTokens, + totalOutputTokens: stats.totalOutputTokens + usage.outputTokens, + totalTokens: stats.totalTokens + usage.totalTokens, + cacheHits: stats.cacheHits + (usage.cacheHit ? 1 : 0), + cacheMisses: stats.cacheMisses + (usage.cacheHit ? 0 : 1), + savedByCache: + stats.savedByCache + (usage.cacheHit ? usage.inputTokens : 0), + }; +} + +/** Human-readable helper for UI display. */ +export function formatTokenCount(n: number): string { + if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(2)}M`; + if (n >= 1_000) return `${(n / 1_000).toFixed(1)}k`; + return String(n); +}