From 6b457b46f52f6253231d6b21b6dead31b5970576 Mon Sep 17 00:00:00 2001 From: shanyuhai123 <864299347@qq.com> Date: Fri, 13 Nov 2026 08:07:18 +0800 Subject: [PATCH] =?UTF-8?q?fix(buddy):=20=E4=BF=AE=E5=A4=8D=E7=94=9F?= =?UTF-8?q?=E5=9B=BE=E5=BC=95=E7=94=A8=E4=B8=8E=E4=BF=9D=E5=AD=98=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E6=81=A2=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/buddy/service/src/BuddyService.ts | 3 + .../buddyComposerInputBoundary.spec.ts | 2 +- .../src/agent/extensions/BuddyCapability.ts | 2 + .../createBuddySessionExtensions.ts | 3 + .../agent/extensions/toolMediaExtension.ts | 2 +- .../src/agent/sessions/BuddySessionFactory.ts | 3 + .../service/src/artifacts/ArtifactService.ts | 45 +- .../__tests__/ArtifactService.spec.ts | 38 ++ .../src/createBuddyCapabilityFactory.ts | 25 +- .../src/images/ImageGenerationGateway.ts | 12 +- .../images/ImageGenerationMaterialStore.ts | 77 +++ .../src/images/ImageGenerationService.ts | 104 +-- .../src/images/ImageReferenceAccess.ts | 219 ++++++ .../service/src/images/ImageRequestStore.ts | 34 +- .../ImageGenerationMaterialStore.spec.ts | 154 +++++ .../__tests__/ImageGenerationService.spec.ts | 626 +++++++++++++----- .../__tests__/ImageReferenceAccess.spec.ts | 278 ++++++++ .../__tests__/ImageRequestStore.spec.ts | 95 +++ .../imageGenerationExtension.spec.ts | 112 +++- .../imageGenerationLifecycle.spec.ts | 48 +- .../imageGenerationSchemaSession.spec.ts | 10 +- .../imageGenerationToolContract.spec.ts | 45 ++ .../imageGenerationToolParameters.spec.ts | 19 +- .../__tests__/imagePublicationFixture.ts | 26 + .../src/images/imageGenerationExtension.ts | 102 ++- .../src/images/imageGenerationToolContract.ts | 17 +- .../images/imageGenerationToolParameters.ts | 21 +- .../src/images/imageReferenceExtension.ts | 50 ++ .../service/src/images/imageToolGroup.ts | 2 +- apps/buddy/shared/images/imageGeneration.ts | 64 ++ 30 files changed, 1918 insertions(+), 320 deletions(-) create mode 100644 apps/buddy/service/src/images/ImageGenerationMaterialStore.ts create mode 100644 apps/buddy/service/src/images/ImageReferenceAccess.ts create mode 100644 apps/buddy/service/src/images/__tests__/ImageGenerationMaterialStore.spec.ts create mode 100644 apps/buddy/service/src/images/__tests__/ImageReferenceAccess.spec.ts create mode 100644 apps/buddy/service/src/images/__tests__/ImageRequestStore.spec.ts create mode 100644 apps/buddy/service/src/images/__tests__/imageGenerationToolContract.spec.ts create mode 100644 apps/buddy/service/src/images/__tests__/imagePublicationFixture.ts create mode 100644 apps/buddy/service/src/images/imageReferenceExtension.ts diff --git a/apps/buddy/service/src/BuddyService.ts b/apps/buddy/service/src/BuddyService.ts index d9cba83e6..a51c65678 100644 --- a/apps/buddy/service/src/BuddyService.ts +++ b/apps/buddy/service/src/BuddyService.ts @@ -1102,6 +1102,9 @@ export async function startBuddyService( webService, conversations, publications: treeRepository, + executions: executionRecords, + runInputs, + runEvents: options.eventLog, references: sessionReferences, referenceSnapshots: extensionReferences, }, petActions), diff --git a/apps/buddy/service/src/agent/context/__tests__/buddyComposerInputBoundary.spec.ts b/apps/buddy/service/src/agent/context/__tests__/buddyComposerInputBoundary.spec.ts index 9307896ed..734f2fbe8 100644 --- a/apps/buddy/service/src/agent/context/__tests__/buddyComposerInputBoundary.spec.ts +++ b/apps/buddy/service/src/agent/context/__tests__/buddyComposerInputBoundary.spec.ts @@ -252,7 +252,7 @@ describe('composer input at the Buddy session boundary', () => { expect(fixture.contexts).toHaveLength(2) expect(userMessages(fixture.contexts[1])[0]?.content).toEqual(materializedContent(fixture)) expect(fixture.contexts[1]?.messages).toContainEqual(expect.objectContaining({ - content: [fixture.nativeToolImage, { type: 'text', text: expect.stringMatching(/^Saved image references \(in order\): image:[a-f0-9]{64}/) }], + content: [fixture.nativeToolImage, { type: 'text', text: expect.stringMatching(/^Inspection-only image handles \(in order\): image:[a-f0-9]{64}/) }], role: 'toolResult', toolName: 'lexora_fixture_image', })) diff --git a/apps/buddy/service/src/agent/extensions/BuddyCapability.ts b/apps/buddy/service/src/agent/extensions/BuddyCapability.ts index 21b9167c1..68c6f1a8f 100644 --- a/apps/buddy/service/src/agent/extensions/BuddyCapability.ts +++ b/apps/buddy/service/src/agent/extensions/BuddyCapability.ts @@ -6,6 +6,7 @@ import type { BuddySessionMode } from '../../../../shared/permissions/sessionMod import type { BuddyToolClassification, BuddyToolClassificationResult } from '../../approvals/toolClassification' import type { CapturedFileWriterFactory } from '../../changes/CapturedFileWriter' import type { DirectoryGrant } from '../../directories/resolveGrantedPath' +import type { ImageGenerationMaterialWriter } from '../../images/ImageGenerationMaterialStore' import type { BuddyInProcessExtension } from './BuddyInProcessExtension' import type { BuddyMutationEffect } from './changeCaptureExtension' import type { BuddyToolDisclosurePolicy } from './discovery/toolDiscoveryContract' @@ -28,6 +29,7 @@ export interface BuddyCapability { } export interface BuddyCapabilityContext { + writeGeneratedImageMaterial?: ImageGenerationMaterialWriter createFileWriter?: CapturedFileWriterFactory authorizeNetwork?: (input: { toolCallId: string, toolName: string, url: string, operation?: NetworkOperation, signal?: AbortSignal }) => Promise conversationId: string diff --git a/apps/buddy/service/src/agent/extensions/createBuddySessionExtensions.ts b/apps/buddy/service/src/agent/extensions/createBuddySessionExtensions.ts index e34c224ff..8852af4b8 100644 --- a/apps/buddy/service/src/agent/extensions/createBuddySessionExtensions.ts +++ b/apps/buddy/service/src/agent/extensions/createBuddySessionExtensions.ts @@ -18,6 +18,7 @@ import type { MutationCoordinator } from '../../changes/MutationCoordinator' import type { DirectoryGrantService } from '../../directories/DirectoryGrantService' import type { DirectoryGrant } from '../../directories/resolveGrantedPath' import type { FileResourceCoordinator } from '../../files/FileResourceCoordinator' +import type { ImageGenerationMaterialWriter } from '../../images/ImageGenerationMaterialStore' import type { ImageReadingMaterialWriter } from '../../images/ImageReadingMaterialStore' import type { ImageReadingService } from '../../images/ImageReadingService' import type { NetworkPermissionsStore } from '../../permissions/NetworkPermissionsClient' @@ -84,6 +85,7 @@ export interface BuddySessionExtensionServices { export interface CreateBuddySessionExtensionsOptions { imageStore?: ImageBlobStore writeImageMaterial?: ImageReadingMaterialWriter + writeGeneratedImageMaterial?: ImageGenerationMaterialWriter actorKind?: 'actor' | 'npc' approvalPolicy: BuddyApprovalPolicy canonicalRoot: string @@ -199,6 +201,7 @@ export async function createBuddySessionExtensions( : { id: options.conversationId, kind: 'conversation' }, }) capabilities.push(codemode, ...await services.createCapabilities({ + writeGeneratedImageMaterial: options.writeGeneratedImageMaterial, createFileWriter: (event) => { const run = runContext.current if (!run) diff --git a/apps/buddy/service/src/agent/extensions/toolMediaExtension.ts b/apps/buddy/service/src/agent/extensions/toolMediaExtension.ts index 1f01a24ac..24bcc40e0 100644 --- a/apps/buddy/service/src/agent/extensions/toolMediaExtension.ts +++ b/apps/buddy/service/src/agent/extensions/toolMediaExtension.ts @@ -22,7 +22,7 @@ export function createToolMediaExtension(store: ImageBlobStore): BuddyInProcessE content.push({ ...block, data: '' }) } if (images.length) { - content.push({ type: 'text' as const, text: `Saved image references (in order): ${images.map(image => `image:${image.sha256}`).join(', ')}. Use lexora_image_read with these references for focused inspection or comparison when needed.` }) + content.push({ type: 'text' as const, text: `Inspection-only image handles (in order): ${images.map(image => `image:${image.sha256}`).join(', ')}. These handles work only with lexora_image_read, not with lexora_image_generate.referenceIds. For generation, keep the original attachmentId/artifactId values or use lexora_image_list; reading an image does not create a generation reference ID.` }) return { message: { ...message, content, buddyToolMedia: { version: 1, images } } satisfies ToolMediaReferenceMessage } } }) diff --git a/apps/buddy/service/src/agent/sessions/BuddySessionFactory.ts b/apps/buddy/service/src/agent/sessions/BuddySessionFactory.ts index 9afc4a407..f07ab34da 100644 --- a/apps/buddy/service/src/agent/sessions/BuddySessionFactory.ts +++ b/apps/buddy/service/src/agent/sessions/BuddySessionFactory.ts @@ -14,6 +14,7 @@ import { isExecutionProfileWithin } from '../../../../shared/permissions/executi import { AttachmentToolWorkspace } from '../../attachments/AttachmentToolWorkspace' import { conversationImageDirectory, ImageBlobStore } from '../../attachments/ImageBlobStore' import { canReadReferenceSnapshots } from '../../conversations/referenceSnapshotToolContract' +import { ImageGenerationMaterialStore } from '../../images/ImageGenerationMaterialStore' import { ImageReadingMaterialStore } from '../../images/ImageReadingMaterialStore' import { supportsModelToolCalls } from '../../providers/modelCapabilities' import { BuddyAgentRunError } from '../../runs/runError' @@ -102,12 +103,14 @@ export class BuddySessionFactory { providerId: effectiveModel?.providerId ?? run.provider, }) const imageMaterials = new ImageReadingMaterialStore(blueprint.scratchRoot, blueprint.conversationId) + const generatedMaterials = new ImageGenerationMaterialStore(blueprint.scratchRoot, blueprint.conversationId) const imageStore = new ImageBlobStore(conversationImageDirectory(join(resolve(this.#options.conversationsDirectory), blueprint.conversationId))) const extensions = await events.operation('session.resources', async () => { if (run.purpose !== 'conversation.compaction') await this.#options.services.prepareForRun?.(input.signal) return createBuddySessionExtensions({ imageStore, + writeGeneratedImageMaterial: (image, grants, writeFiles, signal) => generatedMaterials.write(image, grants, writeFiles, signal), writeImageMaterial: (text, signal) => imageMaterials.write(text, signal), actorKind: blueprint.actor?.kind, toolScope: run.purpose === 'conversation.compaction' ? { mode: 'none' } : blueprint.actor?.settings.tools, diff --git a/apps/buddy/service/src/artifacts/ArtifactService.ts b/apps/buddy/service/src/artifacts/ArtifactService.ts index 08b6a77f1..8f17a75d4 100644 --- a/apps/buddy/service/src/artifacts/ArtifactService.ts +++ b/apps/buddy/service/src/artifacts/ArtifactService.ts @@ -7,7 +7,8 @@ import type { ArtifactRepository, } from '../storage/artifactRepository' import { randomUUID } from 'node:crypto' -import { stat } from 'node:fs/promises' +import { constants } from 'node:fs' +import { access, stat } from 'node:fs/promises' import { basename, dirname, @@ -472,8 +473,8 @@ export class ArtifactService { export class ArtifactError extends Error { readonly code: string - constructor(code: string) { - super('Lexora Buddy artifact operation failed') + constructor(code: string, options?: ErrorOptions) { + super('Lexora Buddy artifact operation failed', options) this.name = 'ArtifactError' this.code = code } @@ -486,7 +487,7 @@ export class ArtifactPublicationError extends ArtifactError { const code = error && typeof error === 'object' && 'code' in error && typeof error.code === 'string' && /^[A-Z][A-Z0-9_]{0,79}$/.test(error.code) ? error.code : 'ARTIFACT_PUBLICATION_FAILED' - super(code) + super(code, { cause: error }) this.name = 'ArtifactPublicationError' this.receipt = copyEventSnapshot(receipt) } @@ -519,6 +520,42 @@ async function resolveArtifactLocation( } } +export async function preflightGeneratedImageOutput(input: Pick, signal?: AbortSignal): Promise { + signal?.throwIfAborted() + if (!input.outputPath.trim()) + throw new ArtifactError('VALIDATION_FAILED') + const basePath = isAbsolute(input.outputPath) ? input.outputPath : resolve(input.cwd, input.outputPath) + const location = await resolveArtifactLocation(input.grants, generatedImagePath(basePath, 0, 'image/png'), 'create') + signal?.throwIfAborted() + const target = await stat(location.canonicalPath).catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOENT') + throw error + return null + }) + if (target) { + if (!target.isFile()) + throw new ArtifactError('VALIDATION_FAILED') + await access(location.canonicalPath, constants.R_OK | constants.W_OK) + } + let parent = dirname(location.canonicalPath) + while (true) { + signal?.throwIfAborted() + const metadata = await stat(parent).catch((error: NodeJS.ErrnoException) => { + if (error.code !== 'ENOENT' || dirname(parent) === parent) + throw error + return null + }) + if (metadata) { + if (!metadata.isDirectory()) + throw new ArtifactError('VALIDATION_FAILED') + await access(parent, constants.W_OK | constants.X_OK) + signal?.throwIfAborted() + return + } + parent = dirname(parent) + } +} + function generatedImagePath(basePath: string, index: number, mimeType: string): string { const expectedExtension = extensionForImageMimeType(mimeType) const extension = extname(basePath).toLowerCase() diff --git a/apps/buddy/service/src/artifacts/__tests__/ArtifactService.spec.ts b/apps/buddy/service/src/artifacts/__tests__/ArtifactService.spec.ts index 07b9337e7..b6a504c21 100644 --- a/apps/buddy/service/src/artifacts/__tests__/ArtifactService.spec.ts +++ b/apps/buddy/service/src/artifacts/__tests__/ArtifactService.spec.ts @@ -1,3 +1,4 @@ +import type { Model } from '@earendil-works/pi-ai' import type { DatabaseSync } from 'node:sqlite' import type { CapturedFileWriter } from '../../changes/CapturedFileWriter' import type { DirectoryGrant } from '../../directories/resolveGrantedPath' @@ -12,6 +13,8 @@ import { afterEach, describe, expect, it } from 'vitest' import { MutationCoordinator } from '../../changes/MutationCoordinator' import { FileResourceCoordinator } from '../../files/FileResourceCoordinator' import { NativeFileEngine } from '../../files/NativeFileEngine' +import { ImageGenerationMaterialStore } from '../../images/ImageGenerationMaterialStore' +import { ImageGenerationService } from '../../images/ImageGenerationService' import { createArtifactRepository } from '../../storage/artifactRepository' import { createConversationDirectoryGrantRepository } from '../../storage/conversationDirectoryGrantRepository' import { createConversationRepository } from '../../storage/conversationRepository' @@ -31,6 +34,41 @@ afterEach(async () => { }) describe('artifactService', () => { + it('generates into missing parents without creating them before submission and preserves numbered PNG outputs', async () => { + const f = await createFixture() + const bytes = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAAFElEQVR4AQEJAPb/AAD/AP//AAD/EPgD/UE7EkoAAAAASUVORK5CYII=', 'base64') + let submissions = 0 + const materials = new ImageGenerationMaterialStore(f.workspace, 'conversation-1') + const service = new ImageGenerationService({ + writeGeneratedImageMaterial: (image, grants, writeFiles, signal) => materials.write(image, grants, writeFiles, signal), + conversationId: 'conversation-1', + cwd: f.workspace, + grants: f.grants, + artifactService: f.service, + attachmentService: { materializeConversationImages: async () => ({ images: [], records: [] }) }, + referenceAccess: { authorize: () => {} }, + imageGenerationGateway: { supports: () => true, generate: async (input) => { + await expect(readFile(join(f.workspace, 'output'))).rejects.toMatchObject({ code: 'ENOENT' }) + await input.onSubmitting?.() + submissions++ + return { images: [{ bytes, mimeType: 'image/png' }, { bytes, mimeType: 'image/png' }], responseId: 'response-1' } + } }, + }) + try { + const result = await service.generate({ outputPath: 'output/nested/heart', prompt: 'Create' }, { provider: 'fixture', id: 'fixture' } as Model<'openai-responses'>, new AbortController().signal, f.grants, f.writeFiles) + const paths = ['heart.png', 'heart-2.png'].map(name => join(f.workspace, 'output', 'nested', name)) + expect(submissions).toBe(1) + expect(result.outputs.map(output => output.path)).toEqual(paths) + expect(result.artifactIds).toHaveLength(2) + expect(f.repository.listForConversation('conversation-1').map(record => record.currentPath).sort()).toEqual([...paths].sort()) + for (const path of paths) + expect(await readFile(path)).toEqual(bytes) + } + finally { + await service.dispose() + } + }) + it('keeps delivered files and extensionless image MIME readable after grant revocation and service recreation without granting new outputs', async () => { const f = await createFixture() const conversationGrants = createConversationDirectoryGrantRepository(f.database) diff --git a/apps/buddy/service/src/createBuddyCapabilityFactory.ts b/apps/buddy/service/src/createBuddyCapabilityFactory.ts index 627e917e7..9a6921a64 100644 --- a/apps/buddy/service/src/createBuddyCapabilityFactory.ts +++ b/apps/buddy/service/src/createBuddyCapabilityFactory.ts @@ -12,6 +12,7 @@ import type { McpConnectorService } from './connectors/mcp/McpConnectorService' import type { SessionReferenceServices } from './conversations/SessionReferenceService' import type { ImageGenerationGateway } from './images/ImageGenerationGateway' import type { ImageGenerationServiceOptions } from './images/ImageGenerationService' +import type { ImageReferenceAccessOptions } from './images/ImageReferenceAccess' import type { ImageRequestStore } from './images/ImageRequestStore' import type { ImageTransformService } from './images/ImageTransformService' import type { PetActionService } from './pet/PetActionService' @@ -31,6 +32,8 @@ import { createSessionReferenceCapability } from './conversations/sessionReferen import { createImageGenerationCapability } from './images/imageGenerationExtension' import { ImageGenerationService } from './images/ImageGenerationService' import { observeImageDiagnostics } from './images/ImageOperationLifecycle' +import { ImageReferenceAccess } from './images/ImageReferenceAccess' +import { createImageReferenceCapability } from './images/imageReferenceExtension' import { createImageTransformCapability } from './images/imageTransformExtension' import { createPetCapability } from './pet/petExtension' import { createPluginAuthoringCapability } from './plugins/pluginAuthoringCapability' @@ -58,7 +61,10 @@ export interface BuddyCapabilityServices { imageTransformService: Pick webService: Pick conversations: SessionReferenceServices['conversations'] - publications: SessionReferenceServices['publications'] + publications: SessionReferenceServices['publications'] & ImageReferenceAccessOptions['publications'] + executions: ImageReferenceAccessOptions['executions'] + runInputs: ImageReferenceAccessOptions['inputs'] + runEvents: ImageReferenceAccessOptions['events'] references: SessionReferenceRepository referenceSnapshots: ExtensionReferenceRepository } @@ -89,6 +95,16 @@ export function createBuddyCapabilityFactory( ? Promise.reject(new Error('EXECUTION_PROFILE_READ_ONLY')) : writeMcpResult(bytes, mimeType, signal)) context.signal.throwIfAborted() + const imageReferences = new ImageReferenceAccess({ + ...context, + conversations: services.conversations, + publications: services.publications, + executions: services.executions, + inputs: services.runInputs, + events: services.runEvents, + attachments: services.attachmentService, + artifacts: services.artifactService, + }) const capabilities = [ createMcpCapability(mcp, context.isCodemodeEnabled ?? (() => false)), createBrowserCapability({ @@ -114,7 +130,8 @@ export function createBuddyCapabilityFactory( : undefined, }), createWebCapability({ authorizeNetwork: context.authorizeNetwork, service: services.webService, conversationId: context.conversationId }), - createImageCapability(context, services), + createImageCapability(context, services, imageReferences), + createImageReferenceCapability(imageReferences), createImageTransformCapability({ ...context, service: services.imageTransformService }), createOutputPresentationCapability({ ...context, artifactService: services.artifactService }), createSessionReferenceCapability({ conversationId: context.conversationId, getRunId: context.getRunId, conversations: services.conversations, attachments: services.attachmentService, artifacts: services.artifactService, publications: services.publications, references: services.references }), @@ -147,10 +164,12 @@ export function createBuddyCapabilityFactory( } } -function createImageCapability(context: BuddyCapabilityContext, services: BuddyCapabilityServices): BuddyCapability { +function createImageCapability(context: BuddyCapabilityContext, services: BuddyCapabilityServices, referenceAccess: ImageReferenceAccess): BuddyCapability { const service = new ImageGenerationService({ + writeGeneratedImageMaterial: context.writeGeneratedImageMaterial, artifactService: services.artifactService, attachmentService: services.attachmentService, + referenceAccess, conversationId: context.conversationId, cwd: context.cwd, get grants() { return context.grants }, diff --git a/apps/buddy/service/src/images/ImageGenerationGateway.ts b/apps/buddy/service/src/images/ImageGenerationGateway.ts index 19fe10882..4a8de852c 100644 --- a/apps/buddy/service/src/images/ImageGenerationGateway.ts +++ b/apps/buddy/service/src/images/ImageGenerationGateway.ts @@ -1,7 +1,7 @@ import type { Api, ImageContent, Model } from '@earendil-works/pi-ai' -import type { ImageGenerationErrorCode, ImageGenerationErrorDiagnostic, ImageGenerationOutcome, ImageRequestRef } from '../../../shared/images/imageGeneration' +import type { ImageGenerationErrorCode, ImageGenerationErrorDiagnostic, ImageGenerationOutcome, ImageGenerationRetainedOutput, ImageRequestRef } from '../../../shared/images/imageGeneration' import type { ImageOutputOptions, ImagePreferencesSnapshot, ImageSource, ImageSupport, ImageTarget } from '../../../shared/images/imagePreferences' -import { readImageDiagnostic } from '../../../shared/images/imageGeneration' +import { imageGenerationRetainedOutputSchema, readImageDiagnostic } from '../../../shared/images/imageGeneration' export { IMAGE_GENERATION_ERROR_CODES } from '../../../shared/images/imageGeneration' export type { ImageGenerationErrorCode, ImageGenerationErrorDiagnostic } from '../../../shared/images/imageGeneration' @@ -51,10 +51,11 @@ export class ImageGenerationError extends Error { readonly diagnostic: ImageGenerationErrorDiagnostic | null readonly request?: ImageRequestRef readonly artifactIds: string[] + readonly retainedOutputs?: ImageGenerationRetainedOutput[] constructor( code: ImageGenerationErrorCode, - options?: ErrorOptions & { diagnostic?: ImageGenerationErrorDiagnostic, retryable?: boolean, outcome?: ImageGenerationOutcome, request?: ImageRequestRef, artifactIds?: readonly string[] }, + options?: ErrorOptions & { diagnostic?: ImageGenerationErrorDiagnostic, retryable?: boolean, outcome?: ImageGenerationOutcome, request?: ImageRequestRef, artifactIds?: readonly string[], retainedOutputs?: readonly ImageGenerationRetainedOutput[] }, ) { super('Lexora Buddy image generation failed', { cause: options?.cause }) this.name = 'ImageGenerationError' @@ -64,6 +65,8 @@ export class ImageGenerationError extends Error { this.diagnostic = readImageDiagnostic(options?.diagnostic) this.request = options?.request this.artifactIds = [...options?.artifactIds ?? []] + if (options?.retainedOutputs) + this.retainedOutputs = options.retainedOutputs.map(output => imageGenerationRetainedOutputSchema.parse(output)) } } @@ -73,8 +76,9 @@ export function readImageGenerationError(error: unknown): { outcome: ImageGenerationOutcome request?: ImageRequestRef artifactIds?: string[] + retainedOutputs?: ImageGenerationRetainedOutput[] } { return error instanceof ImageGenerationError - ? { code: error.code, diagnostic: error.diagnostic, outcome: error.outcome, artifactIds: error.artifactIds, ...(error.request ? { request: error.request } : {}) } + ? { code: error.code, diagnostic: error.diagnostic, outcome: error.outcome, artifactIds: error.artifactIds, ...(error.retainedOutputs ? { retainedOutputs: error.retainedOutputs } : {}), ...(error.request ? { request: error.request } : {}) } : { code: 'IMAGE_GENERATION_FAILED', diagnostic: null, outcome: 'unknown' } } diff --git a/apps/buddy/service/src/images/ImageGenerationMaterialStore.ts b/apps/buddy/service/src/images/ImageGenerationMaterialStore.ts new file mode 100644 index 000000000..c144864cd --- /dev/null +++ b/apps/buddy/service/src/images/ImageGenerationMaterialStore.ts @@ -0,0 +1,77 @@ +import type { ImageGenerationRetainedOutput } from '../../../shared/images/imageGeneration' +import type { CapturedFileWriter } from '../changes/CapturedFileWriter' +import type { DirectoryGrant } from '../directories/resolveGrantedPath' +import { createHash, randomUUID } from 'node:crypto' +import { lstat, realpath } from 'node:fs/promises' +import { join } from 'node:path' +import { resolveGrantedPath } from '../directories/resolveGrantedPath' + +export type ImageGenerationMaterialWriter = ( + image: { bytes: Uint8Array, mimeType: 'image/png' }, + grants: readonly DirectoryGrant[], + writeFiles: CapturedFileWriter, + signal?: AbortSignal, +) => Promise + +export class ImageGenerationMaterialError extends Error { + readonly retainedOutput: ImageGenerationRetainedOutput + + constructor(cause: unknown, retainedOutput: ImageGenerationRetainedOutput) { + super('Image generation material publication cleanup failed', { cause }) + this.name = 'ImageGenerationMaterialError' + this.retainedOutput = retainedOutput + } +} + +export class ImageGenerationMaterialStore { + readonly #scratchRoot: string + readonly #conversationId: string + + constructor(scratchRoot: string, conversationId: string) { + if (!/^[A-Z0-9][\w-]{0,127}$/i.test(conversationId)) + throw new Error('Invalid image generation material owner') + this.#scratchRoot = scratchRoot + this.#conversationId = conversationId + } + + async write(image: Parameters[0], grants: readonly DirectoryGrant[], writeFiles: CapturedFileWriter, signal?: AbortSignal): Promise { + const root = join(this.#scratchRoot, '.image-generation') + const directory = join(root, this.#conversationId) + const path = join(directory, `${randomUUID()}.png`) + const assertCurrent = async () => { + signal?.throwIfAborted() + await resolveGrantedPath(grants, this.#scratchRoot, 'existing') + for (const candidate of [this.#scratchRoot, root, directory]) { + const metadata = await lstat(candidate).catch((error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT' && candidate !== this.#scratchRoot) + return null + throw error + }) + if (metadata && (!metadata.isDirectory() || await realpath(candidate) !== candidate)) + throw identityChanged() + } + if ((await resolveGrantedPath(grants, path, 'create')).canonicalPath !== path) + throw identityChanged() + signal?.throwIfAborted() + } + await assertCurrent() + const output: ImageGenerationRetainedOutput = { path, mimeType: 'image/png', sizeBytes: image.bytes.byteLength, sha256: createHash('sha256').update(image.bytes).digest('hex') } + let retained = false + try { + await writeFiles([path], async (publish) => { + await publish({ path, bytes: image.bytes, size: image.bytes.byteLength, overwrite: false, assertCurrent }) + retained = true + }) + } + catch (error) { + if (retained) + throw new ImageGenerationMaterialError(error, output) + throw error + } + return output + } +} + +function identityChanged(): Error { + return Object.assign(new Error('Image generation material identity changed'), { code: 'IMAGE_MATERIAL_IDENTITY_CHANGED' }) +} diff --git a/apps/buddy/service/src/images/ImageGenerationService.ts b/apps/buddy/service/src/images/ImageGenerationService.ts index fea923292..26ae2bc2c 100644 --- a/apps/buddy/service/src/images/ImageGenerationService.ts +++ b/apps/buddy/service/src/images/ImageGenerationService.ts @@ -1,16 +1,22 @@ import type { Api, ImageContent, Model } from '@earendil-works/pi-ai' +import type { ImageGenerationRetainedOutput } from '../../../shared/images/imageGeneration' import type { ImageOutputOptions, ImagePreferencesSnapshot } from '../../../shared/images/imagePreferences' import type { CapturedFileWriter } from '../changes/CapturedFileWriter' import type { DirectoryGrant } from '../directories/resolveGrantedPath' import type { ImageGenerationGateway } from './ImageGenerationGateway' +import type { ImageGenerationMaterialWriter } from './ImageGenerationMaterialStore' +import type { ImageReferenceAccess } from './ImageReferenceAccess' import type { ImageRequestStore } from './ImageRequestStore' +import { Buffer } from 'node:buffer' import { createHash, randomUUID } from 'node:crypto' import { extname, resolve } from 'node:path' import { diagnosticTransportCode } from '../../../shared/diagnostics/providerRequestDiagnostic' import { copyEventSnapshot } from '../../../shared/events/eventSnapshot' +import { readImageFileErrorCode } from '../../../shared/images/imageGeneration' import { imageOutputOptionsSchema } from '../../../shared/images/imagePreferences' -import { ArtifactPublicationError } from '../artifacts/ArtifactService' +import { ArtifactPublicationError, preflightGeneratedImageOutput } from '../artifacts/ArtifactService' import { ImageGenerationError, readImageGenerationError } from './ImageGenerationGateway' +import { ImageGenerationMaterialError } from './ImageGenerationMaterialStore' import { ImageOperationLifecycle } from './ImageOperationLifecycle' import { validateGeneratedImages } from './imageResponseBody' @@ -18,7 +24,7 @@ export interface GenerateConversationImageInput { output?: Partial outputPath: string prompt: string - reference?: { mode: 'latest' } | { mode: 'resources', resourceIds: readonly string[] } + referenceIds?: readonly string[] } interface ConversationImageSource { @@ -30,9 +36,9 @@ interface ConversationImageSource { } export interface ImageGenerationServiceOptions { + writeGeneratedImageMaterial?: ImageGenerationMaterialWriter requests?: Pick artifactService: ConversationImageSource & { - listConversationArtifacts: (conversationId: string) => Array<{ id: string, kind: string, mimeType: string, createdAt: string, updatedAt: string }> registerGeneratedImages: (input: { conversationId: string cwd: string @@ -42,9 +48,8 @@ export interface ImageGenerationServiceOptions { sourceArtifactId: string | null }, writeFiles?: CapturedFileWriter) => Promise> } - attachmentService: ConversationImageSource & { - listForConversation: (conversationId: string) => Array<{ id: string, mimeType: string, createdAt: string }> - } + attachmentService: ConversationImageSource + referenceAccess: Pick conversationId: string cwd: string grants: readonly DirectoryGrant[] @@ -60,8 +65,10 @@ export class ImageGenerationService { constructor(options: ImageGenerationServiceOptions) { this.#options = { requests: options.requests, + writeGeneratedImageMaterial: options.writeGeneratedImageMaterial, artifactService: options.artifactService, attachmentService: options.attachmentService, + referenceAccess: options.referenceAccess, conversationId: options.conversationId, cwd: options.cwd, get grants() { return options.grants }, @@ -91,16 +98,18 @@ export class ImageGenerationService { throw new ImageGenerationError('IMAGE_GENERATION_UNSUPPORTED', { outcome: 'failed' }) let references: { artifactIds: string[], images: ImageContent[] } = { artifactIds: [], images: [] } try { - if (request.reference) - references = await this.#materializeReferences(request.reference, directories, signal) + if (request.referenceIds) { + if (!request.referenceIds.length || new Set(request.referenceIds).size !== request.referenceIds.length) + throw new Error('INVALID_IMAGE_REFERENCES') + await this.#options.referenceAccess.authorize(request.referenceIds, signal) + references = await this.#materializeReferences(request.referenceIds, directories, signal) + } } catch (cause) { signal.throwIfAborted() throw new ImageGenerationError('IMAGE_GENERATION_REFERENCE_UNAVAILABLE', { cause, outcome: 'failed' }) } signal.throwIfAborted() - if (request.reference && !references.images.length) - throw new ImageGenerationError('IMAGE_GENERATION_REFERENCE_UNAVAILABLE', { outcome: 'failed' }) const output = imageOutputOptionsSchema.parse(request.output ?? {}) const destination = resolve(this.#options.cwd, request.outputPath.trim()) const hash = (value: string) => createHash('sha256').update(value).digest('hex') @@ -108,7 +117,11 @@ export class ImageGenerationService { ? await this.#options.requests.begin(this.#options.conversationId, invocation.runId, invocation.toolCallId, { fingerprint: hash(JSON.stringify({ prompt: request.prompt.trim(), output, references: references.images.map(image => ({ mimeType: image.mimeType, digest: hash(image.data) })) })), destination: hash(extname(destination) ? destination : `${destination}.png`), - }) + }, references.images.map((image, index) => ({ + referenceId: request.referenceIds![index]!, + mimeType: image.mimeType, + sha256: createHash('sha256').update(Buffer.from(image.data, 'base64')).digest('hex'), + }))) : undefined if (begun?.kind === 'completed') return { value: { ...begun.result, request: begun.request }, artifactIds: begun.result.artifactIds } @@ -116,8 +129,20 @@ export class ImageGenerationService { let submitted = false let responseId: string | null = null let artifactIds: string[] = [] + const retainedOutputs: ImageGenerationRetainedOutput[] = [] let stage: 'preparing' | 'submitting' | 'receiving' | 'publishing' = 'preparing' try { + try { + await preflightGeneratedImageOutput({ cwd: this.#options.cwd, grants: directories, outputPath: request.outputPath.trim() }, signal) + } + catch (cause) { + signal.throwIfAborted() + throw new ImageGenerationError('IMAGE_GENERATION_OUTPUT_UNAVAILABLE', { cause, outcome: 'failed', diagnostic: { fileErrorCode: readImageFileErrorCode(cause) ?? 'ARTIFACT_PUBLICATION_FAILED' } }) + } + signal.throwIfAborted() + const writeMaterial = this.#options.writeGeneratedImageMaterial + if (!writeMaterial || !writeFiles) + throw new ImageGenerationError('IMAGE_GENERATION_FAILED', { outcome: 'failed', diagnostic: { fileErrorCode: 'IMAGE_MATERIAL_UNAVAILABLE' } }) progress('processing') const generated = await this.#options.imageGenerationGateway.generate({ requestId: receipt?.request.requestId ?? randomUUID(), @@ -140,6 +165,18 @@ export class ImageGenerationService { progress('result-received') signal.throwIfAborted() stage = 'publishing' + for (const image of generated.images) { + signal.throwIfAborted() + try { + retainedOutputs.push(await writeMaterial(image, directories, writeFiles, signal)) + } + catch (error) { + if (error instanceof ImageGenerationMaterialError) + retainedOutputs.push(error.retainedOutput) + throw error + } + await receipt?.retain(retainedOutputs) + } progress('publishing') signal?.throwIfAborted() const artifacts = await this.#options.artifactService.registerGeneratedImages({ @@ -158,13 +195,15 @@ export class ImageGenerationService { } catch (error) { const publication = error instanceof ArtifactPublicationError ? error.receipt : undefined + const fileErrorCode = readImageFileErrorCode(error) ?? (stage === 'publishing' ? 'ARTIFACT_PUBLICATION_FAILED' : undefined) const failure = error instanceof ImageGenerationError ? readImageGenerationError(error) - : { code: 'IMAGE_GENERATION_FAILED' as const, outcome: stage === 'publishing' ? 'partial' as const : submitted || stage !== 'preparing' ? 'unknown' as const : 'failed' as const, diagnostic: { transportCode: diagnosticTransportCode(error) } } - const diagnostic = { stage, ...(responseId ? { responseId } : {}), ...(receipt ? { localRequestId: receipt.request.requestId } : {}), ...failure.diagnostic } - const normalized = new ImageGenerationError(failure.code, { cause: error, outcome: failure.outcome, diagnostic, request: receipt?.request, artifactIds: publication?.artifactIds ?? artifactIds }) - await receipt?.fail({ code: normalized.code, outcome: normalized.outcome, diagnostic: normalized.diagnostic }, normalized.artifactIds) - signal.throwIfAborted() + : { code: 'IMAGE_GENERATION_FAILED' as const, outcome: stage === 'publishing' ? 'partial' as const : submitted || stage !== 'preparing' ? 'unknown' as const : 'failed' as const, diagnostic: fileErrorCode ? { fileErrorCode } : { transportCode: diagnosticTransportCode(error) } } + const diagnostic = { stage, ...(fileErrorCode ? { fileErrorCode } : {}), ...(responseId ? { responseId } : {}), ...(receipt ? { localRequestId: receipt.request.requestId } : {}), ...failure.diagnostic } + const normalized = new ImageGenerationError(failure.code, { cause: error, outcome: stage === 'publishing' ? 'partial' : failure.outcome, diagnostic, request: receipt?.request, artifactIds: publication?.artifactIds ?? artifactIds, ...(retainedOutputs.length ? { retainedOutputs } : {}) }) + await receipt?.fail({ code: normalized.code, outcome: normalized.outcome, diagnostic: normalized.diagnostic, ...(normalized.retainedOutputs ? { retainedOutputs: normalized.retainedOutputs } : {}) }, normalized.artifactIds).catch(() => {}) + if (!retainedOutputs.length) + signal.throwIfAborted() throw normalized } }) @@ -175,41 +214,28 @@ export class ImageGenerationService { } async #materializeReferences( - reference: NonNullable, + referenceIds: readonly string[], grants: readonly DirectoryGrant[], signal: AbortSignal, ): Promise<{ artifactIds: string[], images: ImageContent[] }> { const { artifactService, attachmentService, conversationId } = this.#options - let resourceIds: readonly string[] - if (reference.mode === 'latest') { - const candidates = [ - ...artifactService.listConversationArtifacts(conversationId) - .filter(record => record.kind === 'file' && record.mimeType.startsWith('image/')) - .map(record => ({ id: record.id, timestamp: Date.parse(record.updatedAt || record.createdAt) || 0 })), - ...attachmentService.listForConversation(conversationId) - .filter(record => record.mimeType.startsWith('image/')) - .map(record => ({ id: record.id, timestamp: Date.parse(record.createdAt) || 0 })), - ] - const latest = candidates.reduce<(typeof candidates)[number] | undefined>((selected, candidate) => !selected || candidate.timestamp >= selected.timestamp ? candidate : selected, undefined) - resourceIds = latest ? [latest.id] : [] - } - else { - resourceIds = reference.resourceIds - } const artifactIds: string[] = [] const images: ImageContent[] = [] - for (const resourceId of resourceIds) { + for (const referenceId of referenceIds) { + signal.throwIfAborted() + let materialized: Awaited> try { - const artifacts = await artifactService.materializeConversationImages(conversationId, [resourceId], { cwd: this.#options.cwd, grants, signal }) - artifactIds.push(resourceId) - images.push(...artifacts.images) + materialized = await artifactService.materializeConversationImages(conversationId, [referenceId], { cwd: this.#options.cwd, grants, signal }) + artifactIds.push(referenceId) } catch (error) { if ((error as { code?: unknown }).code !== 'ARTIFACT_NOT_FOUND') throw error - const attachments = await attachmentService.materializeConversationImages(conversationId, [resourceId]) - images.push(...attachments.images) + materialized = await attachmentService.materializeConversationImages(conversationId, [referenceId]) } + if (materialized.images.length !== 1 || materialized.records.length !== 1 || materialized.records[0]!.id !== referenceId) + throw new Error('IMAGE_REFERENCE_MISMATCH') + images.push(materialized.images[0]!) } return { artifactIds, images } } diff --git a/apps/buddy/service/src/images/ImageReferenceAccess.ts b/apps/buddy/service/src/images/ImageReferenceAccess.ts new file mode 100644 index 000000000..403be6db9 --- /dev/null +++ b/apps/buddy/service/src/images/ImageReferenceAccess.ts @@ -0,0 +1,219 @@ +import type { ArtifactService } from '../artifacts/ArtifactService' +import type { AttachmentService } from '../attachments/AttachmentService' +import type { RunEventReader } from '../events/RunEventPorts' +import type { ConversationRepository } from '../storage/conversationRepository' +import type { ConversationTreeRepository } from '../storage/conversationTreeRepository' +import type { ExecutionRepository } from '../storage/executionRepository' +import type { RunInputRepository } from '../storage/runInputRepository' +import { buddyRunOutputPayloadSchema } from '../../../shared/runs/runOutput' +import { ArtifactError } from '../artifacts/ArtifactService' +import { AttachmentError } from '../attachments/AttachmentService' + +export const IMAGE_REFERENCE_LIST_LIMIT = 50 + +export interface ImageReferenceQuery { + query?: string + offset?: number + limit?: number +} + +export interface ImageReferenceItem { + referenceId: string + sourceKind: 'attachment' | 'artifact' + name: string + path: string + sourceMessageId: string | null + sourceRunId: string | null + mimeType: string + sizeBytes: number + createdAt: string +} + +export interface ImageReferencePage { + items: ImageReferenceItem[] + total: number + nextOffset: number | null +} + +export interface ImageReferenceAccessOptions { + conversationId: string + getRunId: () => string | undefined + signal: AbortSignal + conversations: Pick + publications: Pick + executions: Pick + inputs: Pick + events: Pick + attachments: Pick + artifacts: Pick +} + +export class ImageReferenceAccess { + readonly options: ImageReferenceAccessOptions + + constructor(options: ImageReferenceAccessOptions) { + this.options = options + } + + authorize(referenceIds: readonly string[], signal: AbortSignal): void { + this.options.signal.throwIfAborted() + signal.throwIfAborted() + if (referenceIds.some(id => typeof id !== 'string' || !id.trim() || id.length > 256)) + throw new Error('IMAGE_REFERENCE_INVALID') + if (!referenceIds.length) + return + const visible = new Set(this.#items(signal, new Set(referenceIds)).map(item => item.referenceId)) + for (const id of referenceIds) { + if (!visible.has(id)) + throw new Error(`IMAGE_REFERENCE_NOT_VISIBLE: ${id}; use lexora_image_list to find a visible referenceId`) + } + } + + list(input: ImageReferenceQuery, signal: AbortSignal): ImageReferencePage { + const offset = input.offset ?? 0 + const limit = input.limit ?? 20 + if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isInteger(limit) || limit < 1 || limit > IMAGE_REFERENCE_LIST_LIMIT + || (input.query !== undefined && (typeof input.query !== 'string' || input.query.length > 256))) { + throw new Error('IMAGE_REFERENCE_QUERY_INVALID') + } + const query = input.query?.trim().toLowerCase() ?? '' + const items = this.#items(signal).filter(item => !query || [item.referenceId, item.name, item.sourceMessageId, item.sourceRunId] + .some(value => value?.toLowerCase().includes(query))).sort((left, right) => compare(right.createdAt, left.createdAt) || compare(left.referenceId, right.referenceId) || compare(left.sourceKind, right.sourceKind)) + if (offset > items.length) + throw new Error('IMAGE_REFERENCE_OFFSET_OUT_OF_RANGE') + const page: ImageReferenceItem[] = [] + let bytes = 0 + for (const item of items.slice(offset, offset + limit)) { + const size = new TextEncoder().encode(JSON.stringify(item)).byteLength + if (bytes + size > 64 * 1024) { + if (!page.length) + throw new Error('IMAGE_REFERENCE_METADATA_TOO_LARGE') + break + } + bytes += size + page.push(item) + } + return { items: page, total: items.length, nextOffset: offset + page.length < items.length ? offset + page.length : null } + } + + #capture(signal: AbortSignal) { + this.options.signal.throwIfAborted() + signal.throwIfAborted() + const runId = this.options.getRunId() + const execution = runId ? this.options.executions.find(runId) : null + const task = this.options.conversations.findById(this.options.conversationId) + if (!execution || !task || task.deletedAt || execution.run.conversationId !== task.id || execution.run.status !== 'running' + || execution.cancelRequestedAt || execution.phase === 'closing' || execution.phase === 'closed') { + throw new Error('IMAGE_REFERENCE_SCOPE_UNAVAILABLE') + } + return execution + } + + #items(signal: AbortSignal, requested?: ReadonlySet): ImageReferenceItem[] { + const execution = this.#capture(signal) + const { conversationId, branchId, threadId } = execution.run + const attempts = [execution] + const runIds = new Set([execution.run.id]) + for (let attempt = execution; attempt.work.previousRunId;) { + const previous = this.options.executions.find(attempt.work.previousRunId) + if (!previous || runIds.has(previous.run.id) || previous.run.conversationId !== conversationId + || previous.run.branchId !== branchId || previous.run.threadId !== threadId || previous.work.workId !== execution.work.workId) { + throw new Error('IMAGE_REFERENCE_SCOPE_UNAVAILABLE') + } + runIds.add(previous.run.id) + attempts.push(previous) + attempt = previous + } + const items = new Map() + const addAttachment = (id: string, sourceMessageId: string | null, sourceRunId: string | null) => { + if (items.has(id) || (requested && !requested.has(id))) + return + try { + const record = this.options.attachments.getInputMetadata([id], conversationId)[0] + if (record?.mimeType.startsWith('image/') && record.conversationId === conversationId && record.draftId === null) { + items.set(id, { + referenceId: id, + sourceKind: 'attachment', + name: record.name.slice(0, 256), + path: record.storedPath, + sourceMessageId: sourceMessageId ?? record.messageId, + sourceRunId, + mimeType: record.mimeType, + sizeBytes: record.sizeBytes, + createdAt: record.createdAt, + }) + } + } + catch (error) { + if (!(error instanceof AttachmentError) || !['ATTACHMENT_NOT_FOUND', 'VALIDATION_FAILED'].includes(error.code)) + throw error + } + } + const addArtifact = (id: string, sourceRunId: string | null, createdAt?: string) => { + if (requested && !requested.has(id)) + return + try { + const { resource, canonicalPath } = this.options.artifacts.resolveConversationArtifactLocation(conversationId, id) + if (resource.kind !== 'file' || !resource.mimeType.startsWith('image/') || resource.conversationId !== conversationId) + return + const item: ImageReferenceItem = { + referenceId: id, + sourceKind: 'artifact', + name: resource.name.slice(0, 256), + path: canonicalPath, + sourceMessageId: null, + sourceRunId, + mimeType: resource.mimeType, + sizeBytes: resource.sizeBytes, + createdAt: createdAt ?? resource.createdAt, + } + if (!items.has(id) || items.get(id)!.createdAt < item.createdAt) + items.set(id, item) + } + catch (error) { + if (!(error instanceof ArtifactError) || error.code !== 'ARTIFACT_NOT_FOUND') + throw error + } + } + for (const attempt of attempts) { + signal.throwIfAborted() + const selected = attempt.work.contextMode === 'isolated' ? new Set(attempt.work.contextSourceIds) : null + const materials = this.options.publications.listPublications(conversationId, branchId, this.options.publications.getPublicSnapshot(attempt.run.id)) + for (const material of materials) { + if (selected && !selected.has(material.sourceId)) + continue + if (material.kind === 'output') { + for (const artifact of material.artifacts) + addArtifact(artifact.artifactId, material.runId) + } + else if (material.role === 'user') { + const input = this.options.inputs.findByMessageId(material.messageId) + for (const id of input?.attachmentIds ?? []) + addAttachment(id, material.messageId, input!.runId) + } + } + } + for (const id of this.options.executions.listAttachments(execution.run.id)) + addAttachment(id, null, execution.run.id) + for (const event of this.options.events.listForRuns([...runIds])) { + signal.throwIfAborted() + if (event.type !== 'output.produced' || !runIds.has(event.runId)) + continue + const output = buddyRunOutputPayloadSchema.safeParse(event.payload) + if (output.success) { + for (const id of output.data.artifactIds) + addArtifact(id, event.runId, event.createdAt) + } + } + const current = this.#capture(signal) + if (current.run.id !== execution.run.id || current.run.generation !== execution.run.generation + || current.run.branchId !== branchId || current.run.threadId !== threadId) { + throw new Error('IMAGE_REFERENCE_SCOPE_CHANGED') + } + return [...items.values()] + } +} + +function compare(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0 +} diff --git a/apps/buddy/service/src/images/ImageRequestStore.ts b/apps/buddy/service/src/images/ImageRequestStore.ts index f1ec1b424..3929f6a95 100644 --- a/apps/buddy/service/src/images/ImageRequestStore.ts +++ b/apps/buddy/service/src/images/ImageRequestStore.ts @@ -1,4 +1,4 @@ -import type { ImageGenerationFailure, ImageRequestRef } from '../../../shared/images/imageGeneration' +import type { ImageGenerationFailure, ImageGenerationRetainedOutput, ImageRequestRef } from '../../../shared/images/imageGeneration' import type { ImageTarget } from '../../../shared/images/imagePreferences' import type { BuddyDataPaths } from '../storage/BuddyDataPaths' import { randomUUID } from 'node:crypto' @@ -6,7 +6,7 @@ import { mkdir, open, readdir, readFile, rename, rm } from 'node:fs/promises' import { join } from 'node:path' import { z } from 'zod' import { syncStorageDirectory } from '../../../platform/storage/storageFiles' -import { imageGenerationFailureSchema } from '../../../shared/images/imageGeneration' +import { imageGenerationFailureSchema, imageGenerationRetainedOutputSchema } from '../../../shared/images/imageGeneration' import { imageSourceSchema, imageTargetSchema } from '../../../shared/images/imagePreferences' import { ImageGenerationError } from './ImageGenerationGateway' @@ -19,18 +19,22 @@ const resultSchema = z.object({ export type ImageRequestResult = z.infer const identitySchema = z.object({ fingerprint: z.string().regex(/^[a-f0-9]{64}$/), destination: z.string().regex(/^[a-f0-9]{64}$/) }) export type ImageRequestIdentity = z.infer +const referenceSchema = z.object({ referenceId: z.string().min(1), mimeType: z.string().min(1), sha256: z.string().regex(/^[a-f0-9]{64}$/) }) +export type ImageRequestReference = z.infer const receiptSchema = z.object({ version: z.literal(2).optional(), id: z.uuid(), runId: z.uuid(), toolCallId: z.string(), identity: identitySchema.optional(), + references: z.array(referenceSchema).optional(), state: z.enum(['preparing', 'unknown', 'completed', 'failed', 'partial']), attempts: z.array(z.object({ target: imageTargetSchema, attempt: z.number().int(), backendRequestId: z.string().optional() })), artifactIds: z.array(z.string()), responseId: z.string().nullable(), result: resultSchema.optional(), failure: imageGenerationFailureSchema.optional(), + retainedOutputs: z.array(imageGenerationRetainedOutputSchema).optional(), }) type Receipt = z.infer export interface ImageRequestReceipt { @@ -38,6 +42,7 @@ export interface ImageRequestReceipt { submitted: boolean attempt: (target: ImageTarget, attempt: number, backendRequestId?: string) => Promise submitting: () => Promise + retain: (outputs: readonly ImageGenerationRetainedOutput[]) => Promise complete: (result: ImageRequestResult) => Promise fail: (failure: ImageGenerationFailure, artifactIds?: readonly string[]) => Promise } @@ -48,21 +53,22 @@ export class ImageRequestStore { readonly paths: BuddyDataPaths constructor(paths: BuddyDataPaths) { this.paths = paths } - begin(conversationId: string, runId: string, toolCallId: string, identity: ImageRequestIdentity): Promise<{ kind: 'started', receipt: ImageRequestReceipt } | { kind: 'completed', result: ImageRequestResult, request: ImageRequestRef }> { + begin(conversationId: string, runId: string, toolCallId: string, identity: ImageRequestIdentity, references?: readonly ImageRequestReference[]): Promise<{ kind: 'started', receipt: ImageRequestReceipt } | { kind: 'completed', result: ImageRequestResult, request: ImageRequestRef }> { return this.#serialize(async () => { z.uuid().parse(runId) identitySchema.parse(identity) + const inputs = references ? z.array(referenceSchema).parse(references) : undefined const previous = await this.#readRun(conversationId, runId) const sameInvocation = previous.find(value => value.toolCallId === toolCallId) if (sameInvocation) { const request = this.#reference(conversationId, sameInvocation) if (sameInvocation.identity && (sameInvocation.identity.fingerprint !== identity.fingerprint || sameInvocation.identity.destination !== identity.destination)) - throw new ImageGenerationError('IMAGE_GENERATION_FAILED', { outcome: 'failed', diagnostic: { stage: 'preparing', localRequestId: sameInvocation.id }, request }) + throw new ImageGenerationError('IMAGE_GENERATION_FAILED', { outcome: 'failed', diagnostic: { stage: 'preparing', localRequestId: sameInvocation.id }, request, retainedOutputs: sameInvocation.retainedOutputs }) if (sameInvocation.state === 'completed') return { kind: 'completed' as const, request, result: sameInvocation.result ?? { artifactIds: sameInvocation.artifactIds, outputs: [], responseId: sameInvocation.responseId } } if (sameInvocation.state === 'failed') { const failure = sameInvocation.failure - throw new ImageGenerationError(failure?.code === 'VALIDATION_FAILED' ? 'IMAGE_GENERATION_FAILED' : failure?.code ?? 'IMAGE_GENERATION_FAILED', { outcome: 'failed', diagnostic: failure?.diagnostic ?? { stage: 'preparing', localRequestId: sameInvocation.id }, request }) + throw new ImageGenerationError(failure?.code === 'VALIDATION_FAILED' ? 'IMAGE_GENERATION_FAILED' : failure?.code ?? 'IMAGE_GENERATION_FAILED', { outcome: 'failed', diagnostic: failure?.diagnostic ?? { stage: 'preparing', localRequestId: sameInvocation.id }, request, retainedOutputs: sameInvocation.retainedOutputs }) } if (sameInvocation.state !== 'preparing' || this.#active.has(sameInvocation.id)) this.#block(conversationId, sameInvocation) @@ -76,9 +82,14 @@ export class ImageRequestStore { }) if (unresolved) this.#block(conversationId, unresolved) - if (sameInvocation) + if (sameInvocation) { + if (inputs) { + sameInvocation.references = inputs + await this.#save(conversationId, sameInvocation) + } return { kind: 'started' as const, receipt: this.#activate(conversationId, sameInvocation) } - const value: Receipt = { version: 2, id: randomUUID(), runId, toolCallId, identity, state: 'preparing', attempts: [], artifactIds: [], responseId: null } + } + const value: Receipt = { version: 2, id: randomUUID(), runId, toolCallId, identity, ...(inputs ? { references: inputs } : {}), state: 'preparing', attempts: [], artifactIds: [], responseId: null } await this.#save(conversationId, value) return { kind: 'started' as const, receipt: this.#activate(conversationId, value) } }) @@ -121,12 +132,18 @@ export class ImageRequestStore { value.state = 'unknown' receipt.submitted = true }), + retain: outputs => update(async () => { + const retainedOutputs = z.array(imageGenerationRetainedOutputSchema).parse(outputs) + await this.#save(conversationId, { ...value, state: 'partial', retainedOutputs }) + value.state = 'partial' + value.retainedOutputs = retainedOutputs + }), complete: result => update(async () => { const saved = resultSchema.parse(result) await this.#save(conversationId, { ...value, state: 'completed', artifactIds: saved.artifactIds, responseId: saved.responseId, result: saved }) }, true), fail: (failure, artifactIds = []) => update(async () => { - await this.#save(conversationId, { ...value, state: failure.outcome, failure, responseId: failure.diagnostic?.responseId ?? value.responseId, artifactIds: [...artifactIds] }) + await this.#save(conversationId, { ...value, state: failure.outcome, failure, ...(failure.retainedOutputs ? { retainedOutputs: failure.retainedOutputs } : {}), responseId: failure.diagnostic?.responseId ?? value.responseId, artifactIds: [...artifactIds] }) }, true), } return receipt @@ -137,6 +154,7 @@ export class ImageRequestStore { outcome: value.state === 'partial' ? 'partial' : 'unknown', request: this.#reference(conversationId, value), artifactIds: value.artifactIds, + retainedOutputs: value.retainedOutputs, diagnostic: { ...value.failure?.diagnostic, stage: 'blocked', localRequestId: value.id }, }) } diff --git a/apps/buddy/service/src/images/__tests__/ImageGenerationMaterialStore.spec.ts b/apps/buddy/service/src/images/__tests__/ImageGenerationMaterialStore.spec.ts new file mode 100644 index 000000000..3da1e67a6 --- /dev/null +++ b/apps/buddy/service/src/images/__tests__/ImageGenerationMaterialStore.spec.ts @@ -0,0 +1,154 @@ +import type { DirectoryGrant } from '../../directories/resolveGrantedPath' +import { Buffer } from 'node:buffer' +import { createHash } from 'node:crypto' +import { copyFile, mkdir, mkdtemp, readdir, readFile, realpath, rename, rm, stat, symlink, unlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import process from 'node:process' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ArtifactService } from '../../artifacts/ArtifactService' +import { createArtifactRepository } from '../../storage/artifactRepository' +import { createConversationRepository } from '../../storage/conversationRepository' +import { openBuddyDatabase } from '../../storage/database' +import { ImageGenerationMaterialError, ImageGenerationMaterialStore } from '../ImageGenerationMaterialStore' +import { createImagePublicationFixture } from './imagePublicationFixture' + +const png = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAAFElEQVR4AQEJAPb/AAD/AP//AAD/EPgD/UE7EkoAAAAASUVORK5CYII=', 'base64') +const image = { bytes: png, mimeType: 'image/png' as const } +let fixtureRoot: string +let root: string +let grants: DirectoryGrant[] +let publication: ReturnType + +beforeEach(async () => { + fixtureRoot = await realpath(await mkdtemp(join(tmpdir(), 'image-generation-material-'))) + root = join(fixtureRoot, 'scratch') + await mkdir(root) + grants = [{ canonicalRoot: root, root, kind: 'workspace', grantId: 'scratch' }] + publication = createImagePublicationFixture(fixtureRoot) +}) + +afterEach(async () => { + await publication.dispose() + await rm(fixtureRoot, { recursive: true, force: true }) +}) + +describe('owned image generation materials', () => { + it('keeps independent originals and permits registration through the existing file presentation path', async () => { + const materials = new ImageGenerationMaterialStore(root, 'conversation-1') + const retained = await Promise.all([materials.write(image, grants, publication.writeFiles), materials.write(image, grants, publication.writeFiles)]) + expect(new Set(retained.map(output => output.path)).size).toBe(2) + for (const output of retained) { + expect(output).toEqual({ path: expect.stringContaining(join(root, '.image-generation', 'conversation-1')), mimeType: 'image/png', sizeBytes: png.length, sha256: createHash('sha256').update(png).digest('hex') }) + expect(await readFile(output.path)).toEqual(png) + if (process.platform !== 'win32') + expect((await stat(output.path)).mode & 0o777).toBe(0o600) + } + const destination = join(root, 'delivered.png') + await copyFile(retained[0]!.path, destination) + const database = openBuddyDatabase({ databasePath: ':memory:' }) + createConversationRepository(database).create({ id: 'conversation-1', branchId: 'branch-1', spaceId: null, title: 'Fixture', approvalPolicy: 'policy', executionProfile: 'workspace_write', createdAt: '2026-08-27T00:00:00.000Z' }) + const service = new ArtifactService({ repository: createArtifactRepository(database) }) + try { + const artifacts = await service.presentOutputs({ conversationId: 'conversation-1', cwd: root, grants, paths: [retained[0]!.path, destination] }) + expect(artifacts.map(artifact => ({ path: artifact.currentPath, mimeType: artifact.mimeType, sizeBytes: artifact.sizeBytes }))).toEqual([ + { path: retained[0]!.path, mimeType: 'image/png', sizeBytes: png.length }, + { path: destination, mimeType: 'image/png', sizeBytes: png.length }, + ]) + await writeFile(destination, 'replaced user output') + expect(await readFile(retained[0]!.path)).toEqual(png) + await rm(destination) + expect(await readFile(retained[0]!.path)).toEqual(png) + } + finally { + await service.dispose() + database.close() + } + }) + + it.each(['scratch', 'collection', 'conversation'])('rejects a redirected %s before writing outside owned scratch', async (kind) => { + const outside = await realpath(await mkdtemp(join(tmpdir(), 'image-generation-outside-'))) + try { + let scratch = root + let link: string + if (kind === 'scratch') { + scratch = join(root, 'linked-scratch') + link = scratch + } + else if (kind === 'collection') { + link = join(root, '.image-generation') + } + else { + await mkdir(join(root, '.image-generation')) + link = join(root, '.image-generation', 'conversation-1') + } + await symlink(outside, link, process.platform === 'win32' ? 'junction' : 'dir') + const externalGrant: DirectoryGrant = { canonicalRoot: outside, root: outside, kind: 'granted', grantId: 'external' } + await expect(new ImageGenerationMaterialStore(scratch, 'conversation-1').write(image, [...grants, externalGrant], publication.writeFiles)).rejects.toMatchObject({ code: 'IMAGE_MATERIAL_IDENTITY_CHANGED' }) + expect(await readdir(outside)).toEqual([]) + } + finally { + await rm(outside, { recursive: true, force: true }) + } + }) + + it('rejects a transient parent substitution at the native publication boundary without leaving outside files', async () => { + const directory = join(root, '.image-generation', 'conversation-1') + const outside = join(fixtureRoot, 'outside') + const backup = join(root, '.image-generation', 'original') + await mkdir(directory, { recursive: true }) + await mkdir(outside) + const mutate = publication.engine.mutate.bind(publication.engine) + let swapped = false + vi.spyOn(publication.engine, 'mutate').mockImplementation(async (request) => { + if (request.action.kind !== 'writeFile') + return mutate(request) + await rename(directory, backup) + await symlink(outside, directory, process.platform === 'win32' ? 'junction' : 'dir') + swapped = true + try { + return await mutate(request) + } + finally { + await unlink(directory) + await rename(backup, directory) + } + }) + await expect(new ImageGenerationMaterialStore(root, 'conversation-1').write(image, grants, publication.writeFiles)).rejects.toMatchObject({ code: 'FILE_PERMISSION_REVOKED' }) + expect(swapped).toBe(true) + expect(await readdir(outside)).toEqual([]) + expect(await readdir(directory)).toEqual([]) + }) + + it('does not replace a file which appeared at the chosen original path', async () => { + let path = '' + await expect(new ImageGenerationMaterialStore(root, 'conversation-1').write(image, grants, async (paths, write) => { + path = paths[0]! + await mkdir(dirname(path), { recursive: true }) + await writeFile(path, 'existing original') + await publication.writeFiles(paths, write) + })).rejects.toThrow('FILE_CONFLICT') + expect(await readFile(path, 'utf8')).toBe('existing original') + }) + + it('keeps a confirmed original when native publication cleanup fails after cancellation', async () => { + const controller = new AbortController() + const error = await new ImageGenerationMaterialStore(root, 'conversation-1').write(image, grants, async (paths, write) => { + await publication.writeFiles(paths, write) + controller.abort() + throw new Error('publication cleanup failed') + }, controller.signal).catch(error => error) + expect(error).toBeInstanceOf(ImageGenerationMaterialError) + expect(error.retainedOutput).toMatchObject({ mimeType: 'image/png', sizeBytes: png.length, sha256: createHash('sha256').update(png).digest('hex') }) + expect(await readFile(error.retainedOutput.path)).toEqual(png) + }) + + it('requires the explicit execution grant and rejects cancellation before creating directories', async () => { + const store = new ImageGenerationMaterialStore(root, 'conversation-1') + await expect(store.write(image, [], publication.writeFiles)).rejects.toMatchObject({ code: 'PATH_OUTSIDE_GRANTED_DIRECTORY' }) + expect(await readdir(root)).toEqual([]) + await expect(store.write(image, grants, publication.writeFiles, AbortSignal.abort())).rejects.toMatchObject({ name: 'AbortError' }) + expect(await readdir(root)).toEqual([]) + expect(() => new ImageGenerationMaterialStore(root, '../other')).toThrow('owner') + }) +}) diff --git a/apps/buddy/service/src/images/__tests__/ImageGenerationService.spec.ts b/apps/buddy/service/src/images/__tests__/ImageGenerationService.spec.ts index 7f40a2f46..c20502c59 100644 --- a/apps/buddy/service/src/images/__tests__/ImageGenerationService.spec.ts +++ b/apps/buddy/service/src/images/__tests__/ImageGenerationService.spec.ts @@ -1,19 +1,176 @@ import type { Model } from '@earendil-works/pi-ai' +import type { CapturedFileWriter } from '../../changes/CapturedFileWriter' +import type { DirectoryGrant } from '../../directories/resolveGrantedPath' import type { ProviderService } from '../../providers/ProviderService' import type { ImageGenerationServiceOptions } from '../ImageGenerationService' import type { ImageOperationEvent } from '../ImageOperationLifecycle' import { Buffer } from 'node:buffer' -import { randomUUID } from 'node:crypto' -import { describe, expect, it } from 'vitest' +import { createHash, randomUUID } from 'node:crypto' +import { chmod, mkdir, mkdtemp, readdir, readFile, realpath, rename, rm, symlink, unlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import process from 'node:process' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { readImageDiagnostic, readImageFileErrorCode } from '../../../../shared/images/imageGeneration' import { ArtifactPublicationError } from '../../artifacts/ArtifactService' +import { BuddyDataPaths } from '../../storage/BuddyDataPaths' +import { ImageGenerationError } from '../ImageGenerationGateway' +import { ImageGenerationMaterialStore } from '../ImageGenerationMaterialStore' import { ImageGenerationService } from '../ImageGenerationService' +import { observeImageDiagnostics } from '../ImageOperationLifecycle' +import { ImageRequestStore } from '../ImageRequestStore' import { RoutedImageGateway } from '../RoutedImageGateway' +import { createImagePublicationFixture } from './imagePublicationFixture' const png = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAAFElEQVR4AQEJAPb/AAD/AP//AAD/EPgD/UE7EkoAAAAASUVORK5CYII=', 'base64') const image = { type: 'image' as const, mimeType: 'image/png', data: 'fixture' } const model = { provider: 'fixture', id: 'fixture' } as Model<'openai-responses'> +let workspace: string +let publicationRoot: string +let publication: ReturnType +let grants: DirectoryGrant[] + +beforeEach(async () => { + workspace = await realpath(await mkdtemp(join(tmpdir(), 'lexora-image-generation-'))) + publicationRoot = await realpath(await mkdtemp(join(tmpdir(), 'lexora-image-publication-'))) + publication = createImagePublicationFixture(publicationRoot) + grants = [{ canonicalRoot: workspace, root: workspace, kind: 'workspace', grantId: 'workspace-1' }] +}) + +afterEach(async () => { + await publication.dispose() + await rm(publicationRoot, { recursive: true, force: true }) + await rm(workspace, { recursive: true, force: true }) +}) + describe('conversation image generation', () => { + it.each([' ', 'result.jpg', 'invalid\0.png', 'directory.png', 'parent/result.png'])('rejects an invalid output before submission without changing files: %j', async (outputPath) => { + await mkdir(join(workspace, 'directory.png')) + await writeFile(join(workspace, 'parent'), 'unchanged') + const before = await readdir(workspace) + const fixture = createGenerationFixture() + await expect(generate(fixture.service, { outputPath, prompt: 'Create' }, model, new AbortController().signal)).rejects.toMatchObject({ code: 'IMAGE_GENERATION_OUTPUT_UNAVAILABLE', outcome: 'failed', retryable: false, diagnostic: { stage: 'preparing' } }) + expect(fixture.submissions).toBe(0) + expect(fixture.publications).toBe(0) + expect(fixture.events.map(event => event.phase)).toEqual(['preparing', 'settled']) + expect(fixture.events.at(-1)).toMatchObject({ outcome: 'failed', artifactIds: [] }) + expect(await readdir(workspace)).toEqual(before) + expect(await readFile(join(workspace, 'parent'), 'utf8')).toBe('unchanged') + await fixture.service.dispose() + }) + + it('uses only the supplied execution grants and permits a corrected authorized absolute output', async () => { + const fixture = createGenerationFixture() + const input = { outputPath: join(workspace, 'result.png'), prompt: 'Create' } + await expect(generate(fixture.service, input, model, new AbortController().signal, [])).rejects.toMatchObject({ code: 'IMAGE_GENERATION_OUTPUT_UNAVAILABLE', outcome: 'failed', cause: { cause: { code: 'PATH_OUTSIDE_GRANTED_DIRECTORY' } } }) + expect(fixture.submissions).toBe(0) + expect(fixture.publications).toBe(0) + await expect(generate(fixture.service, input, model, new AbortController().signal, grants)).resolves.toMatchObject({ artifactIds: ['generated-1'] }) + expect(fixture.submissions).toBe(1) + expect(grants).toHaveLength(1) + await fixture.service.dispose() + }) + + it.each(['outside', 'dangling'])('rejects a %s symlink before submitting without following it for writes', async (kind) => { + const allowed = join(workspace, 'allowed') + const outside = join(workspace, 'outside') + await mkdir(allowed) + await mkdir(outside) + const existing = join(outside, 'result.png') + await writeFile(existing, png) + await symlink(kind === 'outside' ? outside : join(workspace, 'missing'), join(allowed, 'linked'), process.platform === 'win32' ? 'junction' : 'dir') + const fixture = createGenerationFixture() + await expect(generate(fixture.service, { outputPath: join(allowed, 'linked', 'result.png'), prompt: 'Create' }, model, new AbortController().signal, [{ canonicalRoot: allowed, root: allowed, grantId: 'allowed', kind: 'granted' }])).rejects.toMatchObject({ code: 'IMAGE_GENERATION_OUTPUT_UNAVAILABLE', outcome: 'failed' }) + expect(fixture.submissions).toBe(0) + expect(fixture.publications).toBe(0) + expect(await readFile(existing)).toEqual(png) + await fixture.service.dispose() + }) + + it('accepts a symlink whose canonical destination stays within the execution grants', async () => { + const target = join(workspace, 'target') + await mkdir(target) + await symlink(target, join(workspace, 'linked'), process.platform === 'win32' ? 'junction' : 'dir') + const fixture = createGenerationFixture() + await expect(generate(fixture.service, { outputPath: 'linked/result.png', prompt: 'Create' }, model, new AbortController().signal)).resolves.toMatchObject({ artifactIds: ['generated-1'] }) + expect(fixture.submissions).toBe(1) + await fixture.service.dispose() + }) + + it.skipIf(process.platform === 'win32' || process.getuid?.() === 0).each(['file', 'parent'])('rejects a host read-only %s before submission', async (kind) => { + const target = join(workspace, kind === 'file' ? 'result.png' : 'locked') + if (kind === 'file') + await writeFile(target, png) + else + await mkdir(target) + await chmod(target, kind === 'file' ? 0o400 : 0o500) + const fixture = createGenerationFixture() + try { + await expect(generate(fixture.service, { outputPath: kind === 'file' ? target : join(target, 'missing', 'nested', 'result.png'), prompt: 'Create' }, model, new AbortController().signal)).rejects.toMatchObject({ code: 'IMAGE_GENERATION_OUTPUT_UNAVAILABLE', outcome: 'failed', diagnostic: { stage: 'preparing' }, cause: { cause: { code: 'EACCES' } } }) + expect(fixture.submissions).toBe(0) + expect(fixture.publications).toBe(0) + if (kind === 'file') + expect(await readFile(target)).toEqual(png) + else + expect(await readdir(target)).toEqual([]) + } + finally { + await chmod(target, kind === 'file' ? 0o600 : 0o700) + await fixture.service.dispose() + } + }) + + it('persists preflight failure as unsubmitted and accepts a new corrected request without automatic retry', async () => { + const paths = new BuddyDataPaths(join(workspace, 'receipts')) + const requests = new ImageRequestStore(paths) + const fixture = createGenerationFixture({ requests }) + const runId = randomUUID() + const signal = new AbortController().signal + await expect(generate(fixture.service, { outputPath: 'result.jpg', prompt: 'Create' }, model, signal, grants, undefined, { runId, toolCallId: 'invalid' })).rejects.toMatchObject({ code: 'IMAGE_GENERATION_OUTPUT_UNAVAILABLE', outcome: 'failed' }) + expect(fixture.submissions).toBe(0) + const directory = join(paths.conversationDirectory('conversation-1'), 'image-requests', runId) + const [name] = await readdir(directory) + const failed = JSON.parse(await readFile(join(directory, name!), 'utf8')) + expect(failed).toMatchObject({ state: 'failed', attempts: [], artifactIds: [], responseId: null, failure: { code: 'IMAGE_GENERATION_OUTPUT_UNAVAILABLE', outcome: 'failed', diagnostic: { stage: 'preparing' } } }) + const result = await generate(fixture.service, { outputPath: 'result.png', prompt: 'Create' }, model, signal, grants, undefined, { runId, toolCallId: 'corrected' }) + expect(result.request?.requestId).not.toBe(failed.id) + expect(fixture.submissions).toBe(1) + expect(fixture.publications).toBe(1) + expect(await requests.completed('conversation-1', runId, 'corrected')).toMatchObject({ result: { artifactIds: ['generated-1'] } }) + await fixture.service.dispose() + }) + + it.each(['unknown', 'partial'] as const)('keeps an unresolved %s receipt ahead of a later invalid output preflight', async (failure) => { + const requests = new ImageRequestStore(new BuddyDataPaths(join(workspace, 'receipts'))) + const fixture = createGenerationFixture({ requests, failure }) + const runId = randomUUID() + const signal = new AbortController().signal + await expect(generate(fixture.service, { outputPath: 'result.png', prompt: 'Create' }, model, signal, grants, undefined, { runId, toolCallId: 'original' })).rejects.toMatchObject({ outcome: failure }) + await expect(generate(fixture.service, { outputPath: 'invalid.jpg', prompt: 'Create' }, model, signal, grants, undefined, { runId, toolCallId: 'later' })).rejects.toMatchObject({ code: 'IMAGE_GENERATION_BLOCKED', outcome: failure, diagnostic: { stage: 'blocked' } }) + expect(fixture.submissions).toBe(1) + await fixture.service.dispose() + }) + + it('does not submit when cancelled before the read-only preflight', async () => { + const controller = new AbortController() + const requests = new ImageRequestStore(new BuddyDataPaths(join(workspace, 'receipts'))) + const fixture = createGenerationFixture({ requests: { + completed: (...args) => requests.completed(...args), + begin: async (...args) => { + const begun = await requests.begin(...args) + controller.abort(new Error('cancelled before preflight')) + return begun + }, + } }) + await expect(generate(fixture.service, { outputPath: 'missing/result.png', prompt: 'Create' }, model, controller.signal, grants, undefined, { runId: randomUUID(), toolCallId: 'cancelled' })).rejects.toThrow('cancelled before preflight') + expect(fixture.submissions).toBe(0) + expect(fixture.publications).toBe(0) + expect(fixture.events.at(-1)).toMatchObject({ outcome: 'cancelled', cancellationRequested: true }) + await expect(readFile(join(workspace, 'missing/result.png'))).rejects.toMatchObject({ code: 'ENOENT' }) + await fixture.service.dispose() + }) + it('settles accepted publication with actual artifacts after cancellation and drains disposal', async () => { const controller = new AbortController() let release!: () => void @@ -25,7 +182,7 @@ describe('conversation image generation', () => { release = resolve }) const source = { listConversationArtifacts: () => [], listForConversation: () => [], materializeConversationImages: async () => ({ images: [], records: [] }) } - const service = new ImageGenerationService({ conversationId: 'conversation-1', cwd: '/workspace', grants: [], attachmentService: source, artifactService: { ...source, async registerGeneratedImages() { + const service = createService({ referenceAccess: { authorize: () => {} }, conversationId: 'conversation-1', cwd: workspace, grants, attachmentService: source, artifactService: { ...source, async registerGeneratedImages() { started() await gate @@ -34,7 +191,7 @@ describe('conversation image generation', () => { } }, imageGenerationGateway: { supports: () => true, generate: async () => ({ images: [{ bytes: new Uint8Array(png), mimeType: 'image/png' }], responseId: 'private-response' }) } }) const events: ImageOperationEvent[] = [] service.onDidChange(event => events.push(event)) - const pending = service.generate({ outputPath: 'private.png', prompt: 'private prompt' }, model, controller.signal) + const pending = generate(service, { outputPath: 'private.png', prompt: 'private prompt' }, model, controller.signal) await publicationStarted controller.abort() let disposed = false @@ -50,17 +207,17 @@ describe('conversation image generation', () => { expect(events.at(-1)).toMatchObject({ outcome: 'completed', cancellationRequested: true, artifactIds: ['generated-1'] }) expect(Object.isFrozen(events.at(-1)!.artifactIds)).toBe(true) expect(JSON.stringify(events)).not.toContain('private') - await expect(service.generate({ outputPath: 'later.png', prompt: 'later' }, model, controller.signal)).rejects.toThrow('IMAGE_SERVICE_STOPPED') + await expect(generate(service, { outputPath: 'later.png', prompt: 'later' }, model, controller.signal)).rejects.toThrow('IMAGE_SERVICE_STOPPED') }) it('preserves publication partial effects in image settlement', async () => { const source = { listConversationArtifacts: () => [], listForConversation: () => [], materializeConversationImages: async () => ({ images: [], records: [] }) } - const service = new ImageGenerationService({ conversationId: 'conversation-1', cwd: '/workspace', grants: [], attachmentService: source, artifactService: { ...source, async registerGeneratedImages() { + const service = createService({ referenceAccess: { authorize: () => {} }, conversationId: 'conversation-1', cwd: workspace, grants, attachmentService: source, artifactService: { ...source, async registerGeneratedImages() { throw new ArtifactPublicationError(new Error('storage failed'), { operationId: 'publication-1', conversationId: 'conversation-1', cause: 'generated', requested: 2, written: 2, unconfirmedWrites: 0, artifactIds: ['generated-1'], stage: 'catalogue', outcome: 'partial' }) } }, imageGenerationGateway: { supports: () => true, generate: async () => ({ images: [{ bytes: new Uint8Array(png), mimeType: 'image/png' }], responseId: 'private-response' }) } }) const events: ImageOperationEvent[] = [] service.onDidChange(event => events.push(event)) - await expect(service.generate({ outputPath: 'private.png', prompt: 'private prompt' }, model, new AbortController().signal)).rejects.toMatchObject({ outcome: 'partial', artifactIds: ['generated-1'], diagnostic: { stage: 'publishing', responseId: 'private-response' }, cause: { receipt: { outcome: 'partial' } } }) + await expect(generate(service, { outputPath: 'private.png', prompt: 'private prompt' }, model, new AbortController().signal)).rejects.toMatchObject({ outcome: 'partial', artifactIds: ['generated-1'], diagnostic: { stage: 'publishing', responseId: 'private-response' }, cause: { receipt: { outcome: 'partial' } } }) expect(events.at(-1)).toMatchObject({ phase: 'settled', outcome: 'partial', artifactIds: ['generated-1'], publication: { written: 2, operationId: 'publication-1' } }) await service.dispose() }) @@ -69,121 +226,263 @@ describe('conversation image generation', () => { const source = { listConversationArtifacts: () => [], listForConversation: () => [], materializeConversationImages: async () => ({ images: [], records: [] }) } const request = { conversationId: randomUUID(), runId: randomUUID(), requestId: randomUUID() } let saved: unknown - const service = new ImageGenerationService({ - conversationId: request.conversationId, - cwd: '/workspace', - grants: [], - attachmentService: source, - artifactService: { ...source, registerGeneratedImages: async () => [{ id: 'generated-1', currentPath: '/workspace/result.png', mimeType: 'image/png' }] }, - imageGenerationGateway: { supports: () => true, generate: async () => ({ images: [{ bytes: new Uint8Array(png), mimeType: 'image/png' }], responseId: 'response-1' }) }, - requests: { completed: async () => null, begin: async () => ({ kind: 'started', receipt: { - request, - submitted: false, - attempt: async () => {}, - submitting: async () => {}, - complete: async () => { throw new Error('receipt disk failure') }, - fail: async (failure, artifactIds) => { saved = { failure, artifactIds } }, - } }) }, - }) - await expect(service.generate({ outputPath: 'result.png', prompt: 'Create' }, model, new AbortController().signal, [], undefined, { runId: request.runId, toolCallId: 'fixture' })).rejects.toMatchObject({ outcome: 'partial', artifactIds: ['generated-1'], request, diagnostic: { stage: 'publishing', responseId: 'response-1' } }) + const service = createService({ referenceAccess: { authorize: () => {} }, conversationId: request.conversationId, cwd: workspace, grants, attachmentService: source, artifactService: { ...source, registerGeneratedImages: async () => [{ id: 'generated-1', currentPath: '/workspace/result.png', mimeType: 'image/png' }] }, imageGenerationGateway: { supports: () => true, generate: async () => ({ images: [{ bytes: new Uint8Array(png), mimeType: 'image/png' }], responseId: 'response-1' }) }, requests: { completed: async () => null, begin: async () => ({ kind: 'started', receipt: { + request, + submitted: false, + attempt: async () => {}, + submitting: async () => {}, + retain: async () => {}, + complete: async () => { throw new Error('receipt disk failure') }, + fail: async (failure, artifactIds) => { saved = { failure, artifactIds } }, + } }) } }) + await expect(generate(service, { outputPath: 'result.png', prompt: 'Create' }, model, new AbortController().signal, grants, undefined, { runId: request.runId, toolCallId: 'fixture' })).rejects.toMatchObject({ outcome: 'partial', artifactIds: ['generated-1'], request, diagnostic: { stage: 'publishing', responseId: 'response-1' } }) expect(saved).toMatchObject({ failure: { outcome: 'partial', diagnostic: { responseId: 'response-1' } }, artifactIds: ['generated-1'] }) await service.dispose() }) - it('replays completed work even when latest references or the selected provider have changed', async () => { + it('retains verified originals before a failed publication and returns identical recovery metadata on blocked calls', async () => { + const paths = new BuddyDataPaths(join(workspace, 'receipts')) + const requests = new ImageRequestStore(paths) + const input = { outputPath: 'result.png', prompt: 'private prompt' } + await writeFile(join(workspace, input.outputPath), 'existing output') + const fixture = createGenerationFixture({ requests, failure: 'partial', publicationError: Object.assign(new Error(`private path: ${workspace}`), { code: 'EACCES' }) }) + const diagnostics: unknown[] = [] + const observer = observeImageDiagnostics(fixture.service, event => diagnostics.push(event)) + const invocation = { runId: randomUUID(), toolCallId: 'original' } + const error = await generate(fixture.service, input, model, new AbortController().signal, grants, undefined, invocation).catch(error => error) + expect(error).toBeInstanceOf(ImageGenerationError) + expect(error).toMatchObject({ code: 'IMAGE_GENERATION_FAILED', outcome: 'partial', diagnostic: { stage: 'publishing', fileErrorCode: 'EACCES' }, retainedOutputs: [{ mimeType: 'image/png', sizeBytes: png.length, sha256: createHash('sha256').update(png).digest('hex') }] }) + expect(error.diagnostic.transportCode).toBeUndefined() + const retained = error.retainedOutputs[0] + expect(retained.path).toContain(join(workspace, '.image-generation', 'conversation-1')) + expect(await readFile(retained.path)).toEqual(png) + expect(await readFile(join(workspace, input.outputPath), 'utf8')).toBe('existing output') + const receiptPath = join(paths.conversationDirectory('conversation-1'), 'image-requests', invocation.runId, `${error.request.requestId}.json`) + expect(JSON.parse(await readFile(receiptPath, 'utf8'))).toMatchObject({ state: 'partial', retainedOutputs: error.retainedOutputs, failure: { retainedOutputs: error.retainedOutputs } }) + await expect(generate(fixture.service, input, model, new AbortController().signal, grants, undefined, { ...invocation, toolCallId: 'blocked' })).rejects.toMatchObject({ code: 'IMAGE_GENERATION_BLOCKED', outcome: 'partial', request: error.request, retainedOutputs: error.retainedOutputs }) + expect(fixture.submissions).toBe(1) + expect(fixture.publications).toBe(1) + expect(JSON.stringify(diagnostics)).not.toContain(workspace) + expect(JSON.stringify(diagnostics)).not.toContain('private prompt') + expect(JSON.stringify(diagnostics)).not.toContain('retainedOutputs') + observer.dispose() + await fixture.service.dispose() + }) + + it.each(['write-failure', 'cancellation'] as const)('keeps completed originals when later retention is interrupted by %s', async (kind) => { + const requests = new ImageRequestStore(new BuddyDataPaths(join(workspace, 'receipts'))) + const materials = new ImageGenerationMaterialStore(workspace, 'conversation-1') + const controller = new AbortController() + let writes = 0 + const fixture = createGenerationFixture({ requests, imageCount: 2, writeGeneratedImageMaterial: async (image, grants, writeFiles, signal) => { + if (++writes === 2) + throw Object.assign(new Error('storage exhausted'), { code: 'ENOSPC' }) + const retained = await materials.write(image, grants, writeFiles, signal) + if (kind === 'cancellation') + controller.abort(new Error('cancelled after retained file')) + return retained + } }) + const invocation = { runId: randomUUID(), toolCallId: 'original' } + const error = await generate(fixture.service, { outputPath: 'result.png', prompt: 'Create' }, model, controller.signal, grants, undefined, invocation).catch(error => error) + expect(error).toMatchObject({ outcome: 'partial', retainedOutputs: [expect.objectContaining({ sizeBytes: png.length })] }) + expect(error.retainedOutputs).toHaveLength(1) + expect(await readFile(error.retainedOutputs[0].path)).toEqual(png) + if (kind === 'write-failure') + expect(error.diagnostic.fileErrorCode).toBe('ENOSPC') + await expect(requests.begin('conversation-1', invocation.runId, invocation.toolCallId, { fingerprint: 'a'.repeat(64), destination: 'b'.repeat(64) })).rejects.toMatchObject({ retainedOutputs: error.retainedOutputs }) + expect(fixture.submissions).toBe(1) + expect(fixture.publications).toBe(0) + await fixture.service.dispose() + }) + + it('keeps a valid generated result partial when the first original cannot be retained', async () => { + const requests = new ImageRequestStore(new BuddyDataPaths(join(workspace, 'receipts'))) + const fixture = createGenerationFixture({ requests, writeGeneratedImageMaterial: async () => { + throw Object.assign(new Error('scratch is full'), { code: 'ENOSPC' }) + } }) + const invocation = { runId: randomUUID(), toolCallId: 'original' } + const input = { outputPath: 'result.png', prompt: 'Create' } + const error = await generate(fixture.service, input, model, new AbortController().signal, grants, undefined, invocation).catch(error => error) + expect(error).toMatchObject({ outcome: 'partial', retryable: false, diagnostic: { stage: 'publishing', fileErrorCode: 'ENOSPC' } }) + expect(error.retainedOutputs).toBeUndefined() + await expect(generate(fixture.service, input, model, new AbortController().signal, grants, undefined, { ...invocation, toolCallId: 'blocked' })).rejects.toMatchObject({ code: 'IMAGE_GENERATION_BLOCKED', outcome: 'partial' }) + expect(fixture.submissions).toBe(1) + expect(fixture.publications).toBe(0) + await fixture.service.dispose() + }) + + it('does not hide retained files when both receipt writes fail', async () => { + const store = new ImageRequestStore(new BuddyDataPaths(join(workspace, 'receipts'))) + const fixture = createGenerationFixture({ requests: { + completed: (...args) => store.completed(...args), + begin: async (...args) => { + const begun = await store.begin(...args) + if (begun.kind === 'started') { + begun.receipt.retain = async () => { + throw Object.assign(new Error('receipt write failed'), { code: 'EROFS' }) + } + begun.receipt.fail = async () => { + throw new Error('receipt still unavailable') + } + } + return begun + }, + } }) + const error = await generate(fixture.service, { outputPath: 'result.png', prompt: 'Create' }, model, new AbortController().signal, grants, undefined, { runId: randomUUID(), toolCallId: 'original' }).catch(error => error) + expect(error).toMatchObject({ outcome: 'partial', diagnostic: { fileErrorCode: 'EROFS' }, retainedOutputs: [expect.objectContaining({ sizeBytes: png.length })] }) + expect(await readFile(error.retainedOutputs[0].path)).toEqual(png) + expect(fixture.publications).toBe(0) + await fixture.service.dispose() + }) + + it.each(['EACCES', 'EPERM', 'EROFS', 'ENOSPC', 'PATH_OUTSIDE_GRANTED_DIRECTORY', 'FILE_IO_FAILED', 'FILE_PERMISSION_REVOKED', 'FILE_ENGINE_INTERRUPTED'])('classifies local %s errors without copying raw messages or paths to diagnostics', (code) => { + const native = Object.assign(new Error(`private path: ${workspace}`), { code }) + const wrapped = new ArtifactPublicationError(native, { operationId: 'publication', conversationId: 'conversation-1', cause: 'generated', requested: 1, written: 0, unconfirmedWrites: 0, artifactIds: [], stage: 'file', outcome: 'failed' }) + expect(readImageFileErrorCode(wrapped)).toBe(code) + expect(readImageDiagnostic({ fileErrorCode: code, message: native.message, stack: native.stack, retainedOutputs: [{ path: workspace }] })).toEqual({ fileErrorCode: code }) + expect(readImageDiagnostic({ fileErrorCode: `${code}: ${workspace}` })).toBeNull() + expect(readImageFileErrorCode(new Error(`FILE_PERMISSION_REVOKED ${workspace}`))).toBeUndefined() + expect(readImageFileErrorCode(new Error('FILE_PERMISSION_REVOKED'))).toBe('FILE_PERMISSION_REVOKED') + }) + + it('rejects a missing host material writer before submitting', async () => { + const fixture = createGenerationFixture({ writeGeneratedImageMaterial: undefined }) + await expect(generate(fixture.service, { outputPath: 'result.png', prompt: 'Create' }, model, new AbortController().signal)).rejects.toMatchObject({ code: 'IMAGE_GENERATION_FAILED', outcome: 'failed', diagnostic: { stage: 'preparing', fileErrorCode: 'IMAGE_MATERIAL_UNAVAILABLE' } }) + expect(fixture.submissions).toBe(0) + expect(fixture.publications).toBe(0) + await fixture.service.dispose() + }) + + it('rejects a missing captured publication writer before submitting', async () => { + const fixture = createGenerationFixture() + await expect(fixture.service.generate({ outputPath: 'result.png', prompt: 'Create' }, model, new AbortController().signal)).rejects.toMatchObject({ code: 'IMAGE_GENERATION_FAILED', outcome: 'failed', diagnostic: { stage: 'preparing', fileErrorCode: 'IMAGE_MATERIAL_UNAVAILABLE' } }) + expect(fixture.submissions).toBe(0) + expect(fixture.publications).toBe(0) + await fixture.service.dispose() + }) + + it('persists only the confirmed original when captured-writer cleanup fails after cancellation', async () => { + const requests = new ImageRequestStore(new BuddyDataPaths(join(workspace, 'receipts'))) + const fixture = createGenerationFixture({ requests, imageCount: 2 }) + const controller = new AbortController() + const writeFiles: CapturedFileWriter = async (paths, write) => { + await publication.writeFiles(paths, write) + controller.abort(new Error('cancelled during cleanup')) + throw Object.assign(new Error('release failed'), { code: 'FILE_ENGINE_INTERRUPTED' }) + } + const invocation = { runId: randomUUID(), toolCallId: 'original' } + const input = { outputPath: 'result.png', prompt: 'Create' } + const error = await generate(fixture.service, input, model, controller.signal, grants, writeFiles, invocation).catch(error => error) + expect(error).toMatchObject({ outcome: 'partial', diagnostic: { fileErrorCode: 'FILE_ENGINE_INTERRUPTED' } }) + expect(error.retainedOutputs).toHaveLength(1) + expect(await readFile(error.retainedOutputs[0].path)).toEqual(png) + await expect(generate(fixture.service, input, model, new AbortController().signal, grants, publication.writeFiles, { ...invocation, toolCallId: 'blocked' })).rejects.toMatchObject({ code: 'IMAGE_GENERATION_BLOCKED', outcome: 'partial', retainedOutputs: error.retainedOutputs }) + expect(fixture.submissions).toBe(1) + expect(fixture.publications).toBe(0) + await fixture.service.dispose() + }) + + it('keeps a native parent-substitution refusal partial without writing outside the authorized scratch', async () => { + const requests = new ImageRequestStore(new BuddyDataPaths(join(workspace, 'receipts'))) + const fixture = createGenerationFixture({ requests }) + const directory = join(workspace, '.image-generation', 'conversation-1') + const backup = join(workspace, '.image-generation', 'original') + const outside = join(publicationRoot, 'outside') + await mkdir(directory, { recursive: true }) + await mkdir(outside) + const mutate = publication.engine.mutate.bind(publication.engine) + let swapped = false + vi.spyOn(publication.engine, 'mutate').mockImplementation(async (request) => { + if (request.action.kind !== 'writeFile') + return mutate(request) + await rename(directory, backup) + await symlink(outside, directory, process.platform === 'win32' ? 'junction' : 'dir') + swapped = true + try { + return await mutate(request) + } + finally { + await unlink(directory) + await rename(backup, directory) + } + }) + const invocation = { runId: randomUUID(), toolCallId: 'original' } + const input = { outputPath: 'result.png', prompt: 'Create' } + const error = await generate(fixture.service, input, model, new AbortController().signal, grants, publication.writeFiles, invocation).catch(error => error) + expect(error).toMatchObject({ outcome: 'partial', diagnostic: { fileErrorCode: 'FILE_PERMISSION_REVOKED' } }) + expect(error.retainedOutputs).toBeUndefined() + expect(swapped).toBe(true) + expect(await readdir(outside)).toEqual([]) + expect(await readdir(directory)).toEqual([]) + await expect(generate(fixture.service, input, model, new AbortController().signal, grants, publication.writeFiles, { ...invocation, toolCallId: 'blocked' })).rejects.toMatchObject({ code: 'IMAGE_GENERATION_BLOCKED', outcome: 'partial' }) + expect(fixture.submissions).toBe(1) + expect(fixture.publications).toBe(0) + await fixture.service.dispose() + }) + + it('replays completed work even when references or the selected provider are unavailable', async () => { const request = { conversationId: randomUUID(), runId: randomUUID(), requestId: randomUUID() } const result = { artifactIds: ['generated-1'], outputs: [{ artifactId: 'generated-1', path: '/workspace/result.png', mimeType: 'image/png' }], responseId: 'response-1' } const unavailable = async (): Promise => { throw new Error('The current provider or reference is unavailable') } - const source = { listConversationArtifacts: () => [], listForConversation: () => [], materializeConversationImages: unavailable } - const service = new ImageGenerationService({ - conversationId: request.conversationId, - cwd: '/workspace', - grants: [], - attachmentService: source, - artifactService: { ...source, registerGeneratedImages: unavailable }, - imageGenerationGateway: { supports: () => false, generate: unavailable }, - requests: { completed: async () => ({ request, result }), begin: unavailable }, - }) - await expect(service.generate({ outputPath: 'result.png', prompt: 'Edit', reference: { mode: 'latest' } }, model, new AbortController().signal, [], undefined, { runId: request.runId, toolCallId: 'fixture' })).resolves.toEqual({ ...result, request }) + const source = { materializeConversationImages: unavailable } + const service = createService({ referenceAccess: { authorize: () => { + throw new Error('Reference is no longer visible') + } }, conversationId: request.conversationId, cwd: workspace, grants, attachmentService: source, artifactService: { ...source, registerGeneratedImages: unavailable }, imageGenerationGateway: { supports: () => false, generate: unavailable }, requests: { completed: async () => ({ request, result }), begin: unavailable } }) + await expect(service.generate({ outputPath: 'invalid.jpg', prompt: 'Edit', referenceIds: ['unavailable-reference'] }, model, new AbortController().signal, [], undefined, { runId: request.runId, toolCallId: 'fixture' })).resolves.toEqual({ ...result, request }) await service.dispose() }) it('preserves reference order and uses the last artifact reference for lineage', async () => { const references: string[] = [] let saved: Parameters[0] | undefined - const service = new ImageGenerationService({ - conversationId: 'conversation-1', - cwd: '/workspace', - grants: [], - artifactService: { - listConversationArtifacts: () => [], - async materializeConversationImages(_conversationId, ids) { - if (ids?.[0] === 'attachment-1') - throw Object.assign(new Error('Not an artifact'), { code: 'ARTIFACT_NOT_FOUND' }) - return { images: [{ ...image, data: ids![0]! }], records: [{ id: ids![0]! }] } - }, - async registerGeneratedImages(input) { - saved = input - return [{ id: 'generated-1', currentPath: '/workspace/result.png', mimeType: 'image/png' }] - }, + const service = createService({ referenceAccess: { authorize: () => {} }, conversationId: 'conversation-1', cwd: workspace, grants, artifactService: { + async materializeConversationImages(_conversationId, ids) { + if (ids?.[0] === 'attachment-1') + throw Object.assign(new Error('Not an artifact'), { code: 'ARTIFACT_NOT_FOUND' }) + return { images: [{ ...image, data: ids![0]! }], records: [{ id: ids![0]! }] } }, - attachmentService: { - listForConversation: () => [], - materializeConversationImages: async () => ({ images: [{ ...image, data: 'attachment-1' }], records: [{ id: 'attachment-1' }] }), + async registerGeneratedImages(input) { + saved = input + return [{ id: 'generated-1', currentPath: '/workspace/result.png', mimeType: 'image/png' }] }, - imageGenerationGateway: { - supports: () => true, - async generate(input) { - references.push(...input.inputImages.map(item => item.data)) - return { images: [{ bytes: new Uint8Array(png), mimeType: 'image/png' }], responseId: 'response-1' } - }, + }, attachmentService: { + materializeConversationImages: async () => ({ images: [{ ...image, data: 'attachment-1' }], records: [{ id: 'attachment-1' }] }), + }, imageGenerationGateway: { + supports: () => true, + async generate(input) { + references.push(...input.inputImages.map(item => item.data)) + return { images: [{ bytes: new Uint8Array(png), mimeType: 'image/png' }], responseId: 'response-1' } }, - }) - await expect(service.generate({ outputPath: ' result.png ', prompt: 'use references', reference: { mode: 'resources', resourceIds: ['artifact-1', 'attachment-1', 'artifact-2'] } }, model, new AbortController().signal)).resolves.toEqual({ artifactIds: ['generated-1'], outputs: [{ artifactId: 'generated-1', path: '/workspace/result.png', mimeType: 'image/png' }], responseId: 'response-1' }) + } }) + await expect(generate(service, { outputPath: ' result.png ', prompt: 'use references', referenceIds: ['artifact-1', 'attachment-1', 'artifact-2'] }, model, new AbortController().signal)).resolves.toEqual({ artifactIds: ['generated-1'], outputs: [{ artifactId: 'generated-1', path: '/workspace/result.png', mimeType: 'image/png' }], responseId: 'response-1' }) expect(references).toEqual(['artifact-1', 'attachment-1', 'artifact-2']) expect(saved).toMatchObject({ sourceArtifactId: 'artifact-2', outputPath: 'result.png', conversationId: 'conversation-1' }) }) it.each([ - ['2026-10-31T00:00:00Z', '2026-10-31T00:01:00Z', 'attachment-1'], - ['2026-10-31T00:02:00Z', '2026-10-31T00:01:00Z', 'artifact-1'], - ['2026-10-31T00:01:00Z', '2026-10-31T00:01:00Z', 'attachment-1'], - ['', '2026-10-31T00:01:00Z', 'attachment-1'], - ])('uses the newest updated artifact or attachment for latest: %s / %s', async (artifactTime, attachmentTime, expected) => { - const read: string[] = [] - let generatedReferences: string[] = [] - const service = new ImageGenerationService({ + { images: [], records: [] }, + { images: [image, image], records: [{ id: 'reference-1' }] }, + { images: [image], records: [{ id: 'different-reference' }] }, + ])('rejects missing or mismatched reference material before submission: %j', async (materialized) => { + let submissions = 0 + const source = { materializeConversationImages: async () => materialized } + const service = createService({ conversationId: 'conversation-1', - cwd: '/workspace', - grants: [], - artifactService: { - listConversationArtifacts: () => [{ id: 'artifact-1', kind: 'file', mimeType: 'image/png', createdAt: artifactTime, updatedAt: artifactTime }], - materializeConversationImages: async (_conversationId, ids) => { - if (ids?.[0] !== 'artifact-1') - throw Object.assign(new Error('Missing'), { code: 'ARTIFACT_NOT_FOUND' }) - read.push('artifact-1') - return { images: [{ ...image, data: 'artifact-1' }], records: [{ id: 'artifact-1' }] } - }, - registerGeneratedImages: async () => [{ id: 'result', currentPath: '/workspace/actual-2.png', mimeType: 'image/png' }], - }, - attachmentService: { - listForConversation: () => [{ id: 'attachment-1', mimeType: 'image/png', createdAt: attachmentTime }], - materializeConversationImages: async () => { - read.push('attachment-1') - return { images: [{ ...image, data: 'attachment-1' }], records: [{ id: 'attachment-1' }] } - }, - }, - imageGenerationGateway: { supports: () => true, generate: async (input) => { - generatedReferences = input.inputImages.map(image => image.data) - return { images: [{ bytes: new Uint8Array(png), mimeType: 'image/png' }], responseId: null } + cwd: workspace, + grants, + referenceAccess: { authorize: () => {} }, + attachmentService: source, + artifactService: { ...source, registerGeneratedImages: async () => [] }, + imageGenerationGateway: { supports: () => true, generate: async () => { + submissions++ + return { images: [], responseId: null } } }, }) - const result = await service.generate({ outputPath: 'requested.png', prompt: 'Edit latest', reference: { mode: 'latest' } }, model, new AbortController().signal) - expect(read).toEqual([expected]) - expect(generatedReferences).toEqual([expected]) - expect(result.outputs).toEqual([{ artifactId: 'result', path: '/workspace/actual-2.png', mimeType: 'image/png' }]) + await expect(generate(service, { outputPath: 'result.png', prompt: 'Edit', referenceIds: ['reference-1'] }, model, new AbortController().signal)).rejects.toMatchObject({ code: 'IMAGE_GENERATION_REFERENCE_UNAVAILABLE', outcome: 'failed' }) + expect(submissions).toBe(0) + await service.dispose() }) it('reports inaccessible references as a known failure before sending an image request', async () => { @@ -191,18 +490,11 @@ describe('conversation image generation', () => { throw Object.assign(new Error('private path and provider body'), { code: 'DIRECTORY_NOT_AUTHORIZED' }) } } let submitted = false - const service = new ImageGenerationService({ - conversationId: 'conversation-1', - cwd: '/workspace', - grants: [], - attachmentService: source, - artifactService: { ...source, registerGeneratedImages: async () => [] }, - imageGenerationGateway: { supports: () => true, generate: async () => { - submitted = true - return { images: [], responseId: null } - } }, - }) - await expect(service.generate({ outputPath: 'result.png', prompt: 'Edit', reference: { mode: 'resources', resourceIds: ['restricted-reference'] } }, model, new AbortController().signal)).rejects.toMatchObject({ code: 'IMAGE_GENERATION_REFERENCE_UNAVAILABLE', outcome: 'failed', retryable: false }) + const service = createService({ referenceAccess: { authorize: () => {} }, conversationId: 'conversation-1', cwd: workspace, grants, attachmentService: source, artifactService: { ...source, registerGeneratedImages: async () => [] }, imageGenerationGateway: { supports: () => true, generate: async () => { + submitted = true + return { images: [], responseId: null } + } } }) + await expect(generate(service, { outputPath: 'result.png', prompt: 'Edit', referenceIds: ['restricted-reference'] }, model, new AbortController().signal)).rejects.toMatchObject({ code: 'IMAGE_GENERATION_REFERENCE_UNAVAILABLE', outcome: 'failed', retryable: false }) expect(submitted).toBe(false) }) @@ -214,18 +506,11 @@ describe('conversation image generation', () => { preferences: async () => ({ generate: { mode: 'native', target: null, fallbacks: [] } }), }) let published: Uint8Array | undefined - const service = new ImageGenerationService({ - conversationId: 'conversation-1', - cwd: '/workspace', - grants: [], - attachmentService: source, - artifactService: { ...source, registerGeneratedImages: async ({ images }) => { - published = images[0]!.bytes - return [{ id: 'generated-1', currentPath: '/workspace/result.png', mimeType: 'image/png' }] - } }, - imageGenerationGateway: gateway, - }) - const result = await service.generate({ outputPath: 'result.png', prompt: 'Create', output: { size: '1024x1024', background: 'transparent', quality: 'high' } }, model, new AbortController().signal) + const service = createService({ referenceAccess: { authorize: () => {} }, conversationId: 'conversation-1', cwd: workspace, grants, attachmentService: source, artifactService: { ...source, registerGeneratedImages: async ({ images }) => { + published = images[0]!.bytes + return [{ id: 'generated-1', currentPath: '/workspace/result.png', mimeType: 'image/png' }] + } }, imageGenerationGateway: gateway }) + const result = await generate(service, { outputPath: 'result.png', prompt: 'Create', output: { size: '1024x1024', background: 'transparent', quality: 'high' } }, model, new AbortController().signal) expect(result).toMatchObject({ artifactIds: ['generated-1'], outputs: [{ artifactId: 'generated-1', path: '/workspace/result.png', mimeType: 'image/png' }], responseId: 'response-different-size' }) expect(Buffer.from(published!)).toEqual(png) expect([png.readUInt32BE(16), png.readUInt32BE(20)]).toEqual([2, 1]) @@ -234,51 +519,86 @@ describe('conversation image generation', () => { it('rejects a PNG signature without decodable pixels before publication with unknown outcome', async () => { const source = { listConversationArtifacts: () => [], listForConversation: () => [], materializeConversationImages: async () => ({ images: [], records: [] }) } let published = false - const service = new ImageGenerationService({ - conversationId: 'conversation-1', - cwd: '/workspace', - grants: [], - attachmentService: source, - artifactService: { ...source, registerGeneratedImages: async () => { - published = true - return [] - } }, - imageGenerationGateway: { supports: () => true, generate: async () => ({ images: [{ bytes: Uint8Array.from([137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 0]), mimeType: 'image/png' }], responseId: null }) }, - }) - await expect(service.generate({ outputPath: 'result.png', prompt: 'Create' }, model, new AbortController().signal)).rejects.toMatchObject({ code: 'IMAGE_GENERATION_INVALID_RESPONSE', outcome: 'unknown', retryable: false }) + const service = createService({ referenceAccess: { authorize: () => {} }, conversationId: 'conversation-1', cwd: workspace, grants, attachmentService: source, artifactService: { ...source, registerGeneratedImages: async () => { + published = true + return [] + } }, imageGenerationGateway: { supports: () => true, generate: async () => ({ images: [{ bytes: Uint8Array.from([137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 0]), mimeType: 'image/png' }], responseId: null }) } }) + await expect(generate(service, { outputPath: 'result.png', prompt: 'Create' }, model, new AbortController().signal)).rejects.toMatchObject({ code: 'IMAGE_GENERATION_INVALID_RESPONSE', outcome: 'unknown', retryable: false }) expect(published).toBe(false) }) - it.each(['provider', 'publishing'])('does not persist generated output when cancellation precedes file publication at %s', async (phase) => { + it.each(['provider', 'publishing'])('does not publish the user output when cancellation precedes publication at %s', async (phase) => { const controller = new AbortController() let saved = false const source = { listConversationArtifacts: () => [], listForConversation: () => [], materializeConversationImages: async () => ({ images: [], records: [] }) } - const service = new ImageGenerationService({ - conversationId: 'conversation-1', - cwd: '/workspace', - grants: [], - artifactService: { - ...source, - async registerGeneratedImages() { - saved = true - return [{ id: 'generated-1', currentPath: '/workspace/result.png', mimeType: 'image/png' }] - }, + const service = createService({ referenceAccess: { authorize: () => {} }, conversationId: 'conversation-1', cwd: workspace, grants, artifactService: { + ...source, + async registerGeneratedImages() { + saved = true + return [{ id: 'generated-1', currentPath: '/workspace/result.png', mimeType: 'image/png' }] }, - attachmentService: source, - imageGenerationGateway: { - supports: () => true, - async generate() { - if (phase === 'provider') - controller.abort(new Error('cancelled')) - return { images: [{ bytes: new Uint8Array(png), mimeType: 'image/png' }], responseId: 'response-1' } - }, + }, attachmentService: source, imageGenerationGateway: { + supports: () => true, + async generate() { + if (phase === 'provider') + controller.abort(new Error('cancelled')) + return { images: [{ bytes: new Uint8Array(png), mimeType: 'image/png' }], responseId: 'response-1' } }, - }) + } }) service.onDidChange((event) => { if (phase === 'publishing' && event.phase === 'publishing') controller.abort(new Error('cancelled')) }) - await expect(service.generate({ outputPath: 'result.png', prompt: 'generate' }, model, controller.signal)).rejects.toThrow('cancelled') + const pending = generate(service, { outputPath: 'result.png', prompt: 'generate' }, model, controller.signal) + if (phase === 'provider') + await expect(pending).rejects.toThrow('cancelled') + else + await expect(pending).rejects.toMatchObject({ outcome: 'partial', retainedOutputs: [expect.objectContaining({ mimeType: 'image/png' })] }) expect(saved).toBe(false) }) }) + +function generate(service: ImageGenerationService, input: Parameters[0], model: Parameters[1], signal: AbortSignal, executionGrants = grants, writeFiles: CapturedFileWriter = publication.writeFiles, invocation?: Parameters[5]) { + return service.generate(input, model, signal, executionGrants, writeFiles, invocation) +} + +function createService(options: ImageGenerationServiceOptions) { + const materials = new ImageGenerationMaterialStore(workspace, options.conversationId) + return new ImageGenerationService({ writeGeneratedImageMaterial: (image, grants, writeFiles, signal) => materials.write(image, grants, writeFiles, signal), ...options }) +} + +function createGenerationFixture(options: { requests?: ImageGenerationServiceOptions['requests'], failure?: 'unknown' | 'partial', publicationError?: Error, writeGeneratedImageMaterial?: ImageGenerationServiceOptions['writeGeneratedImageMaterial'], imageCount?: number } = {}) { + let submissions = 0 + let publications = 0 + const events: ImageOperationEvent[] = [] + const source = { materializeConversationImages: async () => ({ images: [], records: [] }) } + const service = createService({ + conversationId: 'conversation-1', + cwd: workspace, + grants, + requests: options.requests, + ...('writeGeneratedImageMaterial' in options ? { writeGeneratedImageMaterial: options.writeGeneratedImageMaterial } : {}), + referenceAccess: { authorize: () => {} }, + attachmentService: source, + artifactService: { ...source, registerGeneratedImages: async () => { + publications++ + if (options.failure === 'partial') + throw new ArtifactPublicationError(options.publicationError ?? new Error('publication interrupted'), { operationId: 'publication-1', conversationId: 'conversation-1', cause: 'generated', requested: 1, written: 0, unconfirmedWrites: 1, artifactIds: [], stage: 'file', outcome: 'partial' }) + return [{ id: 'generated-1', currentPath: join(workspace, 'result.png'), mimeType: 'image/png' }] + } }, + imageGenerationGateway: { supports: () => true, generate: async (input) => { + await input.onSubmitting?.() + submissions++ + if (options.failure === 'unknown') + throw new ImageGenerationError('IMAGE_GENERATION_FAILED', { outcome: 'unknown' }) + return { images: Array.from({ length: options.imageCount ?? 1 }, () => ({ bytes: png, mimeType: 'image/png' as const })), responseId: 'response-1' } + } }, + }) + service.onDidChange(event => events.push(event)) + return { + service, + events, + get submissions() { return submissions }, + get publications() { return publications }, + } +} diff --git a/apps/buddy/service/src/images/__tests__/ImageReferenceAccess.spec.ts b/apps/buddy/service/src/images/__tests__/ImageReferenceAccess.spec.ts new file mode 100644 index 000000000..bc756c57c --- /dev/null +++ b/apps/buddy/service/src/images/__tests__/ImageReferenceAccess.spec.ts @@ -0,0 +1,278 @@ +import type { ExtensionAPI, ExtensionToolContext, ToolCallEvent, ToolDefinition } from '@earendil-works/pi-coding-agent' +import type { BuddyRunEvent } from '../../events/BuddyRunEvent' +import type { ImageReferencePage } from '../ImageReferenceAccess' +import { afterEach, describe, expect, it } from 'vitest' +import { createBuddyUserContent } from '../../../../shared/conversation/buddyUserContent' +import { ArtifactService } from '../../artifacts/ArtifactService' +import { AttachmentService } from '../../attachments/AttachmentService' +import { RunEventQueries } from '../../events/RunEventQueries' +import { createExecutionStorageFixture } from '../../storage/__tests__/executionFixture' +import { createArtifactRepository } from '../../storage/artifactRepository' +import { createAttachmentRepository } from '../../storage/attachmentRepository' +import { BuddyDataPaths } from '../../storage/BuddyDataPaths' +import { createChatQueueRepository } from '../../storage/chatQueueRepository' +import { createComposerDraftRepository } from '../../storage/composerDraftRepository' +import { projectConversationPublication } from '../../storage/conversationPublicationRepository' +import { createConversationTreeRepository } from '../../storage/conversationTreeRepository' +import { hashChildAdmissionRequest } from '../../storage/executionRepository' +import { createRunInputRepository } from '../../storage/runInputRepository' +import { ImageReferenceAccess } from '../ImageReferenceAccess' +import { createImageReferenceCapability } from '../imageReferenceExtension' + +const fixtures: ReturnType[] = [] +afterEach(() => fixtures.splice(0).forEach(f => f.database.close())) + +function fixture() { + const f = createExecutionStorageFixture() + fixtures.push(f) + const records = createAttachmentRepository(f.database) + const artifactRecords = createArtifactRepository(f.database) + const publications = createConversationTreeRepository(f.database) + const controller = new AbortController() + let currentRunId = 'root' + let counter = 0 + const now = () => new Date(Date.UTC(2026, 9, 24, 0, 0, counter++)).toISOString() + const access = new ImageReferenceAccess({ + conversationId: 'conversation', + getRunId: () => currentRunId, + signal: controller.signal, + conversations: f.conversations, + publications, + executions: f.executions, + inputs: createRunInputRepository(f.database), + events: new RunEventQueries(f.database), + attachments: new AttachmentService({ paths: new BuddyDataPaths('/fixture'), repository: records }), + artifacts: new ArtifactService({ repository: artifactRecords }), + }) + function message(id: string, branchId = 'branch', conversationId = 'conversation') { + return f.conversations.createMessage({ id, conversationId, branchId, role: 'user', runId: null, createdAt: now(), content: { resourceSnapshots: [], userContent: createBuddyUserContent('Reference fixture') } }) + } + function attachment(id: string, messageId = 'goal', mimeType = 'image/png', createdAt = now()) { + const path = `/fixture/${id}.png` + return records.create({ id, conversationId: 'conversation', messageId, draftId: null, storedPath: path, name: `${id}.png`, mimeType, sizeBytes: 1, createdAt }) + } + function inputs(runId: string, ids: string[]) { + f.database.prepare('INSERT INTO run_inputs (run_id, prompt, attachment_ids_json, context_items_json, created_at) VALUES (?, ?, ?, ?, ?)') + .run(runId, 'Reference fixture', JSON.stringify(ids), '[]', now()) + } + function root(id: string, messageId: string, ids: string[] = [], branchId = 'branch', conversationId = 'conversation') { + const run = f.runs.create({ id, conversationId, branchId, triggeringMessageId: messageId, buddyId: 'default', actorSnapshot: f.snapshot('default'), provider: 'fixture', model: 'fixture', purpose: 'chat', status: 'running', piSessionFile: null, startedAt: now(), approvalPolicy: 'policy', executionProfile: 'workspace_write' }) + inputs(id, ids) + currentRunId = id + return run + } + function output(id: string, runId: string, sequence = 1) { + const run = f.runs.findById(runId)! + const createdAt = now() + artifactRecords.save({ id, conversationId: run.conversationId, createdAt, updatedAt: createdAt, currentPath: `/fixture/${id}.png`, directoryGrantId: 'grant', directoryRoot: '/fixture', relativePath: `${id}.png`, name: `${id}.png`, mimeType: 'image/png', sizeBytes: 1, kind: 'file', sourceArtifactId: null }) + const event: BuddyRunEvent = { runId, sequence, type: 'output.produced', payload: { artifactIds: [id], sourceToolCallId: `tool-${id}`, sourceToolName: 'lexora_image_generate' }, createdAt } + f.database.prepare('INSERT INTO run_events (run_id, sequence, event_type, payload_json, created_at) VALUES (?, ?, ?, ?, ?)') + .run(runId, sequence, event.type, JSON.stringify(event.payload), createdAt) + projectConversationPublication(f.database, event) + return `output:${runId}:${sequence}` + } + function child(id: string, options: { callerId?: string, sources?: string[], attachments?: string[], previousRunId?: string, conversation?: boolean, actor?: boolean } = {}) { + const input = f.child(id, options.actor ? f.actor.id : f.npc.id, options.callerId ?? 'root') + input.work = { contextMode: options.conversation ? 'conversation' : 'isolated', contextSourceIds: options.sources ?? [], previousRunId: options.previousRunId } + input.initialInput.attachmentIds = options.attachments ?? [] + input.requestHash = hashChildAdmissionRequest(input) + const record = f.executions.admitChild(input).execution + f.executions.claimWorking(f.fence(id)) + f.runs.markRunning(id, now()) + currentRunId = id + return record + } + message('goal') + return { ...f, access, controller, now, message, attachment, inputs, root, output, child, publications, setRun: (id: string) => { + currentRunId = id + }, ids: () => access.list({}, controller.signal).items.map(item => item.referenceId) } +} + +describe('visible image references', () => { + it('uses frozen branch publications, not the default branch or newer conversation images', () => { + const f = fixture() + f.attachment('old', 'goal') + f.inputs('root', ['old']) + f.output('old-output', 'root') + f.message('accepted') + f.root('accepted-run', 'accepted') + f.message('late') + f.attachment('late', 'late') + f.root('late-run', 'late', ['late']) + f.output('late-output', 'late-run') + f.setRun('accepted-run') + expect(new Set(f.ids())).toEqual(new Set(['old', 'old-output'])) + expect(() => f.access.authorize(['late'], f.controller.signal)).toThrow('IMAGE_REFERENCE_NOT_VISIBLE') + f.conversations.createBranch({ id: 'fork', conversationId: 'conversation', parentBranchId: 'branch', forkedFromMessageId: 'goal', createdAt: f.now(), activate: true }) + expect(new Set(f.ids())).toEqual(new Set(['old', 'old-output'])) + f.message('fork-input', 'fork') + f.root('fork-run', 'fork-input', [], 'fork') + expect(f.ids()).toEqual(['old']) + expect(() => f.access.authorize(['old-output'], f.controller.signal)).toThrow('IMAGE_REFERENCE_NOT_VISIBLE') + }) + + it('does not grant isolated children caller images or peer private outputs', () => { + const f = fixture() + f.attachment('accepted') + f.inputs('root', ['accepted']) + f.output('public-output', 'root') + f.child('private') + f.output('private-output', 'private') + expect(f.ids()).toEqual(['private-output']) + expect(() => f.access.authorize(['accepted', 'public-output'], f.controller.signal)).toThrow('IMAGE_REFERENCE_NOT_VISIBLE') + f.child('peer') + expect(f.ids()).toEqual([]) + expect(() => f.access.authorize(['private-output'], f.controller.signal)).toThrow('IMAGE_REFERENCE_NOT_VISIBLE') + f.setRun('root') + expect(new Set(f.ids())).toEqual(new Set(['accepted', 'public-output'])) + expect(() => f.access.authorize(['private-output'], f.controller.signal)).toThrow('IMAGE_REFERENCE_NOT_VISIBLE') + }) + + it('admits only explicit public selections and handed-off attachments, retaining them on same-work continuation', () => { + const f = fixture() + f.attachment('accepted') + f.inputs('root', ['accepted']) + const selected = f.output('selected-output', 'root') + f.message('new-goal') + f.root('new-root', 'new-goal') + const first = f.child('selected', { callerId: 'new-root', sources: [selected] }) + expect(f.ids()).toEqual(['selected-output']) + f.output('own-output', 'selected') + f.runs.reconcileTerminal('selected', 'completed', f.now(), null) + f.executions.recordCleanup('selected', 'clean', null) + const next = f.child('continued', { callerId: 'new-root', previousRunId: 'selected' }) + expect(next.run.threadId).toBe(first.run.threadId) + expect(new Set(f.ids())).toEqual(new Set(['selected-output', 'own-output'])) + f.child('handed', { attachments: ['accepted'] }) + expect(f.ids()).toEqual(['accepted']) + expect(() => f.access.authorize(['selected-output'], f.controller.signal)).toThrow('IMAGE_REFERENCE_NOT_VISIBLE') + }) + + it('gives conversation-mode actors the accepted public snapshot without private NPC artifacts', () => { + const f = fixture() + f.attachment('accepted') + f.inputs('root', ['accepted']) + f.output('public-output', 'root') + f.child('npc') + f.output('private-output', 'npc') + f.message('new-goal') + f.root('new-root', 'new-goal') + f.child('actor', { callerId: 'new-root', conversation: true, actor: true }) + expect(new Set(f.ids())).toEqual(new Set(['accepted', 'public-output'])) + }) + + it('accepts queued materials only after commit and does not retroactively pass them to children', () => { + const f = fixture() + f.inputs('root', []) + f.child('before-queue') + const queue = createChatQueueRepository(f.database) + const source = f.attachment('queued-source') + const draft = createComposerDraftRepository(f.database).open({ + draftId: 'queued-draft', + scope: { kind: 'global' }, + initialContent: createBuddyUserContent('Queued image'), + initialModelSelection: null, + initialExecutionConfig: { approvalPolicy: 'policy', executionProfile: 'workspace_write' }, + now: f.now(), + }) + queue.enqueue({ + approvalPolicy: 'policy', + executionProfile: 'workspace_write', + branchId: 'branch', + conversationId: 'conversation', + createdAt: f.now(), + draft: { draftId: draft.draftId, expectedRevision: draft.revision }, + model: 'fixture', + provider: 'fixture', + requestId: 'queued-request', + requestFingerprint: 'fingerprint', + runId: 'queued-run', + spaceId: null, + title: 'Fixture', + userMessageId: 'queued', + attachmentBindings: [{ id: 'queued-image', messageId: 'queued', sourceAttachmentId: source.id, sourceDraftId: null, storedPath: '/fixture/queued-image.png', createdAt: source.createdAt, mimeType: source.mimeType }], + userMessageContent: { resourceSnapshots: [], userContent: createBuddyUserContent('Queued image') }, + runInput: { prompt: 'Queued image', attachmentIds: ['queued-image'], contextItems: [], reasoning: null, serviceTier: null }, + }) + f.setRun('root') + expect(f.ids()).toEqual([]) + queue.commitInRun({ ...queue.pending({ conversationId: 'conversation', branchId: 'branch', id: 'queued' })!, actorSnapshot: f.snapshot('default') }, 'root', undefined) + expect(f.ids()).toEqual(['queued-image']) + f.setRun('before-queue') + expect(f.ids()).toEqual([]) + }) + + it('rejects other tasks, unaccepted files, non-images and composer resource identities', () => { + const f = fixture() + f.attachment('accepted') + f.attachment('text', 'goal', 'text/plain') + f.inputs('root', ['accepted', 'text']) + f.message('unaccepted') + f.attachment('unaccepted', 'unaccepted') + f.conversations.create({ id: 'other', branchId: 'other-branch', title: 'Other', spaceId: null, createdAt: f.now(), approvalPolicy: 'policy', executionProfile: 'workspace_write' }) + f.message('other-input', 'other-branch', 'other') + f.attachment('foreign', 'other-input') + f.root('foreign-run', 'other-input', ['foreign'], 'other-branch', 'other') + f.output('foreign-output', 'foreign-run') + f.message('current') + f.root('current-run', 'current') + expect(f.ids()).toEqual(['accepted']) + for (const id of ['foreign', 'foreign-output', 'unaccepted', 'text', 'composer-resource', 'missing', '/fixture/accepted.png']) + expect(() => f.access.authorize([id], f.controller.signal)).toThrow('IMAGE_REFERENCE_NOT_VISIBLE') + expect(() => f.access.authorize([''], f.controller.signal)).toThrow('IMAGE_REFERENCE_INVALID') + const ids = Object.freeze(['accepted', 'accepted']) + f.access.authorize(ids, f.controller.signal) + expect(ids).toEqual(['accepted', 'accepted']) + f.setRun('foreign-run') + expect(() => f.ids()).toThrow('IMAGE_REFERENCE_SCOPE_UNAVAILABLE') + }) + + it('pages deterministically, filters metadata, bounds requests and preserves authorization order', () => { + const f = fixture() + const ids = Array.from({ length: 55 }, (_, index) => `image-${String(index).padStart(2, '0')}`) + const time = f.now() + for (const id of ids.toReversed()) + f.attachment(id, 'goal', 'image/png', time) + f.inputs('root', ids) + const first = f.access.list({ limit: 50 }, f.controller.signal) + const second = f.access.list({ offset: first.nextOffset! }, f.controller.signal) + expect(first).toMatchObject({ total: 55, nextOffset: 50 }) + expect(second.nextOffset).toBeNull() + expect([...first.items, ...second.items].map(item => item.referenceId)).toEqual(ids) + expect(f.access.list({ query: 'IMAGE-03' }, f.controller.signal).items).toEqual([first.items[3]]) + expect(f.access.list({ offset: 55 }, f.controller.signal).items).toEqual([]) + expect(() => f.access.list({ offset: 56 }, f.controller.signal)).toThrow('IMAGE_REFERENCE_OFFSET_OUT_OF_RANGE') + for (const query of [{ limit: 51 }, { limit: 0 }, { offset: -1 }, { offset: 1.5 }, { query: 'x'.repeat(257) }]) + expect(() => f.access.list(query, f.controller.signal)).toThrow('IMAGE_REFERENCE_QUERY_INVALID') + const ordered = Object.freeze(['image-54', 'image-01', 'image-25']) + f.access.authorize(ordered, f.controller.signal) + expect(ordered).toEqual(['image-54', 'image-01', 'image-25']) + }) + + it.each(['abort', 'deleted', 'terminal', 'cancelled'] as const)('revalidates %s scope at execution, with read-only on-demand disclosure', async (change) => { + const f = fixture() + f.attachment('accepted') + f.inputs('root', ['accepted']) + const capability = createImageReferenceCapability(f.access) + const tools: ToolDefinition[] = [] + await capability.extension.factory({ registerTool: (tool: ToolDefinition) => tools.push(tool) } as unknown as ExtensionAPI) + expect(capability.disclosure).toMatchObject([{ exposure: 'on_demand', tools: [{ name: 'lexora_image_list', namespace: { name: 'lexora_image' } }] }]) + const event: ToolCallEvent = { type: 'tool_call', toolName: 'lexora_image_list', toolCallId: 'call', input: {} } + expect(await capability.classify(event, f.controller.signal)).toEqual({ access: 'read', paths: [] }) + const result = await tools[0]!.execute('call', {}, f.controller.signal, undefined, {} as ExtensionToolContext) + const text = result.content.find(content => content.type === 'text')! + const page: ImageReferencePage = JSON.parse(text.text) + expect(page.items[0]).toMatchObject({ referenceId: 'accepted', sourceKind: 'attachment', sourceMessageId: 'goal', mimeType: 'image/png' }) + await expect(tools[0]!.execute('call', { limit: 1000 }, f.controller.signal, undefined, {} as ExtensionToolContext)).rejects.toThrow('VALIDATION_FAILED') + if (change === 'abort') + f.controller.abort() + if (change === 'deleted') + f.conversations.markDeleted('conversation', f.now()) + if (change === 'terminal') + f.runs.reconcileTerminal('root', 'completed', f.now(), null) + if (change === 'cancelled') + f.database.prepare('UPDATE execution_state SET cancel_requested_at = ? WHERE run_id = ?').run(f.now(), 'root') + await expect(tools[0]!.execute('call', {}, f.controller.signal, undefined, {} as ExtensionToolContext)).rejects.toThrow() + expect(() => f.access.authorize(['accepted'], f.controller.signal)).toThrow() + }) +}) diff --git a/apps/buddy/service/src/images/__tests__/ImageRequestStore.spec.ts b/apps/buddy/service/src/images/__tests__/ImageRequestStore.spec.ts new file mode 100644 index 000000000..1e1a31640 --- /dev/null +++ b/apps/buddy/service/src/images/__tests__/ImageRequestStore.spec.ts @@ -0,0 +1,95 @@ +import { randomUUID } from 'node:crypto' +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { BuddyDataPaths } from '../../storage/BuddyDataPaths' +import { ImageRequestStore } from '../ImageRequestStore' + +const roots: string[] = [] +const identity = { fingerprint: 'a'.repeat(64), destination: 'b'.repeat(64) } +const references = [ + { referenceId: 'character', mimeType: 'image/png', sha256: 'c'.repeat(64) }, + { referenceId: 'pose', mimeType: 'image/png', sha256: 'd'.repeat(64) }, +] +const result = { artifactIds: ['output'], outputs: [{ artifactId: 'output', path: '/workspace/output.png', mimeType: 'image/png' }], responseId: 'response' } + +afterEach(async () => { + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))) +}) + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), 'image-request-')) + roots.push(root) + const paths = new BuddyDataPaths(root) + const store = new ImageRequestStore(paths) + const conversationId = randomUUID() + const runId = randomUUID() + const begun = await store.begin(conversationId, runId, 'call', identity, references) + if (begun.kind !== 'started') + throw new Error('Expected new request') + const path = join(paths.conversationDirectory(conversationId), 'image-requests', runId, `${begun.receipt.request.requestId}.json`) + return { store, paths, conversationId, runId, receipt: begun.receipt, path } +} + +describe('image request reference evidence', () => { + it('persists ordered input identities before submission and preserves completed replay', async () => { + const f = await fixture() + expect(JSON.parse(await readFile(f.path, 'utf8'))).toMatchObject({ state: 'preparing', identity, references }) + await f.receipt.submitting() + expect(JSON.parse(await readFile(f.path, 'utf8'))).toMatchObject({ state: 'unknown', references }) + await f.receipt.complete(result) + expect(await new ImageRequestStore(f.paths).completed(f.conversationId, f.runId, 'call')).toEqual({ result, request: f.receipt.request }) + expect(JSON.parse(await readFile(f.path, 'utf8'))).toMatchObject({ state: 'completed', references }) + }) + + it('reads completed legacy receipts without reference evidence and does not rewrite them', async () => { + const f = await fixture() + await f.receipt.complete(result) + const previous = JSON.parse(await readFile(f.path, 'utf8')) + delete previous.references + delete previous.identity + delete previous.version + const bytes = JSON.stringify(previous) + await writeFile(f.path, bytes) + const restored = new ImageRequestStore(f.paths) + expect(await restored.completed(f.conversationId, f.runId, 'call')).toEqual({ result, request: f.receipt.request }) + expect(await readFile(f.path, 'utf8')).toBe(bytes) + }) + + it.each(['unknown', 'partial'] as const)('does not let new reference metadata bypass an unresolved %s request', async (state) => { + const f = await fixture() + await f.receipt.submitting() + if (state === 'partial') + await f.receipt.fail({ code: 'IMAGE_GENERATION_FAILED', outcome: 'partial', diagnostic: null }, ['output']) + const restored = new ImageRequestStore(f.paths) + await expect(restored.begin(f.conversationId, f.runId, 'another-call', identity, [...references].reverse())).rejects.toMatchObject({ code: 'IMAGE_GENERATION_BLOCKED', outcome: state, request: f.receipt.request }) + expect(JSON.parse(await readFile(f.path, 'utf8')).references).toEqual(references) + }) + + it('persists retained outputs before publication and blocks replay after restart without rewriting the receipt', async () => { + const f = await fixture() + const retainedOutputs = [{ path: '/authorized/scratch/original.png', mimeType: 'image/png' as const, sizeBytes: 73, sha256: 'e'.repeat(64) }] + await f.receipt.submitting() + await f.receipt.retain(retainedOutputs) + const bytes = await readFile(f.path, 'utf8') + expect(JSON.parse(bytes)).toMatchObject({ state: 'partial', retainedOutputs }) + const restored = new ImageRequestStore(f.paths) + for (const toolCallId of ['call', 'another-call']) { + await expect(restored.begin(f.conversationId, f.runId, toolCallId, identity)).rejects.toMatchObject({ code: 'IMAGE_GENERATION_BLOCKED', outcome: 'partial', request: f.receipt.request, retainedOutputs }) + } + expect(await readFile(f.path, 'utf8')).toBe(bytes) + expect(await restored.begin(randomUUID(), f.runId, 'call', identity)).toMatchObject({ kind: 'started' }) + await f.receipt.complete(result) + expect(await restored.begin(f.conversationId, f.runId, 'call', identity)).toEqual({ kind: 'completed', request: f.receipt.request, result }) + expect(JSON.parse(await readFile(f.path, 'utf8'))).toMatchObject({ state: 'completed', retainedOutputs }) + }) + + it('records the actual selected identities when resuming an unsubmitted preparation', async () => { + const f = await fixture() + const selected = references.map(reference => ({ ...reference, referenceId: `${reference.referenceId}-copy` })) + const restored = new ImageRequestStore(f.paths) + expect(await restored.begin(f.conversationId, f.runId, 'call', identity, selected)).toMatchObject({ kind: 'started', receipt: { request: f.receipt.request } }) + expect(JSON.parse(await readFile(f.path, 'utf8'))).toMatchObject({ state: 'preparing', identity, references: selected }) + }) +}) diff --git a/apps/buddy/service/src/images/__tests__/imageGenerationExtension.spec.ts b/apps/buddy/service/src/images/__tests__/imageGenerationExtension.spec.ts index eaba0c068..96e3addb2 100644 --- a/apps/buddy/service/src/images/__tests__/imageGenerationExtension.spec.ts +++ b/apps/buddy/service/src/images/__tests__/imageGenerationExtension.spec.ts @@ -2,29 +2,39 @@ import type { Model } from '@earendil-works/pi-ai' import type { ToolDefinition } from '@earendil-works/pi-coding-agent' import type { ImageGenerationServiceOptions } from '../ImageGenerationService' import { Buffer } from 'node:buffer' - -import { describe, expect, it, vi } from 'vitest' +import { mkdtempSync, realpathSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, describe, expect, it, vi } from 'vitest' import { createImageGenerationExtension } from '../imageGenerationExtension' import { ImageGenerationError } from '../ImageGenerationGateway' +import { ImageGenerationMaterialStore } from '../ImageGenerationMaterialStore' import { ImageGenerationService } from '../ImageGenerationService' import { classifyImageGenerationTool, IMAGE_GENERATION_TOOL_NAME, } from '../imageGenerationToolContract' +import { createImagePublicationFixture } from './imagePublicationFixture' const png = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAAFElEQVR4AQEJAPb/AAD/AP//AAD/EPgD/UE7EkoAAAAASUVORK5CYII=', 'base64') +const workspace = realpathSync(mkdtempSync(join(tmpdir(), 'lexora-image-tool-'))) +const publication = createImagePublicationFixture(workspace) +afterAll(async () => { + await publication.dispose() + rmSync(workspace, { recursive: true, force: true }) +}) const workspaceContext = { - cwd: '/workspace', + cwd: workspace, grants: [{ - canonicalRoot: '/workspace', + canonicalRoot: workspace, grantId: 'workspace-1', kind: 'workspace' as const, - root: '/workspace', + root: workspace, }], } describe('imageGenerationExtension', () => { - it('generates from the latest conversation image and returns immutable artifact handles', async () => { + it('generates from explicit references and returns artifact handles', async () => { const materializeConversationImages = vi.fn(async () => ({ images: [{ data: png.toString('base64'), @@ -35,7 +45,7 @@ describe('imageGenerationExtension', () => { })) const registerGeneratedImages = vi.fn(async () => [{ id: 'generated-1', - currentPath: '/workspace/images/像素风角色.png', + currentPath: join(workspace, 'images/像素风角色.png'), mimeType: 'image/png', }]) const generate = vi.fn(async () => ({ @@ -46,12 +56,10 @@ describe('imageGenerationExtension', () => { const extension = createTestImageExtension({ ...workspaceContext, artifactService: { - listConversationArtifacts: () => [{ id: 'reference-1', kind: 'file', mimeType: 'image/png', createdAt: '2026-10-31T00:00:00Z', updatedAt: '2026-10-31T00:00:00Z' }], materializeConversationImages, registerGeneratedImages, }, attachmentService: { - listForConversation: () => [], materializeConversationImages: vi.fn(), }, conversationId: 'conversation-1', @@ -79,7 +87,7 @@ describe('imageGenerationExtension', () => { await expect(execute('tool-call-1', { outputPath: 'images/像素风角色.png', prompt: 'Turn the reference into Q-style pixel art', - reference: { mode: 'latest' }, + referenceIds: ['reference-1'], }, new AbortController().signal, undefined, { model: openAiModel() })).resolves.toMatchObject({ content: [{ type: 'text' }], details: { @@ -94,12 +102,32 @@ describe('imageGenerationExtension', () => { })) expect(registerGeneratedImages).toHaveBeenCalledWith(expect.objectContaining({ conversationId: 'conversation-1', - cwd: '/workspace', + cwd: workspace, grants: workspaceContext.grants, images: [expect.objectContaining({ mimeType: 'image/png' })], outputPath: 'images/像素风角色.png', sourceArtifactId: 'reference-1', - }), undefined) + }), publication.writeFiles) + }) + + it.each([ + { reference: { mode: 'latest' } }, + { reference: { mode: 'resources', resourceIds: ['reference-1'] } }, + { referenceIds: [] }, + { referenceIds: ['reference-1', 'reference-1'] }, + ])('rejects invalid reference selection before submitting: %j', async (selection) => { + let tool: ToolDefinition | undefined + let submitted = 0 + createImageGenerationExtension({ + getRunId: () => 'run-1', + service: { supports: () => true, generate: async () => { + submitted++ + return { artifactIds: [], outputs: [], responseId: null } + } }, + }).factory({ on: () => {}, registerTool: (registered: ToolDefinition) => tool = registered } as never) + const result = await tool!.execute!('legacy-call', { outputPath: 'result.png', prompt: 'Edit the image', ...selection }, undefined, undefined, { model: openAiModel() } as never) + expect(result.details).toMatchObject({ code: 'VALIDATION_FAILED', outcome: 'failed', artifactIds: [] }) + expect(submitted).toBe(0) }) it('declares image generation as a first-party artifact write', () => { @@ -137,9 +165,10 @@ describe('imageGenerationExtension', () => { it.each([ ['IMAGE_GENERATION_REFERENCE_UNAVAILABLE', 'failed', 'No image request was submitted'], + ['IMAGE_GENERATION_OUTPUT_UNAVAILABLE', 'failed', 'preserve the intended referenceIds'], ['IMAGE_GENERATION_INVALID_RESPONSE', 'unknown', 'do not automatically repeat'], - ['IMAGE_GENERATION_BLOCKED', 'unknown', 'Inspect any existing artifacts before choosing another approach'], - ] as const)('distinguishes reference preparation failure from an unknown provider result: %s', async (code, outcome, recovery) => { + ['IMAGE_GENERATION_BLOCKED', 'unknown', 'No new image request was submitted'], + ] as const)('distinguishes preparation failure from an unknown provider result: %s', async (code, outcome, recovery) => { let tool: ToolDefinition | undefined createImageGenerationExtension({ getRunId: () => 'run-1', @@ -151,13 +180,60 @@ describe('imageGenerationExtension', () => { expect(JSON.stringify(result)).not.toContain('private body') }) + it.each(['IMAGE_GENERATION_FAILED', 'IMAGE_GENERATION_BLOCKED'] as const)('provides copy-only recovery of retained results without disguising publication failure: %s', async (code) => { + const retainedOutputs = [{ path: join(workspace, '.image-generation', 'result.png'), mimeType: 'image/png' as const, sizeBytes: png.length, sha256: 'a'.repeat(64) }] + let tool: ToolDefinition | undefined + createImageGenerationExtension({ + getRunId: () => 'run-1', + service: { supports: () => true, generate: async () => { + throw new ImageGenerationError(code, { outcome: 'partial', diagnostic: { stage: 'publishing', fileErrorCode: 'EACCES' }, retainedOutputs, cause: new Error('private filesystem error') }) + } }, + }).factory({ on: () => {}, registerTool: (registered: ToolDefinition) => tool = registered } as never) + const result = await tool!.execute!('publication-call', { outputPath: 'output/image.png', prompt: 'Preserve image 1' }, undefined, undefined, { model: openAiModel() } as never) + expect(result.details).toMatchObject({ code, outcome: 'partial', artifactIds: [], retainedOutputs, diagnostic: { fileErrorCode: 'EACCES' } }) + const content = result.content[0]! + expect(content.type).toBe('text') + expect(JSON.parse(content.type === 'text' ? content.text : '')).toMatchObject({ + outputPath: 'output/image.png', + retainedOutputs, + message: expect.stringContaining('Recover by copying those files'), + }) + expect(JSON.stringify(result)).toContain('do not call lexora_image_generate again') + expect(JSON.stringify(result)).not.toContain('private filesystem error') + }) + + it.each([ + { code: 'IMAGE_GENERATION_REFERENCE_UNAVAILABLE' as const, diagnostic: undefined, referenceIds: [`image:${'a'.repeat(64)}`], message: 'inspection-only handles for lexora_image_read' }, + { code: 'IMAGE_GENERATION_FAILED' as const, diagnostic: { stage: 'preparing' as const, fileErrorCode: 'IMAGE_MATERIAL_UNAVAILABLE' as const }, referenceIds: undefined, message: 'the host cannot retain generated image results' }, + ])('identifies the actual preparation problem instead of suggesting image reinterpretation: $code', async ({ code, diagnostic, referenceIds, message }) => { + let tool: ToolDefinition | undefined + createImageGenerationExtension({ + getRunId: () => 'run-1', + service: { supports: () => true, generate: async () => { throw new ImageGenerationError(code, { outcome: 'failed', diagnostic }) } }, + }).factory({ on: () => {}, registerTool: (registered: ToolDefinition) => tool = registered } as never) + const result = await tool!.execute!('preparation-call', { outputPath: 'image.png', prompt: 'Preserve the reference', ...(referenceIds ? { referenceIds } : {}) }, undefined, undefined, { model: openAiModel() } as never) + expect(result.details).toMatchObject({ code, outcome: 'failed' }) + expect(JSON.stringify(result.content)).toContain(message) + expect(JSON.stringify(result.content)).toContain('No image request was submitted') + }) + + it('does not imply that an unpublished legacy result has an available retained file', async () => { + let tool: ToolDefinition | undefined + createImageGenerationExtension({ + getRunId: () => 'run-1', + service: { supports: () => true, generate: async () => { throw new ImageGenerationError('IMAGE_GENERATION_FAILED', { outcome: 'partial' }) } }, + }).factory({ on: () => {}, registerTool: (registered: ToolDefinition) => tool = registered } as never) + const result = await tool!.execute!('legacy-call', { outputPath: 'image.png', prompt: 'Create' }, undefined, undefined, { model: openAiModel() } as never) + expect(JSON.stringify(result.content)).toContain('No published artifact or retained output is available in this receipt') + expect(result.details).not.toHaveProperty('retainedOutputs') + }) + it('returns safe provider diagnostics to the model', async () => { let tool: ToolDefinition | undefined createTestImageExtension({ ...workspaceContext, artifactService: emptyArtifactService(), attachmentService: { - listForConversation: () => [], materializeConversationImages: vi.fn(), }, conversationId: 'conversation-1', @@ -210,7 +286,6 @@ describe('imageGenerationExtension', () => { function emptyArtifactService() { return { - listConversationArtifacts: () => [], materializeConversationImages: vi.fn(async () => ({ images: [], records: [] })), registerGeneratedImages: vi.fn(async () => []), } @@ -231,6 +306,7 @@ function openAiModel(): Model<'openai-responses'> { } } -function createTestImageExtension(options: ImageGenerationServiceOptions & { getRunId: () => string | undefined }) { - return createImageGenerationExtension({ getRunId: options.getRunId, service: new ImageGenerationService(options) }) +function createTestImageExtension(options: Omit & { getRunId: () => string | undefined }) { + const materials = new ImageGenerationMaterialStore(options.cwd, options.conversationId) + return createImageGenerationExtension({ createFileWriter: () => publication.writeFiles, getRunId: options.getRunId, service: new ImageGenerationService({ ...options, writeGeneratedImageMaterial: (image, grants, writeFiles, signal) => materials.write(image, grants, writeFiles, signal), referenceAccess: { authorize: () => {} } }) }) } diff --git a/apps/buddy/service/src/images/__tests__/imageGenerationLifecycle.spec.ts b/apps/buddy/service/src/images/__tests__/imageGenerationLifecycle.spec.ts index 39b4b32b8..402b8e39f 100644 --- a/apps/buddy/service/src/images/__tests__/imageGenerationLifecycle.spec.ts +++ b/apps/buddy/service/src/images/__tests__/imageGenerationLifecycle.spec.ts @@ -14,13 +14,16 @@ import { describe, expect, it, vi } from 'vitest' import { createPiEventProjectionState, projectPiEvent } from '../../agent/events/projectPiEvent' import { createIsolatedBuddySession } from '../../agent/sessions/__tests__/isolatedBuddySession' import { createImageGenerationExtension } from '../imageGenerationExtension' +import { ImageGenerationMaterialStore } from '../ImageGenerationMaterialStore' import { ImageGenerationService } from '../ImageGenerationService' import { IMAGE_GENERATION_TOOL_NAME } from '../imageGenerationToolContract' import { OpenAiImageGenerationService } from '../OpenAiImageGenerationService' +import { createImagePublicationFixture } from './imagePublicationFixture' describe('image generation through the Pi session', () => { it.each(['incomplete', 'cancel', 'json-disconnect'] as const)('settles a mixed image batch and releases its HTTP requests on %s', async (ending) => { const root = await realpath(await mkdtemp(join(tmpdir(), 'buddy-image-session-'))) + const publication = createImagePublicationFixture(root) const codex = ending !== 'json-disconnect' const requests: string[] = [] const closed: string[] = [] @@ -82,31 +85,25 @@ describe('image generation through the Pi session', () => { vi.spyOn(runtime, 'streamSimple').mockImplementation(target => assistantResponse(target, turns++ === 0)) const source = { listConversationArtifacts: () => [], listForConversation: () => [], materializeConversationImages: async () => ({ images: [], records: [] }) } const published: string[] = [] - service = new ImageGenerationService({ - conversationId: 'conversation-1', - cwd: root, - grants: [], - attachmentService: source, - artifactService: { - ...source, - async registerGeneratedImages(input) { - expect(input.images).toEqual([{ bytes: new Uint8Array(png), mimeType: 'image/png' }]) - published.push(input.outputPath) - return [{ id: 'artifact-1', currentPath: join(root, 'success.png'), mimeType: 'image/png' }] - }, + const materials = new ImageGenerationMaterialStore(root, 'conversation-1') + service = new ImageGenerationService({ writeGeneratedImageMaterial: (image, grants, writeFiles, signal) => materials.write(image, grants, writeFiles, signal), referenceAccess: { authorize: () => {} }, conversationId: 'conversation-1', cwd: root, grants: [{ grantId: 'workspace', kind: 'workspace', root, canonicalRoot: root }], attachmentService: source, artifactService: { + ...source, + async registerGeneratedImages(input) { + expect(input.images).toEqual([{ bytes: new Uint8Array(png), mimeType: 'image/png' }]) + published.push(input.outputPath) + return [{ id: 'artifact-1', currentPath: join(root, 'success.png'), mimeType: 'image/png' }] }, - imageGenerationGateway: new OpenAiImageGenerationService({ - modelRuntime: runtime, - resolveCredentialMetadata: id => id === model.provider ? { type: codex ? 'oauth' : 'api_key', revision: 1 } : null, - fetch: async (url, init) => { - expect(url).toBe(`${model.baseUrl}${codex ? '/codex' : ''}/responses`) - const response = await fetch(url, init) - if (ending === 'json-disconnect' && JSON.parse(String(init?.body)).input[0].content[0].text === 'pending') - pendingResponse!.destroy() - return response - }, - }), - }) + }, imageGenerationGateway: new OpenAiImageGenerationService({ + modelRuntime: runtime, + resolveCredentialMetadata: id => id === model.provider ? { type: codex ? 'oauth' : 'api_key', revision: 1 } : null, + fetch: async (url, init) => { + expect(url).toBe(`${model.baseUrl}${codex ? '/codex' : ''}/responses`) + const response = await fetch(url, init) + if (ending === 'json-disconnect' && JSON.parse(String(init?.body)).input[0].content[0].text === 'pending') + pendingResponse!.destroy() + return response + }, + }) }) const operations: ImageOperationEvent[] = [] service.onDidChange(event => operations.push(event)) created = await createIsolatedBuddySession({ @@ -122,7 +119,7 @@ describe('image generation through the Pi session', () => { modelRuntime: runtime, thinkingLevel: 'off', resources: { skillReadRoots: [], skillReferences: [], approvedSkills: [], context: { agentsFiles: [], diagnostics: [] }, directoryContext: '', revision: 'empty' }, - inProcessExtensions: [createImageGenerationExtension({ getRunId: () => 'run-1', service })], + inProcessExtensions: [createImageGenerationExtension({ createFileWriter: () => publication.writeFiles, getRunId: () => 'run-1', service })], }) created.session.setActiveToolsByName([IMAGE_GENERATION_TOOL_NAME]) const state = createPiEventProjectionState({ canonicalRoot: root }) @@ -167,6 +164,7 @@ describe('image generation through the Pi session', () => { await service?.dispose() await created?.shutdown('quit') await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())) + await publication.dispose() await rm(root, { recursive: true, force: true }) } }) diff --git a/apps/buddy/service/src/images/__tests__/imageGenerationSchemaSession.spec.ts b/apps/buddy/service/src/images/__tests__/imageGenerationSchemaSession.spec.ts index 2f5024fea..ceeae18b6 100644 --- a/apps/buddy/service/src/images/__tests__/imageGenerationSchemaSession.spec.ts +++ b/apps/buddy/service/src/images/__tests__/imageGenerationSchemaSession.spec.ts @@ -36,7 +36,7 @@ describe('generation tool schema through the public Pi extension lifecycle', () preferences: async () => select(candidates[1]!.target), }) const source = { materializeConversationImages: async () => ({ images: [], records: [] }), listConversationArtifacts: () => [], listForConversation: () => [] } - const service = new ImageGenerationService({ conversationId: 'fixture', cwd: root, grants: [], imageGenerationGateway: gateway, getImagePreferences: () => snapshot, artifactService: { ...source, registerGeneratedImages: async () => [] }, attachmentService: source }) + const service = new ImageGenerationService({ referenceAccess: { authorize: () => {} }, conversationId: 'fixture', cwd: root, grants: [], imageGenerationGateway: gateway, getImagePreferences: () => snapshot, artifactService: { ...source, registerGeneratedImages: async () => [] }, attachmentService: source }) const credentials = new InMemoryCredentialStore() await credentials.modify('openai', async () => ({ type: 'api_key', key: 'offline-only' })) const runtime = await ModelRuntime.create({ credentials, modelsStore: new InMemoryModelsStore(), modelsPath: null, refreshOnCreate: false }) @@ -70,18 +70,18 @@ describe('generation tool schema through the public Pi extension lifecycle', () expect(JSON.stringify(payloads[0])).toContain('When the user asks to create or generatively edit an image, use lexora_image_generate') expect(tools[0]!.description).toContain('Generate only: backend_declaration') expect(tools[0]!.description).toContain('edit [unavailable') - expect(tools[0]!.parameters.properties.reference).toBeUndefined() + expect(tools[0]!.parameters.properties.referenceIds).toBeUndefined() expect(tools[0]!.parameters.required).toContain('output') expect(tools[0]!.parameters.properties.output!.properties.size).toMatchObject({ anyOf: [{ const: '1024x1024' }] }) expect(tools[1]!.description).toContain('Reference editor: backend_declaration') - expect(tools[1]!.parameters.properties.reference!.anyOf[1]!.properties.resourceIds!.maxItems).toBe(2) + expect(tools[1]!.parameters.properties.referenceIds!.maxItems).toBe(2) expect(tools[1]!.parameters.properties.output!.properties.background).toMatchObject({ anyOf: [{ const: 'auto' }, { const: 'transparent' }] }) expect(tools[2]!.description).toContain('Custom image model: adapter_constraints') expect(tools[2]!.description).toContain('actual model capabilities are unverified') - expect(tools[2]!.parameters.properties.reference!.anyOf[1]!.properties.resourceIds!.maxItems).toBe(4) + expect(tools[2]!.parameters.properties.referenceIds!.maxItems).toBe(4) expect(tools[3]!.description).toContain('generate [unavailable under the current policy]; edit [unavailable under the current policy]') expect(tools[3]!.parameters.properties.output).toBeUndefined() - expect(tools[3]!.parameters.properties.reference).toBeUndefined() + expect(tools[3]!.parameters.properties.referenceIds).toBeUndefined() } finally { session.dispose() diff --git a/apps/buddy/service/src/images/__tests__/imageGenerationToolContract.spec.ts b/apps/buddy/service/src/images/__tests__/imageGenerationToolContract.spec.ts new file mode 100644 index 000000000..104adb0b3 --- /dev/null +++ b/apps/buddy/service/src/images/__tests__/imageGenerationToolContract.spec.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest' +import { toPublicRunEvent } from '../../../../shared/runs/publicRunEvent' +import { createImageGenerationToolPresentation, readImageGenerationToolDetails } from '../imageGenerationToolContract' + +const output = { details: { artifactIds: ['result'], responseId: 'response' } } + +describe('image generation presentation compatibility', () => { + it('projects explicit ordered reference IDs into the existing image card format', () => { + expect(createImageGenerationToolPresentation({ + toolName: 'lexora_image_generate', + arguments: { prompt: 'Preserve image 1 and use the pose from image 2', outputPath: 'result.png', referenceIds: ['character', 'pose'] }, + result: output, + })).toMatchObject({ + card: 'image', + reference: { mode: 'resources', resourceIds: ['character', 'pose'] }, + artifactIds: ['result'], + generatedCount: 1, + status: 'completed', + }) + }) + + it('keeps retained result metadata in tool details while projecting only safe failure diagnostics', () => { + const retainedOutputs = [{ path: '/owned-scratch/.image-generation/original.png', mimeType: 'image/png', sizeBytes: 100, sha256: 'a'.repeat(64) }] + const result = { details: { artifactIds: [], code: 'IMAGE_GENERATION_FAILED', outcome: 'partial', diagnostic: { stage: 'publishing', fileErrorCode: 'EACCES' }, retainedOutputs } } + expect(readImageGenerationToolDetails(result)).toMatchObject({ retainedOutputs }) + const presentation = createImageGenerationToolPresentation({ toolName: 'lexora_image_generate', arguments: { outputPath: 'output/image.png' }, result, isError: true }) + expect(presentation).toMatchObject({ status: 'partial', failure: { diagnostic: { stage: 'publishing', fileErrorCode: 'EACCES' } } }) + expect(JSON.stringify(presentation)).not.toContain(retainedOutputs[0]!.path) + expect(readImageGenerationToolDetails({ details: { ...result.details, retainedOutputs: [{ ...retainedOutputs[0], sha256: 'invalid' }] } })).not.toHaveProperty('retainedOutputs') + }) + + it.each([{ mode: 'latest' }, { mode: 'resources', resourceIds: ['original', 'pose'] }])('reads stored image cards without reexecuting or rewriting their selection: %j', (reference) => { + const presentation = { + card: 'image', + artifactIds: ['result'], + description: null, + generatedCount: 1, + prompt: 'Edit the reference', + reference, + status: 'completed', + } + const historical = { createdAt: '2026-10-01T00:00:00.000Z', runId: 'run-1', sequence: 1, type: 'tool.completed', payload: { toolCallId: 'call-1', toolName: 'lexora_image_generate', isError: false, presentation } } + expect(toPublicRunEvent(historical).payload.presentation).toEqual(presentation) + }) +}) diff --git a/apps/buddy/service/src/images/__tests__/imageGenerationToolParameters.spec.ts b/apps/buddy/service/src/images/__tests__/imageGenerationToolParameters.spec.ts index 483c3d028..e79e5869f 100644 --- a/apps/buddy/service/src/images/__tests__/imageGenerationToolParameters.spec.ts +++ b/apps/buddy/service/src/images/__tests__/imageGenerationToolParameters.spec.ts @@ -21,17 +21,28 @@ describe('image generation parameter contracts', () => { it('does not advertise omittable output when generation has no default candidate but editing does', () => { const schema = createImageGenerationParameters({ generate: [generator], edit: [editor] }) expect(Check(schema, input)).toBe(false) - expect(Check(schema, { ...input, output: {}, reference: { mode: 'latest' } })).toBe(true) + expect(Check(schema, { ...input, output: {}, referenceIds: ['reference-1'] })).toBe(true) expect(Check(schema, { ...input, output: { size: '1024x1024', quality: 'high', background: 'opaque' } })).toBe(true) }) + it.each([ + { referenceIds: [] }, + { referenceIds: ['one', 'one'] }, + { referenceIds: ['one', 'two', 'three'] }, + { referenceIds: [' '] }, + { reference: { mode: 'latest' } }, + { reference: { mode: 'resources', resourceIds: ['one'] } }, + ])('rejects ambiguous or legacy selections: %j', (selection) => { + expect(Check(createImageGenerationParameters({ generate: [], edit: [editor] }), { ...input, ...selection })).toBe(false) + }) + it('requires references for edit-only routes and excludes them for generation-only routes', () => { const editOnly = createImageGenerationParameters({ generate: [], edit: [editor] }) expect(Check(editOnly, input)).toBe(false) - expect(Check(editOnly, { ...input, reference: { mode: 'latest' } })).toBe(true) + expect(Check(editOnly, { ...input, referenceIds: ['reference-1'] })).toBe(true) const generateOnly = createImageGenerationParameters({ generate: [generator], edit: [] }) - expect(generateOnly.properties).not.toHaveProperty('reference') - expect(Check(generateOnly, { ...input, output: { size: '1024x1024', quality: 'high', background: 'opaque' }, reference: { mode: 'latest' } })).toBe(false) + expect(generateOnly.properties).not.toHaveProperty('referenceIds') + expect(Check(generateOnly, { ...input, output: { size: '1024x1024', quality: 'high', background: 'opaque' }, referenceIds: ['reference-1'] })).toBe(false) }) }) diff --git a/apps/buddy/service/src/images/__tests__/imagePublicationFixture.ts b/apps/buddy/service/src/images/__tests__/imagePublicationFixture.ts new file mode 100644 index 000000000..b24e94560 --- /dev/null +++ b/apps/buddy/service/src/images/__tests__/imagePublicationFixture.ts @@ -0,0 +1,26 @@ +import type { CapturedFileWriter } from '../../changes/CapturedFileWriter' +import { randomUUID } from 'node:crypto' +import { join } from 'node:path' +import { MutationCoordinator } from '../../changes/MutationCoordinator' +import { FileResourceCoordinator } from '../../files/FileResourceCoordinator' +import { NativeFileEngine } from '../../files/NativeFileEngine' + +export function createImagePublicationFixture(root: string, signal = new AbortController().signal) { + const engine = new NativeFileEngine(join(root, 'file-journal')) + const files = new FileResourceCoordinator(engine) + const mutations = new MutationCoordinator({ resources: files }) + const writeFiles: CapturedFileWriter = (paths, write) => files.withStagingDirectory(async (stagingDirectory) => { + const claims = await files.fileWriteClaims(paths) + const lease = await mutations.acquire({ runId: 'image-publication-test', invocationId: randomUUID(), lifetime: 'runtime', claims: [...claims, { kind: 'tree', canonicalPath: stagingDirectory }], signal }) + try { + await mutations.withInvocation(lease, () => write(input => files.writeFile(lease.invocationId, stagingDirectory, input, signal))) + } + finally { + await lease.release() + } + }) + return { engine, files, writeFiles, async dispose() { + await mutations.dispose() + await engine.dispose() + } } +} diff --git a/apps/buddy/service/src/images/imageGenerationExtension.ts b/apps/buddy/service/src/images/imageGenerationExtension.ts index 5adef79ca..05d041793 100644 --- a/apps/buddy/service/src/images/imageGenerationExtension.ts +++ b/apps/buddy/service/src/images/imageGenerationExtension.ts @@ -1,6 +1,6 @@ import type { ExtensionContext, ToolDefinition } from '@earendil-works/pi-coding-agent' import type { TSchema } from 'typebox' -import type { ImageGenerationOutcome, ImageRequestRef } from '../../../shared/images/imageGeneration' +import type { ImageGenerationOutcome, ImageGenerationRetainedOutput, ImageRequestRef } from '../../../shared/images/imageGeneration' import type { BuddyCapability } from '../agent/extensions/BuddyCapability' import type { BuddyInProcessExtension } from '../agent/extensions/BuddyInProcessExtension' import type { CapturedFileWriterFactory } from '../changes/CapturedFileWriter' @@ -84,7 +84,7 @@ function createImageGenerationTool(options: CreateImageGenerationExtensionOption if (!Check(parametersSchema, parameters) || !options.getRunId()) return imageToolFailure('VALIDATION_FAILED') const input = parameters as unknown as GenerateConversationImageInput - if (description && !(input.reference ? description.edit : description.generate).length) + if (description && !(input.referenceIds ? description.edit : description.generate).length) return imageToolFailure('IMAGE_GENERATION_UNSUPPORTED') const outputPath = input.outputPath.trim() if (!outputPath || outputPath === '.' || outputPath === '..') @@ -104,9 +104,14 @@ function createImageGenerationTool(options: CreateImageGenerationExtensionOption } } catch (error) { - executionSignal.throwIfAborted() const failure = readImageGenerationError(error) - return imageToolFailure(failure.code, failure.diagnostic, failure.outcome, failure.request, failure.artifactIds) + if (!failure.retainedOutputs?.length) + executionSignal.throwIfAborted() + return imageToolFailure(failure.code, { + ...failure, + outputPath, + inspectionReferences: input.referenceIds?.some(id => id.startsWith('image:')), + }) } }, label: 'Generate image', @@ -114,45 +119,76 @@ function createImageGenerationTool(options: CreateImageGenerationExtensionOption parameters: parametersSchema, promptGuidelines: [ 'When the user asks to create or generatively edit an image, use lexora_image_generate instead of shell scripts or drawing libraries.', - 'With lexora_image_generate, use reference only when the current schema includes it. For an image already in the conversation, use reference.mode=resources with its attachmentId or artifactId, or reference.mode=latest. Explicit ids are preferable when multiple images are present. In the prompt assign a role to each ordered reference, describe the changes and the elements to preserve.', - 'With lexora_image_generate, set outputPath to the intended path in the current workspace. The tool appends the image extension when omitted and numeric suffixes when one call returns multiple images.', - 'A successful lexora_image_generate receipt proves that an image file was created, not that its appearance meets the request. Use the returned outputs[].path with lexora_image_read when a visual check is needed; do not claim visual verification from a receipt alone. Do not automatically repeat a request whose outcome is unknown.', + 'The image generation backend cannot see chat attachments, conversation history or images inspected with lexora_image_read. For reference-based generation or editing, pass ALL user-designated images in referenceIds, using exact attachmentId or artifactId values in order. A character reference plus a pose or style reference needs both images, not only the character. Assign their roles in prompt; descriptions cannot replace the actual images. Omit referenceIds only for a new image that uses no references.', + 'For edits with references, keep prompt focused on the requested transformation and numbered image roles: preserve image 1 and apply only the requested pose or style from image 2. Do not reconstruct unchanged appearance as a text inventory or guess anatomy, colors, logos or lettering. Such guesses can override the supplied image and change the subject; the actual reference, not your paraphrase, is the source of unchanged details.', + 'A follow-up or confirmation continues the original image task and its relevant references. If IDs are missing, use lexora_image_list and explicitly select the intended images; do not switch to text-only generation. A recent result is only a candidate, not an instruction to use it. Do not guess IDs, pass paths or image: hashes, automatically include unrelated images, or use the removed reference/mode parameters.', + 'For workspace output, use a relative outputPath such as output/edited.png. A leading / means the filesystem root: do not invent /workspace or /output paths. An absolute path must be the actual intended, writable destination. The tool appends the image extension when omitted and numeric suffixes when multiple images are returned.', + 'A successful image receipt proves file creation, not visual correctness. Compare the actual output with the relevant original references before claiming character, pose or style preservation. A partial outcome with retainedOutputs means images were generated and saved, but the requested destination was not fully published: copy the retained files to an authorized destination and present them using existing file tools, keeping the originals. Do not inspect the input images to fix a local save failure. For unknown or partial outcomes, do not call image generation again or change prompt, path or references to evade the pending request.', ], }) } function imageToolFailure( code: ImageGenerationErrorCode | 'VALIDATION_FAILED', - diagnostic: ImageGenerationErrorDiagnostic | null = null, - outcome: ImageGenerationOutcome = 'failed', - request?: ImageRequestRef, - artifactIds: string[] = [], + options: { + diagnostic?: ImageGenerationErrorDiagnostic | null + outcome?: ImageGenerationOutcome + request?: ImageRequestRef + artifactIds?: string[] + retainedOutputs?: ImageGenerationRetainedOutput[] + outputPath?: string + inspectionReferences?: boolean + } = {}, ) { - const message = code === 'IMAGE_GENERATION_BLOCKED' - ? '. No new image request was submitted because a matching request in this run is active or unresolved. Repeating it may create duplicate images or charges. Inspect any existing artifacts before choosing another approach.' - : outcome === 'partial' - ? '. The provider returned a result, but local publication was incomplete. Inspect existing artifacts before continuing. Do not automatically generate again.' - : outcome === 'unknown' || code === 'IMAGE_GENERATION_INCOMPLETE' - ? '. The provider result is unknown; do not automatically repeat this image request.' - : code === 'IMAGE_GENERATION_REFERENCE_UNAVAILABLE' - ? '. The reference is missing, invalid or unreadable under the current directory permissions. No image request was submitted.' - : code === 'IMAGE_GENERATION_UNSUPPORTED' - ? '. No supported image route is available under the current policy. No image request was submitted.' - : code === 'IMAGE_GENERATION_RESOURCE_EXHAUSTED' || code === 'IMAGE_GENERATION_BUSY' - ? '. The image backend is busy or lacks resources.' - : '' + const { diagnostic, outcome = 'failed', request, artifactIds = [], retainedOutputs = [], outputPath } = options + let message: string + if (outcome === 'partial') { + message = retainedOutputs.length + ? 'The provider already generated images. Local publication was incomplete, but the files listed in retainedOutputs were saved. Recover by copying those files with existing file tools to an authorized destination, then present them; keep the retained originals. This does not confirm outputPath was saved. Do not inspect the input references to repair a local save failure, and do not call lexora_image_generate again.' + : `The provider returned a result, but local publication was incomplete. ${artifactIds.length ? 'Only the listed artifactIds identify published results.' : 'No published artifact or retained output is available in this receipt; do not claim that a generated file was saved.'} Explain the local publication failure to the user. Inspecting the input references cannot repair it. Do not automatically generate again.` + } + else if (outcome === 'unknown' || code === 'IMAGE_GENERATION_INCOMPLETE') { + message = 'The provider result is unknown; do not automatically repeat this image request. Report the unresolved result rather than guessing that generation failed.' + } + else if (code === 'IMAGE_GENERATION_OUTPUT_UNAVAILABLE') { + message = 'No image request was submitted: outputPath is invalid, outside the current write authorization or not writable. For workspace output use a relative path such as output/edited.png, not an invented root path such as /output/edited.png. Correct only the destination or obtain the required authorization; preserve the intended referenceIds and image requirements.' + } + else if (diagnostic?.fileErrorCode === 'IMAGE_MATERIAL_UNAVAILABLE') { + message = 'No image request was submitted: the host cannot retain generated image results. This is a local host storage or capability configuration failure, not an image prompt or reference problem. Report it instead of changing referenceIds or retrying generation.' + } + else if (code === 'IMAGE_GENERATION_REFERENCE_UNAVAILABLE') { + message = options.inspectionReferences + ? 'No image request was submitted. image: values are inspection-only handles for lexora_image_read, not generation referenceIds. Use the original attachmentId/artifactId values or lexora_image_list. Re-reading images does not create generation IDs; do not drop the intended references.' + : 'A reference is missing, not visible on this execution branch, invalid or unreadable under the current directory permissions. No image request was submitted. Use lexora_image_list to locate exact visible attachmentId/artifactId values; do not silently drop references.' + } + else if (code === 'IMAGE_GENERATION_UNSUPPORTED') { + message = 'No supported image route is available under the current policy. No image request was submitted.' + } + else if (code === 'IMAGE_GENERATION_RESOURCE_EXHAUSTED' || code === 'IMAGE_GENERATION_BUSY') { + message = 'The image backend is busy or lacks resources.' + } + else if (code === 'VALIDATION_FAILED') { + message = 'Invalid image generation parameters. Use the current schema and explicit ordered referenceIds; legacy reference/mode parameters are not accepted. No image request was submitted.' + } + else { + message = 'Image generation failed. Use the reported stage and error classification to distinguish generation from local file publication.' + } + if (code === 'IMAGE_GENERATION_BLOCKED') + message = `No new image request was submitted because a matching request in this run is active or unresolved. ${message}` + const details = { + artifactIds, + ...(retainedOutputs.length ? { retainedOutputs } : {}), + ...(request ? { request } : {}), + code, + outcome, + ...(diagnostic ? { diagnostic } : {}), + responseId: null, + } return { content: [{ - text: JSON.stringify({ code, outcome, artifactIds, ...(request ? { request } : {}), ...(diagnostic ? { diagnostic } : {}), ...(message ? { message: message.slice(2) } : {}) }), + text: JSON.stringify({ ...details, ...(outputPath ? { outputPath } : {}), message }), type: 'text' as const, }], - details: { - artifactIds, - ...(request ? { request } : {}), - code, - outcome, - ...(diagnostic ? { diagnostic } : {}), - responseId: null, - }, + details, } } diff --git a/apps/buddy/service/src/images/imageGenerationToolContract.ts b/apps/buddy/service/src/images/imageGenerationToolContract.ts index c96b7c437..e533c0ea7 100644 --- a/apps/buddy/service/src/images/imageGenerationToolContract.ts +++ b/apps/buddy/service/src/images/imageGenerationToolContract.ts @@ -1,11 +1,11 @@ import type { ToolCallEvent } from '@earendil-works/pi-coding-agent' -import type { ImageGenerationErrorDiagnostic, ImageGenerationOutcome, ImageRequestRef } from '../../../shared/images/imageGeneration' +import type { ImageGenerationErrorDiagnostic, ImageGenerationOutcome, ImageGenerationRetainedOutput, ImageRequestRef } from '../../../shared/images/imageGeneration' import type { BuddyToolPresentation } from '../../../shared/runs/runEventPresentation' import type { BuddyRunOutputPayload } from '../../../shared/runs/runOutput' import type { BuddyToolClassification } from '../approvals/toolClassification' import type { CreateBuddyToolPresentationInput } from '../events/toolPresentationSupport' import { BUDDY_ATTACHMENT_COUNT_LIMIT } from '../../../shared/conversation/attachmentPolicy' -import { imageGenerationFailureSchema, imageRequestRefSchema, readImageDiagnostic } from '../../../shared/images/imageGeneration' +import { imageGenerationFailureSchema, imageGenerationRetainedOutputSchema, imageRequestRefSchema, readImageDiagnostic } from '../../../shared/images/imageGeneration' import { imageSourceSchema } from '../../../shared/images/imagePreferences' import { readOptionalString, @@ -20,6 +20,7 @@ export interface ImageGenerationToolDetails { code?: string diagnostic?: ImageGenerationErrorDiagnostic outputs?: Array<{ artifactId: string, path: string, mimeType: string }> + retainedOutputs?: ImageGenerationRetainedOutput[] outcome?: ImageGenerationOutcome request?: ImageRequestRef responseId: string | null @@ -97,6 +98,7 @@ export function readImageGenerationToolDetails(value: unknown): ImageGenerationT const diagnostic = readImageDiagnostic(details.diagnostic) const request = imageRequestRefSchema.safeParse(details.request) const source = imageSourceSchema.safeParse(details.source) + const retainedOutputs = imageGenerationRetainedOutputSchema.array().safeParse(details.retainedOutputs) return { artifactIds, ...(['failed', 'unknown', 'partial'].includes(details.outcome as string) ? { outcome: details.outcome as ImageGenerationOutcome } : {}), @@ -104,6 +106,7 @@ export function readImageGenerationToolDetails(value: unknown): ImageGenerationT ...(typeof details.code === 'string' && details.code ? { code: details.code } : {}), ...(diagnostic ? { diagnostic } : {}), ...(source.success ? { source: source.data } : {}), + ...(retainedOutputs.success ? { retainedOutputs: retainedOutputs.data } : {}), responseId: typeof details.responseId === 'string' && details.responseId ? details.responseId : null, @@ -113,15 +116,9 @@ export function readImageGenerationToolDetails(value: unknown): ImageGenerationT function readImageReference( arguments_: Record | null, ): Extract['reference'] { - const reference = readRecord(arguments_?.reference) - if (reference?.mode === 'latest') { - return { - mode: 'latest', - } - } - if (reference?.mode !== 'resources' || !Array.isArray(reference.resourceIds)) + if (!Array.isArray(arguments_?.referenceIds)) return null - const resourceIds = [...new Set(reference.resourceIds.filter( + const resourceIds = [...new Set(arguments_.referenceIds.filter( (value): value is string => typeof value === 'string' && Boolean(value), ))].slice(0, 4) return resourceIds.length > 0 diff --git a/apps/buddy/service/src/images/imageGenerationToolParameters.ts b/apps/buddy/service/src/images/imageGenerationToolParameters.ts index 6a59d9a2c..a7cae4307 100644 --- a/apps/buddy/service/src/images/imageGenerationToolParameters.ts +++ b/apps/buddy/service/src/images/imageGenerationToolParameters.ts @@ -9,13 +9,13 @@ export function createImageGenerationParameters(description?: ImageGenerationDes const editSupports = description ? description.edit.map(route => route.support) : [nativeImageSupport] const properties: Record = { outputPath: Type.String({ - description: 'Output PNG file path in the current workspace. The extension is optional.', + description: 'Output PNG path, relative to the current workspace, for example output/edited.png. A leading / means the filesystem root, not the workspace. Use an absolute path only when it is the actual intended, writable destination. The extension is optional.', maxLength: 4096, minLength: 1, pattern: '\\S', }), prompt: Type.String({ - description: 'Describe the desired result. For ordered references, specify the role of each image, changes to make and elements to preserve.', + description: 'For reference-based edits, write a short edit instruction using numbered image roles, for example: keep image 1 unchanged except apply the pose from image 2. Let the actual images supply unchanged appearance and rendering style; do not rewrite an appearance inventory or guess anatomy, colors, logos or lettering. For generation without references, describe the desired result.', maxLength: 32 * 1024, minLength: 1, pattern: '\\S', @@ -46,20 +46,19 @@ export function createImageGenerationParameters(description?: ImageGenerationDes } const maxReferences = Math.min(4, Math.max(0, ...editSupports.map(support => support.maxReferenceImages))) if (maxReferences > 0) { - const reference = Type.Union([ - Type.Object({ mode: Type.Literal('latest') }, { additionalProperties: false }), - Type.Object({ - resourceIds: Type.Array(Type.String({ maxLength: 256, minLength: 1 }), { maxItems: maxReferences, minItems: 1, uniqueItems: true }), - mode: Type.Literal('resources'), - }, { additionalProperties: false }), - ]) - properties.reference = description?.generate.length === 0 ? reference : Type.Optional(reference) + const referenceIds = Type.Array(Type.String({ maxLength: 256, minLength: 1, pattern: '\\S' }), { + description: 'Required when the task uses reference images or edits an existing image, including references from earlier turns. List ALL intended images in order: a character image plus a pose image requires BOTH IDs. Copy attachmentId or artifactId from attachments, outputs or lexora_image_list; not composer resourceIds, paths or image: hashes. Omit only for a new image that uses no references.', + maxItems: maxReferences, + minItems: 1, + uniqueItems: true, + }) + properties.referenceIds = description?.generate.length === 0 ? referenceIds : Type.Optional(referenceIds) } return Type.Object(properties, { additionalProperties: false }) } export function describeImageGeneration(description?: ImageGenerationDescription): string { - const base = 'Generate images through the configured image policy independently of the chat model. Valid returned images are saved and shown in the conversation even when the backend ignores requested size, background or quality. These parameters are requests, not guarantees. Inspect the saved outputs before claiming exact dimensions, transparency or other requested properties; use post-processing when needed.' + const base = 'Generate or edit images. The image backend receives only this prompt and the images explicitly listed in referenceIds: it cannot see chat attachments, conversation history or lexora_image_read results. For reference-based tasks, pass every intended reference image, not a text description in its place. Without referenceIds this is text-to-image generation, not an edit of the user\'s images. Size, background and quality are requests, not guarantees; inspect actual outputs before claiming those properties. A local publication failure may return retainedOutputs: copy and present those saved files without another generation request.' if (!description) return `${base} Backend capabilities have not been declared; accepted adapter parameters do not guarantee backend support for generation, editing or transparency.` const routes = (operation: 'generate' | 'edit') => description[operation].length diff --git a/apps/buddy/service/src/images/imageReferenceExtension.ts b/apps/buddy/service/src/images/imageReferenceExtension.ts new file mode 100644 index 000000000..f5dca5617 --- /dev/null +++ b/apps/buddy/service/src/images/imageReferenceExtension.ts @@ -0,0 +1,50 @@ +import type { BuddyCapability } from '../agent/extensions/BuddyCapability' +import type { ImageReferenceAccess } from './ImageReferenceAccess' +import { defineTool } from '@earendil-works/pi-coding-agent' +import { Type } from 'typebox' +import { Check } from 'typebox/value' +import { createToolClassificationFailure } from '../approvals/toolClassification' +import { IMAGE_REFERENCE_LIST_LIMIT } from './ImageReferenceAccess' +import { IMAGE_TOOLS } from './imageToolGroup' + +export const imageReferenceListParameters = Type.Object({ + query: Type.Optional(Type.String({ maxLength: 256, description: 'Filter visible image names, referenceIds or source message/run IDs.' })), + offset: Type.Optional(Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER })), + limit: Type.Optional(Type.Integer({ minimum: 1, maximum: IMAGE_REFERENCE_LIST_LIMIT, default: 20 })), +}, { additionalProperties: false }) + +export function createImageReferenceCapability(access: Pick): BuddyCapability { + return { + classify(event, signal) { + if (event.toolName !== IMAGE_TOOLS.list.name) + return null + if (!Check(imageReferenceListParameters, event.input)) + return createToolClassificationFailure('VALIDATION_FAILED') + signal.throwIfAborted() + return { access: 'read', paths: [] } + }, + disclosure: [{ + source: { kind: 'builtin', id: 'image_reference', title: 'Image references' }, + exposure: 'on_demand', + keywords: 'image reference attachment artifact list 图片 引用 附件 产物 目录', + tools: [IMAGE_TOOLS.list], + }], + extension: { + name: 'lexora-image-reference', + factory(pi) { + pi.registerTool(defineTool({ + ...IMAGE_TOOLS.list, + label: 'List visible image references', + description: 'List a bounded page of image attachments and artifacts visible to this execution on its accepted task branch. Returns explicit referenceId (attachmentId or artifactId), sourceKind, name, sourceMessageId, sourceRunId, path and image metadata. path locates the saved attachment or current artifact; reading still requires ordinary file authorization. query searches only names and IDs, not image contents. Does not import files, choose a reference, or expand file permissions.', + parameters: imageReferenceListParameters, + promptGuidelines: ['Use lexora_image_list to recover image reference IDs lost from context or compaction. Pass the exact returned referenceId to image generation; composer resourceId is a different identity. Keep the requested reference order and choose only the images relevant to the user request. sourceMessageId may be null for generated artifacts. The directory is metadata, not visual inspection or file access authorization.'], + async execute(_id, input, signal) { + if (!Check(imageReferenceListParameters, input)) + throw new Error('VALIDATION_FAILED') + return { content: [{ type: 'text', text: JSON.stringify(access.list(input, signal ?? new AbortController().signal)) }], details: {} } + }, + })) + }, + }, + } +} diff --git a/apps/buddy/service/src/images/imageToolGroup.ts b/apps/buddy/service/src/images/imageToolGroup.ts index dd474514c..46c0b081d 100644 --- a/apps/buddy/service/src/images/imageToolGroup.ts +++ b/apps/buddy/service/src/images/imageToolGroup.ts @@ -1,3 +1,3 @@ import { toPiToolGroup } from '../agent/extensions/piToolGroup' -export const IMAGE_TOOLS = toPiToolGroup({ name: 'lexora_image', description: 'Read image content, generate images using configured models, or remove a solid chroma background.', tools: [{ name: 'read' }, { name: 'generate' }, { name: 'chroma_key' }] }) +export const IMAGE_TOOLS = toPiToolGroup({ name: 'lexora_image', description: 'Read images, discover visible image references, generate images, or remove a solid chroma background.', tools: [{ name: 'read' }, { name: 'list' }, { name: 'generate' }, { name: 'chroma_key' }] }) diff --git a/apps/buddy/shared/images/imageGeneration.ts b/apps/buddy/shared/images/imageGeneration.ts index 1ce2eba78..0a8f2e2b6 100644 --- a/apps/buddy/shared/images/imageGeneration.ts +++ b/apps/buddy/shared/images/imageGeneration.ts @@ -10,12 +10,21 @@ export const IMAGE_GENERATION_ERROR_CODES = [ 'IMAGE_GENERATION_INVALID_RESPONSE', 'IMAGE_GENERATION_RESPONSE_TOO_LARGE', 'IMAGE_GENERATION_REFERENCE_UNAVAILABLE', + 'IMAGE_GENERATION_OUTPUT_UNAVAILABLE', 'IMAGE_GENERATION_UNSUPPORTED', 'PROVIDER_ACCESS_DENIED', 'PROVIDER_AUTHENTICATION_FAILED', 'PROVIDER_RATE_LIMITED', ] as const +export const imageGenerationRetainedOutputSchema = z.object({ + path: z.string().min(1), + mimeType: z.literal('image/png'), + sizeBytes: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), + sha256: z.string().regex(/^[a-f0-9]{64}$/), +}).strict() +export type ImageGenerationRetainedOutput = z.infer + export const imageGenerationOutcomeSchema = z.enum(['failed', 'unknown', 'partial']) const diagnosticTokenSchema = z.string().min(1).max(256).regex(/^[\w.\-[\]:/]+$/) export const imageGenerationDiagnosticSchema = z.object({ @@ -28,12 +37,55 @@ export const imageGenerationDiagnosticSchema = z.object({ httpStatus: z.number().int().min(100).max(599).optional(), responseType: providerRequestDiagnosticSchema.shape.responseType, transportCode: providerRequestDiagnosticSchema.shape.transportCode, + fileErrorCode: z.enum([ + 'EACCES', + 'EPERM', + 'EROFS', + 'ENOSPC', + 'EDQUOT', + 'ENOENT', + 'ENOTDIR', + 'EISDIR', + 'EEXIST', + 'ELOOP', + 'ENAMETOOLONG', + 'EIO', + 'EMFILE', + 'ENFILE', + 'INVALID_PATH', + 'PATH_NOT_FOUND', + 'PATH_OUTSIDE_GRANTED_DIRECTORY', + 'DIRECTORY_NOT_AUTHORIZED', + 'DIRECTORY_GRANT_CHANGED', + 'TOOL_AUTHORIZATION_UNAVAILABLE', + 'FILE_CONFLICT', + 'FILE_INVALID_REQUEST', + 'FILE_IO_FAILED', + 'FILE_JOURNAL_FAILED', + 'FILE_LEASE_INVALID', + 'FILE_NOT_FOUND', + 'FILE_OPERATION_CANCELLED', + 'FILE_OPERATION_ID_CONFLICT', + 'FILE_PERMISSION_REVOKED', + 'FILE_PROTOCOL_FAILED', + 'FILE_RECOVERY_REQUIRED', + 'FILE_UNSUPPORTED', + 'FILE_ENGINE_UNAVAILABLE', + 'FILE_ENGINE_INTERRUPTED', + 'FILE_ENGINE_BUSY', + 'IMAGE_MATERIAL_IDENTITY_CHANGED', + 'IMAGE_MATERIAL_UNAVAILABLE', + 'ARTIFACT_PUBLICATION_FAILED', + 'ARTIFACT_SIZE_LIMIT', + 'VALIDATION_FAILED', + ]).optional(), lastEvent: z.enum(['response.created', 'response.in_progress', 'response.output_item.added', 'response.output_item.done', 'response.image_generation_call.in_progress', 'response.image_generation_call.generating', 'response.image_generation_call.partial_image', 'response.image_generation_call.completed', 'response.completed', 'response.done', 'response.failed', 'response.incomplete', 'response.cancelled', 'error', '[DONE]', 'other']).optional(), }).strict() export const imageGenerationFailureSchema = z.object({ code: z.enum([...IMAGE_GENERATION_ERROR_CODES, 'VALIDATION_FAILED']), outcome: imageGenerationOutcomeSchema, diagnostic: imageGenerationDiagnosticSchema.nullable(), + retainedOutputs: z.array(imageGenerationRetainedOutputSchema).optional(), }).strict() export const imageRequestRefSchema = z.object({ conversationId: z.uuid(), runId: z.uuid(), requestId: z.uuid() }).strict() @@ -43,6 +95,18 @@ export type ImageGenerationErrorDiagnostic = z.infer export type ImageRequestRef = z.infer +export function readImageFileErrorCode(error: unknown): ImageGenerationErrorDiagnostic['fileErrorCode'] { + const seen = new Set() + let code: ImageGenerationErrorDiagnostic['fileErrorCode'] + for (let current = error; current && typeof current === 'object' && seen.size < 8 && !seen.has(current); current = 'cause' in current ? current.cause : undefined) { + seen.add(current) + const parsed = imageGenerationDiagnosticSchema.shape.fileErrorCode.safeParse('code' in current ? current.code : 'message' in current ? current.message : undefined) + if (parsed.success && parsed.data) + code = parsed.data + } + return code +} + export function readImageDiagnostic(value: unknown): ImageGenerationErrorDiagnostic | null { if (!value || typeof value !== 'object') return null