Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions apps/buddy/service/src/BuddyService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1102,6 +1102,9 @@ export async function startBuddyService(
webService,
conversations,
publications: treeRepository,
executions: executionRecords,
runInputs,
runEvents: options.eventLog,
references: sessionReferences,
referenceSnapshots: extensionReferences,
}, petActions),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ describe('composer input at the Buddy session boundary', () => {
expect(fixture.contexts).toHaveLength(2)
expect(userMessages(fixture.contexts[1])[0]?.content).toEqual(materializedContent(fixture))
expect(fixture.contexts[1]?.messages).toContainEqual(expect.objectContaining({
content: [fixture.nativeToolImage, { type: 'text', text: expect.stringMatching(/^Saved image references \(in order\): image:[a-f0-9]{64}/) }],
content: [fixture.nativeToolImage, { type: 'text', text: expect.stringMatching(/^Inspection-only image handles \(in order\): image:[a-f0-9]{64}/) }],
role: 'toolResult',
toolName: 'lexora_fixture_image',
}))
Expand Down
2 changes: 2 additions & 0 deletions apps/buddy/service/src/agent/extensions/BuddyCapability.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import type { BuddySessionMode } from '../../../../shared/permissions/sessionMod
import type { BuddyToolClassification, BuddyToolClassificationResult } from '../../approvals/toolClassification'
import type { CapturedFileWriterFactory } from '../../changes/CapturedFileWriter'
import type { DirectoryGrant } from '../../directories/resolveGrantedPath'
import type { ImageGenerationMaterialWriter } from '../../images/ImageGenerationMaterialStore'
import type { BuddyInProcessExtension } from './BuddyInProcessExtension'
import type { BuddyMutationEffect } from './changeCaptureExtension'
import type { BuddyToolDisclosurePolicy } from './discovery/toolDiscoveryContract'
Expand All @@ -28,6 +29,7 @@ export interface BuddyCapability {
}

export interface BuddyCapabilityContext {
writeGeneratedImageMaterial?: ImageGenerationMaterialWriter
createFileWriter?: CapturedFileWriterFactory
authorizeNetwork?: (input: { toolCallId: string, toolName: string, url: string, operation?: NetworkOperation, signal?: AbortSignal }) => Promise<void>
conversationId: string
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import type { MutationCoordinator } from '../../changes/MutationCoordinator'
import type { DirectoryGrantService } from '../../directories/DirectoryGrantService'
import type { DirectoryGrant } from '../../directories/resolveGrantedPath'
import type { FileResourceCoordinator } from '../../files/FileResourceCoordinator'
import type { ImageGenerationMaterialWriter } from '../../images/ImageGenerationMaterialStore'
import type { ImageReadingMaterialWriter } from '../../images/ImageReadingMaterialStore'
import type { ImageReadingService } from '../../images/ImageReadingService'
import type { NetworkPermissionsStore } from '../../permissions/NetworkPermissionsClient'
Expand Down Expand Up @@ -84,6 +85,7 @@ export interface BuddySessionExtensionServices {
export interface CreateBuddySessionExtensionsOptions {
imageStore?: ImageBlobStore
writeImageMaterial?: ImageReadingMaterialWriter
writeGeneratedImageMaterial?: ImageGenerationMaterialWriter
actorKind?: 'actor' | 'npc'
approvalPolicy: BuddyApprovalPolicy
canonicalRoot: string
Expand Down Expand Up @@ -199,6 +201,7 @@ export async function createBuddySessionExtensions(
: { id: options.conversationId, kind: 'conversation' },
})
capabilities.push(codemode, ...await services.createCapabilities({
writeGeneratedImageMaterial: options.writeGeneratedImageMaterial,
createFileWriter: (event) => {
const run = runContext.current
if (!run)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export function createToolMediaExtension(store: ImageBlobStore): BuddyInProcessE
content.push({ ...block, data: '' })
}
if (images.length) {
content.push({ type: 'text' as const, text: `Saved image references (in order): ${images.map(image => `image:${image.sha256}`).join(', ')}. Use lexora_image_read with these references for focused inspection or comparison when needed.` })
content.push({ type: 'text' as const, text: `Inspection-only image handles (in order): ${images.map(image => `image:${image.sha256}`).join(', ')}. These handles work only with lexora_image_read, not with lexora_image_generate.referenceIds. For generation, keep the original attachmentId/artifactId values or use lexora_image_list; reading an image does not create a generation reference ID.` })
return { message: { ...message, content, buddyToolMedia: { version: 1, images } } satisfies ToolMediaReferenceMessage }
}
})
Expand Down
3 changes: 3 additions & 0 deletions apps/buddy/service/src/agent/sessions/BuddySessionFactory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { isExecutionProfileWithin } from '../../../../shared/permissions/executi
import { AttachmentToolWorkspace } from '../../attachments/AttachmentToolWorkspace'
import { conversationImageDirectory, ImageBlobStore } from '../../attachments/ImageBlobStore'
import { canReadReferenceSnapshots } from '../../conversations/referenceSnapshotToolContract'
import { ImageGenerationMaterialStore } from '../../images/ImageGenerationMaterialStore'
import { ImageReadingMaterialStore } from '../../images/ImageReadingMaterialStore'
import { supportsModelToolCalls } from '../../providers/modelCapabilities'
import { BuddyAgentRunError } from '../../runs/runError'
Expand Down Expand Up @@ -102,12 +103,14 @@ export class BuddySessionFactory {
providerId: effectiveModel?.providerId ?? run.provider,
})
const imageMaterials = new ImageReadingMaterialStore(blueprint.scratchRoot, blueprint.conversationId)
const generatedMaterials = new ImageGenerationMaterialStore(blueprint.scratchRoot, blueprint.conversationId)
const imageStore = new ImageBlobStore(conversationImageDirectory(join(resolve(this.#options.conversationsDirectory), blueprint.conversationId)))
const extensions = await events.operation('session.resources', async () => {
if (run.purpose !== 'conversation.compaction')
await this.#options.services.prepareForRun?.(input.signal)
return createBuddySessionExtensions({
imageStore,
writeGeneratedImageMaterial: (image, grants, writeFiles, signal) => generatedMaterials.write(image, grants, writeFiles, signal),
writeImageMaterial: (text, signal) => imageMaterials.write(text, signal),
actorKind: blueprint.actor?.kind,
toolScope: run.purpose === 'conversation.compaction' ? { mode: 'none' } : blueprint.actor?.settings.tools,
Expand Down
45 changes: 41 additions & 4 deletions apps/buddy/service/src/artifacts/ArtifactService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ import type {
ArtifactRepository,
} from '../storage/artifactRepository'
import { randomUUID } from 'node:crypto'
import { stat } from 'node:fs/promises'
import { constants } from 'node:fs'
import { access, stat } from 'node:fs/promises'
import {
basename,
dirname,
Expand Down Expand Up @@ -472,8 +473,8 @@ export class ArtifactService {
export class ArtifactError extends Error {
readonly code: string

constructor(code: string) {
super('Lexora Buddy artifact operation failed')
constructor(code: string, options?: ErrorOptions) {
super('Lexora Buddy artifact operation failed', options)
this.name = 'ArtifactError'
this.code = code
}
Expand All @@ -486,7 +487,7 @@ export class ArtifactPublicationError extends ArtifactError {
const code = error && typeof error === 'object' && 'code' in error && typeof error.code === 'string' && /^[A-Z][A-Z0-9_]{0,79}$/.test(error.code)
? error.code
: 'ARTIFACT_PUBLICATION_FAILED'
super(code)
super(code, { cause: error })
this.name = 'ArtifactPublicationError'
this.receipt = copyEventSnapshot(receipt)
}
Expand Down Expand Up @@ -519,6 +520,42 @@ async function resolveArtifactLocation(
}
}

export async function preflightGeneratedImageOutput(input: Pick<GeneratedImagesInput, 'cwd' | 'grants' | 'outputPath'>, signal?: AbortSignal): Promise<void> {
signal?.throwIfAborted()
if (!input.outputPath.trim())
throw new ArtifactError('VALIDATION_FAILED')
const basePath = isAbsolute(input.outputPath) ? input.outputPath : resolve(input.cwd, input.outputPath)
const location = await resolveArtifactLocation(input.grants, generatedImagePath(basePath, 0, 'image/png'), 'create')
signal?.throwIfAborted()
const target = await stat(location.canonicalPath).catch((error: NodeJS.ErrnoException) => {
if (error.code !== 'ENOENT')
throw error
return null
})
if (target) {
if (!target.isFile())
throw new ArtifactError('VALIDATION_FAILED')
await access(location.canonicalPath, constants.R_OK | constants.W_OK)
}
let parent = dirname(location.canonicalPath)
while (true) {
signal?.throwIfAborted()
const metadata = await stat(parent).catch((error: NodeJS.ErrnoException) => {
if (error.code !== 'ENOENT' || dirname(parent) === parent)
throw error
return null
})
if (metadata) {
if (!metadata.isDirectory())
throw new ArtifactError('VALIDATION_FAILED')
await access(parent, constants.W_OK | constants.X_OK)
signal?.throwIfAborted()
return
}
parent = dirname(parent)
}
}

function generatedImagePath(basePath: string, index: number, mimeType: string): string {
const expectedExtension = extensionForImageMimeType(mimeType)
const extension = extname(basePath).toLowerCase()
Expand Down
38 changes: 38 additions & 0 deletions apps/buddy/service/src/artifacts/__tests__/ArtifactService.spec.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { Model } from '@earendil-works/pi-ai'
import type { DatabaseSync } from 'node:sqlite'
import type { CapturedFileWriter } from '../../changes/CapturedFileWriter'
import type { DirectoryGrant } from '../../directories/resolveGrantedPath'
Expand All @@ -12,6 +13,8 @@ import { afterEach, describe, expect, it } from 'vitest'
import { MutationCoordinator } from '../../changes/MutationCoordinator'
import { FileResourceCoordinator } from '../../files/FileResourceCoordinator'
import { NativeFileEngine } from '../../files/NativeFileEngine'
import { ImageGenerationMaterialStore } from '../../images/ImageGenerationMaterialStore'
import { ImageGenerationService } from '../../images/ImageGenerationService'
import { createArtifactRepository } from '../../storage/artifactRepository'
import { createConversationDirectoryGrantRepository } from '../../storage/conversationDirectoryGrantRepository'
import { createConversationRepository } from '../../storage/conversationRepository'
Expand All @@ -31,6 +34,41 @@ afterEach(async () => {
})

describe('artifactService', () => {
it('generates into missing parents without creating them before submission and preserves numbered PNG outputs', async () => {
const f = await createFixture()
const bytes = Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAYAAAD0In+KAAAAFElEQVR4AQEJAPb/AAD/AP//AAD/EPgD/UE7EkoAAAAASUVORK5CYII=', 'base64')
let submissions = 0
const materials = new ImageGenerationMaterialStore(f.workspace, 'conversation-1')
const service = new ImageGenerationService({
writeGeneratedImageMaterial: (image, grants, writeFiles, signal) => materials.write(image, grants, writeFiles, signal),
conversationId: 'conversation-1',
cwd: f.workspace,
grants: f.grants,
artifactService: f.service,
attachmentService: { materializeConversationImages: async () => ({ images: [], records: [] }) },
referenceAccess: { authorize: () => {} },
imageGenerationGateway: { supports: () => true, generate: async (input) => {
await expect(readFile(join(f.workspace, 'output'))).rejects.toMatchObject({ code: 'ENOENT' })
await input.onSubmitting?.()
submissions++
return { images: [{ bytes, mimeType: 'image/png' }, { bytes, mimeType: 'image/png' }], responseId: 'response-1' }
} },
})
try {
const result = await service.generate({ outputPath: 'output/nested/heart', prompt: 'Create' }, { provider: 'fixture', id: 'fixture' } as Model<'openai-responses'>, new AbortController().signal, f.grants, f.writeFiles)
const paths = ['heart.png', 'heart-2.png'].map(name => join(f.workspace, 'output', 'nested', name))
expect(submissions).toBe(1)
expect(result.outputs.map(output => output.path)).toEqual(paths)
expect(result.artifactIds).toHaveLength(2)
expect(f.repository.listForConversation('conversation-1').map(record => record.currentPath).sort()).toEqual([...paths].sort())
for (const path of paths)
expect(await readFile(path)).toEqual(bytes)
}
finally {
await service.dispose()
}
})

it('keeps delivered files and extensionless image MIME readable after grant revocation and service recreation without granting new outputs', async () => {
const f = await createFixture()
const conversationGrants = createConversationDirectoryGrantRepository(f.database)
Expand Down
25 changes: 22 additions & 3 deletions apps/buddy/service/src/createBuddyCapabilityFactory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import type { McpConnectorService } from './connectors/mcp/McpConnectorService'
import type { SessionReferenceServices } from './conversations/SessionReferenceService'
import type { ImageGenerationGateway } from './images/ImageGenerationGateway'
import type { ImageGenerationServiceOptions } from './images/ImageGenerationService'
import type { ImageReferenceAccessOptions } from './images/ImageReferenceAccess'
import type { ImageRequestStore } from './images/ImageRequestStore'
import type { ImageTransformService } from './images/ImageTransformService'
import type { PetActionService } from './pet/PetActionService'
Expand All @@ -31,6 +32,8 @@ import { createSessionReferenceCapability } from './conversations/sessionReferen
import { createImageGenerationCapability } from './images/imageGenerationExtension'
import { ImageGenerationService } from './images/ImageGenerationService'
import { observeImageDiagnostics } from './images/ImageOperationLifecycle'
import { ImageReferenceAccess } from './images/ImageReferenceAccess'
import { createImageReferenceCapability } from './images/imageReferenceExtension'
import { createImageTransformCapability } from './images/imageTransformExtension'
import { createPetCapability } from './pet/petExtension'
import { createPluginAuthoringCapability } from './plugins/pluginAuthoringCapability'
Expand Down Expand Up @@ -58,7 +61,10 @@ export interface BuddyCapabilityServices {
imageTransformService: Pick<ImageTransformService, 'removeChroma'>
webService: Pick<WebCapabilityService, 'search' | 'fetch'>
conversations: SessionReferenceServices['conversations']
publications: SessionReferenceServices['publications']
publications: SessionReferenceServices['publications'] & ImageReferenceAccessOptions['publications']
executions: ImageReferenceAccessOptions['executions']
runInputs: ImageReferenceAccessOptions['inputs']
runEvents: ImageReferenceAccessOptions['events']
references: SessionReferenceRepository
referenceSnapshots: ExtensionReferenceRepository
}
Expand Down Expand Up @@ -89,6 +95,16 @@ export function createBuddyCapabilityFactory(
? Promise.reject(new Error('EXECUTION_PROFILE_READ_ONLY'))
: writeMcpResult(bytes, mimeType, signal))
context.signal.throwIfAborted()
const imageReferences = new ImageReferenceAccess({
...context,
conversations: services.conversations,
publications: services.publications,
executions: services.executions,
inputs: services.runInputs,
events: services.runEvents,
attachments: services.attachmentService,
artifacts: services.artifactService,
})
const capabilities = [
createMcpCapability(mcp, context.isCodemodeEnabled ?? (() => false)),
createBrowserCapability({
Expand All @@ -114,7 +130,8 @@ export function createBuddyCapabilityFactory(
: undefined,
}),
createWebCapability({ authorizeNetwork: context.authorizeNetwork, service: services.webService, conversationId: context.conversationId }),
createImageCapability(context, services),
createImageCapability(context, services, imageReferences),
createImageReferenceCapability(imageReferences),
createImageTransformCapability({ ...context, service: services.imageTransformService }),
createOutputPresentationCapability({ ...context, artifactService: services.artifactService }),
createSessionReferenceCapability({ conversationId: context.conversationId, getRunId: context.getRunId, conversations: services.conversations, attachments: services.attachmentService, artifacts: services.artifactService, publications: services.publications, references: services.references }),
Expand Down Expand Up @@ -147,10 +164,12 @@ export function createBuddyCapabilityFactory(
}
}

function createImageCapability(context: BuddyCapabilityContext, services: BuddyCapabilityServices): BuddyCapability {
function createImageCapability(context: BuddyCapabilityContext, services: BuddyCapabilityServices, referenceAccess: ImageReferenceAccess): BuddyCapability {
const service = new ImageGenerationService({
writeGeneratedImageMaterial: context.writeGeneratedImageMaterial,
artifactService: services.artifactService,
attachmentService: services.attachmentService,
referenceAccess,
conversationId: context.conversationId,
cwd: context.cwd,
get grants() { return context.grants },
Expand Down
12 changes: 8 additions & 4 deletions apps/buddy/service/src/images/ImageGenerationGateway.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import type { Api, ImageContent, Model } from '@earendil-works/pi-ai'
import type { ImageGenerationErrorCode, ImageGenerationErrorDiagnostic, ImageGenerationOutcome, ImageRequestRef } from '../../../shared/images/imageGeneration'
import type { ImageGenerationErrorCode, ImageGenerationErrorDiagnostic, ImageGenerationOutcome, ImageGenerationRetainedOutput, ImageRequestRef } from '../../../shared/images/imageGeneration'
import type { ImageOutputOptions, ImagePreferencesSnapshot, ImageSource, ImageSupport, ImageTarget } from '../../../shared/images/imagePreferences'
import { readImageDiagnostic } from '../../../shared/images/imageGeneration'
import { imageGenerationRetainedOutputSchema, readImageDiagnostic } from '../../../shared/images/imageGeneration'

export { IMAGE_GENERATION_ERROR_CODES } from '../../../shared/images/imageGeneration'
export type { ImageGenerationErrorCode, ImageGenerationErrorDiagnostic } from '../../../shared/images/imageGeneration'
Expand Down Expand Up @@ -51,10 +51,11 @@ export class ImageGenerationError extends Error {
readonly diagnostic: ImageGenerationErrorDiagnostic | null
readonly request?: ImageRequestRef
readonly artifactIds: string[]
readonly retainedOutputs?: ImageGenerationRetainedOutput[]

constructor(
code: ImageGenerationErrorCode,
options?: ErrorOptions & { diagnostic?: ImageGenerationErrorDiagnostic, retryable?: boolean, outcome?: ImageGenerationOutcome, request?: ImageRequestRef, artifactIds?: readonly string[] },
options?: ErrorOptions & { diagnostic?: ImageGenerationErrorDiagnostic, retryable?: boolean, outcome?: ImageGenerationOutcome, request?: ImageRequestRef, artifactIds?: readonly string[], retainedOutputs?: readonly ImageGenerationRetainedOutput[] },
) {
super('Lexora Buddy image generation failed', { cause: options?.cause })
this.name = 'ImageGenerationError'
Expand All @@ -64,6 +65,8 @@ export class ImageGenerationError extends Error {
this.diagnostic = readImageDiagnostic(options?.diagnostic)
this.request = options?.request
this.artifactIds = [...options?.artifactIds ?? []]
if (options?.retainedOutputs)
this.retainedOutputs = options.retainedOutputs.map(output => imageGenerationRetainedOutputSchema.parse(output))
}
}

Expand All @@ -73,8 +76,9 @@ export function readImageGenerationError(error: unknown): {
outcome: ImageGenerationOutcome
request?: ImageRequestRef
artifactIds?: string[]
retainedOutputs?: ImageGenerationRetainedOutput[]
} {
return error instanceof ImageGenerationError
? { code: error.code, diagnostic: error.diagnostic, outcome: error.outcome, artifactIds: error.artifactIds, ...(error.request ? { request: error.request } : {}) }
? { code: error.code, diagnostic: error.diagnostic, outcome: error.outcome, artifactIds: error.artifactIds, ...(error.retainedOutputs ? { retainedOutputs: error.retainedOutputs } : {}), ...(error.request ? { request: error.request } : {}) }
: { code: 'IMAGE_GENERATION_FAILED', diagnostic: null, outcome: 'unknown' }
}
Loading
Loading