diff --git a/.changeset/empty-image-session-poisoning.md b/.changeset/empty-image-session-poisoning.md new file mode 100644 index 0000000000..6c16a65c92 --- /dev/null +++ b/.changeset/empty-image-session-poisoning.md @@ -0,0 +1,6 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Fix pasted images intermittently failing to reach the model: a zero-byte image from a failed clipboard read poisoned the session, so every later image was dropped with an ambiguous placeholder and the model could hallucinate having seen it. Empty images are now replaced by a clear notice at ingestion and at send time (already-affected sessions recover automatically), and the placeholder tells the model the attachment was removed and must not be guessed at. +web: Show an error chip instead of uploading a zero-byte attachment (e.g. from a failed clipboard image read). diff --git a/apps/kimi-web/src/components/chat/AttachmentChip.vue b/apps/kimi-web/src/components/chat/AttachmentChip.vue index b245664d2c..3d5e650b91 100644 --- a/apps/kimi-web/src/components/chat/AttachmentChip.vue +++ b/apps/kimi-web/src/components/chat/AttachmentChip.vue @@ -30,6 +30,8 @@ const props = withDefaults( uploading?: boolean; /** Composer: upload failed — chip tinted, info icon replaces the badge. */ error?: boolean; + /** Composer: why the attachment failed — appended to the tooltip. */ + errorReason?: string; /** Composer: show a remove button. */ removable?: boolean; /** Accessible label for the remove button. */ @@ -74,6 +76,7 @@ function formatSize(size: number): string { const title = computed(() => { const parts = [displayName.value]; if (props.size !== undefined) parts.push(formatSize(props.size)); + if (props.errorReason !== undefined) parts.push(props.errorReason); return parts.join(' · '); }); diff --git a/apps/kimi-web/src/components/chat/Composer.vue b/apps/kimi-web/src/components/chat/Composer.vue index 5d2e16f6d9..23af308b75 100644 --- a/apps/kimi-web/src/components/chat/Composer.vue +++ b/apps/kimi-web/src/components/chat/Composer.vue @@ -859,6 +859,7 @@ function selectModel(modelId: string): void { :size="att.size" :uploading="att.uploading" :error="att.error" + :error-reason="att.errorReason" removable :remove-label="t('composer.removeNamed', { name: att.name })" @activate="onAttachmentActivate(att)" diff --git a/apps/kimi-web/src/composables/useAttachmentUpload.ts b/apps/kimi-web/src/composables/useAttachmentUpload.ts index 2c97c4d6ce..deea376706 100644 --- a/apps/kimi-web/src/composables/useAttachmentUpload.ts +++ b/apps/kimi-web/src/composables/useAttachmentUpload.ts @@ -13,6 +13,7 @@ import { computed, onMounted, onUnmounted, ref, watch } from 'vue'; import { getKimiWebApi } from '../api'; +import { i18n } from '../i18n'; export interface Attachment { /** Unique local id (used as :key) */ @@ -33,6 +34,8 @@ export interface Attachment { fileId?: string; /** True if upload failed */ error?: boolean; + /** Localized reason shown in the chip tooltip when `error` is set. */ + errorReason?: string; } type UploadImage = ( @@ -89,6 +92,24 @@ export function useAttachmentUpload(deps: AttachmentUploadDeps) { for (const file of files) { const kind = attachmentKind(file.type); const localId = nextLocalId(); + // A zero-byte file means the source (clipboard owner, picker) handed + // over no data — uploading it would attach an empty image that the + // provider later rejects. Mark it failed instead so the user + // re-captures; the chip stays visible but is excluded from submit. + if (file.size === 0) { + const att: Attachment = { + localId, + name: file.name, + kind, + mediaType: file.type || 'application/octet-stream', + size: file.size, + uploading: false, + error: true, + errorReason: i18n.global.t('composer.attachmentEmpty'), + }; + setForSession(sid, [...(attachmentsBySession.value[sid] ?? []), att]); + continue; + } // Only media gets a thumbnail object URL; files render an icon chip. const previewUrl = kind === 'file' ? undefined : URL.createObjectURL(file); const att: Attachment = { @@ -177,6 +198,19 @@ export function useAttachmentUpload(deps: AttachmentUploadDeps) { const key = `${blob.size}:${blob.type}:${name}`; if (seenKeys.has(key)) return; seenKeys.add(key); + // Diagnostics for the transient "empty clipboard image" failure: when + // the source app fails to deliver the selection, the browser hands us a + // named, typed, zero-byte File. Log everything visible at this point so + // the source can be identified the next time it happens. + if (blob.size === 0) { + console.warn('[kimi-web] paste produced a zero-byte file', { + name, + type: blob.type, + clipboardTypes: [...cd.types], + items: Array.from(cd.items).map((item) => ({ kind: item.kind, type: item.type })), + userAgent: navigator.userAgent, + }); + } const ext = blob.type.split('/')[1] ?? 'png'; const safeName = name.includes('.') ? name : `paste-${Date.now()}.${ext}`; files.push(blob instanceof File ? blob : new File([blob], safeName, { type: blob.type })); diff --git a/apps/kimi-web/src/i18n/locales/en/composer.ts b/apps/kimi-web/src/i18n/locales/en/composer.ts index 5f566c6729..c2131a0e60 100644 --- a/apps/kimi-web/src/i18n/locales/en/composer.ts +++ b/apps/kimi-web/src/i18n/locales/en/composer.ts @@ -14,6 +14,7 @@ export default { attachmentVideo: 'Video', attachmentFile: 'File', attachmentOpenUnsupported: 'Can’t open {name} — this file type isn’t supported', + attachmentEmpty: 'Empty file (0 bytes) — not attached. If it came from the clipboard, take the screenshot or copy the image again.', dropToAttach: 'Drop files to attach', remove: 'Remove', removeNamed: 'Remove {name}', diff --git a/apps/kimi-web/src/i18n/locales/zh/composer.ts b/apps/kimi-web/src/i18n/locales/zh/composer.ts index a0ef9f952b..6695b67d8c 100644 --- a/apps/kimi-web/src/i18n/locales/zh/composer.ts +++ b/apps/kimi-web/src/i18n/locales/zh/composer.ts @@ -14,6 +14,7 @@ export default { attachmentVideo: '视频', attachmentFile: '文件', attachmentOpenUnsupported: '无法打开 {name}:暂不支持此文件类型', + attachmentEmpty: '文件为空(0 字节),未添加。如果来自剪贴板,请重新截图或复制后再粘贴。', dropToAttach: '松开鼠标添加附件', remove: '移除', removeNamed: '移除 {name}', diff --git a/apps/kimi-web/test/attachment-upload.test.ts b/apps/kimi-web/test/attachment-upload.test.ts index 6c5ed9b759..73420365e9 100644 --- a/apps/kimi-web/test/attachment-upload.test.ts +++ b/apps/kimi-web/test/attachment-upload.test.ts @@ -86,6 +86,28 @@ describe('useAttachmentUpload', () => { expect(att.attachments.value).toHaveLength(0); }); + it('rejects a zero-byte file as a failed chip without uploading (issue #2209)', () => { + // A clipboard failure can hand the browser a named, typed, zero-byte + // File. Uploading it would attach an empty image that the provider later + // rejects — mark it failed instead so the user re-captures. + const uploadImage = vi.fn(); + const att = setup(uploadImage); + att.handleFileInputChange( + inputEvent([{ name: 'image.png', type: 'image/png', size: 0 } as unknown as File]), + ); + + expect(att.attachments.value).toHaveLength(1); + expect(att.attachments.value[0]).toMatchObject({ + name: 'image.png', + kind: 'image', + uploading: false, + error: true, + }); + expect(att.attachments.value[0].errorReason).toBeTruthy(); + expect(uploadImage).not.toHaveBeenCalled(); + expect(createObjectURL).not.toHaveBeenCalled(); + }); + it('removeAttachment on a file chip has no object URL to revoke', () => { const uploadImage = vi.fn().mockResolvedValue(null); const att = setup(uploadImage); diff --git a/packages/agent-core-v2/src/agent/contextProjector/contextProjectorService.ts b/packages/agent-core-v2/src/agent/contextProjector/contextProjectorService.ts index 88227ba4e0..f80fe59b41 100644 --- a/packages/agent-core-v2/src/agent/contextProjector/contextProjectorService.ts +++ b/packages/agent-core-v2/src/agent/contextProjector/contextProjectorService.ts @@ -23,6 +23,12 @@ * rejected, then replaces only that snapshot on later steps so a newly * generated recovery image remains visible. Both are read-side only — the * history keeps its media. + * + * Every projection also passes each message's content through `media`'s + * image format gate, so a malformed or undeliverable image already persisted + * in history (e.g. an empty data URL from a failed clipboard read) is + * replaced by a text notice on the wire instead of poisoning every later + * request — likewise read-side only. */ import { createHash } from 'node:crypto'; @@ -35,6 +41,7 @@ import { isVacuousContentPart } from '#/agent/contextMemory/vacuousContent'; import { IAgentStateService } from '#/agent/state/agentState'; import { ErrorCodes, Error2 } from '#/errors'; import type { ContentPart, Message } from '#/kosong/contract/message'; +import { gateImageFormatParts } from '#/agent/media/image-format-policy'; import { ITelemetryService } from '#/app/telemetry/telemetry'; import { IAgentContextProjectorService, @@ -188,20 +195,20 @@ export const MEDIA_DEGRADE_KEEP_RECENT = 2; const MEDIA_DEGRADED_PLACEHOLDERS = { image_url: - '[image omitted: dropped to fit the provider request size limit; re-read the file to view it]', + '[An image attached to an earlier message was removed to fit the provider request size limit. You have NOT seen this image — do not describe or guess its contents. If it matters, ask the user to re-send it or to point you at the file so you can read it with ReadMediaFile.]', audio_url: - '[audio omitted: dropped to fit the provider request size limit; re-read the file to hear it]', + '[An audio clip attached to an earlier message was removed to fit the provider request size limit. You have NOT heard it — do not describe or guess its contents.]', video_url: - '[video omitted: dropped to fit the provider request size limit; re-read the file to view it]', + '[A video attached to an earlier message was removed to fit the provider request size limit. You have NOT seen it — do not describe or guess its contents.]', } as const; export const MEDIA_STRIPPED_PLACEHOLDERS = { image_url: - '[image omitted for provider compatibility; re-read the file to view it or get conversion guidance]', + '[An image attached to this message was removed before sending because the provider rejected the request (unsupported or unreadable image data, or the request exceeded its size limit). You have NOT seen this image — do not describe or guess its contents. Tell the user the image failed to reach you and suggest re-sending it as PNG or JPEG, or a smaller copy.]', audio_url: - '[audio omitted for provider compatibility; re-read the file to hear it]', + '[An audio clip attached to this message was removed before sending because the provider rejected the request. You have NOT heard it — do not describe or guess its contents.]', video_url: - '[video omitted for provider compatibility; re-read the file to view it]', + '[A video attached to this message was removed before sending because the provider rejected the request. You have NOT seen it — do not describe or guess its contents.]', } as const; type MediaPlaceholderSet = typeof MEDIA_DEGRADED_PLACEHOLDERS | typeof MEDIA_STRIPPED_PLACEHOLDERS; @@ -540,7 +547,7 @@ function projectedContent(source: ContextMessage, onAnomaly?: OnAnomaly): Conten note: source.note, }) : source.content; - return cleanContent(source, content, onAnomaly); + return cleanContent(source, gateImageFormatParts(content), onAnomaly); } function cleanContent( diff --git a/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts b/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts index aec58a5d4c..becbe1b8d9 100644 --- a/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts +++ b/packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts @@ -73,7 +73,7 @@ import { completionBudgetParams, resolveCompletionBudget } from '#/kosong/model/ import { resolveThinkingKeep, type ThinkingConfig } from '#/kosong/model/thinking'; import { THINKING_SECTION } from '#/app/kosongConfig/configSection'; import type { Protocol } from '#/kosong/protocol/protocol'; -import type { ApiErrorEvent } from '#/app/telemetry/events'; +import type { ApiErrorEvent, MediaStrippedEvent } from '#/app/telemetry/events'; import { ITelemetryService } from '#/app/telemetry/telemetry'; import { IWireService } from '#/wire/wire'; import type { PayloadOf } from '#/wire/types'; @@ -331,6 +331,37 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService { ): Promise { const shaped = this.toolSelect.shapeHistory(request.messages); let mediaStripSnapshot = this.mediaStripSnapshotForTurn(request.source); + const reportMediaStripped = ( + message: string, + reason: MediaStrippedEvent['reason'], + raw: unknown, + ): void => { + const statusCode = raw instanceof APIStatusError ? raw.statusCode : undefined; + const errorMessage = (raw instanceof Error ? raw.message : String(raw)).slice(0, 300); + this.log.warn(message, { + model: request.model.name, + statusCode, + errorMessage, + ...request.logFields, + }); + let mediaCount = 0; + for (const msg of shaped) { + for (const part of msg.content) { + if (part.type === 'image_url' || part.type === 'audio_url' || part.type === 'video_url') { + mediaCount += 1; + } + } + } + const properties: MediaStrippedEvent = { + reason, + model: request.model.name, + alias: request.modelAlias, + status_code: statusCode, + turn_id: request.source?.type === 'turn' ? request.source.turnId : undefined, + media_count: mediaCount, + }; + this.telemetry.track2('media_stripped', properties); + }; const requestInput = (projection: RequestProjection) => { return { systemPrompt: request.systemPrompt, @@ -463,12 +494,10 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService { this.markRecoveryTurn(this.mediaDegradedTurns, request.source); projection = 'media-degraded'; } else { - this.log.warn( + reportMediaStripped( 'provider rejected degraded-media request as too large; resending with rejected media stripped', - { - model: request.model.name, - ...request.logFields, - }, + 'degraded_too_large', + raw, ); mediaStripSnapshot = this.projector.captureMediaStripSnapshot(shaped); this.markMediaStrippedRecoveryTurn(mediaStripSnapshot, request.source); @@ -478,12 +507,10 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService { } if (projection !== 'media-stripped' && isImageFormatError(raw)) { signal?.throwIfAborted(); - this.log.warn( + reportMediaStripped( 'provider rejected an image in the request; resending with rejected media stripped', - { - model: request.model.name, - ...request.logFields, - }, + 'image_format', + raw, ); mediaStripSnapshot = this.projector.captureMediaStripSnapshot(shaped); this.markMediaStrippedRecoveryTurn(mediaStripSnapshot, request.source); diff --git a/packages/agent-core-v2/src/agent/media/image-compress.ts b/packages/agent-core-v2/src/agent/media/image-compress.ts index f3961d1998..3b7146f3d3 100644 --- a/packages/agent-core-v2/src/agent/media/image-compress.ts +++ b/packages/agent-core-v2/src/agent/media/image-compress.ts @@ -38,18 +38,14 @@ import type { ContentPart } from '#/kosong/contract/message'; import { sniffImageDimensions } from './file-type'; import { - buildMalformedImageNotice, - buildUnsupportedImageNotice, - decodeBase64Prefix, - isDataUrl, - isModelAcceptedImageMime, + gateImageFormatParts, normalizeImageMime, parseImageDataUrl, - resolveEffectiveImageMime, - unsupportedImageMimeFromUrl, } from './image-format-policy'; import { decodeWebp, isAnimatedWebp } from './webp-decode'; +export { gateImageFormatParts }; + export const MAX_IMAGE_EDGE_PX = 2000; let configuredMaxImageEdgePx: number | undefined; @@ -316,46 +312,6 @@ export interface CompressedContentParts { readonly captions: readonly string[]; } -export function gateImageFormatParts(parts: readonly ContentPart[]): ContentPart[] { - const out: ContentPart[] = []; - for (const part of parts) { - if (part.type === 'image_url') { - const parsed = parseImageDataUrl(part.imageUrl.url); - if (parsed === null) { - if (isDataUrl(part.imageUrl.url)) { - out.push({ type: 'text', text: buildMalformedImageNotice(part.imageUrl.url) }); - continue; - } - const extMime = unsupportedImageMimeFromUrl(part.imageUrl.url); - if (extMime !== null) { - out.push({ - type: 'text', - text: buildUnsupportedImageNotice(extMime, part.imageUrl.url), - }); - continue; - } - out.push(part); - continue; - } - const effectiveMime = resolveEffectiveImageMime( - parsed.mimeType, - decodeBase64Prefix(parsed.base64), - ); - if (!isModelAcceptedImageMime(effectiveMime)) { - out.push({ type: 'text', text: buildUnsupportedImageNotice(effectiveMime) }); - continue; - } - const canonicalUrl = `data:${normalizeImageMime(effectiveMime)};base64,${parsed.base64}`; - if (part.imageUrl.url !== canonicalUrl) { - out.push({ type: 'image_url', imageUrl: { ...part.imageUrl, url: canonicalUrl } }); - continue; - } - } - out.push(part); - } - return out; -} - export async function compressImageContentParts( parts: readonly ContentPart[], options: CompressImageOptions & { readonly annotate?: CompressAnnotateOptions } = {}, diff --git a/packages/agent-core-v2/src/agent/media/image-format-policy.ts b/packages/agent-core-v2/src/agent/media/image-format-policy.ts index 3c74514126..9aead809ca 100644 --- a/packages/agent-core-v2/src/agent/media/image-format-policy.ts +++ b/packages/agent-core-v2/src/agent/media/image-format-policy.ts @@ -7,7 +7,13 @@ * is rejected by the API — and because prompts and tool results persist in * the session history, that one part makes every subsequent request fail * too ("session poisoning"). Every ingestion point therefore refuses - * unsupported formats instead of passing the bytes through. + * unsupported formats instead of passing the bytes through: ReadMediaFile + * refuses with a conversion command the model can run, and prompt/MCP + * ingestion replaces the image with a text notice. The same applies to an + * empty payload (`data:image/png;base64,` — a clipboard/upload failure that + * captured no bytes), which {@link gateImageFormatParts} replaces with a + * notice both at ingestion and, for histories already carrying one, at + * projection time. * * The policy is deliberately a closed set, not a denylist: a format is only * ever sent when it is known to be accepted. Supporting a new format means @@ -27,6 +33,8 @@ * server-side; those pass through unchanged. */ +import type { ContentPart } from '#/kosong/contract/message'; + import { IMAGE_MIME_BY_SUFFIX, sniffMediaFromMagic } from './file-type'; export const MODEL_ACCEPTED_IMAGE_MIMES: ReadonlySet = new Set([ @@ -167,3 +175,53 @@ export function buildMalformedImageNotice(url: string): string { 'could not be parsed). Re-encode the image as PNG or JPEG and try again.]' ); } + +export function buildEmptyImageNotice(name?: string): string { + const what = name === undefined || name.length === 0 ? 'The attached image' : `"${name}"`; + return ( + `[Image omitted: ${what} contained no image data (0 bytes) — the clipboard ` + + 'or upload captured nothing. Re-paste or re-upload the image and try again.]' + ); +} + +export function gateImageFormatParts(parts: readonly ContentPart[]): ContentPart[] { + const out: ContentPart[] = []; + for (const part of parts) { + if (part.type === 'image_url') { + const parsed = parseImageDataUrl(part.imageUrl.url); + if (parsed === null) { + if (isDataUrl(part.imageUrl.url)) { + out.push({ type: 'text', text: buildMalformedImageNotice(part.imageUrl.url) }); + continue; + } + const extMime = unsupportedImageMimeFromUrl(part.imageUrl.url); + if (extMime !== null) { + out.push({ + type: 'text', + text: buildUnsupportedImageNotice(extMime, part.imageUrl.url), + }); + continue; + } + out.push(part); + continue; + } + const head = decodeBase64Prefix(parsed.base64); + if (head.length === 0) { + out.push({ type: 'text', text: buildEmptyImageNotice() }); + continue; + } + const effectiveMime = resolveEffectiveImageMime(parsed.mimeType, head); + if (!isModelAcceptedImageMime(effectiveMime)) { + out.push({ type: 'text', text: buildUnsupportedImageNotice(effectiveMime) }); + continue; + } + const canonicalUrl = `data:${normalizeImageMime(effectiveMime)};base64,${parsed.base64}`; + if (part.imageUrl.url !== canonicalUrl) { + out.push({ type: 'image_url', imageUrl: { ...part.imageUrl, url: canonicalUrl } }); + continue; + } + } + out.push(part); + } + return out; +} diff --git a/packages/agent-core-v2/src/app/telemetry/events.ts b/packages/agent-core-v2/src/app/telemetry/events.ts index c534d23e17..0371310ee5 100644 --- a/packages/agent-core-v2/src/app/telemetry/events.ts +++ b/packages/agent-core-v2/src/app/telemetry/events.ts @@ -122,6 +122,15 @@ export interface ApiErrorEvent { trace_id?: string; } +export interface MediaStrippedEvent { + reason: 'too_large' | 'degraded_too_large' | 'image_format'; + model: string; + alias?: string; + status_code?: number; + turn_id?: number; + media_count: number; +} + export interface SkillInvokedEvent { skill_name: string; trigger: 'user-slash' | 'model-tool' | 'nested-skill'; @@ -526,6 +535,19 @@ export const telemetryEventDefinitions = { trigger: 'How the skill was triggered', }, }), + media_stripped: defineAgentTelemetryEvent({ + owner: 'kimi-code', + comment: + 'The provider rejected media in the request; every media part present was replaced by a text placeholder for the resend.', + properties: { + reason: 'Which rejection led to the strip', + model: 'Model id the request targeted', + alias: 'Model alias the request targeted', + status_code: 'HTTP status code of the rejecting response when available', + turn_id: 'Per-agent turn index when the request belongs to a turn; omitted for out-of-turn operations', + media_count: 'Number of media parts in context at strip time', + }, + }), flow_invoked: defineAgentTelemetryEvent({ owner: 'kimi-code', comment: 'A flow-type skill is invoked.', diff --git a/packages/agent-core-v2/src/index.ts b/packages/agent-core-v2/src/index.ts index aee6359fb4..453bd8db3d 100644 --- a/packages/agent-core-v2/src/index.ts +++ b/packages/agent-core-v2/src/index.ts @@ -453,6 +453,7 @@ export { } from '#/agent/media/image-compress'; export { MODEL_ACCEPTED_IMAGE_MIMES, + buildEmptyImageNotice, buildImageConversionGuidance, buildUnsupportedImageNotice, decodeBase64Prefix, diff --git a/packages/agent-core-v2/test/agent/contextProjector/projector-tool-exchanges.test.ts b/packages/agent-core-v2/test/agent/contextProjector/projector-tool-exchanges.test.ts index 8fab71d4b3..10fb95dfbe 100644 --- a/packages/agent-core-v2/test/agent/contextProjector/projector-tool-exchanges.test.ts +++ b/packages/agent-core-v2/test/agent/contextProjector/projector-tool-exchanges.test.ts @@ -679,6 +679,36 @@ describe('projector tool-exchange normalization', () => { }); }); + describe('image format gate at projection time', () => { + it('replaces an empty-payload image already in history with a notice (self-heal)', () => { + const history = [ + { + role: 'user' as const, + content: [{ type: 'image_url' as const, imageUrl: { url: 'data:image/png;base64,' } }], + toolCalls: [], + origin: { kind: 'user' as const }, + }, + user('what do you see?'), + ]; + const parts = project(history).flatMap((message) => message.content); + expect(parts.some((part) => part.type === 'image_url')).toBe(false); + const texts = parts.filter((part) => part.type === 'text').map((part) => part.text); + expect(texts.some((text) => text.includes('no image data (0 bytes)'))).toBe(true); + expect(texts.some((text) => text.includes('what do you see?'))).toBe(true); + }); + + it('passes a deliverable image through untouched', () => { + const part = { + type: 'image_url' as const, + imageUrl: { url: 'data:image/png;base64,QUJD' }, + }; + const history = [ + { role: 'user' as const, content: [part], toolCalls: [], origin: { kind: 'user' as const } }, + ]; + expect(project(history).flatMap((message) => message.content)).toContainEqual(part); + }); + }); + describe('projectMediaDegraded', () => { function imageMessage(url: string): ContextMessage { return { @@ -708,7 +738,7 @@ describe('projector tool-exchange normalization', () => { .filter((part) => part.type === 'text') .map((part) => part.text); expect( - markers.filter((text) => text.includes('dropped to fit the provider request size limit')), + markers.filter((text) => text.includes('removed to fit the provider request size limit')), ).toHaveLength(2); }); @@ -760,8 +790,8 @@ describe('projector tool-exchange normalization', () => { const texts = allParts.filter((part) => part.type === 'text').map((part) => part.text); expect(texts).toContain('look at these'); expect(texts).toContain(''); - expect(texts.some((text) => text.includes('omitted for provider compatibility'))).toBe(true); - expect(texts.some((text) => text.includes('get conversion guidance'))).toBe(true); + expect(texts.some((text) => text.includes('provider rejected the request'))).toBe(true); + expect(texts.some((text) => text.includes('You have NOT seen this image'))).toBe(true); }); it('returns the projected messages untouched when there is no media', () => { diff --git a/packages/agent-core-v2/test/agent/media/image-compress.test.ts b/packages/agent-core-v2/test/agent/media/image-compress.test.ts index eec3074636..d235a568a9 100644 --- a/packages/agent-core-v2/test/agent/media/image-compress.test.ts +++ b/packages/agent-core-v2/test/agent/media/image-compress.test.ts @@ -783,6 +783,15 @@ describe('gateImageFormatParts', () => { } }); + it('drops an empty-payload data URL (clipboard/upload captured nothing)', () => { + const out = gateImageFormatParts([ + { type: 'image_url', imageUrl: { url: 'data:image/png;base64,' } }, + ]); + expect(out.some((p) => p.type === 'image_url')).toBe(false); + expect(out[0]).toMatchObject({ type: 'text' }); + expect((out[0] as { text: string }).text).toContain('no image data (0 bytes)'); + }); + it('truncates a long malformed data URL in the notice', () => { const url = `data:image/png${'x'.repeat(500)}`; const out = gateImageFormatParts([{ type: 'image_url', imageUrl: { url } }]); diff --git a/packages/agent-core/src/agent/context/projector.ts b/packages/agent-core/src/agent/context/projector.ts index 14d3970752..35c5587a81 100644 --- a/packages/agent-core/src/agent/context/projector.ts +++ b/packages/agent-core/src/agent/context/projector.ts @@ -3,6 +3,7 @@ import { createHash } from 'node:crypto'; import type { ContentPart, Message, TextPart } from '@moonshot-ai/kosong'; import { ErrorCodes, KimiError } from '../../errors'; +import { gateImageFormatParts } from '../../tools/support/image-format-policy'; import { renderToolResultForModel } from './tool-result-render'; import type { ContextMessage } from './types'; @@ -109,7 +110,15 @@ export type ProjectionAnomaly = | { readonly kind: 'vacuous_message_dropped'; readonly role: string }; export function project(history: readonly ContextMessage[], options?: ProjectOptions): Message[] { - let result = mergeAdjacentUserMessages(history, options?.onAnomaly); + // The image format gate runs at projection time too (not only at ingestion): + // a malformed/undeliverable image already persisted in an old session's + // history is replaced by a text notice on the wire, so it can no longer + // poison every later request. Read-side only — the history keeps its parts. + const gated = history.map((message) => ({ + ...message, + content: gateImageFormatParts(message.content), + })); + let result = mergeAdjacentUserMessages(gated, options?.onAnomaly); if (options?.dedupeDuplicateToolCalls === true) { result = dedupeDuplicateToolCalls(result, options.onAnomaly); } @@ -523,27 +532,28 @@ export const MEDIA_DEGRADE_KEEP_RECENT = 2; const MEDIA_DEGRADED_PLACEHOLDERS = { image_url: - '[image omitted: dropped to fit the provider request size limit; re-read the file to view it]', + '[An image attached to an earlier message was removed to fit the provider request size limit. You have NOT seen this image — do not describe or guess its contents. If it matters, ask the user to re-send it or to point you at the file so you can read it with ReadMediaFile.]', audio_url: - '[audio omitted: dropped to fit the provider request size limit; re-read the file to hear it]', + '[An audio clip attached to an earlier message was removed to fit the provider request size limit. You have NOT heard it — do not describe or guess its contents.]', video_url: - '[video omitted: dropped to fit the provider request size limit; re-read the file to view it]', + '[A video attached to an earlier message was removed to fit the provider request size limit. You have NOT seen it — do not describe or guess its contents.]', } as const; /** * Provider-compatible markers for a resend with every media part stripped. * This projection recovers from both an image-format rejection and a request * that remains too large after retaining recent media, so the wording must - * not diagnose either cause. Re-reading the path gives the model the relevant - * conversion or size-reduction guidance at the tool boundary. + * stay cause-neutral while making the removal and its consequence explicit — + * a vague placeholder reads like a generic compatibility notice and invites + * the model to hallucinate having seen the attachment. */ export const MEDIA_STRIPPED_PLACEHOLDERS = { image_url: - '[image omitted for provider compatibility; re-read the file to view it or get conversion guidance]', + '[An image attached to this message was removed before sending because the provider rejected the request (unsupported or unreadable image data, or the request exceeded its size limit). You have NOT seen this image — do not describe or guess its contents. Tell the user the image failed to reach you and suggest re-sending it as PNG or JPEG, or a smaller copy.]', audio_url: - '[audio omitted for provider compatibility; re-read the file to hear it]', + '[An audio clip attached to this message was removed before sending because the provider rejected the request. You have NOT heard it — do not describe or guess its contents.]', video_url: - '[video omitted for provider compatibility; re-read the file to view it]', + '[A video attached to this message was removed before sending because the provider rejected the request. You have NOT seen it — do not describe or guess its contents.]', } as const; type MediaPlaceholderSet = typeof MEDIA_DEGRADED_PLACEHOLDERS | typeof MEDIA_STRIPPED_PLACEHOLDERS; diff --git a/packages/agent-core/src/index.ts b/packages/agent-core/src/index.ts index 52a5c66dac..73460482a0 100644 --- a/packages/agent-core/src/index.ts +++ b/packages/agent-core/src/index.ts @@ -77,6 +77,7 @@ export { } from './tools/support/image-compress'; export { MODEL_ACCEPTED_IMAGE_MIMES, + buildEmptyImageNotice, buildImageConversionGuidance, buildUnsupportedImageNotice, decodeBase64Prefix, diff --git a/packages/agent-core/src/tools/support/image-compress.ts b/packages/agent-core/src/tools/support/image-compress.ts index 41011e01b7..46e2b70bd9 100644 --- a/packages/agent-core/src/tools/support/image-compress.ts +++ b/packages/agent-core/src/tools/support/image-compress.ts @@ -38,18 +38,14 @@ import type { TelemetryClient } from '#/telemetry'; import { sniffImageDimensions } from './file-type'; import { - buildMalformedImageNotice, - buildUnsupportedImageNotice, - decodeBase64Prefix, - isDataUrl, - isModelAcceptedImageMime, + gateImageFormatParts, normalizeImageMime, parseImageDataUrl, - resolveEffectiveImageMime, - unsupportedImageMimeFromUrl, } from './image-format-policy'; import { decodeWebp, isAnimatedWebp } from './webp-decode'; +export { gateImageFormatParts }; + /** * Built-in longest-edge ceiling (px). Larger images are scaled down to fit. * This is the default only: the effective ceiling is resolved per call by @@ -493,78 +489,6 @@ export interface CompressedContentParts { readonly captions: readonly string[]; } -/** - * Enforce the provider-accepted image format set (see ./image-format-policy) - * on a content-part list. Inline `data:` image parts whose MIME is outside - * the accepted set are dropped and replaced with a text notice, so one - * unsupported image cannot poison the session history. Accepted images are - * forwarded only as the byte-exact canonical data URL: an alias - * (`image/jpg`), case/whitespace variants, or MIME parameters - * (`image/jpeg;charset=utf-8`) all rebuild to the bare canonical form, - * because strict provider whitelists exact-match the full header. Remote - * (non-data) image URLs and non-image parts pass through — a URL carries no - * bytes to inspect. - * - * The BYTES are authoritative, not the declared MIME: the header of each - * inline image is sniffed, and a mismatch (e.g. AVIF bytes an MCP image - * search tool labels `image/png`) is gated on what the image IS — the - * provider decodes bytes, not labels. When the sniffer doesn't recognize - * the bytes (corrupt image, exotic container), the declared MIME stands - * and the 400-recovery path remains the backstop. - * - * This is the format gate shared by every ingestion point; run it BEFORE - * compression so unsupported bytes are never decoded. - */ -export function gateImageFormatParts(parts: readonly ContentPart[]): ContentPart[] { - const out: ContentPart[] = []; - for (const part of parts) { - if (part.type === 'image_url') { - const parsed = parseImageDataUrl(part.imageUrl.url); - if (parsed === null) { - // A `data:` URL that failed to parse (missing `;base64,` separator, - // empty MIME, …) is guaranteed to fail at the provider — Anthropic - // throws on it, OpenAI-compat servers 400. Drop it for a notice at - // ingestion instead of leaving it to poison the session and trigger - // the media-stripped resend on every later turn. - if (isDataUrl(part.imageUrl.url)) { - out.push({ type: 'text', text: buildMalformedImageNotice(part.imageUrl.url) }); - continue; - } - // Remote image URL (no bytes to sniff): reject when its path - // extension names a format providers reject (e.g. a search-tool - // link ending in `.avif`) — the notice keeps the URL so the model - // can still fetch and convert the image. Extensionless / unknown - // URLs pass through to the provider — and to the 400 recovery. - const extMime = unsupportedImageMimeFromUrl(part.imageUrl.url); - if (extMime !== null) { - out.push({ - type: 'text', - text: buildUnsupportedImageNotice(extMime, part.imageUrl.url), - }); - continue; - } - out.push(part); - continue; - } - const effectiveMime = resolveEffectiveImageMime( - parsed.mimeType, - decodeBase64Prefix(parsed.base64), - ); - if (!isModelAcceptedImageMime(effectiveMime)) { - out.push({ type: 'text', text: buildUnsupportedImageNotice(effectiveMime) }); - continue; - } - const canonicalUrl = `data:${normalizeImageMime(effectiveMime)};base64,${parsed.base64}`; - if (part.imageUrl.url !== canonicalUrl) { - out.push({ type: 'image_url', imageUrl: { ...part.imageUrl, url: canonicalUrl } }); - continue; - } - } - out.push(part); - } - return out; -} - /** * Compress any inline base64 image parts in a content-part list — used by * the MCP tool-result path (prompt ingestion compresses per image with diff --git a/packages/agent-core/src/tools/support/image-format-policy.ts b/packages/agent-core/src/tools/support/image-format-policy.ts index db72e4bb18..69653fd003 100644 --- a/packages/agent-core/src/tools/support/image-format-policy.ts +++ b/packages/agent-core/src/tools/support/image-format-policy.ts @@ -8,7 +8,11 @@ * too ("session poisoning"). Every ingestion point therefore refuses * unsupported formats instead of passing the bytes through: ReadMediaFile * refuses with a conversion command the model can run, and prompt/MCP - * ingestion replaces the image with a text notice. + * ingestion replaces the image with a text notice. The same applies to an + * empty payload (`data:image/png;base64,` — a clipboard/upload failure that + * captured no bytes), which {@link gateImageFormatParts} replaces with a + * notice both at ingestion and, for histories already carrying one, at + * projection time. * * The policy is deliberately a closed set, not a denylist: a format is only * ever sent when it is known to be accepted. Supporting a new format means @@ -28,6 +32,8 @@ * server-side; those pass through unchanged. */ +import type { ContentPart } from '@moonshot-ai/kosong'; + import { IMAGE_MIME_BY_SUFFIX, sniffMediaFromMagic } from './file-type'; /** Image MIME types every provider accepts. The closed set. */ @@ -268,3 +274,80 @@ export function buildMalformedImageNotice(url: string): string { 'could not be parsed). Re-encode the image as PNG or JPEG and try again.]' ); } + +/** + * Notice standing in for an image that carried no bytes at all + * (`data:image/png;base64,` — a clipboard/upload failure captured nothing). + * Like a malformed URL it is guaranteed to fail at the provider, so it is + * dropped for a notice instead of poisoning the session. + */ +export function buildEmptyImageNotice(name?: string): string { + const what = name === undefined || name.length === 0 ? 'The attached image' : `"${name}"`; + return ( + `[Image omitted: ${what} contained no image data (0 bytes) — the clipboard ` + + 'or upload captured nothing. Re-paste or re-upload the image and try again.]' + ); +} + +/** + * Content-part format gate shared by every image ingestion point — and by the + * context projector, so a malformed image already sitting in an old session's + * history is replaced on the wire instead of poisoning every later request + * ("session poisoning", see the module doc). Images the provider cannot + * accept never pass through: each is replaced by a text notice that tells the + * model what happened and how to recover. + * + * A parsed data URL is still rejected when its payload is empty + * (`data:image/png;base64,` — a clipboard/upload failure captured no bytes). + */ +export function gateImageFormatParts(parts: readonly ContentPart[]): ContentPart[] { + const out: ContentPart[] = []; + for (const part of parts) { + if (part.type === 'image_url') { + const parsed = parseImageDataUrl(part.imageUrl.url); + if (parsed === null) { + // A `data:` URL that failed to parse (missing `;base64,` separator, + // empty MIME, …) is guaranteed to fail at the provider — Anthropic + // throws on it, OpenAI-compat servers 400. Drop it for a notice at + // ingestion instead of leaving it to poison the session and trigger + // the media-stripped resend on every later turn. + if (isDataUrl(part.imageUrl.url)) { + out.push({ type: 'text', text: buildMalformedImageNotice(part.imageUrl.url) }); + continue; + } + // Remote image URL (no bytes to sniff): reject when its path + // extension names a format providers reject (e.g. a search-tool + // link ending in `.avif`) — the notice keeps the URL so the model + // can still fetch and convert the image. Extensionless / unknown + // URLs pass through to the provider — and to the 400 recovery. + const extMime = unsupportedImageMimeFromUrl(part.imageUrl.url); + if (extMime !== null) { + out.push({ + type: 'text', + text: buildUnsupportedImageNotice(extMime, part.imageUrl.url), + }); + continue; + } + out.push(part); + continue; + } + const head = decodeBase64Prefix(parsed.base64); + if (head.length === 0) { + out.push({ type: 'text', text: buildEmptyImageNotice() }); + continue; + } + const effectiveMime = resolveEffectiveImageMime(parsed.mimeType, head); + if (!isModelAcceptedImageMime(effectiveMime)) { + out.push({ type: 'text', text: buildUnsupportedImageNotice(effectiveMime) }); + continue; + } + const canonicalUrl = `data:${normalizeImageMime(effectiveMime)};base64,${parsed.base64}`; + if (part.imageUrl.url !== canonicalUrl) { + out.push({ type: 'image_url', imageUrl: { ...part.imageUrl, url: canonicalUrl } }); + continue; + } + } + out.push(part); + } + return out; +} diff --git a/packages/agent-core/test/agent/context/projector.test.ts b/packages/agent-core/test/agent/context/projector.test.ts index 09b0500e67..699840ca92 100644 --- a/packages/agent-core/test/agent/context/projector.test.ts +++ b/packages/agent-core/test/agent/context/projector.test.ts @@ -991,8 +991,8 @@ describe('degradeOlderMediaParts', () => { expect(parts.filter((part) => part.type === 'image_url')).toHaveLength(1); expect(parts.filter((part) => part.type === 'video_url')).toHaveLength(0); const texts = parts.filter((part) => part.type === 'text').map((part) => part.text); - expect(texts.some((text) => text.startsWith('[image omitted:'))).toBe(true); - expect(texts.some((text) => text.startsWith('[video omitted:'))).toBe(true); + expect(texts.some((text) => text.startsWith('[An image attached to an earlier message'))).toBe(true); + expect(texts.some((text) => text.startsWith('[A video attached to an earlier message'))).toBe(true); // The path wrapper of the degraded image survives for readback. expect(texts).toContain(''); // The surviving image is the most recent one. @@ -1037,7 +1037,7 @@ describe('media strip snapshots', () => { expect(projected[0]?.content).toEqual([ { type: 'text', - text: '[image omitted for provider compatibility; re-read the file to view it or get conversion guidance]', + text: '[An image attached to this message was removed before sending because the provider rejected the request (unsupported or unreadable image data, or the request exceeded its size limit). You have NOT seen this image — do not describe or guess its contents. Tell the user the image failed to reach you and suggest re-sending it as PNG or JPEG, or a smaller copy.]', }, ]); }); diff --git a/packages/agent-core/test/agent/records/index.test.ts b/packages/agent-core/test/agent/records/index.test.ts index 6bfc1a7713..72859d428e 100644 --- a/packages/agent-core/test/agent/records/index.test.ts +++ b/packages/agent-core/test/agent/records/index.test.ts @@ -171,7 +171,7 @@ describe('AgentRecords persistence metadata', () => { const legacyOutput = [ { type: 'text', text: summary }, { type: 'text', text: '' }, - { type: 'image_url', imageUrl: { url: 'data:image/png;base64,A' } }, + { type: 'image_url', imageUrl: { url: 'data:image/png;base64,AQID' } }, { type: 'text', text: '' }, ]; const persistence = new RecordingInMemoryAgentRecordPersistence([ diff --git a/packages/agent-core/test/agent/turn.test.ts b/packages/agent-core/test/agent/turn.test.ts index 08b29fc194..485c33a519 100644 --- a/packages/agent-core/test/agent/turn.test.ts +++ b/packages/agent-core/test/agent/turn.test.ts @@ -138,7 +138,7 @@ describe('Agent turn flow', () => { .filter((part) => part.type === 'text') .map((part) => part.text) .join('\n'); - expect(retryText).toContain('[image omitted:'); + expect(retryText).toContain('removed to fit the provider request size limit'); expect(retryText).toContain(''); // The real history is untouched. expect( @@ -270,13 +270,15 @@ describe('Agent turn flow', () => { }; // Simulate a legacy/pre-gate history that already carries a poisoned - // image. The turn.prompt gate only sanitizes NEW prompt input, not the - // pre-existing context, so this reaches the provider unmodified. + // image. The prompt/projector gates sanitize unsupported formats and + // empty payloads, but corrupt bytes carrying an ACCEPTED label pass both + // — this still reaches the provider unmodified, so the 400 strip + // recovery remains the backstop. function plantPoisonedImage(ctx: TestAgentContext): void { ctx.agent.context.appendUserMessage( [ - { type: 'text', text: '' }, - { type: 'image_url', imageUrl: { url: 'data:image/avif;base64,QUJD' } }, + { type: 'text', text: '' }, + { type: 'image_url', imageUrl: { url: 'data:image/png;base64,%%%corrupt%%%' } }, { type: 'text', text: '' }, ], { kind: 'user' }, @@ -496,7 +498,7 @@ describe('Agent turn flow', () => { .filter((part) => part.type === 'text') .map((part) => part.text) .join('\n'), - ).toContain('[image omitted for provider compatibility;'); + ).toContain('removed before sending because the provider rejected the request'); }); it('keeps the same media stripped when a later tool result recreates its container', async () => { @@ -509,7 +511,7 @@ describe('Agent turn flow', () => { finalParts .filter((part) => part.type === 'text') .map((part) => part.text) - .filter((text) => text.includes('[image omitted for provider compatibility;')), + .filter((text) => text.includes('removed before sending because the provider rejected the request')), ).toHaveLength(2); }); }); diff --git a/packages/agent-core/test/tools/image-compress.test.ts b/packages/agent-core/test/tools/image-compress.test.ts index a0560a3e6e..b53897acaa 100644 --- a/packages/agent-core/test/tools/image-compress.test.ts +++ b/packages/agent-core/test/tools/image-compress.test.ts @@ -1069,6 +1069,19 @@ describe('gateImageFormatParts', () => { } }); + it('drops an empty-payload data URL (clipboard/upload captured nothing)', () => { + // `data:image/png;base64,` parses fine but carries zero bytes — the + // provider rejects it as an invalid image, so it is dropped for a notice + // at ingestion (and at projection for pre-existing history) instead of + // poisoning every later request. + const out = gateImageFormatParts([ + { type: 'image_url', imageUrl: { url: 'data:image/png;base64,' } }, + ]); + expect(out.some((p) => p.type === 'image_url')).toBe(false); + expect(out[0]).toMatchObject({ type: 'text' }); + expect((out[0] as { text: string }).text).toContain('no image data (0 bytes)'); + }); + it('truncates a long malformed data URL in the notice', () => { const url = `data:image/png${'x'.repeat(500)}`; const out = gateImageFormatParts([{ type: 'image_url', imageUrl: { url } }]); diff --git a/packages/kap-server/src/routes/prompts.ts b/packages/kap-server/src/routes/prompts.ts index d5a0cca394..0376ef04f9 100644 --- a/packages/kap-server/src/routes/prompts.ts +++ b/packages/kap-server/src/routes/prompts.ts @@ -34,6 +34,7 @@ import { ITelemetryService, applyPromptMetadataUpdate, buildImageCompressionCaption, + buildEmptyImageNotice, buildUnsupportedImageNotice, compressBase64ForModel, compressImageForModel, @@ -524,10 +525,17 @@ async function resolvePromptMediaFiles( // itself (best effort — the plain notice stands in when persisting // fails). Inline base64 has no original name, so the file is addressed // by content hash with a name derived from the sniffed format. - const effectiveMime = resolveEffectiveImageMime( - part.source.media_type, - decodeBase64Prefix(part.source.data), - ); + // + // Empty payloads (a clipboard/upload failure captured zero bytes) are + // just as poisonous and cannot be persisted into anything readable, so + // they degrade to a plain notice instead of an attachment. + const head = decodeBase64Prefix(part.source.data); + if (head.length === 0) { + content.push({ type: 'text', text: buildEmptyImageNotice() }); + changed = true; + continue; + } + const effectiveMime = resolveEffectiveImageMime(part.source.media_type, head); if (!isModelAcceptedImageMime(effectiveMime)) { const bytes = Buffer.from(part.source.data, 'base64'); const name = `image.${imageExtensionForMime(effectiveMime)}`; @@ -624,6 +632,14 @@ async function resolvePromptMediaFiles( assertMediaFile(file, part.type); if (part.type === 'image') { const data = await readFileOrStream(file); + // A zero-byte upload (e.g. a clipboard failure that captured nothing) + // would inline as an empty data URL and poison every later request — + // degrade it to a notice instead. + if (data.length === 0) { + content.push({ type: 'text', text: buildEmptyImageNotice(file.meta.name) }); + changed = true; + continue; + } let mediaType = file.meta.media_type; let bytes: Uint8Array = data; // Same format gate as the inline path above, and again the bytes are diff --git a/packages/kap-server/test/prompts.test.ts b/packages/kap-server/test/prompts.test.ts index 859c66148c..6794c755c5 100644 --- a/packages/kap-server/test/prompts.test.ts +++ b/packages/kap-server/test/prompts.test.ts @@ -445,6 +445,28 @@ describe('server-v2 /api/v1 prompts', () => { expect(notice.text).toContain('photo.avif'); }); + it('replaces a zero-byte uploaded image file with a text notice instead of an empty data URL', async () => { + // A clipboard failure can hand the client a zero-byte File (issue #2209): + // inlining it as `data:image/png;base64,` would poison every later + // request in the session. The route degrades it to a notice instead. + const id = await createSession(home as string); + await createMainAgent(id); + const uploaded = await uploadFile(Buffer.alloc(0), 'image/png', 'image.png'); + expect(uploaded.size).toBe(0); + + const submitted = await call('POST', `/api/v1/sessions/${id}/prompts`, { + content: [{ type: 'image', source: { kind: 'file', file_id: uploaded.id } }], + }); + expect(submitted.body.code).toBe(0); + + const content = submitted.body.data.content as PromptContentPart[]; + expect(content).toHaveLength(1); + const notice = content[0]; + if (notice?.type !== 'text') throw new Error('expected a text notice'); + expect(notice.text).toContain('no image data (0 bytes)'); + expect(notice.text).toContain('image.png'); + }); + it('replaces a remote image URL with an unsupported extension with a text notice', async () => { const id = await createSession(home as string); await createMainAgent(id); diff --git a/packages/klient/test/e2e/invalid-input-matrix.test.ts b/packages/klient/test/e2e/invalid-input-matrix.test.ts index 10de4eeec2..f586ba34a2 100644 --- a/packages/klient/test/e2e/invalid-input-matrix.test.ts +++ b/packages/klient/test/e2e/invalid-input-matrix.test.ts @@ -599,7 +599,9 @@ describe('image blocks with invalid data', () => { expect(secondContent.some((part) => (part as { type?: string }).type === 'image_url')).toBe( false, ); - expect(JSON.stringify(secondContent)).toContain('image omitted for provider compatibility'); + expect(JSON.stringify(secondContent)).toContain( + 'removed before sending because the provider rejected the request', + ); expect(ctx.payloads('prompt.completed')[0]?.['reason']).toBe('completed'); }, 30_000);