Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 62 additions & 13 deletions app/components/molecules/ChatInput.vue
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ const emit = defineEmits<{
}>()

const { t } = useContent()
const toast = useToast()
// "Upload to media library" is an ee/media-gated capability; hidden otherwise.
const canUploadMedia = useFeature('media.upload')
// The picker lives in the composer's action strip; the command palette writes
Expand All @@ -40,7 +41,10 @@ const pickerAccept = computed(() => pendingIntent.value === 'media' ? IMAGE_ACCE
const URL_RE = /^https?:\/\/\S+$/i

const hasUploading = computed(() => attachments.value.some(a => a.status === 'uploading'))
const canSend = computed(() => !!input.value.trim() && !props.disabled && !hasUploading.value)
// A failed attachment blocks sending until it is removed — otherwise the
// message would silently go out without the file the user believes is attached.
const hasErrored = computed(() => attachments.value.some(a => a.status === 'error'))
const canSend = computed(() => !!input.value.trim() && !props.disabled && !props.streaming && !hasUploading.value && !hasErrored.value)

interface ServerRef {
id: string
Expand Down Expand Up @@ -77,15 +81,23 @@ function formatBytes(bytes?: number): string {
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`
}

/**
* Mark an attachment failed AND say so out loud. The red chip alone proved
* too quiet — users sent messages believing the file was attached.
*/
function failAttachment(att: UIAttachment, message: string) {
att.status = 'error'
att.error = message
toast.error(message)
}

function applyRef(att: UIAttachment, ref: ServerRef | undefined) {
if (!ref) {
att.status = 'error'
att.error = t('chat.attachment_failed')
failAttachment(att, t('chat.attachment_failed'))
return
}
if (ref.error) {
att.status = 'error'
att.error = ref.error
failAttachment(att, ref.error)
return
}
att.status = 'ready'
Expand Down Expand Up @@ -121,8 +133,7 @@ async function uploadFile(file: File, intent: 'context' | 'media') {
applyRef(att, res.attachments?.[0])
}
catch (e) {
att.status = 'error'
att.error = resolveApiError(e, t('chat.attachment_failed'))
failAttachment(att, resolveApiError(e, t('chat.attachment_failed')))
}
}

Expand All @@ -144,8 +155,7 @@ async function attachLink(url: string) {
applyRef(att, res.attachments?.[0])
}
catch (e) {
att.status = 'error'
att.error = resolveApiError(e, t('chat.attachment_failed'))
failAttachment(att, resolveApiError(e, t('chat.attachment_failed')))
}
}

Expand All @@ -158,7 +168,7 @@ function handleFiles(files: FileList | File[] | null | undefined, intent: 'conte
function addFiles(files: FileList | File[] | null) {
handleFiles(files, 'context')
}
defineExpose({ addFiles })
defineExpose({ addFiles, attachLink })

function openPicker(intent: 'context' | 'media') {
pendingIntent.value = intent
Expand Down Expand Up @@ -196,6 +206,17 @@ function onPaste(e: ClipboardEvent) {
handleFiles(dt.files, 'context')
return
}
// Some clipboard sources expose the file only through `items`, never
// `files` — without this fallback the paste was silently discarded.
const itemFiles = Array.from(dt.items ?? [])
.filter(item => item.kind === 'file')
.map(item => item.getAsFile())
.filter((f): f is File => f !== null)
if (itemFiles.length > 0) {
e.preventDefault()
handleFiles(itemFiles, 'context')
return
}
// A clipboard whose entire text is a single URL → attach as a link.
const text = dt.getData('text')?.trim()
if (text && URL_RE.test(text)) {
Expand All @@ -212,12 +233,39 @@ function attachmentIcon(att: UIAttachment): string {
}

function handleSend() {
// Mirrors `canSend` exactly. The streaming check matters for Enter: the
// send button is swapped for stop while streaming, but the keydown path
// used to fall through, wipe the composer + tray, and send nothing
// (useChat's isStreaming guard rejected it downstream).
const text = input.value.trim()
if (!text || props.disabled || hasUploading.value) return
if (!text || props.disabled || props.streaming || hasUploading.value || hasErrored.value) return
emit('send', text, attachments.value)
input.value = ''
attachments.value = []
nextTick(() => autoResize())
nextTick(() => {
autoResize()
textareaRef.value?.focus()
})
}

/**
* The card is styled as one big input, but only the inner textarea accepts
* typing/paste — and nothing used to focus it, so a paste after clicking the
* card's padding landed on <body> and vanished. Clicks on interactive
* children keep their own focus.
*/
function focusComposer(e: MouseEvent) {
const target = e.target as HTMLElement | null
if (target?.closest('button, a, input, textarea, select, [role="menuitem"]')) return
textareaRef.value?.focus()
}

/** Radix returns focus to the trigger on close; hand it back to the textarea. */
function onAttachMenuToggle(open: boolean) {
if (open) return
nextTick(() => {
if (!showLinkInput.value) textareaRef.value?.focus()
})
}

function handleKeydown(e: KeyboardEvent) {
Expand Down Expand Up @@ -248,6 +296,7 @@ function autoResize() {
the whole card lights up as a unit. -->
<div
class="rounded-2xl border border-secondary-200 bg-white shadow-sm transition-colors focus-within:border-primary-500 focus-within:ring-2 focus-within:ring-primary-500/30 dark:border-secondary-700 dark:bg-secondary-900"
@click="focusComposer"
>
<!-- Attachment tray -->
<ul v-if="attachments.length > 0" class="flex flex-wrap gap-2 px-3 pt-3">
Expand Down Expand Up @@ -321,7 +370,7 @@ function autoResize() {
<!-- Action strip -->
<div class="flex items-center gap-1 px-2 pb-2">
<!-- Attach (+) menu -->
<DropdownMenuRoot>
<DropdownMenuRoot @update:open="onAttachMenuToggle">
<DropdownMenuTrigger as-child>
<button
type="button" :disabled="disabled"
Expand Down
29 changes: 24 additions & 5 deletions app/components/organisms/ChatPanel.vue
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,19 @@ const isEmptyActive = computed(() =>
messages.value.length === 0 && !!props.projectStatus && props.projectStatus !== 'setup',
)

// Panel-wide file drag-drop → forwarded to the composer (always as context).
const chatInputRef = ref<{ addFiles: (files: FileList | File[] | null) => void } | null>(null)
// Panel-wide drag-drop → forwarded to the composer (always as context).
// Accepts real files AND `text/uri-list` drags — an image dragged in from
// another browser tab carries no `Files` entry, only its URL; without the
// uri-list branch that drop navigated the page away with zero feedback.
const chatInputRef = ref<{ addFiles: (files: FileList | File[] | null) => void, attachLink: (url: string) => Promise<void> } | null>(null)
const isDragOver = ref(false)

function isAcceptedDrag(dt: DataTransfer | null): boolean {
return !!dt && (dt.types.includes('Files') || dt.types.includes('text/uri-list'))
}

function onPanelDragOver(e: DragEvent) {
if (!e.dataTransfer?.types.includes('Files')) return // ignore context-chip drags
if (!isAcceptedDrag(e.dataTransfer)) return // ignore context-chip drags
e.preventDefault()
isDragOver.value = true
}
Expand All @@ -50,9 +57,21 @@ function onPanelDragLeave(e: DragEvent) {

function onPanelDrop(e: DragEvent) {
isDragOver.value = false
if (!e.dataTransfer?.types.includes('Files')) return
const dt = e.dataTransfer
if (!isAcceptedDrag(dt)) return
e.preventDefault()
chatInputRef.value?.addFiles(e.dataTransfer.files)
if (dt!.files.length > 0) {
chatInputRef.value?.addFiles(dt!.files)
return
}
// uri-list: first non-comment line is the dragged resource's URL.
const uri = dt!.getData('text/uri-list')
.split('\n')
.map(line => line.trim())
.find(line => line && !line.startsWith('#'))
if (uri && /^https?:\/\//i.test(uri)) {
void chatInputRef.value?.attachLink(uri)
}
}
const { chips, toContextItems, clear: clearContext } = useChatContext()
const { state: authState } = useAuth()
Expand Down
6 changes: 4 additions & 2 deletions server/utils/agent-system-prompt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import type { AISystemBlock } from '../providers/ai'
import type { Branch } from '../providers/git'
import type { AgentPermissions } from './agent-permissions'
import type { ChatUIContext, ClassifiedIntent, ProjectPhase } from './agent-types'
import { extractMediaStoragePath } from './media-rewrite'

/**
* Bounded Task Executor system prompt.
Expand Down Expand Up @@ -138,10 +139,11 @@ function buildAttachmentSection(attachments: PromptAttachment[]): string {
]
for (const a of attachments) {
if (a.kind === 'image' && a.url) {
lines.push(`- ${a.filename} (image — already uploaded to the media library at ${a.url}. Reuse this URL directly in image/media fields; do NOT call upload_media for it.)`)
const storagePath = extractMediaStoragePath(a.url)
lines.push(`- ${a.filename} (image — already uploaded to the media library at ${a.url}${storagePath ? `, storage path ${storagePath}` : ''}. Reuse this URL directly in image/media fields; do NOT call upload_media for it.)`)
}
else if (a.kind === 'image') {
lines.push(`- ${a.filename} (image, included in this message for you to view)`)
lines.push(`- ${a.filename} (image, included in this message for you to view — ephemeral, NOT in the media library; it has no URL or path to reference in content, so never invent one)`)
}
else if (a.kind === 'document') {
lines.push(`- ${a.filename} (PDF document, included in this message)`)
Expand Down
111 changes: 93 additions & 18 deletions server/utils/attachment-ingest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import { fileTypeFromBuffer } from 'file-type'
import mammoth from 'mammoth'
import sharp from 'sharp'
import type { AIContentBlock, AIImageMediaType } from '../providers/ai'
import { extractMediaStoragePath } from './media-rewrite'
import { publicMediaBase, toDeliveryUrl } from './media-url'
import { resolveVariantConfig } from './media-variants'
import { useMediaProvider } from './providers'
Expand Down Expand Up @@ -199,26 +200,40 @@ export function validateAttachmentBlocks(
for (const raw of attachments) {
const att = raw as { blocks?: unknown, filename?: unknown }
const rawBlocks = Array.isArray(att?.blocks) ? att.blocks : []
const filename = typeof att?.filename === 'string' ? att.filename : 'attachment'
const filename = typeof att?.filename === 'string' ? att.filename.slice(0, 200) : 'attachment'
const attBlocks: AIContentBlock[] = []
let attKind: AttachmentSummary['kind'] = 'text'
let attUrl: string | undefined
let hasBase64Image = false

for (const candidate of rawBlocks) {
const block = sanitizeBlock(candidate, deliveryPrefix)
if (!block) continue
blocks.push(block)
attBlocks.push(block)
totalChars += JSON.stringify(block).length
if (block.type === 'image') {
attKind = 'image'
if (block.source.type === 'url') attUrl = block.source.url
else hasBase64Image = true
}
else if (block.type === 'document') {
attKind = 'document'
}
}

if (blocks.length > 0)
summary.push({ kind: attKind, filename, url: attUrl })
// An attachment whose blocks were all dropped gets no summary line —
// the prompt must not claim an image the model cannot actually see.
if (attBlocks.length === 0) continue

// The model perceives image blocks as pixels only; the URL inside the
// block is invisible to it. A server-authored reference line ahead of
// the image is the only way the agent can know the asset's address —
// or know that an ephemeral image has none.
const descriptor = buildImageDescriptor(filename, attUrl, hasBase64Image)
if (descriptor) blocks.push(descriptor)

blocks.push(...attBlocks)
summary.push({ kind: attKind, filename, url: attUrl })
}

if (totalChars > MAX_TOTAL_ATTACHMENT_CHARS)
Expand All @@ -227,6 +242,34 @@ export function validateAttachmentBlocks(
return { blocks, summary }
}

/**
* Reference line injected ahead of an image attachment's block. Image blocks
* reach the model as pixels — a media-library image's delivery URL is not
* visible to it, which historically made the agent invent URLs/UUIDs or
* re-ask the user for something it was already given.
*/
function buildImageDescriptor(filename: string, url: string | undefined, hasBase64Image: boolean): AIContentBlock | null {
if (url) {
const storagePath = extractMediaStoragePath(url)
return {
type: 'text',
text: `[Attached image "${filename}" — stored in the media library.`
+ ` Delivery URL: ${url}${storagePath ? ` — storage path: ${storagePath}` : ''}.`
+ ` Use this exact URL in image/video/file fields via save_content. Do NOT call upload_media for it.]`,
}
}
if (hasBase64Image) {
return {
type: 'text',
text: `[Attached image "${filename}" — ephemeral, NOT stored in the media library.`
+ ` You can view it, but it has no URL or path to reference in content fields.`
+ ` To place it in content, ask the user to re-attach it with "Add to media library",`
+ ` or find an existing asset with search_media. Never invent media URLs or paths.]`,
}
}
return null
}

function sanitizeBlock(candidate: unknown, deliveryPrefix: string): AIContentBlock | null {
if (!candidate || typeof candidate !== 'object') return null
const b = candidate as Record<string, unknown>
Expand Down Expand Up @@ -355,19 +398,8 @@ async function ingestImage(input: IngestFileInput, mime: AIImageMediaType): Prom

// Context (default): downscale + size-cap, emit base64 webp. Ephemeral.
try {
let optimized = await sharp(input.buffer)
.rotate()
.resize({ width: IMAGE_MAX_DIM, height: IMAGE_MAX_DIM, fit: 'inside', withoutEnlargement: true })
.webp({ quality: 80 })
.toBuffer()
if (optimized.length > MAX_BASE64_IMAGE_BYTES) {
optimized = await sharp(input.buffer)
.rotate()
.resize({ width: 1024, height: 1024, fit: 'inside', withoutEnlargement: true })
.webp({ quality: 65 })
.toBuffer()
}
if (optimized.length > MAX_BASE64_IMAGE_BYTES)
const optimized = await optimizeContextImage(input.buffer)
if (!optimized)
return errorRef({ filename: input.filename, mime, source: 'upload', kind: 'image', error: errorMessage('attachment.image_too_large') })
return {
id,
Expand All @@ -385,6 +417,26 @@ async function ingestImage(input: IngestFileInput, mime: AIImageMediaType): Prom
}
}

/**
* Downscale + size-cap an image for the ephemeral context path. Returns
* null when even the aggressive pass cannot fit the base64 budget.
*/
async function optimizeContextImage(buffer: Buffer): Promise<Buffer | null> {
let optimized = await sharp(buffer)
.rotate()
.resize({ width: IMAGE_MAX_DIM, height: IMAGE_MAX_DIM, fit: 'inside', withoutEnlargement: true })
.webp({ quality: 80 })
.toBuffer()
if (optimized.length > MAX_BASE64_IMAGE_BYTES) {
optimized = await sharp(buffer)
.rotate()
.resize({ width: 1024, height: 1024, fit: 'inside', withoutEnlargement: true })
.webp({ quality: 65 })
.toBuffer()
}
return optimized.length > MAX_BASE64_IMAGE_BYTES ? null : optimized
}

/**
* Ingest one uploaded file → AttachmentRef. Routing is by extension
* with a magic-byte sanity check for binaries (anti-spoofing).
Expand Down Expand Up @@ -550,8 +602,31 @@ export async function fetchLinkContent(rawUrl: string): Promise<AttachmentRef> {
else if (contentType.startsWith('text/') || contentType.includes('json')) {
body = decodeText(buffer) ?? ''
}
else if (contentType.startsWith('image/')) {
// An image URL (pasted, or dragged in from another browser tab) becomes
// an ephemeral context image — same treatment as a pasted file.
try {
const optimized = await optimizeContextImage(buffer)
if (!optimized)
return errorRef({ filename: url, mime: contentType, source: 'link', kind: 'image', error: errorMessage('attachment.image_too_large') })
const name = new URL(url).pathname.split('/').pop() || url
return {
id: makeId(),
source: 'link',
filename: name,
mime: 'image/webp',
kind: 'image',
destination: 'context',
blocks: [{ type: 'image', source: { type: 'base64', mediaType: 'image/webp', data: optimized.toString('base64') } }],
bytes: optimized.length,
}
}
catch {
return errorRef({ filename: url, mime: contentType, source: 'link', kind: 'image', error: errorMessage('attachment.image_decode_failed') })
}
}
else {
// Binary (image/pdf/etc.) pasted as a link → not the link channel's job.
// Binary (pdf/zip/etc.) pasted as a link → not the link channel's job.
return errorRef({ filename: url, mime: contentType, source: 'link', kind: 'text', error: errorMessage('attachment.link_not_text', { type: contentType }) })
}

Expand Down
Loading
Loading