Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 38 additions & 5 deletions apps/buddy/platform/extensions/ExtensionInference.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { InferenceBinding, InferenceCapability, InferenceHardware, InferenceObservation, InferenceRuntimeReference } from '../../shared/extensions/extensionInference'
import type { DiscoveredModelMetadata } from '../../shared/providers/providerDiscovery'
import type { ExtensionInferenceCredentials } from './ExtensionInferenceCredentials'
import type { InferenceResourceLease, InferenceResourceSource } from './ExtensionInferenceResources'
import type { ExtensionPackage, ExtensionPackageStore } from './ExtensionPackageStore'
Expand Down Expand Up @@ -38,6 +39,8 @@ interface RuntimeLifetime {
}
export class ExtensionInference {
readonly #bindings = new Map<string, InferenceBinding>()
readonly #modelMetadata = new WeakMap<InferenceBinding, Map<string, DiscoveredModelMetadata>>()
readonly #modelsChanged: () => void
readonly #instances = new Map<string, Instance>()
readonly #calls = new Set<Promise<void>>()
readonly #rebinding = new Set<string>()
Expand All @@ -52,12 +55,13 @@ export class ExtensionInference {
readonly ports: ExtensionInferencePorts
readonly source: InferenceResourceSource
readonly assertCurrent: () => void
constructor(store: ExtensionPackageStore, pkg: ExtensionPackage, ports: ExtensionInferencePorts, source: InferenceResourceSource, assertCurrent: () => void) {
constructor(store: ExtensionPackageStore, pkg: ExtensionPackage, ports: ExtensionInferencePorts, source: InferenceResourceSource, assertCurrent: () => void, modelsChanged: () => void = () => {}) {
this.store = store
this.pkg = pkg
this.ports = ports
this.source = source
this.assertCurrent = assertCurrent
this.#modelsChanged = modelsChanged
}

async prepareResources(...[files, options, signal, event]: Parameters<NonNullable<InferenceResourceSource['prepare']>>): Promise<void> {
Expand Down Expand Up @@ -155,7 +159,34 @@ export class ExtensionInference {
}

async models() {
return (await this.#availableBindings()).flatMap(binding => binding.models.map(model => ({ ...model, bindingRevision: createHash('sha256').update(JSON.stringify(binding)).digest('hex') })))
return (await this.#availableBindings()).flatMap(binding => binding.models.map((model) => {
const declared = this.#modelMetadata.get(binding)?.get(model.id)
const metadata = model.metadata || declared ? { ...model.metadata, ...declared } : undefined
if (metadata?.contextWindow !== undefined && metadata.maxTokens !== undefined)
metadata.maxTokens = Math.min(metadata.maxTokens, metadata.contextWindow)
return { ...model, ...(metadata ? { metadata: structuredClone(metadata) } : {}), bindingRevision: createHash('sha256').update(JSON.stringify(binding)).digest('hex') }
}))
}

#acceptModelMetadata(binding: InferenceBinding, observation: InferenceObservation): void {
const model = observation.modelId === undefined
? binding.models.length === 1 ? binding.models[0] : undefined
: binding.models.find(model => model.id === observation.modelId)
if (!model || !observation.modelMetadata)
return
this.assertCurrent()
const models = this.#modelMetadata.get(binding) ?? new Map<string, DiscoveredModelMetadata>()
const previous = models.get(model.id)
const metadata = { ...previous }
for (const [key, value] of Object.entries(observation.modelMetadata)) {
if (value !== undefined)
Object.assign(metadata, { [key]: value })
}
if (!Object.keys(metadata).length || JSON.stringify(previous) === JSON.stringify(metadata))
return
models.set(model.id, metadata)
this.#modelMetadata.set(binding, models)
this.#modelsChanged()
}

async imageModels() {
Expand Down Expand Up @@ -281,13 +312,13 @@ export class ExtensionInference {
const leases: InferenceResourceLease[] = []
const observation: Instance['observation'] = {}
const loading = AbortSignal.any([abort.signal, this.#stop.signal, AbortSignal.timeout(inferenceLimits.startTimeoutMs)])
const ready = (async () => {
const ready = Promise.resolve().then(async () => {
for (const resource of binding.resources)
leases.push(await this.store.inference.acquire(this.pkg.manifest.id, resource, this.source, loading))
this.assertCurrent()
return NativeInferenceProcess.start({ ...this.ports, record: (event) => {
safeDiagnosticReporter(this.ports.record)(event)
if (abort.signal.aborted || event.component !== 'models.inference' || event.extensionId !== this.pkg.manifest.id || !inferenceStderrDiagnosticSchema.shape.event.safeParse(event.event).success)
if (abort.signal.aborted || this.#bindings.get(binding.runtime) !== binding || this.#instances.get(key)?.abort !== abort || event.component !== 'models.inference' || event.extensionId !== this.pkg.manifest.id || !inferenceStderrDiagnosticSchema.shape.event.safeParse(event.event).success)
return
const error = inferenceDiagnosticCodeSchema.safeParse(event.errorCode)
if (event.event === 'chat.gpu.exit') {
Expand All @@ -301,6 +332,8 @@ export class ExtensionInference {
const parsed = inferenceObservationSchema.safeParse({ ...event.inference, ...(error.success ? { error: error.data } : {}), observedAt: Date.now() })
if (!parsed.success || Object.keys(parsed.data).length === 1)
return
if (event.event === 'chat.gpu.ready')
this.#acceptModelMetadata(binding, parsed.data)
const previous = observation.current
observation.current = {
...previous,
Expand All @@ -326,7 +359,7 @@ export class ExtensionInference {
this.assertCurrent()
return this.ports.request(endpoint.url, { method: 'POST', headers, body, signal })
} }, this.store, this.pkg, runtime, binding, leases, loading)
})()
})
const created: Instance = { abort, runtime: runtime.id, ready, observation, capacity: new ExtensionInvocationQueue({ concurrent: runtime.capacity, concurrentPerExtension: runtime.capacity, queued: 1024 }), waiters: 0, active: 0, leases }
instance = created
this.#instances.set(key, created)
Expand Down
2 changes: 1 addition & 1 deletion apps/buddy/platform/extensions/ExtensionService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1961,7 +1961,7 @@ export class ExtensionService {
running = { package: pkg, generation, abort, host, ready: Promise.resolve(), active: false, inFlight: 0, inferencePreparations: new Set(), drained: new Set() }
if (pkg.manifest.contributes.inference && this.#ports.inference) {
const owner = running
running.inference = new ExtensionInference(this.store, pkg, this.#ports.inference, this.#inferenceSource(owner), () => this.#assertCurrent(owner))
running.inference = new ExtensionInference(this.store, pkg, this.#ports.inference, this.#inferenceSource(owner), () => this.#assertCurrent(owner), () => this.#publish({ kind: 'inference-models', extensionId: pkg.manifest.id }))
}
this.#running.set(dependency, running)
const start = performance.now()
Expand Down
36 changes: 36 additions & 0 deletions apps/buddy/platform/extensions/__tests__/ExtensionService.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -911,6 +911,42 @@ it.each(['ready', 'checksum'] as const)('delivers resource progress before %s se
expect(progress).toHaveLength(count)
})

it('publishes deployment metadata without replacing the active model binding or rejecting its original invocation identity', async () => {
const f = await fixture({ inference: { bootstrap: '', executable: '', admit: async () => async () => {} } }, manifest({
id: 'fixture.inference',
apiVersion: 3,
permissions: { inference: true },
contributes: { inference: { runtimes: [{ id: 'local', entry: 'native/fixture.mjs', platforms: ['linux-x64'], idleTimeoutMs: 60000 }], capabilities: [{ id: 'chat', title: 'Chat', runtime: 'local', kind: 'chat.completion' }] } },
}), { 'native/fixture.mjs': 'export function initialize() {}' })
await f.service.inferenceModels()
const host = f.hosts[0]!
const client = createExtensionInferenceClient(host.broker)
expect(client.capabilities.chatModelMetadata).toBe(true)
await client.bind({ runtime: 'local', resources: [], libraryPaths: [], endpoints: [], configuration: {}, imageModels: [], imageReaders: [], models: [{ id: 'model', title: 'Model', capability: 'chat', tools: 'auto', contextWindow: 8192, maxTokens: 512, metadata: { toolCall: true, reasoningOptions: ['off', 'medium'] } }] })
const before = (await f.service.inferenceModels())[0]!
const native: ReturnType<typeof memoryInference>[] = []
vi.spyOn(NativeInferenceProcess, 'start').mockImplementation(async (options) => {
const current = memoryInference({ record: options.record })
native.push(current)
return current.worker
})
cleanup.push(async () => {
for (const current of native) current.close()
await Promise.all(native.map(current => current.reading))
})
const invoke = () => f.service.invokeInference(before.extensionId, before.revision, invocation({ id: randomUUID(), capability: before.capability, generation: before.generation, bindingRevision: before.bindingRevision, input: { model: before.id } }))
await invoke()
const updates: unknown[] = []
const subscription = f.service.onDidChange(change => updates.push(change))
cleanup.push(async () => subscription.dispose())
native[0]!.stderr.write(`${JSON.stringify({ event: 'chat.gpu.ready', modelId: 'model', modelMetadata: { toolCall: false } })}\n`)
expect(updates).toContainEqual(expect.objectContaining({ kind: 'inference-models', extensionId: before.extensionId }))
expect(await f.service.inferenceModels()).toEqual([{ ...before, metadata: { ...before.metadata, toolCall: false } }])
await invoke()
expect(native).toHaveLength(1)
expect(host.disposed).toBe(false)
})

it.each(['host', 'view'] as const)('keeps downloads, inference and original grants alive across concurrent private releases from %s', async (owner) => {
const downloadStarted = deferred<void>()
const finishDownload = deferred<void>()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,18 @@ async function fixture(record?: ApplicationDiagnosticReporter) {
]))).token)
const pkg = store.installed[owner]!.current
const native: ReturnType<typeof memoryInference>[] = []
const reports: ApplicationDiagnosticReporter[] = []
const changes: unknown[] = []
vi.spyOn(NativeInferenceProcess, 'start').mockImplementation(async (options) => {
reports.push(options.record!)
const current = memoryInference({ record: options.record })
native.push(current)
return current.worker
})
const runtime = new ExtensionInference(store, pkg, { bootstrap: '', executable: '', admit: async () => async () => {}, record }, {
response: async () => { throw new Error('Unexpected download') },
openReference: async () => { throw new Error('Unexpected resource') },
}, () => {})
}, () => {}, () => changes.push('models-changed'))
onTestFinished(async () => {
await runtime.dispose()
await Promise.all(native.map(item => item.reading))
Expand All @@ -42,9 +45,81 @@ async function fixture(record?: ApplicationDiagnosticReporter) {
const invoke = (id = 'first') => runtime.invoke(pkg.manifest.contributes.inference!.capabilities.find(item => item.id === id)!, invocation({ capability: id }))
const observation = () => runtime.status()[0]!.instances[0]?.observation
const emit = (data: unknown, index = 0) => native[index]!.stderr.write(`${JSON.stringify(data)}\n`)
return { runtime, native, invoke, observation, emit }
return { runtime, native, invoke, observation, emit, reports, changes }
}

it('projects successful deployment declarations without changing bindings, and retains sparse fields until the binding changes', async () => {
const f = await fixture()
const binding = inferenceBindingSchema.parse({ runtime: 'first', resources: [], models: [{
id: 'local',
title: 'Local',
capability: 'first',
tools: 'auto',
reasoning: 'auto',
contextWindow: 8192,
maxTokens: 512,
metadata: { toolCall: true, reasoningOptions: ['off', 'medium'], input: ['text', 'image'], contextWindow: 32768, maxTokens: 1024 },
}] })
await f.runtime.bind(binding)
const before = (await f.runtime.models())[0]!
await f.invoke()
f.emit({ event: 'chat.gpu.ready', context: { actual: 16384 }, modelMetadata: { toolCall: false, contextWindow: 4096 } })
const expected = { ...before, metadata: { ...before.metadata, toolCall: false, contextWindow: 4096 } }
expect(await f.runtime.models()).toEqual([expected])
expect(f.changes).toEqual(['models-changed'])
f.emit({ event: 'chat.gpu.ready', modelMetadata: { reasoningOptions: [] } })
expected.metadata.reasoningOptions = []
expect(await f.runtime.models()).toEqual([expected])
expect(f.changes).toHaveLength(2)
f.emit({ event: 'chat.gpu.ready', modelMetadata: { reasoningOptions: [] } })
f.emit({ event: 'chat.gpu.error', code: 'EXTENSION_INFERENCE_PROCESS_FAILED', modelMetadata: { toolCall: true } })
f.emit({ event: 'chat.gpu.ready', modelMetadata: { toolCall: 'unknown' } })
f.emit({ event: 'chat.gpu.ready', context: { actual: 65536, train: 262144 } })
expect(await f.runtime.models()).toEqual([expected])
expect(f.changes).toHaveLength(2)
expect(f.native).toHaveLength(1)
expect(binding.models[0]!.metadata?.toolCall).toBe(true)
expect((await f.runtime.models())[0]!.bindingRevision).toBe(before.bindingRevision)
await f.runtime.bind(binding)
await f.invoke()
expect(await f.runtime.models()).toEqual([expected])
expect(f.native).toHaveLength(1)
await f.runtime.stop({ runtime: 'first' })
expect(await f.runtime.models()).toEqual([expected])
await f.invoke()
f.reports[0]!({ component: 'models.inference', extensionId: 'fixture.inference', event: 'chat.gpu.ready', level: 'info', inference: { modelMetadata: { toolCall: true } } })
expect(await f.runtime.models()).toEqual([expected])
f.emit({ event: 'chat.gpu.ready', modelId: 'local', modelMetadata: { input: ['text'] } }, 1)
expected.metadata.input = ['text']
expect(await f.runtime.models()).toEqual([expected])
await f.runtime.bind({ ...binding, configuration: { engine: 'replacement' } })
const replaced = (await f.runtime.models())[0]!
expect(replaced.metadata).toEqual(before.metadata)
expect(replaced.bindingRevision).not.toBe(before.bindingRevision)
f.reports[1]!({ component: 'models.inference', extensionId: 'fixture.inference', event: 'chat.gpu.ready', level: 'info', inference: { modelId: 'local', modelMetadata: { toolCall: false } } })
expect((await f.runtime.models())[0]).toEqual(replaced)
await f.invoke()
f.emit({ event: 'chat.gpu.ready', context: { actual: 4096 } }, 2)
expect((await f.runtime.models())[0]).toEqual(replaced)
})

it('requires an exact model ID for multi-model deployment declarations and never changes sibling capabilities', async () => {
const f = await fixture()
const binding = inferenceBindingSchema.parse({ runtime: 'first', resources: [], models: ['one', 'two'].map(id => ({ id, title: id, capability: 'first', contextWindow: 8192, maxTokens: 512, tools: false })) })
await f.runtime.bind(binding)
const before = await f.runtime.models()
await f.invoke()
f.emit({ event: 'chat.gpu.ready', modelMetadata: { toolCall: true } })
f.emit({ event: 'chat.gpu.ready', modelId: 'unknown', modelMetadata: { toolCall: true } })
expect(await f.runtime.models()).toEqual(before)
expect(f.changes).toEqual([])
f.emit({ event: 'chat.gpu.ready', modelId: 'two', modelMetadata: { toolCall: true } })
expect(await f.runtime.models()).toEqual([before[0], { ...before[1], metadata: { toolCall: true } }])
await f.runtime.unbind({ runtime: 'first' })
await f.runtime.bind(binding)
expect(await f.runtime.models()).toEqual(before)
})

it('projects validated incremental diagnostics per instance and forwards the original report', async () => {
const records: unknown[] = []
const f = await fixture(event => records.push(event))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -645,6 +645,8 @@ export interface InferenceHardware {
export interface InferenceTiming { tokens: number, ms: number, tokensPerSecond?: number }
export interface InferenceObservation {
observedAt: number
modelId?: string
modelMetadata?: DiscoveredModelMetadata
engine?: { name: 'llama.cpp', version?: string }
backend?: { requested: 'cpu' | 'vulkan' | 'cuda' | 'metal' | 'unknown', observed: ('cpu' | 'vulkan' | 'cuda' | 'metal')[] }
device?: { backend: 'cpu' | 'vulkan' | 'cuda' | 'metal', index: number }[]
Expand Down Expand Up @@ -695,7 +697,16 @@ export interface InferenceResourceProgress {
export interface InferenceResourceEvents {
'inference:resources:progress': { readonly id: string, readonly progress: InferenceResourceProgress }
}
export interface InferenceModel { id: string, title: string, capability: string, contextWindow: number, maxTokens: number, tools: boolean | 'auto', reasoning?: boolean | 'auto', catalogModels?: { providerId: string, modelId: string }[], input?: ('text' | 'image')[] }
export interface DiscoveredModelMetadata {
type?: 'chat' | 'transcription' | 'speech' | 'image-generation'
name?: string
input?: ('text' | 'image' | 'pdf' | 'audio' | 'video')[]
toolCall?: boolean
reasoningOptions?: string[]
contextWindow?: number
maxTokens?: number
}
export interface InferenceModel { id: string, title: string, capability: string, contextWindow: number, maxTokens: number, tools: boolean | 'auto', reasoning?: boolean | 'auto', metadata?: DiscoveredModelMetadata, catalogModels?: { providerId: string, modelId: string }[], input?: ('text' | 'image')[] }
export interface InferenceImageModel {
id: string
title: string
Expand All @@ -719,7 +730,7 @@ export interface InferenceBinding {
configuration?: Record<string, Json>
}
export interface InferenceApi {
readonly capabilities: { readonly nativeRuntime: true, readonly runtimeIsolation: true, readonly chatImageInput: true, readonly chatToolMatching: true, readonly chatReasoning: true, readonly resumableDownloads: true }
readonly capabilities: { readonly nativeRuntime: true, readonly runtimeIsolation: true, readonly chatImageInput: true, readonly chatToolMatching: true, readonly chatReasoning: true, readonly chatModelMetadata: true, readonly resumableDownloads: true }
environment: () => Promise<InferenceEnvironment>
status: () => Promise<{ runtime: string, configured: boolean, platforms: string[], instances: { id: string | null, state: 'loading' | 'ready' | 'stopping' | 'stopped', active: number, pending: number, cancelling: number, observation?: InferenceObservation }[] }[]>
stop: (input?: { runtime: string }) => Promise<void>
Expand Down
Loading
Loading