From badaf0c49bcbbcf806a2b1bf32436212cdf1c801 Mon Sep 17 00:00:00 2001 From: dallasbpeters Date: Tue, 25 Aug 2026 01:10:32 -0500 Subject: [PATCH] Implement insert_asset_clip, the one timeline op that could not run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every other operation rearranges what is already on the timeline: trim narrows a clip, duplicate copies one, move reorders them, drop_range cuts a span out. None of them can introduce footage. So an agent asked to put a sting on the front or cut to the B-roll had no operation that could do it — and the operation that was supposed to, `insert_asset_clip`, existed only as a line in the schema. It could be described, parsed, and then silently not performed. The recipe already existed twice over. useTimeline's drop handler inlined it (splice, resequence, rederive) while the comment on its own `moveClip` explains why that belongs in document/timeline.ts: two façades over one recipe is how the two drift into disagreeing about clip widths and anchored pills. So the pure `insertAssetClip` goes there, the op dispatcher calls it, and the drop handler now delegates to it rather than keeping its own copy. Details worth stating: - Neighbours, not an index. The schema names beforeClipId/afterClipId, which is the right contract for a model: "after the intro" survives the list changing under it and a number does not. `insertIndexFor` resolves the pair, preferring `before` as the more deliberate of the two. - It returns the new clip's id beside the document, unlike its neighbours here which return a bare one, because the caller needs it — the drop handler selects what it inserted and corrects its length when the duration probe lands. Stapling the id onto the document would put a field on it the schema does not have, and that document gets parsed, and saved. - An unprobed asset takes the placeholder duration, not zero: resequenceClips floors a clip at 0.001s, which is a clip you cannot see, select or drag. - The clip is marked `agent`. The timeline shows the origin, and "user" would make a model's proposal look like a cut somebody made on purpose. - An edit that cannot be performed throws instead of doing nothing quietly — unknown asset, unknown neighbour, empty source range. Doing nothing quietly is what this op did for its entire existence. Co-Authored-By: Claude Opus 5 --- .../ai-edition/document/operations.test.ts | 127 ++++++++++++++++++ src/lib/ai-edition/document/operations.ts | 39 ++++++ src/lib/ai-edition/document/timeline.ts | 117 ++++++++++++++++ src/lib/ai-edition/store/useTimeline.ts | 40 ++---- 4 files changed, 295 insertions(+), 28 deletions(-) diff --git a/src/lib/ai-edition/document/operations.test.ts b/src/lib/ai-edition/document/operations.test.ts index a4f58128a..72871e80f 100644 --- a/src/lib/ai-edition/document/operations.test.ts +++ b/src/lib/ai-edition/document/operations.test.ts @@ -398,3 +398,130 @@ describe("applyTimelineOperation.update_clip_range", () => { }); }); }); + +describe("applyTimelineOperation.insert_asset_clip", () => { + /** A second asset to bring in — the whole point of this operation. */ + function withSting(): AxcutDocument { + const doc = makeDoc(); + return { + ...doc, + assets: [ + ...doc.assets, + { + id: "asset_sting", + kind: "video" as const, + label: "Title sting", + originalPath: "C:/videos/sting.mp4", + durationSec: 4, + cameraTrack: null, + }, + ], + }; + } + + it("puts the new clip before a named neighbour and pushes it later", () => { + const result = applyTimelineOperation(withSting(), { + type: "insert_asset_clip", + assetId: "asset_sting", + beforeClipId: "clip_1", + reason: "title sting on the front", + }); + expect(result.summary).toMatch(/inserted Title sting at 0:00\.0/); + const clips = result.document.timeline.clips; + expect(clips).toHaveLength(2); + expect(clips[0]).toMatchObject({ + assetId: "asset_sting", + timelineStartSec: 0, + timelineEndSec: 4, + }); + // The recording did not move in its own source, only on the timeline. + expect(clips[1]).toMatchObject({ assetId: "asset_1", timelineStartSec: 4, sourceStartSec: 0 }); + }); + + it("puts it after a named neighbour, and at the end when neither is named", () => { + const after = applyTimelineOperation(withSting(), { + type: "insert_asset_clip", + assetId: "asset_sting", + afterClipId: "clip_1", + }).document; + expect(after.timeline.clips[1]).toMatchObject({ assetId: "asset_sting", timelineStartSec: 60 }); + + const appended = applyTimelineOperation(withSting(), { + type: "insert_asset_clip", + assetId: "asset_sting", + }).document; + expect(appended.timeline.clips[1]).toMatchObject({ + assetId: "asset_sting", + timelineStartSec: 60, + }); + }); + + // The reason B-roll is possible at all: a span of a file, not the whole file. + it("inserts a trimmed span, and lays it out by the trimmed length", () => { + const result = applyTimelineOperation(withSting(), { + type: "insert_asset_clip", + assetId: "asset_1", + afterClipId: "clip_1", + sourceStartSec: 40, + sourceEndSec: 50, + }); + const inserted = result.document.timeline.clips[1]; + expect(inserted).toMatchObject({ + sourceStartSec: 40, + sourceEndSec: 50, + timelineStartSec: 60, + timelineEndSec: 70, + }); + expect(sourceLength(inserted)).toBe(10); + }); + + // The timeline shows the origin, and "user" would make a model's proposal look + // like a cut somebody made on purpose. + it("marks the clip as the agent's", () => { + const result = applyTimelineOperation(withSting(), { + type: "insert_asset_clip", + assetId: "asset_sting", + }); + expect(result.document.timeline.clips[1].origin).toBe("agent"); + }); + + // An edit that cannot be performed has to say so. Silently doing nothing is + // what `insert_asset_clip` did for its whole existence as a declared-only op. + it("refuses an unknown asset, an unknown neighbour and an empty range", () => { + expect(() => + applyTimelineOperation(withSting(), { type: "insert_asset_clip", assetId: "nope" }), + ).toThrow(/Unknown asset nope/); + expect(() => + applyTimelineOperation(withSting(), { + type: "insert_asset_clip", + assetId: "asset_sting", + beforeClipId: "ghost", + }), + ).toThrow(/Unknown clip ghost/); + expect(() => + applyTimelineOperation(withSting(), { + type: "insert_asset_clip", + assetId: "asset_sting", + sourceStartSec: 3, + sourceEndSec: 3, + }), + ).toThrow(/Empty source range/); + }); + + // An asset dropped in and inserted in the same breath has no probed duration. + // resequenceClips floors a clip at 0.001s, which is one you cannot see or grab. + it("gives an unprobed asset a visible length rather than none", () => { + const doc = withSting(); + const unprobed: AxcutDocument = { + ...doc, + assets: doc.assets.map((a) => + a.id === "asset_sting" ? { ...a, durationSec: undefined } : a, + ), + }; + const inserted = applyTimelineOperation(unprobed, { + type: "insert_asset_clip", + assetId: "asset_sting", + }).document.timeline.clips[1]; + expect(sourceLength(inserted)).toBeGreaterThan(1); + }); +}); diff --git a/src/lib/ai-edition/document/operations.ts b/src/lib/ai-edition/document/operations.ts index 94c3ca395..9fdaa3936 100644 --- a/src/lib/ai-edition/document/operations.ts +++ b/src/lib/ai-edition/document/operations.ts @@ -13,6 +13,8 @@ import type { AxcutDocument } from "../schema"; import { formatSec } from "../timeline/format"; import { duplicateClip, + insertAssetClip, + insertIndexFor, moveClip, normalizeIntervals, primaryAssetDuration, @@ -77,6 +79,23 @@ export type AxcutTimelineOperation = type: "duplicate_clip"; clipId: string; reason?: string; + } + | { + /* + * The one operation that brings a file in. + * + * Everything else here rearranges what is already on the timeline, so an + * agent asked to put a sting on the front or cut to B-roll had nothing that + * could do it. Neighbours are named rather than an index, because "after the + * intro" survives the list changing under it and a number does not. + */ + type: "insert_asset_clip"; + assetId: string; + beforeClipId?: string | null; + afterClipId?: string | null; + sourceStartSec?: number; + sourceEndSec?: number; + reason?: string; }; export interface AppliedTimelineOperation { @@ -306,6 +325,26 @@ export function applyTimelineOperation( const next = duplicateClip(document, op.clipId, "user", op.reason ?? ""); return { document: next, summary: `duplicated clip ${op.clipId}` }; } + case "insert_asset_clip": { + const index = insertIndexFor(document, op.beforeClipId, op.afterClipId); + const { document: next, clipId } = insertAssetClip(document, { + assetId: op.assetId, + index, + sourceStartSec: op.sourceStartSec, + sourceEndSec: op.sourceEndSec, + // "agent", because this dispatcher is how a model's edits land. The + // timeline shows the origin, and calling it "user" makes a proposed cut + // look like one somebody made on purpose. + origin: "agent", + reason: op.reason ?? "", + }); + const asset = next.assets.find((a) => a.id === op.assetId); + const clip = next.timeline.clips.find((c) => c.id === clipId); + return { + document: next, + summary: `inserted ${asset?.label ?? op.assetId} at ${formatSec(clip?.timelineStartSec ?? 0)}`, + }; + } default: { // ponytail: exhaustive — TS errors here when a new variant is added. const exhaustive: never = op; diff --git a/src/lib/ai-edition/document/timeline.ts b/src/lib/ai-edition/document/timeline.ts index 232bcb6ee..1093fa327 100644 --- a/src/lib/ai-edition/document/timeline.ts +++ b/src/lib/ai-edition/document/timeline.ts @@ -777,6 +777,123 @@ export function moveClip( return rederiveRegionMs(next, newClips); } +/** + * Put a NEW asset's clip on the timeline — the one timeline op that brings in a file. + * + * Every other operation rearranges what is already there: trim narrows a clip, + * duplicate copies one, move reorders them, drop_range cuts a span out. None of + * them can introduce footage, so an agent asked to "put the logo sting on the + * front" or "cut to the B-roll here" had no operation that could do it. + * `insert_asset_clip` was declared in the schema and applied nowhere — the edit + * could be described, parsed, and then silently not performed. + * + * Here rather than in the store, for the reason `moveClip` gives above: the drop + * handler and the op dispatcher are two façades over one recipe (splice, + * resequence, rederive), and a second copy of it is how the two drift into + * disagreeing about clip widths and anchored pills. + * + * `index` is a position in the clip list, not a time. A caller holding a + * before/after clip id resolves it with `insertIndexFor` — laying clips + * back-to-back is `resequenceClips`'s job, and a timeline position passed in here + * would be overwritten by it anyway. + */ +export function insertAssetClip( + document: AxcutDocument, + { + assetId, + index, + sourceStartSec = 0, + sourceEndSec, + origin = "user", + reason = "", + }: { + assetId: string; + index: number; + sourceStartSec?: number; + sourceEndSec?: number; + origin?: "system" | "agent" | "user"; + reason?: string; + }, + /* + * Returns the new clip's id beside the document, unlike its neighbours here + * which return a bare one. The caller needs it: the drop handler selects what + * it just inserted, and corrects its length when the duration probe lands. + * Stapling the id onto the document instead would put a field on it that the + * schema does not have — and that document gets parsed, and saved. + */ +): { document: AxcutDocument; clipId: string } { + const asset = document.assets.find((a) => a.id === assetId); + if (!asset) { + throw new Error(`Unknown asset ${assetId}.`); + } + + /* + * An unprobed asset gets the placeholder, not zero. + * + * `durationSec` is filled in by a background probe, so a file dropped in and + * inserted in the same breath has none yet — and `resequenceClips` floors a + * clip at 0.001s, which is a clip you cannot see, select or drag. The drop + * handler already corrects the length when the probe lands. + */ + const known = asset.durationSec ?? PLACEHOLDER_DURATION_SEC; + const from = Math.max(0, Math.min(sourceStartSec, known)); + const to = Math.min(known, Math.max(from, sourceEndSec ?? known)); + if (to <= from) { + throw new Error(`Empty source range for asset ${assetId}: ${from}s to ${to}s.`); + } + + const clip: AxcutClip = { + id: createId("clip"), + assetId, + sourceStartSec: from, + sourceEndSec: to, + // Overwritten by resequenceClips; set so the clip is well-formed on its own. + timelineStartSec: 0, + timelineEndSec: to - from, + wordRefs: [], + origin, + reason, + }; + + const arr = [...document.timeline.clips]; + arr.splice(Math.max(0, Math.min(index, arr.length)), 0, clip); + const newClips = resequenceClips(arr); + const next: AxcutDocument = { + ...document, + timeline: { ...document.timeline, clips: newClips }, + }; + return { document: rederiveRegionMs(next, newClips), clipId: clip.id }; +} + +/** + * Where a before/after pair means, as an index. + * + * The schema names neighbours rather than a position, which is the right contract + * for an agent — "after the intro" survives the list changing under it, and an + * index does not. `beforeClipId` wins when both are given: it is the more specific + * of the two, and an agent that supplies a contradictory pair has said something + * about the earlier edge more deliberately. + */ +export function insertIndexFor( + document: AxcutDocument, + beforeClipId?: string | null, + afterClipId?: string | null, +): number { + const clips = document.timeline.clips; + if (beforeClipId) { + const i = clips.findIndex((c) => c.id === beforeClipId); + if (i < 0) throw new Error(`Unknown clip ${beforeClipId}.`); + return i; + } + if (afterClipId) { + const i = clips.findIndex((c) => c.id === afterClipId); + if (i < 0) throw new Error(`Unknown clip ${afterClipId}.`); + return i + 1; + } + // Neither named: the end, which is what "add this footage" means. + return clips.length; +} + // ponytail: duplicate a clip (preserves the original). Used for "split this // clip into two" or "make a copy". Mirrors axcut's // apps/server/src/lib/timeline.ts#duplicateClip. diff --git a/src/lib/ai-edition/store/useTimeline.ts b/src/lib/ai-edition/store/useTimeline.ts index 7823459ec..acdda9115 100644 --- a/src/lib/ai-edition/store/useTimeline.ts +++ b/src/lib/ai-edition/store/useTimeline.ts @@ -10,10 +10,10 @@ import { useScopedT } from "@/contexts/I18nContext"; import { createId } from "../document/ids"; import { duplicateClip as duplicateClipInDocument, + insertAssetClip, moveClip as moveClipInDocument, PLACEHOLDER_DURATION_SEC, type RegionKind, - rederiveRegionMs, removeClip as removeClipInDocument, removeRegion as removeRegionInDocument, resequenceClips, @@ -58,8 +58,6 @@ interface RegionHandle { id: string; } -type Clip = AxcutDocument["timeline"]["clips"][number]; - /** * Patch every region under the pill `id` belongs to. A payload edit must hit them all, * or the pieces of one pill would disagree — and then, by the merge rule, visibly split. @@ -1062,33 +1060,19 @@ export function useTimeline() { if (!currentDoc) return; const asset = currentDoc.assets.find((a) => a.id === assetId); if (!asset) return; - // Insert immediately at whatever we know. If the asset has a cached - // durationSec we use it; otherwise we fall back to the placeholder - // and let the background probe correct it. - const knownDuration = asset.durationSec ?? PLACEHOLDER_DURATION_SEC; - const newClip: Clip = { - id: createId("clip"), + // Delegates to the shared document/timeline.ts implementation — the same + // function the agent tool-executor uses for "insert_asset_clip" ops — for + // the reason `moveClip` below gives: the splice/resequence/rederive recipe + // in two places is how the two paths drift into disagreeing about clip + // widths and anchored pills. The placeholder duration for an unprobed + // asset lives there now too, and the probe below still corrects it. + const { document: finalDoc, clipId } = insertAssetClip(currentDoc, { assetId, - sourceStartSec: 0, - sourceEndSec: knownDuration, - timelineStartSec: 0, - timelineEndSec: knownDuration, - wordRefs: [], - origin: "user", + index, reason: "Inserted from media panel", - }; - const oldClips = currentDoc.timeline.clips; - const arr = [...oldClips]; - const at = Math.max(0, Math.min(arr.length, index)); - arr.splice(at, 0, newClip); - const newClips = resequenceClips(arr); - const next: AxcutDocument = { - ...currentDoc, - timeline: { ...currentDoc.timeline, clips: newClips }, - }; - const finalDoc = rederiveRegionMs(next, newClips); + }); if (!(await saveDocument(finalDoc, { history: true }))) return; - setClipSelection(newClip.id); + setClipSelection(clipId); // If we used the placeholder, kick off the probe in the background. // Don't await — the drop is already responsive; the probe will @@ -1099,7 +1083,7 @@ export function useTimeline() { // that THROWS on a failed write. Losing a background duration correction // is survivable — the clip keeps its placeholder length; an unhandled // rejection is not. - void probeAndCorrectClip(assetId, newClip.id, asset.originalPath).catch((err) => { + void probeAndCorrectClip(assetId, clipId, asset.originalPath).catch((err) => { console.warn("[timeline] background duration probe failed to save:", err); }); }