diff --git a/src/lib/ai-edition/document/operations.test.ts b/src/lib/ai-edition/document/operations.test.ts index a4f58128a..72871e80f 100644 --- a/src/lib/ai-edition/document/operations.test.ts +++ b/src/lib/ai-edition/document/operations.test.ts @@ -398,3 +398,130 @@ describe("applyTimelineOperation.update_clip_range", () => { }); }); }); + +describe("applyTimelineOperation.insert_asset_clip", () => { + /** A second asset to bring in — the whole point of this operation. */ + function withSting(): AxcutDocument { + const doc = makeDoc(); + return { + ...doc, + assets: [ + ...doc.assets, + { + id: "asset_sting", + kind: "video" as const, + label: "Title sting", + originalPath: "C:/videos/sting.mp4", + durationSec: 4, + cameraTrack: null, + }, + ], + }; + } + + it("puts the new clip before a named neighbour and pushes it later", () => { + const result = applyTimelineOperation(withSting(), { + type: "insert_asset_clip", + assetId: "asset_sting", + beforeClipId: "clip_1", + reason: "title sting on the front", + }); + expect(result.summary).toMatch(/inserted Title sting at 0:00\.0/); + const clips = result.document.timeline.clips; + expect(clips).toHaveLength(2); + expect(clips[0]).toMatchObject({ + assetId: "asset_sting", + timelineStartSec: 0, + timelineEndSec: 4, + }); + // The recording did not move in its own source, only on the timeline. + expect(clips[1]).toMatchObject({ assetId: "asset_1", timelineStartSec: 4, sourceStartSec: 0 }); + }); + + it("puts it after a named neighbour, and at the end when neither is named", () => { + const after = applyTimelineOperation(withSting(), { + type: "insert_asset_clip", + assetId: "asset_sting", + afterClipId: "clip_1", + }).document; + expect(after.timeline.clips[1]).toMatchObject({ assetId: "asset_sting", timelineStartSec: 60 }); + + const appended = applyTimelineOperation(withSting(), { + type: "insert_asset_clip", + assetId: "asset_sting", + }).document; + expect(appended.timeline.clips[1]).toMatchObject({ + assetId: "asset_sting", + timelineStartSec: 60, + }); + }); + + // The reason B-roll is possible at all: a span of a file, not the whole file. + it("inserts a trimmed span, and lays it out by the trimmed length", () => { + const result = applyTimelineOperation(withSting(), { + type: "insert_asset_clip", + assetId: "asset_1", + afterClipId: "clip_1", + sourceStartSec: 40, + sourceEndSec: 50, + }); + const inserted = result.document.timeline.clips[1]; + expect(inserted).toMatchObject({ + sourceStartSec: 40, + sourceEndSec: 50, + timelineStartSec: 60, + timelineEndSec: 70, + }); + expect(sourceLength(inserted)).toBe(10); + }); + + // The timeline shows the origin, and "user" would make a model's proposal look + // like a cut somebody made on purpose. + it("marks the clip as the agent's", () => { + const result = applyTimelineOperation(withSting(), { + type: "insert_asset_clip", + assetId: "asset_sting", + }); + expect(result.document.timeline.clips[1].origin).toBe("agent"); + }); + + // An edit that cannot be performed has to say so. Silently doing nothing is + // what `insert_asset_clip` did for its whole existence as a declared-only op. + it("refuses an unknown asset, an unknown neighbour and an empty range", () => { + expect(() => + applyTimelineOperation(withSting(), { type: "insert_asset_clip", assetId: "nope" }), + ).toThrow(/Unknown asset nope/); + expect(() => + applyTimelineOperation(withSting(), { + type: "insert_asset_clip", + assetId: "asset_sting", + beforeClipId: "ghost", + }), + ).toThrow(/Unknown clip ghost/); + expect(() => + applyTimelineOperation(withSting(), { + type: "insert_asset_clip", + assetId: "asset_sting", + sourceStartSec: 3, + sourceEndSec: 3, + }), + ).toThrow(/Empty source range/); + }); + + // An asset dropped in and inserted in the same breath has no probed duration. + // resequenceClips floors a clip at 0.001s, which is one you cannot see or grab. + it("gives an unprobed asset a visible length rather than none", () => { + const doc = withSting(); + const unprobed: AxcutDocument = { + ...doc, + assets: doc.assets.map((a) => + a.id === "asset_sting" ? { ...a, durationSec: undefined } : a, + ), + }; + const inserted = applyTimelineOperation(unprobed, { + type: "insert_asset_clip", + assetId: "asset_sting", + }).document.timeline.clips[1]; + expect(sourceLength(inserted)).toBeGreaterThan(1); + }); +}); diff --git a/src/lib/ai-edition/document/operations.ts b/src/lib/ai-edition/document/operations.ts index 94c3ca395..9fdaa3936 100644 --- a/src/lib/ai-edition/document/operations.ts +++ b/src/lib/ai-edition/document/operations.ts @@ -13,6 +13,8 @@ import type { AxcutDocument } from "../schema"; import { formatSec } from "../timeline/format"; import { duplicateClip, + insertAssetClip, + insertIndexFor, moveClip, normalizeIntervals, primaryAssetDuration, @@ -77,6 +79,23 @@ export type AxcutTimelineOperation = type: "duplicate_clip"; clipId: string; reason?: string; + } + | { + /* + * The one operation that brings a file in. + * + * Everything else here rearranges what is already on the timeline, so an + * agent asked to put a sting on the front or cut to B-roll had nothing that + * could do it. Neighbours are named rather than an index, because "after the + * intro" survives the list changing under it and a number does not. + */ + type: "insert_asset_clip"; + assetId: string; + beforeClipId?: string | null; + afterClipId?: string | null; + sourceStartSec?: number; + sourceEndSec?: number; + reason?: string; }; export interface AppliedTimelineOperation { @@ -306,6 +325,26 @@ export function applyTimelineOperation( const next = duplicateClip(document, op.clipId, "user", op.reason ?? ""); return { document: next, summary: `duplicated clip ${op.clipId}` }; } + case "insert_asset_clip": { + const index = insertIndexFor(document, op.beforeClipId, op.afterClipId); + const { document: next, clipId } = insertAssetClip(document, { + assetId: op.assetId, + index, + sourceStartSec: op.sourceStartSec, + sourceEndSec: op.sourceEndSec, + // "agent", because this dispatcher is how a model's edits land. The + // timeline shows the origin, and calling it "user" makes a proposed cut + // look like one somebody made on purpose. + origin: "agent", + reason: op.reason ?? "", + }); + const asset = next.assets.find((a) => a.id === op.assetId); + const clip = next.timeline.clips.find((c) => c.id === clipId); + return { + document: next, + summary: `inserted ${asset?.label ?? op.assetId} at ${formatSec(clip?.timelineStartSec ?? 0)}`, + }; + } default: { // ponytail: exhaustive — TS errors here when a new variant is added. const exhaustive: never = op; diff --git a/src/lib/ai-edition/document/timeline.ts b/src/lib/ai-edition/document/timeline.ts index 232bcb6ee..1093fa327 100644 --- a/src/lib/ai-edition/document/timeline.ts +++ b/src/lib/ai-edition/document/timeline.ts @@ -777,6 +777,123 @@ export function moveClip( return rederiveRegionMs(next, newClips); } +/** + * Put a NEW asset's clip on the timeline — the one timeline op that brings in a file. + * + * Every other operation rearranges what is already there: trim narrows a clip, + * duplicate copies one, move reorders them, drop_range cuts a span out. None of + * them can introduce footage, so an agent asked to "put the logo sting on the + * front" or "cut to the B-roll here" had no operation that could do it. + * `insert_asset_clip` was declared in the schema and applied nowhere — the edit + * could be described, parsed, and then silently not performed. + * + * Here rather than in the store, for the reason `moveClip` gives above: the drop + * handler and the op dispatcher are two façades over one recipe (splice, + * resequence, rederive), and a second copy of it is how the two drift into + * disagreeing about clip widths and anchored pills. + * + * `index` is a position in the clip list, not a time. A caller holding a + * before/after clip id resolves it with `insertIndexFor` — laying clips + * back-to-back is `resequenceClips`'s job, and a timeline position passed in here + * would be overwritten by it anyway. + */ +export function insertAssetClip( + document: AxcutDocument, + { + assetId, + index, + sourceStartSec = 0, + sourceEndSec, + origin = "user", + reason = "", + }: { + assetId: string; + index: number; + sourceStartSec?: number; + sourceEndSec?: number; + origin?: "system" | "agent" | "user"; + reason?: string; + }, + /* + * Returns the new clip's id beside the document, unlike its neighbours here + * which return a bare one. The caller needs it: the drop handler selects what + * it just inserted, and corrects its length when the duration probe lands. + * Stapling the id onto the document instead would put a field on it that the + * schema does not have — and that document gets parsed, and saved. + */ +): { document: AxcutDocument; clipId: string } { + const asset = document.assets.find((a) => a.id === assetId); + if (!asset) { + throw new Error(`Unknown asset ${assetId}.`); + } + + /* + * An unprobed asset gets the placeholder, not zero. + * + * `durationSec` is filled in by a background probe, so a file dropped in and + * inserted in the same breath has none yet — and `resequenceClips` floors a + * clip at 0.001s, which is a clip you cannot see, select or drag. The drop + * handler already corrects the length when the probe lands. + */ + const known = asset.durationSec ?? PLACEHOLDER_DURATION_SEC; + const from = Math.max(0, Math.min(sourceStartSec, known)); + const to = Math.min(known, Math.max(from, sourceEndSec ?? known)); + if (to <= from) { + throw new Error(`Empty source range for asset ${assetId}: ${from}s to ${to}s.`); + } + + const clip: AxcutClip = { + id: createId("clip"), + assetId, + sourceStartSec: from, + sourceEndSec: to, + // Overwritten by resequenceClips; set so the clip is well-formed on its own. + timelineStartSec: 0, + timelineEndSec: to - from, + wordRefs: [], + origin, + reason, + }; + + const arr = [...document.timeline.clips]; + arr.splice(Math.max(0, Math.min(index, arr.length)), 0, clip); + const newClips = resequenceClips(arr); + const next: AxcutDocument = { + ...document, + timeline: { ...document.timeline, clips: newClips }, + }; + return { document: rederiveRegionMs(next, newClips), clipId: clip.id }; +} + +/** + * Where a before/after pair means, as an index. + * + * The schema names neighbours rather than a position, which is the right contract + * for an agent — "after the intro" survives the list changing under it, and an + * index does not. `beforeClipId` wins when both are given: it is the more specific + * of the two, and an agent that supplies a contradictory pair has said something + * about the earlier edge more deliberately. + */ +export function insertIndexFor( + document: AxcutDocument, + beforeClipId?: string | null, + afterClipId?: string | null, +): number { + const clips = document.timeline.clips; + if (beforeClipId) { + const i = clips.findIndex((c) => c.id === beforeClipId); + if (i < 0) throw new Error(`Unknown clip ${beforeClipId}.`); + return i; + } + if (afterClipId) { + const i = clips.findIndex((c) => c.id === afterClipId); + if (i < 0) throw new Error(`Unknown clip ${afterClipId}.`); + return i + 1; + } + // Neither named: the end, which is what "add this footage" means. + return clips.length; +} + // ponytail: duplicate a clip (preserves the original). Used for "split this // clip into two" or "make a copy". Mirrors axcut's // apps/server/src/lib/timeline.ts#duplicateClip. diff --git a/src/lib/ai-edition/store/useTimeline.ts b/src/lib/ai-edition/store/useTimeline.ts index 7823459ec..acdda9115 100644 --- a/src/lib/ai-edition/store/useTimeline.ts +++ b/src/lib/ai-edition/store/useTimeline.ts @@ -10,10 +10,10 @@ import { useScopedT } from "@/contexts/I18nContext"; import { createId } from "../document/ids"; import { duplicateClip as duplicateClipInDocument, + insertAssetClip, moveClip as moveClipInDocument, PLACEHOLDER_DURATION_SEC, type RegionKind, - rederiveRegionMs, removeClip as removeClipInDocument, removeRegion as removeRegionInDocument, resequenceClips, @@ -58,8 +58,6 @@ interface RegionHandle { id: string; } -type Clip = AxcutDocument["timeline"]["clips"][number]; - /** * Patch every region under the pill `id` belongs to. A payload edit must hit them all, * or the pieces of one pill would disagree — and then, by the merge rule, visibly split. @@ -1062,33 +1060,19 @@ export function useTimeline() { if (!currentDoc) return; const asset = currentDoc.assets.find((a) => a.id === assetId); if (!asset) return; - // Insert immediately at whatever we know. If the asset has a cached - // durationSec we use it; otherwise we fall back to the placeholder - // and let the background probe correct it. - const knownDuration = asset.durationSec ?? PLACEHOLDER_DURATION_SEC; - const newClip: Clip = { - id: createId("clip"), + // Delegates to the shared document/timeline.ts implementation — the same + // function the agent tool-executor uses for "insert_asset_clip" ops — for + // the reason `moveClip` below gives: the splice/resequence/rederive recipe + // in two places is how the two paths drift into disagreeing about clip + // widths and anchored pills. The placeholder duration for an unprobed + // asset lives there now too, and the probe below still corrects it. + const { document: finalDoc, clipId } = insertAssetClip(currentDoc, { assetId, - sourceStartSec: 0, - sourceEndSec: knownDuration, - timelineStartSec: 0, - timelineEndSec: knownDuration, - wordRefs: [], - origin: "user", + index, reason: "Inserted from media panel", - }; - const oldClips = currentDoc.timeline.clips; - const arr = [...oldClips]; - const at = Math.max(0, Math.min(arr.length, index)); - arr.splice(at, 0, newClip); - const newClips = resequenceClips(arr); - const next: AxcutDocument = { - ...currentDoc, - timeline: { ...currentDoc.timeline, clips: newClips }, - }; - const finalDoc = rederiveRegionMs(next, newClips); + }); if (!(await saveDocument(finalDoc, { history: true }))) return; - setClipSelection(newClip.id); + setClipSelection(clipId); // If we used the placeholder, kick off the probe in the background. // Don't await — the drop is already responsive; the probe will @@ -1099,7 +1083,7 @@ export function useTimeline() { // that THROWS on a failed write. Losing a background duration correction // is survivable — the clip keeps its placeholder length; an unhandled // rejection is not. - void probeAndCorrectClip(assetId, newClip.id, asset.originalPath).catch((err) => { + void probeAndCorrectClip(assetId, clipId, asset.originalPath).catch((err) => { console.warn("[timeline] background duration probe failed to save:", err); }); }