From a55c731978f5159cb1fced747af777608f6ecde2 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Sun, 30 Aug 2026 21:03:52 -0400 Subject: [PATCH 01/19] =?UTF-8?q?feat(webv2):=20Ref2VA=20reference-extend?= =?UTF-8?q?=20=E2=80=94=20append=20the=20generation=20to=20an=20initial=20?= =?UTF-8?q?video?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Ref2VA panel gains an Initial Video section: the generated clip is appended to the uploaded video at its End Frame cutpoint (same extract + crossfade video_concat join as FL2VA extend; video_concat rebuilds the soundtrack from every input, so both clips' audio survives). Continuity comes from a linked video reference the panel derives from the clip — its trim defaults to the last 141 frames (~5s at 24 fps, exactly on the 17n+5 grid) before the cutpoint and re-derives whenever the cutpoints change, while remaining an ordinary reference card: trimmable, reorderable, conditioning-selectable, counted against the 3-video cap, and badged as linked. No frame conditioning and no backend changes; generation_mode stays minimax_h3_ref2v with the extend-style source_video extras riding along. State-wise, references and sourceVideo now coexist (validation still rejects the pair on models without reference-extend, and references remain exclusive with first/last frames). Switching FL2VA -> Ref2VA carries an initial video over and derives its tail reference; switching back clears references and keeps the clip for FL2VA extend. Recall restores both the references and the source video with its recorded trim, and the linkage re-establishes by clip identity when the recalled trims are next edited. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01S4B5exWsbC2z2Uu2tA167A --- .../frontend/webv2/public/locales/en.json | 2 + .../src/features/video/core/graph.test.ts | 62 ++++++++++++++ .../webv2/src/features/video/core/graph.ts | 51 +++++++++--- .../src/features/video/core/settings.test.ts | 75 ++++++++++++++++- .../webv2/src/features/video/core/settings.ts | 81 +++++++++++++++++-- .../webv2/src/features/video/core/types.ts | 27 ++++++- .../features/video/core/videoPolicies.test.ts | 63 ++++++++++++++- .../src/features/video/core/videoPolicies.ts | 33 ++++++-- .../video/ui/VideoReferenceListField.tsx | 5 ++ .../src/features/video/ui/VideoWidgetView.tsx | 43 ++++++++-- .../image-actions/executeVideoRecall.ts | 17 ++-- .../image-actions/videoRecall.test.ts | 25 ++++++ .../workbench/image-actions/videoRecall.ts | 10 ++- 13 files changed, 449 insertions(+), 45 deletions(-) diff --git a/invokeai/frontend/webv2/public/locales/en.json b/invokeai/frontend/webv2/public/locales/en.json index a98c3237249..09e35472ee4 100644 --- a/invokeai/frontend/webv2/public/locales/en.json +++ b/invokeai/frontend/webv2/public/locales/en.json @@ -3163,6 +3163,8 @@ "referenceConditioningVideoAudio": "Video + audio", "referenceDetailMatch": "Match generation size", "referenceDetailMax": "Max detail (2048px, slower)", + "referenceExtendHelp": "The generated video is appended to this clip at its End Frame. A linked video reference samples the last ~5 seconds (141 frames) before that cutpoint for continuity — its trim re-derives whenever the cutpoints change.", + "referenceFromInitialVideo": "Initial video", "references": "References", "referencesHelp": "Up to 3 videos and 9 images condition the generation, in order — reordering references changes the result. A video reference can contribute its image track, its soundtrack, or both.", "removeReference": "Remove reference", diff --git a/invokeai/frontend/webv2/src/features/video/core/graph.test.ts b/invokeai/frontend/webv2/src/features/video/core/graph.test.ts index 448ed53f836..d2f4acbba91 100644 --- a/invokeai/frontend/webv2/src/features/video/core/graph.test.ts +++ b/invokeai/frontend/webv2/src/features/video/core/graph.test.ts @@ -563,4 +563,66 @@ describe('compileVideoGraph — MiniMax H3 Ref2VA', () => { expect(nodesOfType(backendGraph, 'minimax_h3_reference_conditioning')).toHaveLength(0); expect(nodesOfType(backendGraph, 'collect')).toHaveLength(0); }); + + it('reference-extend: appends the new clip to the initial video without frame conditioning', () => { + const initialVideo = { + endFrame: 400, + fps: 24, + height: 480, + numFrames: 402, + startFrame: 10, + video_name: 'long.mp4', + width: 832, + }; + const settings = { + ...referenceSettings, + references: [ + // The linked tail reference (as the setter derives it) plus a user reference. + { + clip: { ...initialVideo, endFrame: 400, startFrame: 260 }, + conditioning: 'video_audio' as const, + fromSourceVideo: true, + kind: 'video' as const, + }, + ...referenceSettings.references, + ], + sourceVideo: initialVideo, + }; + const { backendGraph } = compileVideoGraph(settings, model); + + // The new clip is intermediate; the crossfade concat is the output, fed + // [trimmed source, new clip]; the source is retimed to H3's fixed 24 fps. + expect(nodeOfType(backendGraph, 'minimax_h3_latents_to_video')).toMatchObject({ + id: 'extension_clip', + is_intermediate: true, + }); + expect(nodeOfType(backendGraph, 'video_concat')).toMatchObject({ id: 'video_output', transition: 'crossfade' }); + expect(nodeOfType(backendGraph, 'extract_video_range')).toMatchObject({ end_frame: -2, fps: 24, start_frame: 10 }); + expect(hasEdge(backendGraph, 'source_video', 'video', 'source_clip_collect', 'item')).toBe(true); + expect(hasEdge(backendGraph, 'extension_clip', 'video', 'clips_to_join', 'item')).toBe(true); + + // Continuity comes from the references — no frame conditioning, no last-frame extraction. + expect(nodesOfType(backendGraph, 'minimax_h3_frame_conditioning')).toHaveLength(0); + expect(nodesOfType(backendGraph, 'video_frame_extract')).toHaveLength(0); + + // The linked reference is an ordinary first reference; the flag never reaches metadata. + const videoReferences = nodesOfType(backendGraph, 'minimax_h3_video_reference'); + + expect(videoReferences[0]).toMatchObject({ id: 'reference_1', start_frame: 260 }); + const metadata = nodeOfType(backendGraph, 'core_metadata'); + + expect(metadata.generation_mode).toBe('minimax_h3_ref2v'); + expect(metadata).toMatchObject({ + source_video: { video_name: 'long.mp4' }, + source_video_end_frame: 400, + source_video_start_frame: 10, + }); + expect((metadata.minimax_h3_references as Record[])[0]).toEqual({ + conditioning: 'video_audio', + end_frame: 260 + 140, + kind: 'video', + start_frame: 260, + video_name: 'long.mp4', + }); + }); }); diff --git a/invokeai/frontend/webv2/src/features/video/core/graph.ts b/invokeai/frontend/webv2/src/features/video/core/graph.ts index 08fc2509444..d31ac4bb66f 100644 --- a/invokeai/frontend/webv2/src/features/video/core/graph.ts +++ b/invokeai/frontend/webv2/src/features/video/core/graph.ts @@ -131,7 +131,12 @@ const toTailAwareIndex = (frame: number, estimatedNumFrames: number): number => return tailOffset <= 3 ? -(tailOffset + 1) : frame; }; -const addExtendScaffolding = ( +/** + * Trimmed source extraction + [source, new clip] crossfade join — the shared + * core of FL2VA/Wan extend and Ref2VA reference-extend. video_concat rebuilds + * the soundtrack from every input, so both clips' audio survives the join. + */ +const addSourceJoin = ( graph: BackendGraphContract, sourceVideo: NonNullable, newClip: BackendInvocationContract, @@ -147,13 +152,6 @@ const addExtendScaffolding = ( video: { video_name: sourceVideo.video_name }, ...(options.extractFps === undefined ? {} : { fps: options.extractFps }), }); - const lastFrame = addNode(graph, { - frame_index: -1, - id: 'source_last_frame', - is_intermediate: true, - type: 'video_frame_extract', - use_cache: false, - }); const sourceCollect = addNode(graph, { id: 'source_clip_collect', type: 'collect' }); const clipsCollect = addNode(graph, { id: 'clips_to_join', type: 'collect' }); const concat = addNode(graph, { @@ -166,13 +164,32 @@ const addExtendScaffolding = ( use_cache: false, }); - addEdge(graph, extract, 'video', lastFrame, 'video'); // Chained collectors keep the join order deterministic: [trimmed source, new clip]. addEdge(graph, extract, 'video', sourceCollect, 'item'); addEdge(graph, sourceCollect, 'collection', clipsCollect, 'collection'); addEdge(graph, newClip, 'video', clipsCollect, 'item'); addEdge(graph, clipsCollect, 'collection', concat, 'videos'); + return { concat, extract }; +}; + +const addExtendScaffolding = ( + graph: BackendGraphContract, + sourceVideo: NonNullable, + newClip: BackendInvocationContract, + options: { extractFps?: number } = {} +) => { + const { concat, extract } = addSourceJoin(graph, sourceVideo, newClip, options); + const lastFrame = addNode(graph, { + frame_index: -1, + id: 'source_last_frame', + is_intermediate: true, + type: 'video_frame_extract', + use_cache: false, + }); + + addEdge(graph, extract, 'video', lastFrame, 'video'); + return { concat, extract, lastFrame }; }; @@ -527,6 +544,7 @@ const buildMiniMaxH3VideoGraph = (settings: VideoSettings, model: MainModelConfi let output: BackendInvocationContract; let extendParts: { lastFrame: BackendInvocationContract; newClip: BackendInvocationContract } | null = null; + let referenceExtendClip: BackendInvocationContract | null = null; if (mode === 'extend' && settings.sourceVideo && frameConditioning) { const newClip = addLatentsToVideo('extension_clip', true); @@ -538,6 +556,15 @@ const buildMiniMaxH3VideoGraph = (settings: VideoSettings, model: MainModelConfi addEdge(graph, lastFrame, 'image', frameConditioning, 'first_image'); output = concat; extendParts = { lastFrame, newClip }; + } else if (mode === 'reference' && settings.sourceVideo) { + // Reference-extend: the new clip is appended to the trimmed Initial + // Video. Continuity comes from the references (typically the linked tail + // reference), not from frame conditioning, so no last frame is extracted. + const newClip = addLatentsToVideo('extension_clip', true); + const { concat } = addSourceJoin(graph, settings.sourceVideo, newClip, { extractFps: 24 }); + + output = concat; + referenceExtendClip = newClip; } else { output = addLatentsToVideo('video_output', false); } @@ -567,7 +594,11 @@ const buildMiniMaxH3VideoGraph = (settings: VideoSettings, model: MainModelConfi height: dimensions.height, model, negativeWired: false, - outputs: extendParts ? [output, extendParts.newClip] : [output], + outputs: extendParts + ? [output, extendParts.newClip] + : referenceExtendClip + ? [output, referenceExtendClip] + : [output], settings, width: dimensions.width, }); diff --git a/invokeai/frontend/webv2/src/features/video/core/settings.test.ts b/invokeai/frontend/webv2/src/features/video/core/settings.test.ts index e217bf361c3..3a73a63fa25 100644 --- a/invokeai/frontend/webv2/src/features/video/core/settings.test.ts +++ b/invokeai/frontend/webv2/src/features/video/core/settings.test.ts @@ -3,9 +3,11 @@ import { describe, expect, it } from 'vitest'; import type { VideoSettings } from './types'; import { + applyReferenceExtendSourceVideo, clearDeletedVideoMedia, cloneVideoWidgetValues, createVideoSourceClip, + deriveReferenceExtendClip, isVideoSettings, isVideoSourceClip, normalizeVideoSettings, @@ -319,14 +321,16 @@ describe('references', () => { ); }); - it('normalization drops frame/source media when references are present', () => { + it('normalization drops frame media when references are present, keeping the source video', () => { const normalized = normalizeVideoSettings( createSettings({ firstFrameImage: FIRST_FRAME, references: [IMAGE_REFERENCE], sourceVideo: SOURCE_VIDEO }) ); expect(normalized?.references).toEqual([IMAGE_REFERENCE]); expect(normalized?.firstFrameImage).toBeNull(); - expect(normalized?.sourceVideo).toBeNull(); + // References + source video is the Ref2VA reference-extend shape; + // validation rejects the pair on models that cannot consume it. + expect(normalized?.sourceVideo).toEqual(SOURCE_VIDEO); }); it('normalization drops malformed entries and enforces the caps, preserving order', () => { @@ -378,3 +382,70 @@ describe('references', () => { expect(sweptVideo.references).toEqual([IMAGE_REFERENCE]); }); }); + +describe('reference-extend linkage', () => { + const longSource = { ...SOURCE_VIDEO, endFrame: 400, numFrames: 402, video_name: 'long.mp4' }; + + it('derives the tail trim: 141 frames inclusive ending at the cutpoint, clamped at 0', () => { + expect(deriveReferenceExtendClip(longSource)).toMatchObject({ endFrame: 400, startFrame: 260 }); + // Shorter than the tail window: sample from the clip's own start. + expect(deriveReferenceExtendClip(SOURCE_VIDEO)).toMatchObject({ endFrame: 79, startFrame: 0 }); + }); + + it('prepends a linked video+audio reference and re-derives it on cutpoint changes', () => { + const added = applyReferenceExtendSourceVideo([IMAGE_REFERENCE], longSource, 3); + + expect(added).toHaveLength(2); + expect(added[0]).toMatchObject({ + clip: { endFrame: 400, startFrame: 260, video_name: 'long.mp4' }, + conditioning: 'video_audio', + fromSourceVideo: true, + kind: 'video', + }); + + // The user tunes the conditioning, then moves the cutpoint: the trim + // re-derives, the position and conditioning survive. + const tuned = added.map((entry, index) => + index === 0 && entry.kind === 'video' ? { ...entry, conditioning: 'video' as const } : entry + ); + const retrimmed = applyReferenceExtendSourceVideo(tuned, { ...longSource, endFrame: 300 }, 3); + + expect(retrimmed[0]).toMatchObject({ + clip: { endFrame: 300, startFrame: 160 }, + conditioning: 'video', + fromSourceVideo: true, + }); + expect(retrimmed[1]).toBe(added[1]); + }); + + it('clearing the initial video removes only the linked reference (identity-preserving when none)', () => { + const list = applyReferenceExtendSourceVideo([VIDEO_REFERENCE, IMAGE_REFERENCE], longSource, 3); + + expect(applyReferenceExtendSourceVideo(list, null, 3)).toEqual([VIDEO_REFERENCE, IMAGE_REFERENCE]); + + const unlinked = [VIDEO_REFERENCE, IMAGE_REFERENCE]; + + expect(applyReferenceExtendSourceVideo(unlinked, null, 3)).toBe(unlinked); + }); + + it('adopts an unflagged reference for the same clip instead of duplicating it (recall shape)', () => { + const recalled = { ...VIDEO_REFERENCE, clip: { ...VIDEO_REFERENCE.clip, video_name: 'long.mp4' } }; + const result = applyReferenceExtendSourceVideo([IMAGE_REFERENCE, recalled], longSource, 3); + + expect(result).toHaveLength(2); + expect(result[1]).toMatchObject({ + clip: { endFrame: 400, startFrame: 260, video_name: 'long.mp4' }, + fromSourceVideo: true, + }); + }); + + it('leaves a full video-reference list unchanged instead of overflowing the cap', () => { + const full = [ + VIDEO_REFERENCE, + { ...VIDEO_REFERENCE, clip: { ...VIDEO_REFERENCE.clip, video_name: 'b.mp4' } }, + { ...VIDEO_REFERENCE, clip: { ...VIDEO_REFERENCE.clip, video_name: 'c.mp4' } }, + ]; + + expect(applyReferenceExtendSourceVideo(full, longSource, 3)).toBe(full); + }); +}); diff --git a/invokeai/frontend/webv2/src/features/video/core/settings.ts b/invokeai/frontend/webv2/src/features/video/core/settings.ts index 9aa2152a9c1..983769957bd 100644 --- a/invokeai/frontend/webv2/src/features/video/core/settings.ts +++ b/invokeai/frontend/webv2/src/features/video/core/settings.ts @@ -145,7 +145,9 @@ const areAcceleratorLorasPresent = (keys: readonly string[], loras: readonly Gen * frame and a source video never coexist (normalization and the setters both * enforce it), and a last frame refines whichever mode its partner implies: * with a first frame it becomes FLF2V interpolation, with a source video it is - * the destination the extension should land on. + * the destination the extension should land on. References always win: with a + * source video alongside them (Ref2VA reference-extend) the mode stays + * `reference` and the graph appends the new clip to the source. */ export const resolveVideoMode = ( settings: Pick @@ -199,16 +201,16 @@ export const normalizeVideoSettings = (values: unknown): VideoSettings | null => return null; } - // References are mutually exclusive with every frame/source-video slot; when a stale - // record somehow holds both, the references win deterministically (mirroring the - // first-frame-beats-source rule below). + // References are mutually exclusive with the frame slots; when a stale + // record somehow holds both, the references win deterministically. A source + // video COEXISTS with references (Ref2VA reference-extend) — validation + // rejects the pair on models that cannot consume it. const references = sanitizeVideoReferences(values.references); const hasReferences = references.length > 0; const firstFrameImage = !hasReferences && isImageWithDims(values.firstFrameImage) ? values.firstFrameImage : null; // A first frame and a source video are mutually exclusive; if a stale // project somehow holds both, the first frame wins deterministically. - const sourceVideo = - !firstFrameImage && !hasReferences && isVideoSourceClip(values.sourceVideo) ? values.sourceVideo : null; + const sourceVideo = !firstFrameImage && isVideoSourceClip(values.sourceVideo) ? values.sourceVideo : null; const loras = Array.isArray(values.loras) ? values.loras.filter(isVideoLora) : []; const acceleratorLoraKeys = getStringArray(values.acceleratorLoraKeys); // The flag means "the accelerator LoRAs the toggle added are active": if any @@ -294,7 +296,7 @@ export const isVideoSettings = (values: unknown): values is VideoSettings => { values.references.every(isVideoReferenceItem) && !( (values.references as unknown[]).length > 0 && - (values.firstFrameImage !== null || values.lastFrameImage !== null || values.sourceVideo !== null) + (values.firstFrameImage !== null || values.lastFrameImage !== null) ) && Array.isArray(values.loras) && values.loras.every(isVideoLora) && @@ -384,6 +386,71 @@ export const createVideoSourceClip = (item: { /** The minimum frames a trim must keep — video_concat's crossfade consumes a 2-frame tail. */ export const MIN_VIDEO_TRIM_FRAMES = 2; +/** + * Frames the reference-extend tail reference samples ahead of the cutpoint: + * ~5s of lead-in at H3's fixed 24 fps, and exactly on the 17n+5 frame grid + * (17*8+5) so the backend's snap-down keeps all of it. + */ +export const VIDEO_REFERENCE_EXTEND_TAIL_FRAMES = 141; + +/** The tail reference's default trim: the last `VIDEO_REFERENCE_EXTEND_TAIL_FRAMES` frames before the cutpoint. */ +export const deriveReferenceExtendClip = (sourceVideo: VideoSourceClip): VideoSourceClip => ({ + ...sourceVideo, + endFrame: sourceVideo.endFrame, + // Deliberately unclamped by the Initial Video's START cutpoint: the + // reference samples the original clip for continuity, independent of which + // portion the extension keeps. + startFrame: Math.max(0, sourceVideo.endFrame - (VIDEO_REFERENCE_EXTEND_TAIL_FRAMES - 1)), +}); + +/** + * Keeps the reference list in step with the Initial Video on a reference-extend + * panel (pure; the setter and the model-selection transition both use it): + * + * - clearing the Initial Video removes its linked reference; + * - setting or re-trimming it re-derives the linked reference's default trim + * (`[end - 140, end]`) — a manually tuned trim therefore holds only until + * the next cutpoint change, which the section's help text says; + * - with no linked entry yet, an existing video reference for the same clip + * is adopted (recall restores the pair without the linkage flag; adopting + * avoids a duplicate), else a new one is PREPENDED — the continuity anchor + * leads the request order — unless the video cap is already full, in which + * case the list is returned unchanged and the extension simply runs without + * a tail reference. + */ +export const applyReferenceExtendSourceVideo = ( + references: VideoReferenceItem[], + sourceVideo: VideoSourceClip | null, + maxVideos: number +): VideoReferenceItem[] => { + if (!sourceVideo) { + const kept = references.filter((entry) => !(entry.kind === 'video' && entry.fromSourceVideo === true)); + + return kept.length === references.length ? references : kept; + } + + const linked: VideoReferenceItem = { + clip: deriveReferenceExtendClip(sourceVideo), + conditioning: 'video_audio', + fromSourceVideo: true, + kind: 'video', + }; + const linkedIndex = references.findIndex( + (entry) => + entry.kind === 'video' && (entry.fromSourceVideo === true || entry.clip.video_name === sourceVideo.video_name) + ); + + if (linkedIndex >= 0) { + return references.map((entry, index) => + index === linkedIndex && entry.kind === 'video' ? { ...linked, conditioning: entry.conditioning } : entry + ); + } + + const videoCount = references.filter((entry) => entry.kind === 'video').length; + + return videoCount >= maxVideos ? references : [linked, ...references]; +}; + /** * Clears conditioning media that no longer exists in the gallery. Returns the * input object untouched when nothing changes. diff --git a/invokeai/frontend/webv2/src/features/video/core/types.ts b/invokeai/frontend/webv2/src/features/video/core/types.ts index 09ec62cd988..b3bd80442bd 100644 --- a/invokeai/frontend/webv2/src/features/video/core/types.ts +++ b/invokeai/frontend/webv2/src/features/video/core/types.ts @@ -40,7 +40,20 @@ export type VideoReferenceImageDetail = 'max' | 'match'; * kind. A video reference reuses `VideoSourceClip` for its trim bounds. */ export type VideoReferenceItem = - | { kind: 'video'; clip: VideoSourceClip; conditioning: VideoReferenceConditioning } + | { + kind: 'video'; + clip: VideoSourceClip; + conditioning: VideoReferenceConditioning; + /** + * True on the reference the panel derives from the Initial Video in + * Ref2VA extend mode: it tracks that clip's identity, and its trim + * defaults re-derive from the cutpoint (`[end - 140, end]`, ~5s of + * lead-in at 24 fps) whenever the Initial Video trim changes. It is an + * ordinary reference otherwise — reorderable, trimmable, removable — + * and the flag is panel state only, never recorded in metadata. + */ + fromSourceVideo?: boolean; + } | { kind: 'image'; image: ImageWithDims; detail: VideoReferenceImageDetail }; export type WanTargetResolution = '480p' | '720p' | '1080p'; @@ -72,11 +85,17 @@ export interface VideoSettings { * either `firstFrameImage` or `sourceVideo`. */ lastFrameImage: ImageWithDims | null; - /** The clip to extend. Mutually exclusive with `firstFrameImage`. */ + /** + * The clip to extend. Mutually exclusive with `firstFrameImage`. On an + * FL2VA model this drives extend mode; on a Ref2VA model it coexists with + * `references` (reference-extend: the new clip is appended to it, and a + * linked tail reference provides continuity). + */ sourceVideo: VideoSourceClip | null; /** - * Ref2VA references, in conditioning order (up to 3 videos and 9 images). Mutually - * exclusive with `firstFrameImage`/`lastFrameImage`/`sourceVideo`; only a Ref2VA + * Ref2VA references, in conditioning order (up to 3 videos and 9 images). + * Mutually exclusive with `firstFrameImage`/`lastFrameImage`; `sourceVideo` + * may coexist on a Ref2VA model (reference-extend). Only a Ref2VA * transformer consumes them — see `resolveVideoMode` and the `reference` mode. */ references: VideoReferenceItem[]; diff --git a/invokeai/frontend/webv2/src/features/video/core/videoPolicies.test.ts b/invokeai/frontend/webv2/src/features/video/core/videoPolicies.test.ts index 2fe220b4cd8..5e552280a1d 100644 --- a/invokeai/frontend/webv2/src/features/video/core/videoPolicies.test.ts +++ b/invokeai/frontend/webv2/src/features/video/core/videoPolicies.test.ts @@ -1093,7 +1093,7 @@ describe('reference mode policy', () => { const settings = settingsFor(model); expect(getVideoModelPolicy(model, settings).modes).toEqual(['reference']); - expect(getVideoModelPolicy(model, settings).references).toEqual({ maxImages: 9, maxVideos: 3 }); + expect(getVideoModelPolicy(model, settings).references).toEqual({ extend: true, maxImages: 9, maxVideos: 3 }); expect(getVideoModelPolicy(h3Model(), settingsFor(h3Model())).references).toBeNull(); }); @@ -1247,3 +1247,64 @@ describe('H3 component-source seeding', () => { expect(explicit.settings.componentSourceModel?.key).toBe(componentsOnly.key); }); }); + +describe('reference-extend policy', () => { + const initialVideo = { + endFrame: 400, + fps: 24, + height: 480, + numFrames: 402, + startFrame: 0, + video_name: 'long.mp4', + width: 832, + }; + + it('accepts references alongside an initial video on ref2va, rejecting the pair elsewhere', () => { + const ref2va = ref2vaTransformer(); + const combined = settingsFor(ref2va, { + componentSourceModel: h3Model(), + references: [imageReference], + sourceVideo: initialVideo, + }); + + expect(getVideoValidationReasons(ref2va, combined)).toEqual([]); + + // FL2VA (and any non-reference model) still rejects the combination. + const fl2va = h3Model(); + const reasons = getVideoValidationReasons( + fl2va, + settingsFor(fl2va, { references: [imageReference], sourceVideo: initialVideo }) + ); + + expect(reasons).toContainEqual(expect.stringContaining('cannot be combined with an initial video on this model')); + }); + + it('an FL2VA -> Ref2VA switch keeps the initial video and derives its linked tail reference', () => { + const fl2va = h3Model(); + const toRef = getVideoModelSelectionResult({ + currentSettings: settingsFor(fl2va, { modelKey: fl2va.key, sourceVideo: initialVideo }), + model: ref2vaTransformer(), + models: [fl2va], + }); + + expect(toRef.settings.sourceVideo).toEqual(initialVideo); + expect(toRef.clearedLabels).not.toContain('Initial video'); + expect(toRef.settings.references[0]).toMatchObject({ + clip: { endFrame: 400, startFrame: 260, video_name: 'long.mp4' }, + conditioning: 'video_audio', + fromSourceVideo: true, + kind: 'video', + }); + + // And back: the references (linked one included) clear; the clip stays + // for FL2VA's own extend mode. + const backToFl = getVideoModelSelectionResult({ + currentSettings: toRef.settings, + model: fl2va, + models: [fl2va], + }); + + expect(backToFl.settings.references).toEqual([]); + expect(backToFl.settings.sourceVideo).toEqual(initialVideo); + }); +}); diff --git a/invokeai/frontend/webv2/src/features/video/core/videoPolicies.ts b/invokeai/frontend/webv2/src/features/video/core/videoPolicies.ts index 8bda24cadf9..56d32118e77 100644 --- a/invokeai/frontend/webv2/src/features/video/core/videoPolicies.ts +++ b/invokeai/frontend/webv2/src/features/video/core/videoPolicies.ts @@ -38,7 +38,12 @@ import { WAN_TI2V_PIXEL_MULTIPLE, type VideoDimensions, } from './dimensions'; -import { MIN_VIDEO_TRIM_FRAMES, resolveVideoMode, VIDEO_ASPECT_RATIO_IDS } from './settings'; +import { + applyReferenceExtendSourceVideo, + MIN_VIDEO_TRIM_FRAMES, + resolveVideoMode, + VIDEO_ASPECT_RATIO_IDS, +} from './settings'; // Video capabilities registry keyed by model base AND variant: unlike still-image // generation, Wan's variants differ structurally (which conditioning modes exist, @@ -118,7 +123,7 @@ interface VideoVariantConfig { accelerator: VideoAcceleratorConfig | null; audioOutput: boolean; /** Ref2VA reference caps; present only on variants whose modes include 'reference'. */ - references?: { maxVideos: number; maxImages: number }; + references?: { maxVideos: number; maxImages: number; extend?: boolean }; } export const WAN_LIGHTNING_ACCELERATOR: VideoAcceleratorConfig = { @@ -232,7 +237,10 @@ const MINIMAX_H3_REF2VA: VideoVariantConfig = { ...MINIMAX_H3_FL2VA, accelerator: MINIMAX_H3_REF2V_TURBO_ACCELERATOR, modes: ['reference'], - references: { maxImages: 9, maxVideos: 3 }, + // `extend` = reference-extend: an Initial Video the new clip is appended to, + // with a linked tail reference for continuity (no frame conditioning — the + // panel derives a reference from the clip instead). + references: { extend: true, maxImages: 9, maxVideos: 3 }, }; export const VIDEO_GENERATION: Record< @@ -455,7 +463,7 @@ export interface VideoModelPolicy { negativeHelpText?: string; }; /** Ref2VA reference caps; null unless the effective variant has a reference mode. */ - references: { maxVideos: number; maxImages: number } | null; + references: { maxVideos: number; maxImages: number; extend?: boolean } | null; ui: { cfgVisible: boolean; cfgLowNoiseVisible: boolean; @@ -1400,11 +1408,18 @@ export const getVideoModelSelectionResult = ({ addClearedLabel(clearedLabels, 'References'); } - if (next.sourceVideo && !modes.includes('extend')) { + if (next.sourceVideo && !modes.includes('extend') && !config.references?.extend) { next.sourceVideo = null; addClearedLabel(clearedLabels, 'Initial video'); } + // Reference-extend: a surviving Initial Video (e.g. carried over from an + // FL2VA extend setup) gets its linked tail reference derived/re-derived, so + // the switch lands on a generatable panel. + if (config.references?.extend && next.sourceVideo) { + next.references = applyReferenceExtendSourceVideo(next.references, next.sourceVideo, config.references.maxVideos); + } + if (next.firstFrameImage && !modes.includes('first-frame') && !modes.includes('first-last')) { next.firstFrameImage = null; addClearedLabel(clearedLabels, 'First frame'); @@ -1575,8 +1590,12 @@ export const getVideoValidationReasons = (model: MainModelConfig, settings: Vide reasons.push('A first frame and an initial video cannot be combined. Clear one of them.'); } - if (settings.references.length > 0 && (settings.firstFrameImage || settings.lastFrameImage || settings.sourceVideo)) { - reasons.push('References cannot be combined with first/last frames or an initial video. Clear one side.'); + if (settings.references.length > 0 && (settings.firstFrameImage || settings.lastFrameImage)) { + reasons.push('References cannot be combined with first/last frames. Clear one side.'); + } + + if (settings.references.length > 0 && settings.sourceVideo && !config.references?.extend) { + reasons.push('References cannot be combined with an initial video on this model. Clear one side.'); } if (!config.modes.includes(mode)) { diff --git a/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx b/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx index 438617a1c15..5af4792db69 100644 --- a/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx +++ b/invokeai/frontend/webv2/src/features/video/ui/VideoReferenceListField.tsx @@ -150,6 +150,11 @@ const ReferenceCard = memo(function ReferenceCard({ {reference.kind === 'video' ? : } + {reference.kind === 'video' && reference.fromSourceVideo === true ? ( + + {t('widgets.video.referenceFromInitialVideo')} + + ) : null}