feat(webv2): Ref2VA reference-extend — append the generation to an initial video

invokeai/frontend/webv2/public/locales/en.json

@@ -3110,6 +3110,7 @@
       "initialFrames": "Start & End Images",
       "initialVideo": "Initial Video",
       "initialVideoBlocked": "Clear the first frame to extend a video instead.",
+      "initialVideoCapBlocked": "All 3 video reference slots are in use, so there is no room for the continuity reference this clip needs. Remove a video reference first.",
       "lastFrame": "Last Frame",
       "lastFrameExtendHelp": "Optional destination: the extension ends on this image.",
       "lastFrameHelp": "Optional: the video ends on this image (interpolates from the first frame).",
@@ -3163,6 +3164,12 @@
       "referenceConditioningVideoAudio": "Video + audio",
       "referenceDetailMatch": "Match generation size",
       "referenceDetailMax": "Max detail (2048px, slower)",
+      "referenceExtendCapFull": "Initial video not set",
+      "referenceExtendCapFullDescription": "All 3 video reference slots are in use, so there is no room for the continuity reference this clip needs. Remove a video reference, then drop the clip again.",
+      "referenceExtendHelp": "The generated video is appended to this clip at its End Frame. A linked video reference samples up to the last ~5 seconds before that cutpoint for continuity — never more than the generated clip's own length, since the model discards the overrun at the seam. Its trim re-derives whenever the cutpoints or the frame count change.",
+      "referenceImageCapRace": "All {{max}} image reference slots filled while this one was loading, so it was not added.",
+      "referenceVideoCapRace": "All {{max}} video reference slots filled while this one was loading, so it was not added.",
+      "referenceFromInitialVideo": "Initial video",
       "references": "References",
       "referencesHelp": "Up to 3 videos and 9 images condition the generation, in order — reordering references changes the result. A video reference can contribute its image track, its soundtrack, or both.",
       "removeReference": "Remove reference",

invokeai/frontend/webv2/src/features/upscale/ui/UpscaleWidgetView.tsx

@@ -52,7 +52,7 @@ import { Combobox, Field, IconButton, Select, Tooltip } from '@platform/ui';
 import { SliderNumberField } from '@platform/ui/SliderNumberField';
 import { toaster } from '@platform/ui/toaster';
 import { DicesIcon } from 'lucide-react';
-import { memo, useCallback, useMemo } from 'react';
+import { memo, useCallback, useId, useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 
 import { areInputImagesEquivalent, valuesAreEqual } from './upscaleComparators';
@@ -284,6 +284,13 @@ export const UpscaleWidgetView = () => {
     [t, values]
   );
   const patch = useCallback((next: Partial<UpscaleWidgetValues>) => patchValues(next), [patchValues]);
+  // Chakra's `Field.Root` hands its single `ids.control` to EVERY control
+  // inside it, and the seed Field holds the NumberInput, the shuffle button
+  // and this switch. Without an id of its own the switch's hidden input
+  // collides with the seed input, so the `<label>` Switch.Root renders points
+  // at the seed field and clicking the toggle only moved focus.
+  const seedSwitchId = useId();
+  const seedSwitchIds = useMemo(() => ({ hiddenInput: `${seedSwitchId}-randomize-seed` }), [seedSwitchId]);
   const patchPromptDraft = useCallback((next: ProjectPromptDraftPatch) => patchDraft(next), [patchDraft]);
 
   useMountEffect(() => {
@@ -639,7 +646,12 @@ export const UpscaleWidgetView = () => {
                   <DicesIcon />
                 </IconButton>
               </Tooltip>
-              <Switch.Root checked={values.shouldRandomizeSeed} size="sm" onCheckedChange={set.randomizeSeed}>
+              <Switch.Root
+                checked={values.shouldRandomizeSeed}
+                ids={seedSwitchIds}
+                size="sm"
+                onCheckedChange={set.randomizeSeed}
+              >
                 <Switch.HiddenInput />
                 <Switch.Control _checked={SWITCH_CHECKED_PROPS}>
                   <Switch.Thumb />

invokeai/frontend/webv2/src/features/video/core/graph.test.ts

@@ -557,10 +557,108 @@ describe('compileVideoGraph — MiniMax H3 Ref2VA', () => {
     );
   });
 
+  it('reference-extend: only the linked tail window is anchored to the clip end', () => {
+    const linked = (clip: Record<string, unknown>, flag = true) => ({
+      clip: { fps: 24, height: 480, numFrames: 402, video_name: 'long.mp4', width: 832, ...clip },
+      conditioning: 'video_audio' as const,
+      ...(flag ? { fromSourceVideo: true } : {}),
+      kind: 'video' as const,
+    });
+    const startOf = (reference: unknown) =>
+      nodeOfType(
+        compileVideoGraph({ ...referenceSettings, references: [reference] } as never, model).backendGraph,
+        'minimax_h3_video_reference'
+      ).start_frame;
+
+    // A user's own reference keeps an absolute start: their trim is a position,
+    // not a length, and re-anchoring it would drift with the estimate.
+    expect(startOf(linked({ endFrame: 400, startFrame: 260 }, false))).toBe(260);
+    // A start at or inside the estimate's slop stays ABSOLUTE: the relative
+    // form resolves to `startFrame + (real - estimate)`, and the backend
+    // rejects a negative index rather than clamping, so an estimate that
+    // overshoots by more than `startFrame` would fail the whole generation.
+    expect(startOf(linked({ endFrame: 400, startFrame: 0 }))).toBe(0);
+    expect(startOf(linked({ endFrame: 400, startFrame: 1 }))).toBe(1);
+    expect(startOf(linked({ endFrame: 400, startFrame: 3 }))).toBe(3);
+    // Clear of the slop, the window rides the negative anchor again.
+    expect(startOf(linked({ endFrame: 400, startFrame: 4 }))).toBe(-398);
+    // A cutpoint far enough from the end that BOTH bounds keep the estimate.
+    expect(startOf(linked({ endFrame: 300, startFrame: 160 }))).toBe(160);
+    // The tail case: end went negative, so the start follows it.
+    expect(startOf(linked({ endFrame: 400, startFrame: 260 }))).toBe(-142);
+  });
+
   it('fl2va graphs are unchanged by the ref2va machinery', () => {
     const { backendGraph } = compileVideoGraph(settingsFor(componentSource), componentSource);
 
     expect(nodesOfType(backendGraph, 'minimax_h3_reference_conditioning')).toHaveLength(0);
     expect(nodesOfType(backendGraph, 'collect')).toHaveLength(0);
   });
+
+  it('reference-extend: appends the new clip to the initial video without frame conditioning', () => {
+    const initialVideo = {
+      endFrame: 400,
+      fps: 24,
+      height: 480,
+      numFrames: 402,
+      startFrame: 10,
+      video_name: 'long.mp4',
+      width: 832,
+    };
+    const settings = {
+      ...referenceSettings,
+      references: [
+        // The linked tail reference (as the setter derives it) plus a user reference.
+        {
+          clip: { ...initialVideo, endFrame: 400, startFrame: 260 },
+          conditioning: 'video_audio' as const,
+          fromSourceVideo: true,
+          kind: 'video' as const,
+        },
+        ...referenceSettings.references,
+      ],
+      sourceVideo: initialVideo,
+    };
+    const { backendGraph } = compileVideoGraph(settings, model);
+
+    // The new clip is intermediate; the crossfade concat is the output, fed
+    // [trimmed source, new clip]; the source is retimed to H3's fixed 24 fps.
+    expect(nodeOfType(backendGraph, 'minimax_h3_latents_to_video')).toMatchObject({
+      id: 'extension_clip',
+      is_intermediate: true,
+    });
+    expect(nodeOfType(backendGraph, 'video_concat')).toMatchObject({ id: 'video_output', transition: 'crossfade' });
+    expect(nodeOfType(backendGraph, 'extract_video_range')).toMatchObject({ end_frame: -2, fps: 24, start_frame: 10 });
+    expect(hasEdge(backendGraph, 'source_video', 'video', 'source_clip_collect', 'item')).toBe(true);
+    expect(hasEdge(backendGraph, 'extension_clip', 'video', 'clips_to_join', 'item')).toBe(true);
+
+    // Continuity comes from the references — no frame conditioning, no last-frame extraction.
+    expect(nodesOfType(backendGraph, 'minimax_h3_frame_conditioning')).toHaveLength(0);
+    expect(nodesOfType(backendGraph, 'video_frame_extract')).toHaveLength(0);
+
+    // The linked reference is an ordinary first reference; the flag never reaches metadata.
+    const videoReferences = nodesOfType(backendGraph, 'minimax_h3_video_reference');
+
+    // Both bounds ride the SAME negative anchor, so the extracted window keeps
+    // its exact length whatever the clip's real frame count turns out to be.
+    // A positive start would have made it `tail + (real - estimate)` frames,
+    // and the overrun is discarded at the seam.
+    expect(videoReferences[0]).toMatchObject({ end_frame: -2, id: 'reference_1', start_frame: -142 });
+    expect((videoReferences[0].end_frame as number) - (videoReferences[0].start_frame as number)).toBe(140);
+    const metadata = nodeOfType(backendGraph, 'core_metadata');
+
+    expect(metadata.generation_mode).toBe('minimax_h3_ref2v');
+    expect(metadata).toMatchObject({
+      source_video: { video_name: 'long.mp4' },
+      source_video_end_frame: 400,
+      source_video_start_frame: 10,
+    });
+    expect((metadata.minimax_h3_references as Record<string, unknown>[])[0]).toEqual({
+      conditioning: 'video_audio',
+      end_frame: 260 + 140,
+      kind: 'video',
+      start_frame: 260,
+      video_name: 'long.mp4',
+    });
+  });
 });

invokeai/frontend/webv2/src/features/video/core/graph.ts

@@ -66,24 +66,7 @@ const addReferenceNodes = (
 ): BackendInvocationContract => {
   let chain: BackendInvocationContract | null = null;
   references.forEach((reference, index) => {
-    const node =
-      reference.kind === 'image'
-        ? addNode(graph, {
-            detail: reference.detail,
-            id: `reference_${index + 1}`,
-            image: toImageField(reference.image),
-            type: 'minimax_h3_image_reference',
-          })
-        : addNode(graph, {
-            conditioning: reference.conditioning,
-            // Same estimate-overshoot protection as the extend path: tail-window bounds
-            // compile as negative indices the backend resolves against the REAL count.
-            end_frame: toTailAwareIndex(reference.clip.endFrame, reference.clip.numFrames),
-            id: `reference_${index + 1}`,
-            start_frame: toTailAwareIndex(reference.clip.startFrame, reference.clip.numFrames),
-            type: 'minimax_h3_video_reference',
-            video: { video_name: reference.clip.video_name },
-          });
+    const node = addReferenceNode(graph, reference, index);
     const collect = addNode(graph, { id: `reference_collect_${index + 1}`, type: 'collect' });
 
     if (chain) {
@@ -100,6 +83,78 @@ const addReferenceNodes = (
   return chain;
 };
 
+const addReferenceNode = (
+  graph: BackendGraphContract,
+  reference: VideoReferenceItem,
+  index: number
+): BackendInvocationContract => {
+  if (reference.kind === 'image') {
+    return addNode(graph, {
+      detail: reference.detail,
+      id: `reference_${index + 1}`,
+      image: toImageField(reference.image),
+      type: 'minimax_h3_image_reference',
+    });
+  }
+  // Same estimate-overshoot protection as the extend path: tail-window bounds
+  // compile as negative indices the backend resolves against the REAL count.
+  const endFrame = toTailAwareIndex(reference.clip.endFrame, reference.clip.numFrames);
+
+  return addNode(graph, {
+    conditioning: reference.conditioning,
+    end_frame: endFrame,
+    id: `reference_${index + 1}`,
+    start_frame: toReferenceStartIndex(reference, endFrame),
+    type: 'minimax_h3_video_reference',
+    video: { video_name: reference.clip.video_name },
+  });
+};
+
+/**
+ * A video reference's start bound, measured from the same end as its other bound.
+ *
+ * `toTailAwareIndex` sends a near-the-end bound out NEGATIVE, resolved against
+ * the clip's REAL frame count, and leaves anything further back POSITIVE, taken
+ * from the panel's ESTIMATE. That split is right for a hand-picked trim, whose
+ * start is an absolute position the estimate must not drift — but the linked
+ * tail window straddles it: the cutpoint end goes negative while the start,
+ * ~124 frames back, stays positive, so what the backend extracts is
+ * `tail + (real - estimate)` frames rather than `tail`. That length is a budget
+ * the backend enforces by discarding the overrun at the SEAM (see
+ * `deriveReferenceExtendClip`), so it has to survive an inexact estimate.
+ *
+ * The linked entry's start is not an absolute pick — it is defined as
+ * `tail - 1` frames before the cutpoint — so it rides the same negative anchor
+ * and the window keeps its length whatever the real count turns out to be.
+ *
+ * Two cases stay absolute. A cutpoint far enough from the end leaves both
+ * bounds on the estimate already. And a start within `TAIL_INDEX_SLOP` of the
+ * clip's own beginning stays absolute because the relative form resolves to
+ * `startFrame + (real - estimate)`, which goes NEGATIVE once the estimate
+ * overshoots by more than `startFrame` — and `_ResolvedVideoRange.resolve`
+ * rejects an out-of-range index outright rather than clamping, failing the
+ * whole generation. That can only arise when the window fills nearly the entire
+ * clip, where its length cannot be honoured anyway; below the slop the absolute
+ * form is always in range, and the drift it costs is the estimate error itself,
+ * a frame or two.
+ *
+ * DECIDED: the slop is not widened beyond 3. An estimate error past the slop
+ * can still fail the relative form, but only on a clip barely longer than the
+ * window (it needs `error > startFrame`), and `TAIL_INDEX_SLOP` is this file's
+ * declared bound on how wrong the estimate gets. A wider guard would not
+ * remove the cliff — it would move it, and pay for the move with silent length
+ * drift on every clip inside the wider margin.
+ */
+const toReferenceStartIndex = (reference: Extract<VideoReferenceItem, { kind: 'video' }>, endIndex: number): number => {
+  const { clip } = reference;
+
+  if (reference.fromSourceVideo !== true || endIndex >= 0 || clip.startFrame <= TAIL_INDEX_SLOP) {
+    return toTailAwareIndex(clip.startFrame, clip.numFrames);
+  }
+
+  return endIndex - (clip.endFrame - clip.startFrame);
+};
+
 const addPromptAndSeedNodes = (graph: BackendGraphContract) => ({
   negativePrompt: addNode(graph, { id: 'negative_prompt', type: 'string' }),
   positivePrompt: addNode(graph, { id: 'positive_prompt', type: 'string' }),
@@ -125,13 +180,25 @@ const toImageField = (image: { image_name: string }) => ({ image_name: image.ima
 // start is always below the end, so the pair keeps its order when both go
 // negative). Mid-clip picks stay positive: a negative offset computed from
 // an overshooting estimate would drift them instead.
+/**
+ * How wrong the panel's `duration x fps` frame-count estimate is allowed to be.
+ * VFR containers overshoot it by a frame or two, so bounds within this many
+ * frames of the estimated end are emitted relative to the REAL end instead.
+ */
+const TAIL_INDEX_SLOP = 3;
+
 const toTailAwareIndex = (frame: number, estimatedNumFrames: number): number => {
   const tailOffset = estimatedNumFrames - 1 - frame;
 
-  return tailOffset <= 3 ? -(tailOffset + 1) : frame;
+  return tailOffset <= TAIL_INDEX_SLOP ? -(tailOffset + 1) : frame;
 };
 
-const addExtendScaffolding = (
+/**
+ * Trimmed source extraction + [source, new clip] crossfade join — the shared
+ * core of FL2VA/Wan extend and Ref2VA reference-extend. video_concat rebuilds
+ * the soundtrack from every input, so both clips' audio survives the join.
+ */
+const addSourceJoin = (
   graph: BackendGraphContract,
   sourceVideo: NonNullable<VideoSettings['sourceVideo']>,
   newClip: BackendInvocationContract,
@@ -147,13 +214,6 @@ const addExtendScaffolding = (
     video: { video_name: sourceVideo.video_name },
     ...(options.extractFps === undefined ? {} : { fps: options.extractFps }),
   });
-  const lastFrame = addNode(graph, {
-    frame_index: -1,
-    id: 'source_last_frame',
-    is_intermediate: true,
-    type: 'video_frame_extract',
-    use_cache: false,
-  });
   const sourceCollect = addNode(graph, { id: 'source_clip_collect', type: 'collect' });
   const clipsCollect = addNode(graph, { id: 'clips_to_join', type: 'collect' });
   const concat = addNode(graph, {
@@ -166,13 +226,32 @@ const addExtendScaffolding = (
     use_cache: false,
   });
 
-  addEdge(graph, extract, 'video', lastFrame, 'video');
   // Chained collectors keep the join order deterministic: [trimmed source, new clip].
   addEdge(graph, extract, 'video', sourceCollect, 'item');
   addEdge(graph, sourceCollect, 'collection', clipsCollect, 'collection');
   addEdge(graph, newClip, 'video', clipsCollect, 'item');
   addEdge(graph, clipsCollect, 'collection', concat, 'videos');
 
+  return { concat, extract };
+};
+
+const addExtendScaffolding = (
+  graph: BackendGraphContract,
+  sourceVideo: NonNullable<VideoSettings['sourceVideo']>,
+  newClip: BackendInvocationContract,
+  options: { extractFps?: number } = {}
+) => {
+  const { concat, extract } = addSourceJoin(graph, sourceVideo, newClip, options);
+  const lastFrame = addNode(graph, {
+    frame_index: -1,
+    id: 'source_last_frame',
+    is_intermediate: true,
+    type: 'video_frame_extract',
+    use_cache: false,
+  });
+
+  addEdge(graph, extract, 'video', lastFrame, 'video');
+
   return { concat, extract, lastFrame };
 };
 
@@ -527,6 +606,7 @@ const buildMiniMaxH3VideoGraph = (settings: VideoSettings, model: MainModelConfi
 
   let output: BackendInvocationContract;
   let extendParts: { lastFrame: BackendInvocationContract; newClip: BackendInvocationContract } | null = null;
+  let referenceExtendClip: BackendInvocationContract | null = null;
 
   if (mode === 'extend' && settings.sourceVideo && frameConditioning) {
     const newClip = addLatentsToVideo('extension_clip', true);
@@ -538,6 +618,15 @@ const buildMiniMaxH3VideoGraph = (settings: VideoSettings, model: MainModelConfi
     addEdge(graph, lastFrame, 'image', frameConditioning, 'first_image');
     output = concat;
     extendParts = { lastFrame, newClip };
+  } else if (mode === 'reference' && settings.sourceVideo) {
+    // Reference-extend: the new clip is appended to the trimmed Initial
+    // Video. Continuity comes from the references (typically the linked tail
+    // reference), not from frame conditioning, so no last frame is extracted.
+    const newClip = addLatentsToVideo('extension_clip', true);
+    const { concat } = addSourceJoin(graph, settings.sourceVideo, newClip, { extractFps: 24 });
+
+    output = concat;
+    referenceExtendClip = newClip;
   } else {
     output = addLatentsToVideo('video_output', false);
   }
@@ -567,7 +656,11 @@ const buildMiniMaxH3VideoGraph = (settings: VideoSettings, model: MainModelConfi
     height: dimensions.height,
     model,
     negativeWired: false,
-    outputs: extendParts ? [output, extendParts.newClip] : [output],
+    outputs: extendParts
+      ? [output, extendParts.newClip]
+      : referenceExtendClip
+        ? [output, referenceExtendClip]
+        : [output],
     settings,
     width: dimensions.width,
   });