Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 26 additions & 53 deletions server/services/videoGen/generateVideo.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import { getVideoModels, getDefaultVideoModelId, getTextEncoderRepo } from '../.
import { hardwareUnavailableReason, isHardwareCompatible } from '../../lib/systemCapabilities.js';
import { resolveVideoModelSelection } from './modelSelection.js';
import { findFfmpeg, findFfprobe } from '../../lib/ffmpeg.js';
import { inspectModelCache, findCachedRepoFile, findCachedRepoFiles } from '../../lib/hfCache.js';
import { findCachedRepoFile, findCachedRepoFiles } from '../../lib/hfCache.js';
import { safeChildProcessOptions } from '../../lib/processEnv.js';
import { describeRenderConditioning, RENDER_INPUTS_VERSION } from './generateVideoHelpers.js';
import { readTriggerWordsByFilename, readLoraLicensesByFilename } from '../loras.js';
Expand Down Expand Up @@ -66,6 +66,7 @@ import {
resolveT2vTwoStageOverride,
} from './renderArgs.js';
import { spawnAndWatchVideo } from './spawnWatch.js';
import { resolvePinnedSnapshotPath } from './pinnedSnapshot.js';
import {
resolveVideoSpeedProfile, speedProfileDeclineReason, resolveVideoSampler,
inferEffectiveVideoMode,
Expand Down Expand Up @@ -207,20 +208,11 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', m
{ status: 400, code: 'WAN22_INVALID_FRAME_COUNT' },
);
}
if (typeof model.revision !== 'string' || !model.revision) {
throw new ServerError(
`Wan model "${modelId}" is missing an immutable Hugging Face revision.`,
{ status: 500, code: 'VIDEO_MODEL_MISCONFIGURED' },
);
}
const baseCache = await inspectModelCache(model.repo, { revision: model.revision });
if (!baseCache.cached || !baseCache.snapshotPath) {
throw new ServerError(
`${model.name} revision ${model.revision.slice(0, 8)} is not fully cached. Download or repair it in Video Gen before rendering.`,
{ status: 400, code: 'WAN22_MODEL_NOT_CACHED' },
);
}
wanModelPath = baseCache.snapshotPath;
wanModelPath = await resolvePinnedSnapshotPath(model, {
notCachedCode: 'WAN22_MODEL_NOT_CACHED',
onMissingRevision: 'throw',
missingRevisionMessage: `Wan model "${modelId}" is missing an immutable Hugging Face revision.`,
});
for (const dep of Array.isArray(model.requiredWeights) ? model.requiredWeights : []) {
const files = Array.isArray(dep?.files) ? dep.files : [];
const roles = Array.isArray(dep?.targetRoles) ? dep.targetRoles : [];
Expand All @@ -244,54 +236,34 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', m
}
}
}
let fastvideoModelPath = null;
if (model.runtime === 'fastvideo') {
if (typeof model.revision === 'string' && model.revision) {
const baseCache = await inspectModelCache(model.repo, { revision: model.revision });
if (!baseCache.cached || !baseCache.snapshotPath) {
throw new ServerError(
`${model.name} revision ${model.revision.slice(0, 8)} is not fully cached. Download or repair it in Video Gen before rendering.`,
{ status: 400, code: 'FASTVIDEO_MODEL_NOT_CACHED' },
);
}
wanModelPath = baseCache.snapshotPath;
} else {
const baseCache = await inspectModelCache(model.repo);
if (baseCache.cached && baseCache.snapshotPath) {
wanModelPath = baseCache.snapshotPath;
}
}
fastvideoModelPath = await resolvePinnedSnapshotPath(model, {
notCachedCode: 'FASTVIDEO_MODEL_NOT_CACHED',
onMissingRevision: 'best-effort',
// A null miss preserves the legacy handoff: buildFastVideoArgs falls
// back to the repo id only when no unpinned snapshot is resident.
});
}
// Pinned LTX family entries (LTX-2.5 today) must render the verified
// snapshot, not whatever `main` snapshot_download would follow. Unpinned
// 2.3 entries keep passing the repo id so the helper's existing Hub resolve
// stays unchanged.
let ltxModelPath = model.repo;
if (isLtx2FamilyRuntime(model.runtime) && typeof model.revision === 'string' && model.revision) {
const cache = await inspectModelCache(model.repo, { revision: model.revision });
if (!cache.cached || !cache.snapshotPath) {
throw new ServerError(
`${model.name} revision ${model.revision.slice(0, 8)} is not fully cached. Download or repair it in Video Gen before rendering.`,
{ status: 400, code: 'LTX2_MODEL_NOT_CACHED' },
);
}
ltxModelPath = cache.snapshotPath;
if (isLtx2FamilyRuntime(model.runtime)) {
ltxModelPath = await resolvePinnedSnapshotPath(model, {
notCachedCode: 'LTX2_MODEL_NOT_CACHED',
onMissingRevision: 'passthrough-repo',
fallback: model.repo,
});
}
let ref2vaModelPath = null;
if (model.runtime === 'minimax_h3_ref2va') {
if (typeof model.revision !== 'string' || !model.revision) {
throw new ServerError(
`MiniMax H3 Ref2VA model "${modelId}" is missing an immutable Hugging Face revision.`,
{ status: 500, code: 'VIDEO_MODEL_MISCONFIGURED' },
);
}
const cache = await inspectModelCache(model.repo, { revision: model.revision });
if (!cache.cached || !cache.snapshotPath) {
throw new ServerError(
`${model.name} revision ${model.revision.slice(0, 8)} is not fully cached. Download or repair it in Video Gen before rendering.`,
{ status: 400, code: 'MINIMAX_H3_REF2VA_MODEL_NOT_CACHED' },
);
}
ref2vaModelPath = cache.snapshotPath;
ref2vaModelPath = await resolvePinnedSnapshotPath(model, {
notCachedCode: 'MINIMAX_H3_REF2VA_MODEL_NOT_CACHED',
onMissingRevision: 'throw',
missingRevisionMessage: `MiniMax H3 Ref2VA model "${modelId}" is missing an immutable Hugging Face revision.`,
});
}
// Substituted prompt conditioner (#4081). `resolveVideoTextEncoder` returns
// null for the stock choice — the whole override path stays dormant then —
Expand Down Expand Up @@ -844,6 +816,7 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', m
modelId,
model: loraCapableModel,
wanModelPath,
fastvideoModelPath,
wanRequiredWeights,
ltxModelPath,
ref2vaModelPath,
Expand Down
144 changes: 144 additions & 0 deletions server/services/videoGen/local.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,25 @@ vi.mock('../../lib/mediaModels.js', async () => {
termsGate: { id: 'minimax-h3-community-license-2026-08-02' },
memoryProfiles: [{ id: 'int8-lean', name: 'int8, leaf-level', minMemoryGb: 1, minVramGb: 12, unified: false }],
},
{
id: 'minimax_h3_ref2va_8bit', name: 'MiniMax H3 Ref2VA MLX 8-bit', runtime: 'minimax_h3_ref2va',
repo: 'Sawfwair/MiniMax-H3-Ref2VA-MLX-8bit',
revision: '61dc387ef1a7166425cdacd63c2340598dcc364f',
supportedModes: ['a2v'], requiresSourceImageForA2v: true,
defaultFrames: 124, frameOptions: [107, 124, 141, 158], fpsOptions: [24],
defaultWidth: 512, defaultHeight: 320, resolutionStep: 32,
steps: 9, guidance: 0, samplerLocked: true,
supportsNegativePrompt: false, supportsTiling: false, supportsDisableAudio: false,
},
{
id: 'fasth3_dense_datafree_int8', name: 'FastH3 Preview v1 Dense Data-Free', runtime: 'fastvideo',
repo: 'FastVideo/FastVideo-FastH3-4-step-Preview-v1-Dense-DataFree',
revision: 'f624f08c6c279ab43534c003e556fc5b295b6558',
fastvideoFamily: 'fasth3', fastvideoMlxFormat: 'int8', supportedModes: ['text'],
defaultWidth: 832, defaultHeight: 480, defaultFrames: 124,
frameOptions: [107, 124, 141, 158], fpsOptions: [24],
steps: 4, guidance: 1, samplerLocked: true, supportsNegativePrompt: false,
},
{
id: 'ltx25_cuda_distilled', name: 'LTX-2.5 CUDA Distilled', runtime: 'ltx25_cuda',
repo: 'Lightricks/LTX-2.5',
Expand Down Expand Up @@ -1970,6 +1989,131 @@ describe('generateVideo — LTX-2.5 sibling runtime spawn', () => {
});
});

describe('generateVideo — pinned snapshot policies', () => {
const withModel = async (modelId, update, run) => {
const mediaModels = await import('../../lib/mediaModels.js');
const getVideoModelsMock = vi.mocked(mediaModels.getVideoModels);
const catalog = getVideoModelsMock();
getVideoModelsMock.mockReturnValue(catalog.map((model) => (
model.id === modelId ? update(model) : model
)));
try {
return await run();
} finally {
getVideoModelsMock.mockReturnValue(catalog);
}
};

const fastvideoRender = (fields = {}) => generateVideo({
jobId: 'fastvideo-snapshot-policy',
modelId: 'fasth3_dense_datafree_int8',
prompt: 'a quiet street at dusk',
width: 832, height: 480, numFrames: 124, fps: 24, mode: 'text',
...fields,
});

const ref2vaRender = () => generateVideo({
jobId: 'ref2va-snapshot-policy',
modelId: 'minimax_h3_ref2va_8bit',
prompt: 'a fox listens to the rain',
width: 512, height: 320, numFrames: 124, fps: 24, mode: 'a2v',
sourceImagePath: '/mock/source.png', audioFilePath: '/mock/audio.wav',
});

it('passes a cached pinned FastVideo snapshot to the runner', async () => {
const { spawnDetached } = await import('../../lib/detachedSpawn.js');
const spawnMock = vi.mocked(spawnDetached);
spawnMock.mockClear();
mockInspectModelCache.mockResolvedValueOnce({
cached: true,
snapshotPath: '/mock/hf/fastvideo-snapshot',
sizeBytes: 1000,
});

await fastvideoRender();

expect(mockInspectModelCache).toHaveBeenCalledWith(
'FastVideo/FastVideo-FastH3-4-step-Preview-v1-Dense-DataFree',
{ revision: 'f624f08c6c279ab43534c003e556fc5b295b6558' },
);
const call = spawnMock.mock.calls.find(([, args]) => (
Array.isArray(args) && args.some((arg) => basename(String(arg)) === 'generate_fastvideo.py')
));
expect(call).toBeDefined();
expect(call[1][call[1].indexOf('--model-root') + 1])
.toBe('/mock/hf/fastvideo-snapshot');
});

it('rejects a pinned FastVideo model when its snapshot is not cached', async () => {
const { spawnDetached } = await import('../../lib/detachedSpawn.js');
vi.mocked(spawnDetached).mockClear();
mockInspectModelCache.mockResolvedValueOnce({ cached: false, snapshotPath: null, sizeBytes: 0 });

await expect(fastvideoRender()).rejects.toMatchObject({
message: 'FastH3 Preview v1 Dense Data-Free revision f624f08c is not fully cached. Download or repair it in Video Gen before rendering.',
status: 400,
code: 'FASTVIDEO_MODEL_NOT_CACHED',
});
expect(mockInspectModelCache).toHaveBeenCalledWith(
'FastVideo/FastVideo-FastH3-4-step-Preview-v1-Dense-DataFree',
{ revision: 'f624f08c6c279ab43534c003e556fc5b295b6558' },
);
expect(spawnDetached).not.toHaveBeenCalled();
});

it('keeps an unpinned, uncached FastVideo model on its best-effort repo fallback', async () => {
const { spawnDetached } = await import('../../lib/detachedSpawn.js');
const spawnMock = vi.mocked(spawnDetached);
spawnMock.mockClear();
mockInspectModelCache.mockResolvedValueOnce({ cached: false, snapshotPath: null, sizeBytes: 0 });

await withModel('fasth3_dense_datafree_int8', ({ revision: _revision, ...model }) => model, fastvideoRender);

expect(mockInspectModelCache).toHaveBeenCalledWith(
'FastVideo/FastVideo-FastH3-4-step-Preview-v1-Dense-DataFree',
);
const call = spawnMock.mock.calls.find(([, args]) => (
Array.isArray(args) && args.some((arg) => basename(String(arg)) === 'generate_fastvideo.py')
));
expect(call).toBeDefined();
expect(call[1][call[1].indexOf('--model-root') + 1])
.toBe('FastVideo/FastVideo-FastH3-4-step-Preview-v1-Dense-DataFree');
});

it('rejects a Ref2VA model with no immutable revision', async () => {
const { spawnDetached } = await import('../../lib/detachedSpawn.js');
vi.mocked(spawnDetached).mockClear();

await withModel(
'minimax_h3_ref2va_8bit',
({ revision: _revision, ...model }) => model,
async () => expect(ref2vaRender()).rejects.toMatchObject({
message: 'MiniMax H3 Ref2VA model "minimax_h3_ref2va_8bit" is missing an immutable Hugging Face revision.',
status: 500,
code: 'VIDEO_MODEL_MISCONFIGURED',
}),
);
expect(spawnDetached).not.toHaveBeenCalled();
});

it('rejects a pinned Ref2VA model when its snapshot is not cached', async () => {
const { spawnDetached } = await import('../../lib/detachedSpawn.js');
vi.mocked(spawnDetached).mockClear();
mockInspectModelCache.mockResolvedValueOnce({ cached: false, snapshotPath: null, sizeBytes: 0 });

await expect(ref2vaRender()).rejects.toMatchObject({
message: 'MiniMax H3 Ref2VA MLX 8-bit revision 61dc387e is not fully cached. Download or repair it in Video Gen before rendering.',
status: 400,
code: 'MINIMAX_H3_REF2VA_MODEL_NOT_CACHED',
});
expect(mockInspectModelCache).toHaveBeenCalledWith(
'Sawfwair/MiniMax-H3-Ref2VA-MLX-8bit',
{ revision: '61dc387ef1a7166425cdacd63c2340598dcc364f' },
);
expect(spawnDetached).not.toHaveBeenCalled();
});
});

describe('FFLF/ltx2 pixel-budget helpers', () => {
const DEFAULT_BUDGET = 704 * 448 * 25; // 7,884,800 — 48 GB floor
const BUDGET_128GB = 768 * 512 * 97; // 38,141,952 — 128 GB anchor
Expand Down
40 changes: 40 additions & 0 deletions server/services/videoGen/pinnedSnapshot.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/**
* Resolve the local path for a model whose registry entry may pin an immutable
* Hugging Face revision. Each runtime call site selects its missing-revision
* policy explicitly because the legacy behaviors intentionally differ.
*/

import { ServerError } from '../../lib/errorHandler.js';
import { inspectModelCache } from '../../lib/hfCache.js';

export async function resolvePinnedSnapshotPath(model, {
notCachedCode,
onMissingRevision,
fallback = null,
missingRevisionMessage = null,
}) {
const hasPinnedRevision = typeof model.revision === 'string' && model.revision;
if (!hasPinnedRevision) {
if (onMissingRevision === 'throw') {
throw new ServerError(missingRevisionMessage, {
status: 500,
code: 'VIDEO_MODEL_MISCONFIGURED',
});
}
if (onMissingRevision === 'passthrough-repo') return fallback;
if (onMissingRevision === 'best-effort') {
const cache = await inspectModelCache(model.repo);
return cache.cached && cache.snapshotPath ? cache.snapshotPath : fallback;
}
throw new TypeError(`Unknown missing-revision policy: ${onMissingRevision}`);
}

const cache = await inspectModelCache(model.repo, { revision: model.revision });
if (!cache.cached || !cache.snapshotPath) {
throw new ServerError(
`${model.name} revision ${model.revision.slice(0, 8)} is not fully cached. Download or repair it in Video Gen before rendering.`,
{ status: 400, code: notCachedCode },
);
}
return cache.snapshotPath;
}
4 changes: 2 additions & 2 deletions server/services/videoGen/renderArgs.js
Original file line number Diff line number Diff line change
Expand Up @@ -1126,7 +1126,7 @@ export const buildMiniMaxH3Ref2vaArgs = ({
return { bin: process.execPath, args };
};

export const buildArgs = ({ upscale, pythonPath, modelId, model, wanModelPath, wanRequiredWeights, ltxModelPath, ref2vaModelPath, prompt, negativePrompt, width, height, numFrames, fps, steps, stage2Steps, guidance, seed, tiling, disableAudio, sourceImagePath, lastImagePath, keyframes, extendFromVideoPath, audioFilePath, audioStartSec, mode, imageStrength, i2vReferenceMode, textEncoderRepo, textEncoder, outputPath, previewDir, loras, icReferencePaths, icLoraWeightPath, icStrength, icAttentionStrength, icSkipStage2, speedProfile, draftDecoder, streamingMode, ffmpegPath, ffprobePath }) => {
export const buildArgs = ({ upscale, pythonPath, modelId, model, wanModelPath, fastvideoModelPath, wanRequiredWeights, ltxModelPath, ref2vaModelPath, prompt, negativePrompt, width, height, numFrames, fps, steps, stage2Steps, guidance, seed, tiling, disableAudio, sourceImagePath, lastImagePath, keyframes, extendFromVideoPath, audioFilePath, audioStartSec, mode, imageStrength, i2vReferenceMode, textEncoderRepo, textEncoder, outputPath, previewDir, loras, icReferencePaths, icLoraWeightPath, icStrength, icAttentionStrength, icSkipStage2, speedProfile, draftDecoder, streamingMode, ffmpegPath, ffprobePath }) => {
// Generative upscale (#6511) declines FIRST. It is not a text/image render:
// it carries no video model, no prompt and no reference mode, so every guard
// below would either dereference a model it was never given or reject it for
Expand Down Expand Up @@ -1186,7 +1186,7 @@ export const buildArgs = ({ upscale, pythonPath, modelId, model, wanModelPath, w
: videoLoraUnsupportedError(model, modelId);
}
if (model.runtime === 'fastvideo') {
return buildFastVideoArgs({ model, fastvideoModelPath: wanModelPath, prompt, negativePrompt, width, height, numFrames, fps, steps, guidance, seed, sourceImagePath, mode, outputPath });
return buildFastVideoArgs({ model, fastvideoModelPath, prompt, negativePrompt, width, height, numFrames, fps, steps, guidance, seed, sourceImagePath, mode, outputPath });
}
if (model.runtime === 'wan22') {
return buildWan22Args({ model, wanModelPath, wanRequiredWeights, prompt, negativePrompt, width, height, numFrames, fps, steps, guidance, seed, sourceImagePath, mode, outputPath });
Expand Down