From 7d4c759f8a494ae5ed5d945f0593a8335654e365 Mon Sep 17 00:00:00 2001 From: vveerrgg Date: Mon, 9 Nov 2026 12:39:24 -0700 Subject: [PATCH 1/5] Release tooling: install-path test (clean ComfyUI, CPU), one release command, site deploy on publish, stability note; private-methods scrub; demo zip re-stamped to a publishable 0.1.1 - scripts/install_path_check.py + .github/workflows/install-path.yml: every LSDub node registers in ComfyUI v0.42.0 with only the user requirements; Align ctc from an empty models folder (four pinned files, no safetensors, no phonemizer); a Dub at defaults; a Dub with the four mouth Combos. PRs to main, release/** pushes, weekly, by hand; publish.yml calls it first. - scripts/release.py X.Y.Z [--dry-run]: CONTRIBUTING, Releases. - publish.yml: dispatches lsdub-release to the microsite after a good publish (skips with a notice without MICROSITE_DISPATCH_TOKEN). - tests/test_registry_rules.py: refuses ledger dirs, docs/research|plans and the private lane names in the shipped tree. - fetch_demo.py pointed at a zip that was never published (pinned a431085d, the v0.1.0 asset is 7981fef8): every user's demo fetch failed the checksum. Re-rooted as lsdub-demo-0.1.1 (same files, reproducible build), tag demo-0.1.1 (to be attached). - Engine tree f36b7410 vendored (methods scrub; two comments reworded). --- .github/workflows/ci.yml | 2 +- .github/workflows/install-path.yml | 73 +++++++ .github/workflows/publish.yml | 21 ++ CONTRIBUTING.md | 29 +++ README.md | 21 +- lsdub_engine/VERSION | 4 +- lsdub_engine/audits/qa.py | 2 +- scripts/fetch_demo.py | 6 +- scripts/install_path_check.py | 324 +++++++++++++++++++++++++++++ scripts/release.py | 261 +++++++++++++++++++++++ tests/engine/test_watermark.py | 2 +- tests/test_registry_rules.py | 26 ++- 12 files changed, 760 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/install-path.yml create mode 100644 scripts/install_path_check.py create mode 100644 scripts/release.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43dff2e..c6c6a06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: pull_request: push: - branches: [main, rack] + branches: [main, rack, "release/**"] permissions: contents: read jobs: diff --git a/.github/workflows/install-path.yml b/.github/workflows/install-path.yml new file mode 100644 index 0000000..28c36cd --- /dev/null +++ b/.github/workflows/install-path.yml @@ -0,0 +1,73 @@ +# The install-path test: LSDub in a clean ComfyUI the way a user gets it (ComfyUI's requirements + requirements.txt, no dev +# extras), CPU only. Every LSDub node registers; Align with aligner ctc from an empty models folder (the real recogniser +# download); a Dub at defaults; a Dub with the mouth Combos. scripts/install_path_check.py does the work; it runs the same on a +# laptop. publish.yml calls this workflow first, so a tag that fails it is never published. +name: Install path +on: + pull_request: + branches: [main] + push: + branches: ["release/**"] # scripts/release.py waits for this run before it tags + workflow_dispatch: + workflow_call: + schedule: + - cron: "17 6 * * 1" # weekly: ComfyUI's pin stays put, but PyPI and the Hub do not +permissions: + contents: read +concurrency: + group: install-path-${{ github.ref }} + cancel-in-progress: true +env: + COMFYUI_REPO: https://github.com/comfyanonymous/ComfyUI.git + COMFYUI_COMMIT: 355cd6b14c92c19ac3188fe528e47a3f21ef5178 # v0.42.0 + RECOGNISER_REVISION: ae45363bf3413b374fecd9dc8bc1df0e24c3b7f4 # facebook/wav2vec2-lv-60-espeak-cv-ft, as lsdub_engine/_models.py pins it + # only used if scripts/fetch_demo.py itself fails (that still fails the job): the released v0.1.0 demo zip at its real checksum + DEMO_FALLBACK_URL: https://github.com/HumanjavaEnterprises/comfyui-lsdub/releases/download/v0.1.0/lsdub-demo-0.1.0.zip + DEMO_FALLBACK_SHA: 7981fef8f453b0bf29e47b3ffc00b3adcba1d22d8dbe259504f66c2beb00de17 +jobs: + install-path: + runs-on: ubuntu-latest + timeout-minutes: 150 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + path: pack + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - name: System deps (what the README asks a user for) + run: sudo apt-get update && sudo apt-get install -y ffmpeg libgles2 libegl1 + - name: The pinned recogniser revision is the one the pack pins + run: grep -q "$RECOGNISER_REVISION" pack/lsdub_engine/_models.py + - name: ComfyUI at the pinned commit + run: | + git init -q comfy && cd comfy + git fetch -q --depth 1 "$COMFYUI_REPO" "$COMFYUI_COMMIT" && git checkout -q FETCH_HEAD + - name: A user's venv (ComfyUI requirements + requirements.txt, CPU torch) + run: | + python -m venv venv + venv/bin/pip install -q --upgrade pip + venv/bin/pip install -q torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + venv/bin/pip install -q -r comfy/requirements.txt -r pack/requirements.txt + venv/bin/pip freeze | grep -iE '^(torch|transformers|huggingface.hub|mediapipe|numpy|phonemizer)=' || true + if venv/bin/python -c "import phonemizer" 2>/dev/null; then echo "::error::phonemizer is installed; the test needs a venv without it"; exit 1; fi + - name: Recogniser cache (fetched from the Hub once per revision, then served to the same client path) + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: hf-mirror + key: lsdub-recogniser-${{ env.RECOGNISER_REVISION }} + - name: Demo fallback zip + run: curl -sSfL -o demo-fallback.zip "$DEMO_FALLBACK_URL" + - name: Install path (nodes, align ctc from empty, dub at defaults, dub with the mouth Combos) + run: | + venv/bin/python pack/scripts/install_path_check.py --comfyui comfy --pack pack \ + --hf-mirror hf-mirror --save-mirror hf-mirror \ + --demo-fallback demo-fallback.zip --demo-fallback-sha "$DEMO_FALLBACK_SHA" + - name: ComfyUI log + if: always() + run: tail -n 300 comfy/install-path-comfyui.log || true + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: failure() + with: + name: install-path-log + path: comfy/install-path-comfyui.log diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index be6c0b1..c4329eb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -5,7 +5,11 @@ on: permissions: contents: read jobs: + install-path: # the gate: a tag whose tree fails the clean-ComfyUI install test is never published + if: github.repository == 'HumanjavaEnterprises/comfyui-lsdub' + uses: ./.github/workflows/install-path.yml publish: + needs: install-path runs-on: ubuntu-latest if: github.repository == 'HumanjavaEnterprises/comfyui-lsdub' steps: @@ -18,3 +22,20 @@ jobs: with: personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }} # set this secret in the repo settings skip_checkout: 'true' + site: # after a good publish: ask the microsite to deploy its main branch (it serves /changelog) + needs: publish + runs-on: ubuntu-latest + steps: + - name: Dispatch lsdub-release to the microsite + env: + TOKEN: ${{ secrets.MICROSITE_DISPATCH_TOKEN }} + TAG: ${{ github.ref_name }} + run: | + if [ -z "$TOKEN" ]; then + echo "::notice title=Site deploy skipped::MICROSITE_DISPATCH_TOKEN is not set, so the microsite was not asked to deploy. Add it (CONTRIBUTING.md, Releases) or deploy the site by hand." + exit 0 + fi + curl -sSf -X POST -H "Authorization: Bearer $TOKEN" -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/HumanjavaEnterprises/lsdub.web.microsite.src/dispatches \ + -d "{\"event_type\":\"lsdub-release\",\"client_payload\":{\"tag\":\"$TAG\"}}" + echo "::notice title=Site deploy requested::lsdub-release $TAG sent to the microsite" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index db1f292..7f5b94e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,3 +29,32 @@ By contributing you agree your work is licensed under the Apache License 2.0, th - No `## Unreleased` heading in the file; add the section when the version is released. - The heading names the version in pyproject.toml. `python scripts/release_notes.py` checks all of this; CI runs it (`tests/test_release_notes.py`) and the publish job fails without it. + +## Stability policy +- Releases are batched; patch versions only for breakage. Features go into the next minor version. +- Defaults stay byte-identical across a minor series unless the changelog says otherwise. +- Every release passes the install-path test (`.github/workflows/install-path.yml`, `scripts/install_path_check.py`) on a + clean ComfyUI before it is tagged. Run it yourself: `python scripts/install_path_check.py --comfyui PATH/TO/ComfyUI` with + that ComfyUI's venv (its requirements plus `requirements.txt`, nothing else). +- Models are pinned by revision and sha256. +- 0.2.0 to 0.2.2 landed in one day while the install-path test did not yet exist; it has run on every release since. + +## Releases (maintainers) +One command, from a clean pack checkout on main with the engine and the microsite checked out beside it: + + python scripts/release.py X.Y.Z --dry-run # every check, in a throwaway worktree; pushes nothing + python scripts/release.py X.Y.Z + +Before it: write the `## X.Y.Z` section in CHANGELOG.md (see Release notes) and the matching `
` in the +microsite's `docs/changelog.html` on its main. The command refuses unless both trees are clean and on main and engine CI is +green; bumps the version, the Icon/Banner URLs and CITATION.cff; regenerates the node tables and the bug form; vendors the +engine; runs the pack checks; pushes `release/X.Y.Z` and waits for CI and Install path; then fast-forwards main, tags `vX.Y.Z`, +watches the publish (which runs Install path again first), confirms the Registry lists the version and checks the site +deploy. Any failure before the tag stops with a plain message and leaves nothing changed remotely. + +Repository secrets: +- `REGISTRY_ACCESS_TOKEN` (here): the Comfy Registry publisher key, already set. +- `MICROSITE_DISPATCH_TOKEN` (here): a fine-grained GitHub token, resource owner HumanjavaEnterprises, repository access only + `lsdub.web.microsite.src`, permission Contents: Read and write (what `POST /repos/.../dispatches` needs). Without it the + publish job's site step prints a notice and skips, and `release.py` starts the deploy itself with your `gh` login. +- `CLOUDFLARE_API_TOKEN` (in the microsite repo): see its README, Deploys. diff --git a/README.md b/README.md index 1e2d24f..f106f0c 100644 --- a/README.md +++ b/README.md @@ -202,12 +202,29 @@ Python SDK (`pip install comfy-sdk`). Qwen-Image-Edit Lightning LoRA). - **Cost:** cloud runs use your Comfy credits. +## Stability and releases +LSDub aims to be stable software that is easy to install and build on, not a stream of updates. + +- Releases are batched. A patch version (0.2.x) only fixes breakage; new features wait for the next minor version. +- Across a minor series the defaults stay byte-identical: the same inputs at the defaults give the same render, unless the + changelog says otherwise. +- Every release passes the install-path test before it is tagged: a clean ComfyUI on CPU, only this pack's + requirements, every node registered, the phone recogniser downloaded into an empty models folder, and a Dub at the + defaults and with the mouth switches on the demo clip. +- Models are pinned by revision and checked by sha256 ([MODELS.md](MODELS.md)). +- What changed: [the changelog](https://lsdub.sndbnch.app/changelog). Something broke: the + [issue forms](https://github.com/HumanjavaEnterprises/comfyui-lsdub/issues/new/choose). + +In honesty: 0.2.0, 0.2.1 and 0.2.2 landed in one day, before the install-path test existed. Two of them fixed breakage a +clean install would have caught (the phone recogniser in a fresh ComfyUI, and the Dub with the mouth switches on). The test +now runs on every pull request, before every publish, and weekly. + ## Troubleshooting - **The first Align run sits there.** It is downloading Whisper large-v3 (about 3 GB) with no progress bar in the UI. Let it finish once. -- **The demo download fails.** Get `lsdub-demo-0.1.0.zip` from the GitHub Releases page by hand and run - `python scripts/fetch_demo.py --zip lsdub-demo-0.1.0.zip`. +- **The demo download fails.** Get `lsdub-demo-0.1.1.zip` from the GitHub Releases page by hand and run + `python scripts/fetch_demo.py --zip lsdub-demo-0.1.1.zip`. - **Align warns about the lyrics.** If what is sung matches the lyrics poorly (below 0.5), it warns and places the lyrics on the voice. Below 0.2 it stops and asks you to check the lyrics: they probably belong to another song. - **A shot failed to dub.** It is reported, and by default the cut keeps its original frames. Run workflow 3 on the diff --git a/lsdub_engine/VERSION b/lsdub_engine/VERSION index 2e2a75f..291b2f7 100644 --- a/lsdub_engine/VERSION +++ b/lsdub_engine/VERSION @@ -1,4 +1,4 @@ pack_version: 0.2.2 uncommitted: 0 -tree_sha256: 13cb87b83854a0f23de5f00ff3184811a78e3f004cd7fa6f5b01cb714066cb46 -tests_sha256: 70cb132afd1ac5ac9aef39a95d11e23041668f2776c9d64f187b0a7fb48be82f +tree_sha256: f36b7410d7dc3461fba57844603c81a5ba5ec37adb160148f9dc50595e4f19ba +tests_sha256: 59f5e441dbefa42caf1faf35824ac93cfec99b3e58b280a0e522d2f0f8759edb diff --git a/lsdub_engine/audits/qa.py b/lsdub_engine/audits/qa.py index 860675c..6c4a410 100644 --- a/lsdub_engine/audits/qa.py +++ b/lsdub_engine/audits/qa.py @@ -526,7 +526,7 @@ def asked_travel(s): EV_KEYS = ("onset_hit", "onset_n", "closure_hit", "closure_n", "spurious", "spurious_take", "open_ratio", "open_n") -LAG_WINDOW_S = 0.2 # a mouth rise this near a sung onset, either side, is that onset's rise (G10 lane 04's per-onset reading) +LAG_WINDOW_S = 0.2 # a mouth rise this near a sung onset, either side, is that onset's rise (the per-onset reading) def vocal_events(vocal, align=None, phone_models=None): diff --git a/scripts/fetch_demo.py b/scripts/fetch_demo.py index 37472db..3bf5bbd 100644 --- a/scripts/fetch_demo.py +++ b/scripts/fetch_demo.py @@ -20,10 +20,10 @@ import urllib.request import zipfile -VERSION = '0.1.0' -TAG = f'v{VERSION}' +VERSION = '0.1.1' +TAG = 'demo-0.1.1' URL = f'https://github.com/HumanjavaEnterprises/comfyui-lsdub/releases/download/{TAG}/lsdub-demo-{VERSION}.zip' -SHA256 = 'a431085d931f2b983b620d31710ac868a459d461a6dcb158177d199637790edb' +SHA256 = '0b15dfcf069b6dc2cac25bbe312137eda435e4ac350cce6b61436419fd2b518c' def unpack(zf, dst): diff --git a/scripts/install_path_check.py b/scripts/install_path_check.py new file mode 100644 index 0000000..3056ec3 --- /dev/null +++ b/scripts/install_path_check.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Humanjava Enterprises. LSDub: https://github.com/HumanjavaEnterprises/comfyui-lsdub +"""The install-path test: LSDub in a clean ComfyUI, the way a user gets it. CPU only. CI runs it on every PR to main, before +every publish, weekly, and by hand (.github/workflows/install-path.yml); scripts/release.py waits for it. + + python scripts/install_path_check.py --comfyui PATH/TO/ComfyUI [--pack .] [--port 8199] [--demo-zip lsdub-demo-0.1.0.zip] + [--hf-mirror DIR] [--save-mirror DIR] [--steps nodes,align,dub,mouth] + +Run it with the ComfyUI venv's python (ComfyUI's requirements + this pack's requirements.txt, nothing else). It: + 1. copies the pack (this checkout, without .git) into ComfyUI/custom_nodes/comfyui-lsdub and fetches the demo assets; + 2. starts ComfyUI headless (--cpu) and checks that every LSDub node in the source registered; + 3. align: LSDub Align with aligner ctc from an EMPTY models/lsdub/wav2vec2-espeak. The recogniser is downloaded for real + (exactly the four pinned files at the pinned revision, no safetensors), and phonemizer is never needed; + 4. dub: the demo workflow (BYO Clip and Song) at its defaults: Dub, Verify, Save Cut; + 5. mouth: the same Dub with Mouth Tweaks warp_events vocal, warp_hold nucleus-follow, warp_closures aligned-press, + warp_aperture p95 (the Dub's auto path). +A dubbed shot that errors fails the test even though the Dub itself carries on with the source (on_fail=source). + +--hf-mirror DIR serves the four recogniser files from DIR on a local Hugging Face endpoint (CI's cache, so the files are +fetched from the Hub once per revision); the client path (hf_hub_download into the empty folder) is unchanged. +--save-mirror DIR copies the downloaded files there after a real download, for that cache. Exit 0 = pass.""" +import argparse +import glob +import hashlib +import http.server +import json +import os +import re +import shutil +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request + +REPO = 'facebook/wav2vec2-lv-60-espeak-cv-ft' +RECOGNISER_DIR = os.path.join('lsdub', 'wav2vec2-espeak') +MOUTH = dict(warp_events='vocal', warp_hold='nucleus-follow', warp_closures='aligned-press', warp_aperture='p95') +BAD_LOG = [(re.compile(r'phonemizer', re.I), 'phonemizer was needed'), + (re.compile(r'phone network not loaded'), 'the phone recogniser did not load (vocal events fell back to flux)'), + (re.compile(r'shot \d+ failed: '), 'a dubbed shot raised (the Dub fell back to the source)'), + (re.compile(r'IMPORT FAILED.*lsdub', re.I), 'the pack failed to import')] + + +class Fail(SystemExit): + def __init__(self, msg): super().__init__(f'INSTALL-PATH FAIL: {msg}') + + +def say(msg): print(f'[install-path] {msg}', flush=True) + + +def pinned(pack): + """the recogniser's pinned revision and file shas, read from the pack under test (lsdub_engine/_models.py).""" + src = open(os.path.join(pack, 'lsdub_engine', '_models.py'), encoding='utf-8').read() + i = src.index(f'"{REPO}"') + rev = re.search(r'"revision":\s*"([0-9a-f]{40})"', src[i:]).group(1) + files = dict(re.findall(r'"([\w.]+)":\s*"([0-9a-f]{64})"', src[i:src.index('}', src.index('"files"', i)) + 1])) + return rev, files + + +def stand_in(fd, z, sha): + """unpack a demo zip the way fetch_demo.py does, with the zip's own checksum (only after fetch_demo itself has failed).""" + h = hashlib.sha256(open(z, 'rb').read()).hexdigest() + if h != sha: raise Fail(f'demo stand-in {z}: sha256 {h} != {sha}') + src = open(fd[1], encoding='utf-8').read() + tmp = fd[1] + '.standin.py' + import zipfile + root = {n.split('/')[0] for n in zipfile.ZipFile(z).namelist()}.pop() # lsdub-demo- + src = re.sub(r"^VERSION = '.*'$", f"VERSION = '{root.rsplit('-', 1)[-1]}'", src, flags=re.M) + open(tmp, 'w', encoding='utf-8').write(re.sub(r"^SHA256 = '.*'$", f"SHA256 = '{sha}'", src, flags=re.M)) + try: subprocess.run([fd[0], tmp] + fd[2:] + ['--zip', z], check=True, stdout=subprocess.DEVNULL) + finally: os.remove(tmp) + + +def node_ids(pack): + ids = set() + for p in glob.glob(os.path.join(pack, '*.py')): + ids |= set(re.findall(r"node_id='(LSDub\w+)'", open(p, encoding='utf-8').read())) + return ids + + +# ------------------------------------------------------------------------------------------------ the local hub mirror +def serve_mirror(folder, rev, files): + class H(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): pass + + def _file(self): + m = re.match(rf'^/{re.escape(REPO)}/resolve/([^/]+)/([\w.]+)$', self.path.split('?')[0]) + if not m or m.group(1) not in (rev, 'main') or m.group(2) not in files: return None + return os.path.join(folder, m.group(2)) + + def _head(self, p): + self.send_response(200) + self.send_header('X-Repo-Commit', rev) + self.send_header('ETag', f'"{files[os.path.basename(p)]}"') + self.send_header('Content-Length', str(os.path.getsize(p))) + self.send_header('Content-Type', 'application/octet-stream') + self.end_headers() + + def do_HEAD(self): + p = self._file() + if not p or not os.path.isfile(p): + self.send_error(404) + return + self._head(p) + + def do_GET(self): + p = self._file() + if not p or not os.path.isfile(p): + self.send_error(404) + return + self._head(p) + with open(p, 'rb') as f: shutil.copyfileobj(f, self.wfile) + + srv = http.server.ThreadingHTTPServer(('127.0.0.1', 0), H) + threading.Thread(target=srv.serve_forever, daemon=True).start() + return f'http://127.0.0.1:{srv.server_address[1]}' + + +# ------------------------------------------------------------------------------------------------ ComfyUI +def api(port, path, body=None, timeout=30): + req = urllib.request.Request(f'http://127.0.0.1:{port}{path}', data=json.dumps(body).encode() if body is not None else None, + headers={'Content-Type': 'application/json'}) + with urllib.request.urlopen(req, timeout=timeout) as r: # noqa: S310 (loopback) + return json.loads(r.read()) + + +def run_prompt(port, prompt, label, timeout): + try: + pid = api(port, '/prompt', {'prompt': prompt, 'client_id': 'install-path'})['prompt_id'] + except urllib.error.HTTPError as e: + raise Fail(f'{label}: ComfyUI refused the workflow: {e.read().decode(errors="replace")[:2000]}') from None + t0 = time.time() + while time.time() - t0 < timeout: + h = api(port, f'/history/{pid}').get(pid) + if h and h.get('status', {}).get('completed') is not None and (h['status'].get('completed') or h['status'].get('status_str') == 'error'): + if h['status'].get('status_str') != 'success': + msgs = [m for m in h['status'].get('messages', []) if m[0] == 'execution_error'] + raise Fail(f'{label}: the workflow failed: {json.dumps(msgs)[:3000]}') + say(f'{label}: done in {time.time() - t0:.0f} s') + return h['outputs'] + time.sleep(3) + raise Fail(f'{label}: no result after {timeout} s') + + +def texts(outputs, node): + t = outputs.get(node, {}).get('text', []) + return '\n'.join(t if isinstance(t, list) else [str(t)]) + + +def workflow(name, aligner='sphinx', mouth=False, align_only=False): + w = { + '1': ('LSDubProject', dict(name=name, root='', mode='run', fps=25, song_offset=0, preset='landing', seed=0, force_shots='', backend='local')), + '2': ('LoadVideo', dict(file='lsdub_demo_landing_clip.mp4')), + '3': ('LSDubVideoSlot', dict(prompt='', negative_prompt='', seconds=5, project=['1', 0], video=['2', 0])), + '4': ('LoadAudio', dict(audio='lsdub_demo_landing_song.wav')), + '5': ('LoadAudio', dict(audio='lsdub_demo_landing_lead.wav')), + '6': ('LSDubSongSlot', dict(tags='', seconds=60, bpm=120, keyscale='C major', project=['1', 0], audio=['4', 0], vocal=['5', 0])), + '8': ('LSDubLyricsSlot', dict(text='Pleiadians bending space and time\nFour hundred light-years far away\nAlways so serious, never dance a day', + project=['1', 0])), + '9': ('LSDubAlign', dict(pronunciations='', stt_model='large-v3', transcript_json='lsdub_demo_landing_stt.json', aligner=aligner, + project=['1', 0], song=['6', 0], lyrics=['8', 0])), + '13': ('PreviewAny', dict(source=['9', 1])), + } + if not align_only: + w.update({ + '7': ('LSDubShotSplit', dict(project=['1', 0], source=['3', 0])), + '10': ('LSDubTriage', dict(samples=6, project=['1', 0], source=['3', 0], shots=['7', 0], song=['6', 0])), + '11': ('PreviewAny', dict(source=['10', 2])), + '14': ('LSDubBuildLipKit', dict(identity='red_nova', kit_dir='lsdub_demo_kits/red_nova.zip', shot_id=-1, + keep='person, same face shape, skin tone, hair, eyewear and makeup', seeds=3, project=['1', 0])), + '15': ('LSDubDub', dict(only_shots='', quality=18, on_fail='source', audio='source', vocalisations='skip', project=['1', 0], + source=['3', 0], shots=['10', 0], song=['6', 0], align=['9', 0], kit=['14', 0])), + '16': ('LSDubVerify', dict(project=['1', 0], source=['3', 0], render=['15', 0], shots=['10', 0], song=['6', 0], align=['9', 0])), + '17': ('PreviewAny', dict(source=['16', 1])), + '20': ('LSDubSaveCut', dict(watermark=True, quality=18, project=['1', 0], render=['15', 0])), + }) + if mouth: + w['21'] = ('LSDubTweaks', dict(lead=0.06, word_close=0.06, face_target=512.0, snap=True, auto=True, continuity=True, settings='', **MOUTH)) + w['15'][1]['tweaks'] = ['21', 0] + return {k: {'class_type': c, 'inputs': i} for k, (c, i) in w.items()} + + +def check_log(log, since, label): + with open(log, encoding='utf-8', errors='replace') as f: + f.seek(since) + new = f.read() + for rx, why in BAD_LOG: + m = rx.search(new) + if m: + ln = new[max(0, new.rfind('\n', 0, m.start()) + 1):new.find('\n', m.end())] + raise Fail(f'{label}: {why}: {ln.strip()[:500]}') + return new + + +def check_dub(comfy, name, outputs, label): + proj = os.path.join(comfy, 'output', 'lsdub', name) + renders = glob.glob(os.path.join(proj, 'shots', '*', '*', 'render.mp4')) + glob.glob(os.path.join(proj, 'shots', '*', '*', '*.mp4')) + errors = [] + for d in glob.glob(os.path.join(proj, 'shots', '*', '*', 'done.json')): + j = json.load(open(d)) + if j.get('error'): errors.append(f"{os.path.relpath(d, proj)}: {j['error']}") + if errors: raise Fail(f'{label}: dubbed shots raised: {errors}') + if not renders: raise Fail(f'{label}: no shot was dubbed (no render under {proj}/shots)') + cuts = [f for f in glob.glob(os.path.join(comfy, 'output', '**', '*.mp4'), recursive=True) if name in f and 'final' in f] + if not cuts: raise Fail(f'{label}: Save Cut wrote no file') + rep = texts(outputs, '17') + if 'shot' not in rep.lower(): raise Fail(f'{label}: Verify gave no report: {rep[:300]!r}') + say(f'{label}: {len(renders)} render file(s), cut {os.path.relpath(cuts[0], comfy)}; Verify: {rep.splitlines()[0][:120] if rep else ""}') + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split('\n')[0]) + ap.add_argument('--comfyui', required=True) + ap.add_argument('--pack', default=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + ap.add_argument('--port', type=int, default=8199) + ap.add_argument('--demo-zip') + ap.add_argument('--hf-mirror') + ap.add_argument('--save-mirror') + ap.add_argument('--demo-fallback', help='a demo zip to use if fetch_demo.py fails (the step still fails)') + ap.add_argument('--demo-fallback-sha', default='', help="the fallback zip's sha256") + ap.add_argument('--steps', default='nodes,align,dub,mouth') + ap.add_argument('--timeout', type=int, default=3600, help='seconds per workflow') + a = ap.parse_args() + comfy, pack, steps = os.path.abspath(a.comfyui), os.path.abspath(a.pack), set(a.steps.split(',')) + rev, files = pinned(pack) + say(f'pack {pack}; recogniser {REPO}@{rev[:8]} ({len(files)} files)') + + dst = os.path.join(comfy, 'custom_nodes', 'comfyui-lsdub') + shutil.rmtree(dst, ignore_errors=True) + shutil.copytree(pack, dst, ignore=shutil.ignore_patterns('.git', '__pycache__', '.pytest_cache', '.ruff_cache', 'dist', '*.zip')) + failures = [] + fd = [sys.executable, os.path.join(dst, 'scripts', 'fetch_demo.py'), '--comfyui', comfy] + r = subprocess.run(fd + (['--zip', os.path.abspath(a.demo_zip)] if a.demo_zip else []), capture_output=True, text=True) + if r.returncode: + failures.append(f'demo: scripts/fetch_demo.py fails for a user: {(r.stderr or r.stdout).strip().splitlines()[-1][:400]}') + say(failures[-1]) + if not a.demo_fallback: raise Fail(failures[-1]) + say(f'demo: carrying on with the stand-in {a.demo_fallback} (sha256 {a.demo_fallback_sha[:12]}) so the other steps still report') + stand_in(fd, a.demo_fallback, a.demo_fallback_sha) + else: + say('demo: fetch_demo.py ok') + rec = os.path.join(comfy, 'models', RECOGNISER_DIR) + if os.path.exists(rec): raise Fail(f'{rec} is not empty: the align step needs an empty models folder') + + env = dict(os.environ, PYTHONUNBUFFERED='1') + if a.hf_mirror and all(os.path.isfile(os.path.join(a.hf_mirror, f)) for f in files): + env['HF_ENDPOINT'] = serve_mirror(os.path.abspath(a.hf_mirror), rev, files) + say(f'recogniser files from the cached mirror {a.hf_mirror} on {env["HF_ENDPOINT"]}') + else: + say('recogniser files from the Hugging Face Hub') + log = os.path.join(comfy, 'install-path-comfyui.log') + lf = open(log, 'w') + proc = subprocess.Popen([sys.executable, 'main.py', '--cpu', '--listen', '127.0.0.1', '--port', str(a.port), '--disable-auto-launch'], + cwd=comfy, env=env, stdout=lf, stderr=subprocess.STDOUT) + try: + t0 = time.time() + while True: + if proc.poll() is not None: raise Fail(f'ComfyUI exited ({proc.returncode}); see {log}') + try: + info = api(a.port, '/object_info', timeout=60) + break + except (OSError, urllib.error.URLError): + if time.time() - t0 > 600: raise Fail('ComfyUI did not start in 600 s') from None + time.sleep(2) + want, got = node_ids(pack), {k for k in info if k.startswith('LSDub')} + check_log(log, 0, 'startup') + if want - got: raise Fail(f'nodes not registered: {sorted(want - got)} (see {log})') + say(f'nodes: all {len(want)} LSDub nodes registered') + + def step(label, fn): + try: fn() + except Fail as e: + failures.append(str(e.code)) + say(str(e.code)) + + def align(): + pos = os.path.getsize(log) + out = run_prompt(a.port, workflow('ip-align', aligner='ctc', align_only=True), 'align (ctc, empty models folder)', a.timeout) + check_log(log, pos, 'align') + have = sorted(os.path.relpath(p, rec) for p in glob.glob(os.path.join(rec, '**', '*'), recursive=True) if os.path.isfile(p)) + snap = os.path.join(rec, 'models--' + REPO.replace('/', '--'), 'snapshots', rev) + if any(p.endswith('.safetensors') for p in have): raise Fail(f'align: an unpinned safetensors was downloaded: {have}') + for f, sha in files.items(): + p = os.path.join(snap, f) + if not os.path.isfile(p): raise Fail(f'align: pinned file {f} missing from {snap}; have {have}') + h = hashlib.sha256() + with open(p, 'rb') as fh: + for b in iter(lambda: fh.read(1 << 20), b''): h.update(b) + if h.hexdigest() != sha: raise Fail(f'align: {f} sha256 {h.hexdigest()} != pinned {sha}') + blobs = [p for p in have if '/blobs/' in p or p.startswith('models--') and '/blobs' in p] + if len(blobs) > len(files): raise Fail(f'align: more than the {len(files)} pinned files were downloaded: {blobs}') + rep = texts(out, '13') + if 'aligner ctc' not in rep: raise Fail(f'align: the lyric check does not say aligner ctc: {rep[:400]!r}') + say(f'align: four pinned files at {rev[:8]}, sha256 ok, no safetensors; {rep.splitlines()[2] if len(rep.splitlines()) > 2 else rep[:100]}') + if a.save_mirror and 'HF_ENDPOINT' not in env: + os.makedirs(a.save_mirror, exist_ok=True) + for f in files: shutil.copyfile(os.path.join(snap, f), os.path.join(a.save_mirror, f)) + say(f'align: saved the four files to {a.save_mirror} for the cache') + + def dub(name, label, kw): + pos = os.path.getsize(log) + out = run_prompt(a.port, workflow(name, **kw), label, a.timeout) + check_log(log, pos, label) + check_dub(comfy, name, out, label) + + if 'align' in steps: step('align', align) + if 'dub' in steps: step('dub', lambda: dub('ip-dub-defaults', 'dub at defaults', {})) + if 'mouth' in steps: + step('mouth', lambda: dub('ip-dub-mouth', 'dub with the mouth Combos (' + ', '.join(f'{k} {v}' for k, v in MOUTH.items()) + ')', + dict(aligner='ctc', mouth=True))) + if failures: raise Fail(f'{len(failures)} step(s) failed:\n ' + '\n '.join(failures)) + say('PASS') + finally: + proc.terminate() + try: proc.wait(30) + except subprocess.TimeoutExpired: proc.kill() + lf.close() + + +if __name__ == '__main__': + main() diff --git a/scripts/release.py b/scripts/release.py new file mode 100644 index 0000000..e79d209 --- /dev/null +++ b/scripts/release.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Humanjava Enterprises. LSDub: https://github.com/HumanjavaEnterprises/comfyui-lsdub +"""release.py: the one release command. Maintainers only (it needs the private engine checkout and gh signed in). + + python scripts/release.py X.Y.Z [--dry-run] [--engine ../lsdub] [--site ../lsdub.web.microsite.src] + +In order, stopping at the first failure with a plain message (nothing remote changes before step 9): + 1. refuses unless the engine and pack trees are clean, on main and equal to origin/main, and engine CI is green on its HEAD; + 2. refuses unless CHANGELOG.md has a `## X.Y.Z` section that passes release_notes.py, the microsite's main has a + matching section in docs/changelog.html, and the demo zip fetch_demo.py points at is published with its checksum; + 3. bumps pyproject (version, Icon/Banner tag URLs) and CITATION.cff; regenerates the README node tables and the bug form; + 4. vendors the engine (a vendor record commit in the engine, pushed only in step 9); + 5. runs the pack checks CI runs (ruff registry rules, pytest, docs and models checks), including the private-methods scrub; + 6. commits "X.Y.Z: ..." on a release/X.Y.Z branch; + 7. pushes release/X.Y.Z and waits for CI and Install path to pass on that commit; + 8. (dry run stops before this: it works in a throwaway worktree and pushes nothing) + 9. fast-forwards main to it, tags vX.Y.Z, pushes main and the tag, pushes the engine record; + 10. watches the publish run, confirms the Comfy Registry lists X.Y.Z, and makes sure the site deploy ran. +""" +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.request + +PACK = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PACK_REPO = 'HumanjavaEnterprises/comfyui-lsdub' +ENGINE_REPO = 'HumanjavaEnterprises/lsdub' +SITE_REPO = 'HumanjavaEnterprises/lsdub.web.microsite.src' +REGISTRY = 'https://api.comfy.org/nodes/lsdub/versions' +EVIDENCE = ('Evidence: release-check source=lsdub/core/lyric_align.py read the vendored tree: vendor_engine scrub CHECK clean, ' + 'pack tests and install-path green on release/{v}') + + +class Stop(SystemExit): + def __init__(self, msg): super().__init__(f'\nrelease: STOPPED. {msg}\nNothing remote was changed.' if not REMOTE_TOUCHED[0] else + f'\nrelease: STOPPED. {msg}') + + +REMOTE_TOUCHED = [False] +STEP = [0] + + +def step(msg): + STEP[0] += 1 + print(f'\n[{STEP[0]}] {msg}', flush=True) + + +def sh(*cmd, cwd=PACK, check=True, quiet=False, env=None): + if not quiet: print(' $', ' '.join(cmd), flush=True) + r = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) + if check and r.returncode: + raise Stop(f'`{" ".join(cmd)}` failed (exit {r.returncode}) in {cwd}:\n{(r.stdout + r.stderr).strip()[-3000:]}') + return r.stdout.strip() + + +def git(repo, *a, **k): return sh('git', *a, cwd=repo, **k) + + +def gh_json(*a): return json.loads(sh('gh', *a, quiet=True) or 'null') + + +def clean_main(repo, name): + if git(repo, 'status', '--porcelain', quiet=True): raise Stop(f'the {name} tree at {repo} has uncommitted changes. Commit or stash them first.') + br = git(repo, 'rev-parse', '--abbrev-ref', 'HEAD', quiet=True) + if br != 'main': raise Stop(f'the {name} checkout is on {br!r}, not main.') + git(repo, 'fetch', '-q', 'origin', 'main', quiet=True) + if git(repo, 'rev-parse', 'HEAD', quiet=True) != git(repo, 'rev-parse', 'origin/main', quiet=True): + raise Stop(f'the {name} main is not origin/main (pull or push first).') + return git(repo, 'rev-parse', 'HEAD', quiet=True) + + +def runs_green(repo, sha, workflows, wait, timeout=4 * 3600): + """every named workflow has a successful run on sha (waits for them when wait=True).""" + t0 = time.time() + while True: + runs = gh_json('run', 'list', '-R', repo, '--commit', sha, '--json', 'workflowName,status,conclusion,url', '-L', '50') + state = {} + for w in workflows: + mine = [r for r in runs if r['workflowName'] == w] + if any(r['conclusion'] == 'success' for r in mine): state[w] = 'success' + elif any(r['status'] != 'completed' for r in mine): state[w] = 'running' + elif mine: state[w] = f"{mine[0]['conclusion']} ({mine[0]['url']})" + else: state[w] = 'no run yet' + bad = {w: s for w, s in state.items() if s not in ('success', 'running', 'no run yet')} + if bad: raise Stop(f'{repo} @ {sha[:9]}: ' + '; '.join(f'{w} {s}' for w, s in bad.items())) + if all(s == 'success' for s in state.values()): return + if not wait: raise Stop(f'{repo} @ {sha[:9]} is not green yet: {state}') + if time.time() - t0 > timeout: raise Stop(f'{repo} @ {sha[:9]}: gave up waiting: {state}') + print(f' waiting: {state}', flush=True) + time.sleep(60) + + +def read(p): return open(p, encoding='utf-8').read() + + +def write(p, s): open(p, 'w', encoding='utf-8').write(s) + + +def bump(tree, v): + pp = os.path.join(tree, 'pyproject.toml') + s = read(pp) + s = re.sub(r'^version = "[^"]+"', f'version = "{v}"', s, count=1, flags=re.M) + s = re.sub(r'(comfyui-lsdub/)v\d+\.\d+\.\d+(/assets/)', rf'\g<1>v{v}\g<2>', s) + write(pp, s) + cf = os.path.join(tree, 'CITATION.cff') + write(cf, re.sub(r'^version: "[^"]+"', f'version: "{v}"', read(cf), flags=re.M)) + + +def demo_published(tree): + src = read(os.path.join(tree, 'scripts', 'fetch_demo.py')) + ver = re.search(r"^VERSION = '([^']+)'", src, re.M).group(1) + tag = re.search(r"^TAG = (.+)$", src, re.M).group(1).strip() + tag = f'v{ver}' if tag == "f'v{VERSION}'" else tag.strip("'") + sha = re.search(r"^SHA256 = '([0-9a-f]{64})'", src, re.M).group(1) + url = f'https://github.com/{PACK_REPO}/releases/download/{tag}/lsdub-demo-{ver}.zip' + try: + with urllib.request.urlopen(url, timeout=120) as r: got = hashlib.sha256(r.read()).hexdigest() # noqa: S310 (fixed https URL) + except OSError as e: + got = f'unreachable ({e})' + if got != sha: + raise Stop(f'the demo zip fetch_demo.py points at is not published as pinned: {url} -> {got}, pinned {sha}.\n' + f'Attach it first (a new tag for a new zip, never a replaced asset):\n' + f' gh release create {tag} dist/lsdub-demo-{ver}.zip -R {PACK_REPO} --latest=false ' + f'--title "LSDub demo assets {ver}" --notes "Demo assets for the example workflows (scripts/fetch_demo.py)."') + return url + + +def main(): + ap = argparse.ArgumentParser(description='LSDub: the one release command', epilog=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('version') + ap.add_argument('--dry-run', action='store_true', help='every check, in a throwaway worktree; pushes nothing') + ap.add_argument('--engine', default=os.path.join(os.path.dirname(PACK), 'lsdub')) + ap.add_argument('--site', default=os.path.join(os.path.dirname(PACK), 'lsdub.web.microsite.src')) + a = ap.parse_args() + v, dry = a.version, a.dry_run + eng, site = os.path.abspath(a.engine), os.path.abspath(a.site) + print(f'LSDub release {v}{" (DRY RUN: nothing is pushed)" if dry else ""}') + if not re.fullmatch(r'\d+\.\d+\.\d+', v): raise Stop(f'{v!r} is not X.Y.Z') + cur = re.search(r'^version = "([^"]+)"', read(os.path.join(PACK, 'pyproject.toml')), re.M).group(1) + if tuple(map(int, v.split('.'))) <= tuple(map(int, cur.split('.'))): raise Stop(f'{v} is not newer than the current {cur}') + for t in ('git', 'gh'): + if not shutil.which(t): raise Stop(f'{t} is not on PATH') + sh('gh', 'auth', 'status', quiet=True) + + step('engine and pack trees clean, on main, equal to origin/main; engine CI green on its HEAD') + eng_sha = clean_main(eng, 'engine') + pack_sha = clean_main(PACK, 'pack') + runs_green(ENGINE_REPO, eng_sha, ['ci'], wait=False) + if git(PACK, 'ls-remote', '--tags', 'origin', f'refs/tags/v{v}', quiet=True): raise Stop(f'tag v{v} already exists on origin') + if git(PACK, 'ls-remote', '--heads', 'origin', f'release/{v}', quiet=True): raise Stop(f'branch release/{v} already exists on origin') + print(f' engine {eng_sha[:9]} (ci green), pack {pack_sha[:9]}') + + step(f'CHANGELOG section {v} (release_notes.py), the microsite changelog section, the demo zip') + sys.path.insert(0, os.path.join(PACK, 'scripts')) + import release_notes as RN + try: RN.check(read(os.path.join(PACK, 'CHANGELOG.md')), v) + except ValueError as e: raise Stop(f'CHANGELOG.md: {e}. Write the {v} section first (CONTRIBUTING.md, Release notes).') from None + git(site, 'fetch', '-q', 'origin', 'main', quiet=True) + page = git(site, 'show', 'origin/main:docs/changelog.html', quiet=True) + if f'id="v{v}"' not in page: + raise Stop(f'the microsite main has no
in docs/changelog.html. Add it on the site\'s main first.') + print(f' demo zip ok: {demo_published(PACK)}') + + tree, made_worktree = PACK, False + if dry: + tree = tempfile.mkdtemp(prefix=f'lsdub-release-{v}-') + os.rmdir(tree) + git(PACK, 'worktree', 'add', '-q', '--detach', tree, 'HEAD') + made_worktree = True + records = os.path.join(eng, 'scripts', 'vendor_records.tsv') + records_before = read(records) + try: + step(f'bump to {v}: pyproject version + Icon/Banner URLs, CITATION.cff; regenerate README node tables and the bug form') + if not dry: git(PACK, 'checkout', '-q', '-b', f'release/{v}') + bump(tree, v) + sh(sys.executable, 'scripts/make_readme_nodes.py', cwd=tree) + + step('vendor the engine') + sh(sys.executable, os.path.join(eng, 'scripts', 'vendor_engine.py'), '--pack', tree, cwd=eng) + sh(sys.executable, os.path.join(eng, 'scripts', 'vendor_engine.py'), '--pack', tree, '--check', cwd=eng) + + step('pack checks (what CI runs, plus the private-methods scrub in tests/test_registry_rules.py)') + sh(sys.executable, '-m', 'ruff', 'check', '--select', 'S102,S307,E702,F', '--target-version', 'py310', '.', cwd=tree) + sh(sys.executable, '-m', 'pytest', '-q', 'tests', cwd=tree) + sh(sys.executable, 'scripts/make_readme_nodes.py', '--check', cwd=tree) + sh(sys.executable, 'scripts/check_models_md.py', cwd=tree) + sh(sys.executable, 'scripts/release_notes.py', cwd=tree, quiet=True) + + step(f'commit {v} on release/{v}') + title = RN.top_section(read(os.path.join(tree, 'CHANGELOG.md')))[1] + first = next((ln[2:].strip() for ln in title if ln.startswith('- ')), '') + git(tree, 'add', '-A') + if dry: + print(f' would commit "{v}: {first[:100]}" with: ' + git(tree, 'diff', '--cached', '--stat', quiet=True).replace('\n', '\n ')) + print(f' would commit the engine vendor record ({EVIDENCE.format(v=v)[:60]}...)') + print(f'\n[{STEP[0] + 1}..] would push release/{v}, wait for CI + Install path, fast-forward main, tag v{v}, push the tag and the ' + 'engine record, watch publish, confirm the Registry, check the site deploy.\nDRY RUN OK: every check passed.') + return + git(tree, 'commit', '-q', '-m', f'{v}: {first[:120]}') + rel_sha = git(tree, 'rev-parse', 'HEAD', quiet=True) + if read(records) != records_before: + git(eng, 'commit', '-q', '-m', f'{v} vendor record\n\n{EVIDENCE.format(v=v)}', '--', 'scripts/vendor_records.tsv') + + step(f'push release/{v} and wait for CI + Install path on {rel_sha[:9]}') + git(PACK, 'push', '-q', 'origin', f'release/{v}') + print(' (a branch only; main, tags and the Registry are untouched until both pass)') + time.sleep(20) + runs_green(PACK_REPO, rel_sha, ['CI', 'Install path'], wait=True) + + step(f'fast-forward main, tag v{v}, push; push the engine record') + REMOTE_TOUCHED[0] = True + git(PACK, 'checkout', '-q', 'main') + git(PACK, 'merge', '-q', '--ff-only', f'release/{v}') + git(PACK, 'push', '-q', 'origin', 'main') + git(PACK, 'tag', '-a', f'v{v}', '-m', f'LSDub {v}') + git(PACK, 'push', '-q', 'origin', f'v{v}') + tag_time = time.time() + git(eng, 'push', '-q', 'origin', 'main') + + step('watch publish (it re-runs Install path on the tag first)') + time.sleep(20) + runs_green(PACK_REPO, rel_sha, ['Publish to the Comfy Registry'], wait=True) + + step(f'the Comfy Registry lists {v}') + for _ in range(30): + with urllib.request.urlopen(REGISTRY, timeout=60) as r: vs = {x.get('version'): x.get('status') for x in json.loads(r.read())} # noqa: S310 + if v in vs: + print(f' {v}: {vs[v]}') + break + time.sleep(30) + else: + raise Stop(f'publish passed but {REGISTRY} does not list {v} after 15 minutes; check the publisher page.') + + step('site deploy (the publish workflow dispatches it; started here if it did not)') + time.sleep(30) + runs = gh_json('run', 'list', '-R', SITE_REPO, '--workflow', 'deploy.yml', '--json', 'databaseId,createdAt,event,url', '-L', '10') + fresh = [r for r in runs if time.mktime(time.strptime(r['createdAt'], '%Y-%m-%dT%H:%M:%SZ')) - time.timezone >= tag_time - 60] + if not fresh: + sh('gh', 'workflow', 'run', 'deploy.yml', '-R', SITE_REPO, '--ref', 'main') + time.sleep(15) + fresh = gh_json('run', 'list', '-R', SITE_REPO, '--workflow', 'deploy.yml', '--json', 'databaseId,createdAt,event,url', '-L', '1') + if fresh: + sh('gh', 'run', 'watch', str(fresh[0]['databaseId']), '-R', SITE_REPO, '--exit-status', check=False) + print(f' site deploy run: {fresh[0]["url"]} (a skipped deploy means CLOUDFLARE_API_TOKEN is not set there)') + print(f'\nreleased {v}.') + finally: + if made_worktree: + git(PACK, 'worktree', 'remove', '--force', tree, check=False, quiet=True) + if read(records) != records_before: write(records, records_before) # the dry run's vendor record, undone + + +if __name__ == '__main__': + main() diff --git a/tests/engine/test_watermark.py b/tests/engine/test_watermark.py index 92583fc..c744d24 100644 --- a/tests/engine/test_watermark.py +++ b/tests/engine/test_watermark.py @@ -72,7 +72,7 @@ def _top_atoms(path): def test_the_watermarked_file_is_faststart(clip, tmp_path): - """moov ahead of mdat (G10 lane 11: the saved cut, the site's redub.mp4, was the one file with its moov at the end).""" + """moov ahead of mdat (the saved cut, the site's redub.mp4, was the one file with its moov at the end).""" out = str(tmp_path / "wm.mp4") watermark.apply(clip, out) a = _top_atoms(out) diff --git a/tests/test_registry_rules.py b/tests/test_registry_rules.py index 4d2bee4..c7cbb42 100644 --- a/tests/test_registry_rules.py +++ b/tests/test_registry_rules.py @@ -22,7 +22,7 @@ def _files(exts=('.py', '.md', '.json', '.txt', '.toml', '.cff', '.yaml', '.yml' ds[:] = [x for x in ds if x not in SKIP_DIRS] for f in fs: if f in SKIP_DIRS: continue # a worktree's .git is a FILE - if os.path.splitext(f)[1] in exts: yield os.path.join(d, f) + if exts is None or os.path.splitext(f)[1] in exts: yield os.path.join(d, f) def test_no_machine_paths(): @@ -34,6 +34,30 @@ def test_no_machine_paths(): assert not hits, hits +# the pack ships product code and user docs; the methods stay private (the engine's ledger tools, review reports, +# research and plan docs). release.py runs this file, so a tree that carries any of them is never tagged. +METHODS = re.compile('|'.join(['G1' + r'\d\b', 'truth' + '-fix', 'lsdub-tryit' + '-assets', 'review/' + 'G1'])) +METHODS_PATHS = re.compile(r'(^|/)ledger(/|$)|^tests/ledger|^docs/(research|plans)/') + + +def _tracked(): + try: + out = subprocess.run(['git', '-C', ROOT, 'ls-files', '-z'], capture_output=True, check=True).stdout.decode() + return [f for f in out.split('\0') if f] + except (OSError, subprocess.CalledProcessError): + return [os.path.relpath(p, ROOT).replace(os.sep, '/') for p in _files(exts=None)] + + +def test_no_private_methods_in_shipped_tree(): + files = [f for f in _tracked() if os.path.isfile(os.path.join(ROOT, f))] + hits = [f for f in files if METHODS_PATHS.search(f)] + for f in files: + try: txt = open(os.path.join(ROOT, f), encoding='utf-8').read() + except (UnicodeDecodeError, OSError): continue + hits += [f'{f}:{i + 1}' for i, ln in enumerate(txt.splitlines()) if METHODS.search(ln)] + assert not hits, f'private methods in the shipped tree (keep them in the engine repo): {hits}' + + def _ruff(): exe = shutil.which('ruff') if exe: return [exe] From ec1c236b30ae00cc935ea8ee3dc4807f87a95509 Mon Sep 17 00:00:00 2001 From: vveerrgg Date: Mon, 9 Nov 2026 13:24:24 -0700 Subject: [PATCH 2/5] release.py: vendor --check runs after the engine record is committed (the engine is clean again); dry run says so --- scripts/install_path_check.py | 24 ++++++++++++++++++++++-- scripts/release.py | 3 ++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/scripts/install_path_check.py b/scripts/install_path_check.py index 3056ec3..0891733 100644 --- a/scripts/install_path_check.py +++ b/scripts/install_path_check.py @@ -99,16 +99,36 @@ def _head(self, p): self.send_header('Content-Type', 'application/octet-stream') self.end_headers() + def _proxy(self, method): + """anything but the four recogniser files (DINOv2, ...) goes to the real Hub, redirects passed through as they come.""" + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *a, **k): return None + req = urllib.request.Request('https://huggingface.co' + self.path, method=method, + headers={k: v for k, v in self.headers.items() if k.lower() not in ('host', 'connection')}) + try: r = urllib.request.build_opener(NoRedirect).open(req, timeout=300) # noqa: S310 (fixed https host) + except urllib.error.HTTPError as e: r = e + self.send_response(r.status if hasattr(r, 'status') else r.code) + for k, v in r.headers.items(): + if k.lower() not in ('transfer-encoding', 'connection'): self.send_header(k, v) + self.end_headers() + if method == 'GET': shutil.copyfileobj(r, self.wfile) + def do_HEAD(self): p = self._file() - if not p or not os.path.isfile(p): + if not p: + self._proxy('HEAD') + return + if not os.path.isfile(p): self.send_error(404) return self._head(p) def do_GET(self): p = self._file() - if not p or not os.path.isfile(p): + if not p: + self._proxy('GET') + return + if not os.path.isfile(p): self.send_error(404) return self._head(p) diff --git a/scripts/release.py b/scripts/release.py index e79d209..c7f54b6 100644 --- a/scripts/release.py +++ b/scripts/release.py @@ -185,7 +185,6 @@ def main(): step('vendor the engine') sh(sys.executable, os.path.join(eng, 'scripts', 'vendor_engine.py'), '--pack', tree, cwd=eng) - sh(sys.executable, os.path.join(eng, 'scripts', 'vendor_engine.py'), '--pack', tree, '--check', cwd=eng) step('pack checks (what CI runs, plus the private-methods scrub in tests/test_registry_rules.py)') sh(sys.executable, '-m', 'ruff', 'check', '--select', 'S102,S307,E702,F', '--target-version', 'py310', '.', cwd=tree) @@ -200,6 +199,7 @@ def main(): git(tree, 'add', '-A') if dry: print(f' would commit "{v}: {first[:100]}" with: ' + git(tree, 'diff', '--cached', '--stat', quiet=True).replace('\n', '\n ')) + print(' would then run vendor_engine.py --check on the committed record') print(f' would commit the engine vendor record ({EVIDENCE.format(v=v)[:60]}...)') print(f'\n[{STEP[0] + 1}..] would push release/{v}, wait for CI + Install path, fast-forward main, tag v{v}, push the tag and the ' 'engine record, watch publish, confirm the Registry, check the site deploy.\nDRY RUN OK: every check passed.') @@ -208,6 +208,7 @@ def main(): rel_sha = git(tree, 'rev-parse', 'HEAD', quiet=True) if read(records) != records_before: git(eng, 'commit', '-q', '-m', f'{v} vendor record\n\n{EVIDENCE.format(v=v)}', '--', 'scripts/vendor_records.tsv') + sh(sys.executable, os.path.join(eng, 'scripts', 'vendor_engine.py'), '--pack', tree, '--check', cwd=eng) # clean engine again step(f'push release/{v} and wait for CI + Install path on {rel_sha[:9]}') git(PACK, 'push', '-q', 'origin', f'release/{v}') From cadd25b9b6857eada98bb7af141c4bc62909a917 Mon Sep 17 00:00:00 2001 From: vveerrgg Date: Mon, 9 Nov 2026 15:47:11 -0700 Subject: [PATCH 3/5] Install path: the mouth Dub runs as a user's own face (a name the preset does not list), so the plain stack's drive stages run; that path is where 0.2.1's closures crash was; the hub mirror passes every other repo through to the Hub --- scripts/install_path_check.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/install_path_check.py b/scripts/install_path_check.py index 0891733..435400a 100644 --- a/scripts/install_path_check.py +++ b/scripts/install_path_check.py @@ -14,7 +14,7 @@ (exactly the four pinned files at the pinned revision, no safetensors), and phonemizer is never needed; 4. dub: the demo workflow (BYO Clip and Song) at its defaults: Dub, Verify, Save Cut; 5. mouth: the same Dub with Mouth Tweaks warp_events vocal, warp_hold nucleus-follow, warp_closures aligned-press, - warp_aperture p95 (the Dub's auto path). + warp_aperture p95, on the demo kit named as a user's own face (the Dub's auto path, the plain mouth stack). A dubbed shot that errors fails the test even though the Dub itself carries on with the source (on_fail=source). --hf-mirror DIR serves the four recogniser files from DIR on a local Hugging Face endpoint (CI's cache, so the files are @@ -37,6 +37,9 @@ REPO = 'facebook/wav2vec2-lv-60-espeak-cv-ft' RECOGNISER_DIR = os.path.join('lsdub', 'wav2vec2-espeak') +# the mouth Dub runs as a user's own face: a name the preset does not list, so the plain mouth stack and its drive stages +# (events, closures, hold) run, as they do for every face but the demo's (0.2.1's closures crash was on that path) +USER_FACE = 'my_singer' MOUTH = dict(warp_events='vocal', warp_hold='nucleus-follow', warp_closures='aligned-press', warp_aperture='p95') BAD_LOG = [(re.compile(r'phonemizer', re.I), 'phonemizer was needed'), (re.compile(r'phone network not loaded'), 'the phone recogniser did not load (vocal events fell back to flux)'), @@ -170,7 +173,7 @@ def texts(outputs, node): return '\n'.join(t if isinstance(t, list) else [str(t)]) -def workflow(name, aligner='sphinx', mouth=False, align_only=False): +def workflow(name, aligner='sphinx', mouth=False, align_only=False, identity='red_nova'): w = { '1': ('LSDubProject', dict(name=name, root='', mode='run', fps=25, song_offset=0, preset='landing', seed=0, force_shots='', backend='local')), '2': ('LoadVideo', dict(file='lsdub_demo_landing_clip.mp4')), @@ -189,7 +192,7 @@ def workflow(name, aligner='sphinx', mouth=False, align_only=False): '7': ('LSDubShotSplit', dict(project=['1', 0], source=['3', 0])), '10': ('LSDubTriage', dict(samples=6, project=['1', 0], source=['3', 0], shots=['7', 0], song=['6', 0])), '11': ('PreviewAny', dict(source=['10', 2])), - '14': ('LSDubBuildLipKit', dict(identity='red_nova', kit_dir='lsdub_demo_kits/red_nova.zip', shot_id=-1, + '14': ('LSDubBuildLipKit', dict(identity=identity, kit_dir='lsdub_demo_kits/red_nova.zip', shot_id=-1, keep='person, same face shape, skin tone, hair, eyewear and makeup', seeds=3, project=['1', 0])), '15': ('LSDubDub', dict(only_shots='', quality=18, on_fail='source', audio='source', vocalisations='skip', project=['1', 0], source=['3', 0], shots=['10', 0], song=['6', 0], align=['9', 0], kit=['14', 0])), @@ -330,7 +333,7 @@ def dub(name, label, kw): if 'dub' in steps: step('dub', lambda: dub('ip-dub-defaults', 'dub at defaults', {})) if 'mouth' in steps: step('mouth', lambda: dub('ip-dub-mouth', 'dub with the mouth Combos (' + ', '.join(f'{k} {v}' for k, v in MOUTH.items()) + ')', - dict(aligner='ctc', mouth=True))) + dict(aligner='ctc', mouth=True, identity=USER_FACE))) if failures: raise Fail(f'{len(failures)} step(s) failed:\n ' + '\n '.join(failures)) say('PASS') finally: From 6c1d2b3646b58cf0ca696557fdeec7678d3f7f94 Mon Sep 17 00:00:00 2001 From: vveerrgg Date: Mon, 9 Nov 2026 16:19:15 -0700 Subject: [PATCH 4/5] Install path: save the recogniser cache even when another step fails --- .github/workflows/install-path.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/install-path.yml b/.github/workflows/install-path.yml index 28c36cd..4d6f0fa 100644 --- a/.github/workflows/install-path.yml +++ b/.github/workflows/install-path.yml @@ -52,7 +52,8 @@ jobs: venv/bin/pip freeze | grep -iE '^(torch|transformers|huggingface.hub|mediapipe|numpy|phonemizer)=' || true if venv/bin/python -c "import phonemizer" 2>/dev/null; then echo "::error::phonemizer is installed; the test needs a venv without it"; exit 1; fi - name: Recogniser cache (fetched from the Hub once per revision, then served to the same client path) - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + id: hfcache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: hf-mirror key: lsdub-recogniser-${{ env.RECOGNISER_REVISION }} @@ -63,6 +64,12 @@ jobs: venv/bin/python pack/scripts/install_path_check.py --comfyui comfy --pack pack \ --hf-mirror hf-mirror --save-mirror hf-mirror \ --demo-fallback demo-fallback.zip --demo-fallback-sha "$DEMO_FALLBACK_SHA" + - name: Save the recogniser cache (also when another step failed: the files are checked by sha256 either way) + if: always() && steps.hfcache.outputs.cache-hit != 'true' && hashFiles('hf-mirror/pytorch_model.bin') != '' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: hf-mirror + key: lsdub-recogniser-${{ env.RECOGNISER_REVISION }} - name: ComfyUI log if: always() run: tail -n 300 comfy/install-path-comfyui.log || true From c85e86f9dd777dc2b59798dc39069aab7d69d304 Mon Sep 17 00:00:00 2001 From: vveerrgg Date: Mon, 9 Nov 2026 16:43:39 -0700 Subject: [PATCH 5/5] Round 2: install-path.yml valid YAML (quoted step name); release.py names what it pushed when it stops, pushes main and the tag atomically, plain stops for a site main without the changelog page, Registry Pending vs Active; recovery in CONTRIBUTING; stability wording --- .github/workflows/install-path.yml | 2 +- CONTRIBUTING.md | 15 +++++++++++++-- README.md | 3 +-- scripts/release.py | 23 +++++++++++++++-------- 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/.github/workflows/install-path.yml b/.github/workflows/install-path.yml index 4d6f0fa..a851a41 100644 --- a/.github/workflows/install-path.yml +++ b/.github/workflows/install-path.yml @@ -64,7 +64,7 @@ jobs: venv/bin/python pack/scripts/install_path_check.py --comfyui comfy --pack pack \ --hf-mirror hf-mirror --save-mirror hf-mirror \ --demo-fallback demo-fallback.zip --demo-fallback-sha "$DEMO_FALLBACK_SHA" - - name: Save the recogniser cache (also when another step failed: the files are checked by sha256 either way) + - name: "Save the recogniser cache (also when another step failed; the files are checked by sha256 either way)" if: always() && steps.hfcache.outputs.cache-hit != 'true' && hashFiles('hf-mirror/pytorch_model.bin') != '' uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f5b94e..3af30f5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,12 +32,12 @@ By contributing you agree your work is licensed under the Apache License 2.0, th ## Stability policy - Releases are batched; patch versions only for breakage. Features go into the next minor version. -- Defaults stay byte-identical across a minor series unless the changelog says otherwise. +- Defaults stay unchanged within a minor series unless the changelog says otherwise. - Every release passes the install-path test (`.github/workflows/install-path.yml`, `scripts/install_path_check.py`) on a clean ComfyUI before it is tagged. Run it yourself: `python scripts/install_path_check.py --comfyui PATH/TO/ComfyUI` with that ComfyUI's venv (its requirements plus `requirements.txt`, nothing else). - Models are pinned by revision and sha256. -- 0.2.0 to 0.2.2 landed in one day while the install-path test did not yet exist; it has run on every release since. +- 0.2.0 to 0.2.2 landed in one day while the install-path test did not yet exist; it runs on every release from 0.2.3 on. ## Releases (maintainers) One command, from a clean pack checkout on main with the engine and the microsite checked out beside it: @@ -52,6 +52,17 @@ engine; runs the pack checks; pushes `release/X.Y.Z` and waits for CI and Instal watches the publish (which runs Install path again first), confirms the Registry lists the version and checks the site deploy. Any failure before the tag stops with a plain message and leaves nothing changed remotely. +If release.py stops part way: +- Before "push release/X.Y.Z": nothing remote changed. Fix the cause; delete the local `release/X.Y.Z` branch + (`git checkout main && git branch -D release/X.Y.Z`) and drop a local engine record commit if one was made + (`git -C ../lsdub reset --hard origin/main`), then run it again. +- After the release branch is pushed but before the tag: only `release/X.Y.Z` is on origin. Fix on that branch and push + again (release.py waits on the new commit), or delete it (`git push origin --delete release/X.Y.Z`) and start over. +- After the tag: main and `vX.Y.Z` were pushed together (`--atomic`), so publish is running. Never re-tag or delete the tag; + watch the publish run, and if it failed, fix and release the next patch version. Push the engine record by hand if + release.py stopped before it did (`git -C ../lsdub push origin main`). +release.py names what it already pushed whenever it stops. + Repository secrets: - `REGISTRY_ACCESS_TOKEN` (here): the Comfy Registry publisher key, already set. - `MICROSITE_DISPATCH_TOKEN` (here): a fine-grained GitHub token, resource owner HumanjavaEnterprises, repository access only diff --git a/README.md b/README.md index f106f0c..39ccb03 100644 --- a/README.md +++ b/README.md @@ -206,8 +206,7 @@ Python SDK (`pip install comfy-sdk`). LSDub aims to be stable software that is easy to install and build on, not a stream of updates. - Releases are batched. A patch version (0.2.x) only fixes breakage; new features wait for the next minor version. -- Across a minor series the defaults stay byte-identical: the same inputs at the defaults give the same render, unless the - changelog says otherwise. +- Within a minor series the defaults stay unchanged unless the changelog says otherwise. - Every release passes the install-path test before it is tagged: a clean ComfyUI on CPU, only this pack's requirements, every node registered, the phone recogniser downloaded into an empty models folder, and a Dub at the defaults and with the mouth switches on the demo clip. diff --git a/scripts/release.py b/scripts/release.py index c7f54b6..cb86409 100644 --- a/scripts/release.py +++ b/scripts/release.py @@ -40,11 +40,14 @@ class Stop(SystemExit): - def __init__(self, msg): super().__init__(f'\nrelease: STOPPED. {msg}\nNothing remote was changed.' if not REMOTE_TOUCHED[0] else - f'\nrelease: STOPPED. {msg}') + def __init__(self, msg): + pushed = '\n'.join(f' - {p}' for p in PUSHED) + tail = ('\nNothing remote was changed.' if not PUSHED else + f'\nAlready pushed:\n{pushed}\nRecovery: CONTRIBUTING.md, Releases, "If release.py stops part way".') + super().__init__(f'\nrelease: STOPPED. {msg}{tail}') -REMOTE_TOUCHED = [False] +PUSHED = [] STEP = [0] @@ -164,7 +167,9 @@ def main(): try: RN.check(read(os.path.join(PACK, 'CHANGELOG.md')), v) except ValueError as e: raise Stop(f'CHANGELOG.md: {e}. Write the {v} section first (CONTRIBUTING.md, Release notes).') from None git(site, 'fetch', '-q', 'origin', 'main', quiet=True) - page = git(site, 'show', 'origin/main:docs/changelog.html', quiet=True) + r = subprocess.run(['git', 'show', 'origin/main:docs/changelog.html'], cwd=site, capture_output=True, text=True) + if r.returncode: raise Stop(f'the microsite main ({site}) has no docs/changelog.html. The changelog page has to be on the site\'s main first.') + page = r.stdout if f'id="v{v}"' not in page: raise Stop(f'the microsite main has no
in docs/changelog.html. Add it on the site\'s main first.') print(f' demo zip ok: {demo_published(PACK)}') @@ -212,19 +217,20 @@ def main(): step(f'push release/{v} and wait for CI + Install path on {rel_sha[:9]}') git(PACK, 'push', '-q', 'origin', f'release/{v}') + PUSHED.append(f'pack branch release/{v} ({rel_sha[:9]}); main, tags and the Registry untouched') print(' (a branch only; main, tags and the Registry are untouched until both pass)') time.sleep(20) runs_green(PACK_REPO, rel_sha, ['CI', 'Install path'], wait=True) step(f'fast-forward main, tag v{v}, push; push the engine record') - REMOTE_TOUCHED[0] = True git(PACK, 'checkout', '-q', 'main') git(PACK, 'merge', '-q', '--ff-only', f'release/{v}') - git(PACK, 'push', '-q', 'origin', 'main') git(PACK, 'tag', '-a', f'v{v}', '-m', f'LSDub {v}') - git(PACK, 'push', '-q', 'origin', f'v{v}') + git(PACK, 'push', '-q', '--atomic', 'origin', 'main', f'v{v}') # main and the tag land together or not at all + PUSHED.append(f'pack main -> {rel_sha[:9]} and tag v{v} (publish starts from the tag)') tag_time = time.time() git(eng, 'push', '-q', 'origin', 'main') + PUSHED.append('engine main: the vendor record commit') step('watch publish (it re-runs Install path on the tag first)') time.sleep(20) @@ -234,7 +240,8 @@ def main(): for _ in range(30): with urllib.request.urlopen(REGISTRY, timeout=60) as r: vs = {x.get('version'): x.get('status') for x in json.loads(r.read())} # noqa: S310 if v in vs: - print(f' {v}: {vs[v]}') + st = {'NodeVersionStatusActive': 'Active', 'NodeVersionStatusPending': 'Pending'}.get(vs[v], vs[v]) + print(f' {v}: {st}' + (' (listed; the Registry still has to review it before it is Active)' if st == 'Pending' else '')) break time.sleep(30) else: