diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43dff2e..c6c6a06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: pull_request: push: - branches: [main, rack] + branches: [main, rack, "release/**"] permissions: contents: read jobs: diff --git a/.github/workflows/install-path.yml b/.github/workflows/install-path.yml new file mode 100644 index 0000000..a851a41 --- /dev/null +++ b/.github/workflows/install-path.yml @@ -0,0 +1,80 @@ +# The install-path test: LSDub in a clean ComfyUI the way a user gets it (ComfyUI's requirements + requirements.txt, no dev +# extras), CPU only. Every LSDub node registers; Align with aligner ctc from an empty models folder (the real recogniser +# download); a Dub at defaults; a Dub with the mouth Combos. scripts/install_path_check.py does the work; it runs the same on a +# laptop. publish.yml calls this workflow first, so a tag that fails it is never published. +name: Install path +on: + pull_request: + branches: [main] + push: + branches: ["release/**"] # scripts/release.py waits for this run before it tags + workflow_dispatch: + workflow_call: + schedule: + - cron: "17 6 * * 1" # weekly: ComfyUI's pin stays put, but PyPI and the Hub do not +permissions: + contents: read +concurrency: + group: install-path-${{ github.ref }} + cancel-in-progress: true +env: + COMFYUI_REPO: https://github.com/comfyanonymous/ComfyUI.git + COMFYUI_COMMIT: 355cd6b14c92c19ac3188fe528e47a3f21ef5178 # v0.42.0 + RECOGNISER_REVISION: ae45363bf3413b374fecd9dc8bc1df0e24c3b7f4 # facebook/wav2vec2-lv-60-espeak-cv-ft, as lsdub_engine/_models.py pins it + # only used if scripts/fetch_demo.py itself fails (that still fails the job): the released v0.1.0 demo zip at its real checksum + DEMO_FALLBACK_URL: https://github.com/HumanjavaEnterprises/comfyui-lsdub/releases/download/v0.1.0/lsdub-demo-0.1.0.zip + DEMO_FALLBACK_SHA: 7981fef8f453b0bf29e47b3ffc00b3adcba1d22d8dbe259504f66c2beb00de17 +jobs: + install-path: + runs-on: ubuntu-latest + timeout-minutes: 150 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + path: pack + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.12" + - name: System deps (what the README asks a user for) + run: sudo apt-get update && sudo apt-get install -y ffmpeg libgles2 libegl1 + - name: The pinned recogniser revision is the one the pack pins + run: grep -q "$RECOGNISER_REVISION" pack/lsdub_engine/_models.py + - name: ComfyUI at the pinned commit + run: | + git init -q comfy && cd comfy + git fetch -q --depth 1 "$COMFYUI_REPO" "$COMFYUI_COMMIT" && git checkout -q FETCH_HEAD + - name: A user's venv (ComfyUI requirements + requirements.txt, CPU torch) + run: | + python -m venv venv + venv/bin/pip install -q --upgrade pip + venv/bin/pip install -q torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + venv/bin/pip install -q -r comfy/requirements.txt -r pack/requirements.txt + venv/bin/pip freeze | grep -iE '^(torch|transformers|huggingface.hub|mediapipe|numpy|phonemizer)=' || true + if venv/bin/python -c "import phonemizer" 2>/dev/null; then echo "::error::phonemizer is installed; the test needs a venv without it"; exit 1; fi + - name: Recogniser cache (fetched from the Hub once per revision, then served to the same client path) + id: hfcache + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: hf-mirror + key: lsdub-recogniser-${{ env.RECOGNISER_REVISION }} + - name: Demo fallback zip + run: curl -sSfL -o demo-fallback.zip "$DEMO_FALLBACK_URL" + - name: Install path (nodes, align ctc from empty, dub at defaults, dub with the mouth Combos) + run: | + venv/bin/python pack/scripts/install_path_check.py --comfyui comfy --pack pack \ + --hf-mirror hf-mirror --save-mirror hf-mirror \ + --demo-fallback demo-fallback.zip --demo-fallback-sha "$DEMO_FALLBACK_SHA" + - name: "Save the recogniser cache (also when another step failed; the files are checked by sha256 either way)" + if: always() && steps.hfcache.outputs.cache-hit != 'true' && hashFiles('hf-mirror/pytorch_model.bin') != '' + uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: hf-mirror + key: lsdub-recogniser-${{ env.RECOGNISER_REVISION }} + - name: ComfyUI log + if: always() + run: tail -n 300 comfy/install-path-comfyui.log || true + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: failure() + with: + name: install-path-log + path: comfy/install-path-comfyui.log diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index be6c0b1..c4329eb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -5,7 +5,11 @@ on: permissions: contents: read jobs: + install-path: # the gate: a tag whose tree fails the clean-ComfyUI install test is never published + if: github.repository == 'HumanjavaEnterprises/comfyui-lsdub' + uses: ./.github/workflows/install-path.yml publish: + needs: install-path runs-on: ubuntu-latest if: github.repository == 'HumanjavaEnterprises/comfyui-lsdub' steps: @@ -18,3 +22,20 @@ jobs: with: personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }} # set this secret in the repo settings skip_checkout: 'true' + site: # after a good publish: ask the microsite to deploy its main branch (it serves /changelog) + needs: publish + runs-on: ubuntu-latest + steps: + - name: Dispatch lsdub-release to the microsite + env: + TOKEN: ${{ secrets.MICROSITE_DISPATCH_TOKEN }} + TAG: ${{ github.ref_name }} + run: | + if [ -z "$TOKEN" ]; then + echo "::notice title=Site deploy skipped::MICROSITE_DISPATCH_TOKEN is not set, so the microsite was not asked to deploy. Add it (CONTRIBUTING.md, Releases) or deploy the site by hand." + exit 0 + fi + curl -sSf -X POST -H "Authorization: Bearer $TOKEN" -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/HumanjavaEnterprises/lsdub.web.microsite.src/dispatches \ + -d "{\"event_type\":\"lsdub-release\",\"client_payload\":{\"tag\":\"$TAG\"}}" + echo "::notice title=Site deploy requested::lsdub-release $TAG sent to the microsite" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index db1f292..3af30f5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,3 +29,43 @@ By contributing you agree your work is licensed under the Apache License 2.0, th - No `## Unreleased` heading in the file; add the section when the version is released. - The heading names the version in pyproject.toml. `python scripts/release_notes.py` checks all of this; CI runs it (`tests/test_release_notes.py`) and the publish job fails without it. + +## Stability policy +- Releases are batched; patch versions only for breakage. Features go into the next minor version. +- Defaults stay unchanged within a minor series unless the changelog says otherwise. +- Every release passes the install-path test (`.github/workflows/install-path.yml`, `scripts/install_path_check.py`) on a + clean ComfyUI before it is tagged. Run it yourself: `python scripts/install_path_check.py --comfyui PATH/TO/ComfyUI` with + that ComfyUI's venv (its requirements plus `requirements.txt`, nothing else). +- Models are pinned by revision and sha256. +- 0.2.0 to 0.2.2 landed in one day while the install-path test did not yet exist; it runs on every release from 0.2.3 on. + +## Releases (maintainers) +One command, from a clean pack checkout on main with the engine and the microsite checked out beside it: + + python scripts/release.py X.Y.Z --dry-run # every check, in a throwaway worktree; pushes nothing + python scripts/release.py X.Y.Z + +Before it: write the `## X.Y.Z` section in CHANGELOG.md (see Release notes) and the matching `
` in the +microsite's `docs/changelog.html` on its main. The command refuses unless both trees are clean and on main and engine CI is +green; bumps the version, the Icon/Banner URLs and CITATION.cff; regenerates the node tables and the bug form; vendors the +engine; runs the pack checks; pushes `release/X.Y.Z` and waits for CI and Install path; then fast-forwards main, tags `vX.Y.Z`, +watches the publish (which runs Install path again first), confirms the Registry lists the version and checks the site +deploy. Any failure before the tag stops with a plain message and leaves nothing changed remotely. + +If release.py stops part way: +- Before "push release/X.Y.Z": nothing remote changed. Fix the cause; delete the local `release/X.Y.Z` branch + (`git checkout main && git branch -D release/X.Y.Z`) and drop a local engine record commit if one was made + (`git -C ../lsdub reset --hard origin/main`), then run it again. +- After the release branch is pushed but before the tag: only `release/X.Y.Z` is on origin. Fix on that branch and push + again (release.py waits on the new commit), or delete it (`git push origin --delete release/X.Y.Z`) and start over. +- After the tag: main and `vX.Y.Z` were pushed together (`--atomic`), so publish is running. Never re-tag or delete the tag; + watch the publish run, and if it failed, fix and release the next patch version. Push the engine record by hand if + release.py stopped before it did (`git -C ../lsdub push origin main`). +release.py names what it already pushed whenever it stops. + +Repository secrets: +- `REGISTRY_ACCESS_TOKEN` (here): the Comfy Registry publisher key, already set. +- `MICROSITE_DISPATCH_TOKEN` (here): a fine-grained GitHub token, resource owner HumanjavaEnterprises, repository access only + `lsdub.web.microsite.src`, permission Contents: Read and write (what `POST /repos/.../dispatches` needs). Without it the + publish job's site step prints a notice and skips, and `release.py` starts the deploy itself with your `gh` login. +- `CLOUDFLARE_API_TOKEN` (in the microsite repo): see its README, Deploys. diff --git a/README.md b/README.md index 1e2d24f..39ccb03 100644 --- a/README.md +++ b/README.md @@ -202,12 +202,28 @@ Python SDK (`pip install comfy-sdk`). Qwen-Image-Edit Lightning LoRA). - **Cost:** cloud runs use your Comfy credits. +## Stability and releases +LSDub aims to be stable software that is easy to install and build on, not a stream of updates. + +- Releases are batched. A patch version (0.2.x) only fixes breakage; new features wait for the next minor version. +- Within a minor series the defaults stay unchanged unless the changelog says otherwise. +- Every release passes the install-path test before it is tagged: a clean ComfyUI on CPU, only this pack's + requirements, every node registered, the phone recogniser downloaded into an empty models folder, and a Dub at the + defaults and with the mouth switches on the demo clip. +- Models are pinned by revision and checked by sha256 ([MODELS.md](MODELS.md)). +- What changed: [the changelog](https://lsdub.sndbnch.app/changelog). Something broke: the + [issue forms](https://github.com/HumanjavaEnterprises/comfyui-lsdub/issues/new/choose). + +In honesty: 0.2.0, 0.2.1 and 0.2.2 landed in one day, before the install-path test existed. Two of them fixed breakage a +clean install would have caught (the phone recogniser in a fresh ComfyUI, and the Dub with the mouth switches on). The test +now runs on every pull request, before every publish, and weekly. + ## Troubleshooting - **The first Align run sits there.** It is downloading Whisper large-v3 (about 3 GB) with no progress bar in the UI. Let it finish once. -- **The demo download fails.** Get `lsdub-demo-0.1.0.zip` from the GitHub Releases page by hand and run - `python scripts/fetch_demo.py --zip lsdub-demo-0.1.0.zip`. +- **The demo download fails.** Get `lsdub-demo-0.1.1.zip` from the GitHub Releases page by hand and run + `python scripts/fetch_demo.py --zip lsdub-demo-0.1.1.zip`. - **Align warns about the lyrics.** If what is sung matches the lyrics poorly (below 0.5), it warns and places the lyrics on the voice. Below 0.2 it stops and asks you to check the lyrics: they probably belong to another song. - **A shot failed to dub.** It is reported, and by default the cut keeps its original frames. Run workflow 3 on the diff --git a/lsdub_engine/VERSION b/lsdub_engine/VERSION index 2e2a75f..291b2f7 100644 --- a/lsdub_engine/VERSION +++ b/lsdub_engine/VERSION @@ -1,4 +1,4 @@ pack_version: 0.2.2 uncommitted: 0 -tree_sha256: 13cb87b83854a0f23de5f00ff3184811a78e3f004cd7fa6f5b01cb714066cb46 -tests_sha256: 70cb132afd1ac5ac9aef39a95d11e23041668f2776c9d64f187b0a7fb48be82f +tree_sha256: f36b7410d7dc3461fba57844603c81a5ba5ec37adb160148f9dc50595e4f19ba +tests_sha256: 59f5e441dbefa42caf1faf35824ac93cfec99b3e58b280a0e522d2f0f8759edb diff --git a/lsdub_engine/audits/qa.py b/lsdub_engine/audits/qa.py index 860675c..6c4a410 100644 --- a/lsdub_engine/audits/qa.py +++ b/lsdub_engine/audits/qa.py @@ -526,7 +526,7 @@ def asked_travel(s): EV_KEYS = ("onset_hit", "onset_n", "closure_hit", "closure_n", "spurious", "spurious_take", "open_ratio", "open_n") -LAG_WINDOW_S = 0.2 # a mouth rise this near a sung onset, either side, is that onset's rise (G10 lane 04's per-onset reading) +LAG_WINDOW_S = 0.2 # a mouth rise this near a sung onset, either side, is that onset's rise (the per-onset reading) def vocal_events(vocal, align=None, phone_models=None): diff --git a/scripts/fetch_demo.py b/scripts/fetch_demo.py index 37472db..3bf5bbd 100644 --- a/scripts/fetch_demo.py +++ b/scripts/fetch_demo.py @@ -20,10 +20,10 @@ import urllib.request import zipfile -VERSION = '0.1.0' -TAG = f'v{VERSION}' +VERSION = '0.1.1' +TAG = 'demo-0.1.1' URL = f'https://github.com/HumanjavaEnterprises/comfyui-lsdub/releases/download/{TAG}/lsdub-demo-{VERSION}.zip' -SHA256 = 'a431085d931f2b983b620d31710ac868a459d461a6dcb158177d199637790edb' +SHA256 = '0b15dfcf069b6dc2cac25bbe312137eda435e4ac350cce6b61436419fd2b518c' def unpack(zf, dst): diff --git a/scripts/install_path_check.py b/scripts/install_path_check.py new file mode 100644 index 0000000..435400a --- /dev/null +++ b/scripts/install_path_check.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Humanjava Enterprises. LSDub: https://github.com/HumanjavaEnterprises/comfyui-lsdub +"""The install-path test: LSDub in a clean ComfyUI, the way a user gets it. CPU only. CI runs it on every PR to main, before +every publish, weekly, and by hand (.github/workflows/install-path.yml); scripts/release.py waits for it. + + python scripts/install_path_check.py --comfyui PATH/TO/ComfyUI [--pack .] [--port 8199] [--demo-zip lsdub-demo-0.1.0.zip] + [--hf-mirror DIR] [--save-mirror DIR] [--steps nodes,align,dub,mouth] + +Run it with the ComfyUI venv's python (ComfyUI's requirements + this pack's requirements.txt, nothing else). It: + 1. copies the pack (this checkout, without .git) into ComfyUI/custom_nodes/comfyui-lsdub and fetches the demo assets; + 2. starts ComfyUI headless (--cpu) and checks that every LSDub node in the source registered; + 3. align: LSDub Align with aligner ctc from an EMPTY models/lsdub/wav2vec2-espeak. The recogniser is downloaded for real + (exactly the four pinned files at the pinned revision, no safetensors), and phonemizer is never needed; + 4. dub: the demo workflow (BYO Clip and Song) at its defaults: Dub, Verify, Save Cut; + 5. mouth: the same Dub with Mouth Tweaks warp_events vocal, warp_hold nucleus-follow, warp_closures aligned-press, + warp_aperture p95, on the demo kit named as a user's own face (the Dub's auto path, the plain mouth stack). +A dubbed shot that errors fails the test even though the Dub itself carries on with the source (on_fail=source). + +--hf-mirror DIR serves the four recogniser files from DIR on a local Hugging Face endpoint (CI's cache, so the files are +fetched from the Hub once per revision); the client path (hf_hub_download into the empty folder) is unchanged. +--save-mirror DIR copies the downloaded files there after a real download, for that cache. Exit 0 = pass.""" +import argparse +import glob +import hashlib +import http.server +import json +import os +import re +import shutil +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request + +REPO = 'facebook/wav2vec2-lv-60-espeak-cv-ft' +RECOGNISER_DIR = os.path.join('lsdub', 'wav2vec2-espeak') +# the mouth Dub runs as a user's own face: a name the preset does not list, so the plain mouth stack and its drive stages +# (events, closures, hold) run, as they do for every face but the demo's (0.2.1's closures crash was on that path) +USER_FACE = 'my_singer' +MOUTH = dict(warp_events='vocal', warp_hold='nucleus-follow', warp_closures='aligned-press', warp_aperture='p95') +BAD_LOG = [(re.compile(r'phonemizer', re.I), 'phonemizer was needed'), + (re.compile(r'phone network not loaded'), 'the phone recogniser did not load (vocal events fell back to flux)'), + (re.compile(r'shot \d+ failed: '), 'a dubbed shot raised (the Dub fell back to the source)'), + (re.compile(r'IMPORT FAILED.*lsdub', re.I), 'the pack failed to import')] + + +class Fail(SystemExit): + def __init__(self, msg): super().__init__(f'INSTALL-PATH FAIL: {msg}') + + +def say(msg): print(f'[install-path] {msg}', flush=True) + + +def pinned(pack): + """the recogniser's pinned revision and file shas, read from the pack under test (lsdub_engine/_models.py).""" + src = open(os.path.join(pack, 'lsdub_engine', '_models.py'), encoding='utf-8').read() + i = src.index(f'"{REPO}"') + rev = re.search(r'"revision":\s*"([0-9a-f]{40})"', src[i:]).group(1) + files = dict(re.findall(r'"([\w.]+)":\s*"([0-9a-f]{64})"', src[i:src.index('}', src.index('"files"', i)) + 1])) + return rev, files + + +def stand_in(fd, z, sha): + """unpack a demo zip the way fetch_demo.py does, with the zip's own checksum (only after fetch_demo itself has failed).""" + h = hashlib.sha256(open(z, 'rb').read()).hexdigest() + if h != sha: raise Fail(f'demo stand-in {z}: sha256 {h} != {sha}') + src = open(fd[1], encoding='utf-8').read() + tmp = fd[1] + '.standin.py' + import zipfile + root = {n.split('/')[0] for n in zipfile.ZipFile(z).namelist()}.pop() # lsdub-demo- + src = re.sub(r"^VERSION = '.*'$", f"VERSION = '{root.rsplit('-', 1)[-1]}'", src, flags=re.M) + open(tmp, 'w', encoding='utf-8').write(re.sub(r"^SHA256 = '.*'$", f"SHA256 = '{sha}'", src, flags=re.M)) + try: subprocess.run([fd[0], tmp] + fd[2:] + ['--zip', z], check=True, stdout=subprocess.DEVNULL) + finally: os.remove(tmp) + + +def node_ids(pack): + ids = set() + for p in glob.glob(os.path.join(pack, '*.py')): + ids |= set(re.findall(r"node_id='(LSDub\w+)'", open(p, encoding='utf-8').read())) + return ids + + +# ------------------------------------------------------------------------------------------------ the local hub mirror +def serve_mirror(folder, rev, files): + class H(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): pass + + def _file(self): + m = re.match(rf'^/{re.escape(REPO)}/resolve/([^/]+)/([\w.]+)$', self.path.split('?')[0]) + if not m or m.group(1) not in (rev, 'main') or m.group(2) not in files: return None + return os.path.join(folder, m.group(2)) + + def _head(self, p): + self.send_response(200) + self.send_header('X-Repo-Commit', rev) + self.send_header('ETag', f'"{files[os.path.basename(p)]}"') + self.send_header('Content-Length', str(os.path.getsize(p))) + self.send_header('Content-Type', 'application/octet-stream') + self.end_headers() + + def _proxy(self, method): + """anything but the four recogniser files (DINOv2, ...) goes to the real Hub, redirects passed through as they come.""" + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *a, **k): return None + req = urllib.request.Request('https://huggingface.co' + self.path, method=method, + headers={k: v for k, v in self.headers.items() if k.lower() not in ('host', 'connection')}) + try: r = urllib.request.build_opener(NoRedirect).open(req, timeout=300) # noqa: S310 (fixed https host) + except urllib.error.HTTPError as e: r = e + self.send_response(r.status if hasattr(r, 'status') else r.code) + for k, v in r.headers.items(): + if k.lower() not in ('transfer-encoding', 'connection'): self.send_header(k, v) + self.end_headers() + if method == 'GET': shutil.copyfileobj(r, self.wfile) + + def do_HEAD(self): + p = self._file() + if not p: + self._proxy('HEAD') + return + if not os.path.isfile(p): + self.send_error(404) + return + self._head(p) + + def do_GET(self): + p = self._file() + if not p: + self._proxy('GET') + return + if not os.path.isfile(p): + self.send_error(404) + return + self._head(p) + with open(p, 'rb') as f: shutil.copyfileobj(f, self.wfile) + + srv = http.server.ThreadingHTTPServer(('127.0.0.1', 0), H) + threading.Thread(target=srv.serve_forever, daemon=True).start() + return f'http://127.0.0.1:{srv.server_address[1]}' + + +# ------------------------------------------------------------------------------------------------ ComfyUI +def api(port, path, body=None, timeout=30): + req = urllib.request.Request(f'http://127.0.0.1:{port}{path}', data=json.dumps(body).encode() if body is not None else None, + headers={'Content-Type': 'application/json'}) + with urllib.request.urlopen(req, timeout=timeout) as r: # noqa: S310 (loopback) + return json.loads(r.read()) + + +def run_prompt(port, prompt, label, timeout): + try: + pid = api(port, '/prompt', {'prompt': prompt, 'client_id': 'install-path'})['prompt_id'] + except urllib.error.HTTPError as e: + raise Fail(f'{label}: ComfyUI refused the workflow: {e.read().decode(errors="replace")[:2000]}') from None + t0 = time.time() + while time.time() - t0 < timeout: + h = api(port, f'/history/{pid}').get(pid) + if h and h.get('status', {}).get('completed') is not None and (h['status'].get('completed') or h['status'].get('status_str') == 'error'): + if h['status'].get('status_str') != 'success': + msgs = [m for m in h['status'].get('messages', []) if m[0] == 'execution_error'] + raise Fail(f'{label}: the workflow failed: {json.dumps(msgs)[:3000]}') + say(f'{label}: done in {time.time() - t0:.0f} s') + return h['outputs'] + time.sleep(3) + raise Fail(f'{label}: no result after {timeout} s') + + +def texts(outputs, node): + t = outputs.get(node, {}).get('text', []) + return '\n'.join(t if isinstance(t, list) else [str(t)]) + + +def workflow(name, aligner='sphinx', mouth=False, align_only=False, identity='red_nova'): + w = { + '1': ('LSDubProject', dict(name=name, root='', mode='run', fps=25, song_offset=0, preset='landing', seed=0, force_shots='', backend='local')), + '2': ('LoadVideo', dict(file='lsdub_demo_landing_clip.mp4')), + '3': ('LSDubVideoSlot', dict(prompt='', negative_prompt='', seconds=5, project=['1', 0], video=['2', 0])), + '4': ('LoadAudio', dict(audio='lsdub_demo_landing_song.wav')), + '5': ('LoadAudio', dict(audio='lsdub_demo_landing_lead.wav')), + '6': ('LSDubSongSlot', dict(tags='', seconds=60, bpm=120, keyscale='C major', project=['1', 0], audio=['4', 0], vocal=['5', 0])), + '8': ('LSDubLyricsSlot', dict(text='Pleiadians bending space and time\nFour hundred light-years far away\nAlways so serious, never dance a day', + project=['1', 0])), + '9': ('LSDubAlign', dict(pronunciations='', stt_model='large-v3', transcript_json='lsdub_demo_landing_stt.json', aligner=aligner, + project=['1', 0], song=['6', 0], lyrics=['8', 0])), + '13': ('PreviewAny', dict(source=['9', 1])), + } + if not align_only: + w.update({ + '7': ('LSDubShotSplit', dict(project=['1', 0], source=['3', 0])), + '10': ('LSDubTriage', dict(samples=6, project=['1', 0], source=['3', 0], shots=['7', 0], song=['6', 0])), + '11': ('PreviewAny', dict(source=['10', 2])), + '14': ('LSDubBuildLipKit', dict(identity=identity, kit_dir='lsdub_demo_kits/red_nova.zip', shot_id=-1, + keep='person, same face shape, skin tone, hair, eyewear and makeup', seeds=3, project=['1', 0])), + '15': ('LSDubDub', dict(only_shots='', quality=18, on_fail='source', audio='source', vocalisations='skip', project=['1', 0], + source=['3', 0], shots=['10', 0], song=['6', 0], align=['9', 0], kit=['14', 0])), + '16': ('LSDubVerify', dict(project=['1', 0], source=['3', 0], render=['15', 0], shots=['10', 0], song=['6', 0], align=['9', 0])), + '17': ('PreviewAny', dict(source=['16', 1])), + '20': ('LSDubSaveCut', dict(watermark=True, quality=18, project=['1', 0], render=['15', 0])), + }) + if mouth: + w['21'] = ('LSDubTweaks', dict(lead=0.06, word_close=0.06, face_target=512.0, snap=True, auto=True, continuity=True, settings='', **MOUTH)) + w['15'][1]['tweaks'] = ['21', 0] + return {k: {'class_type': c, 'inputs': i} for k, (c, i) in w.items()} + + +def check_log(log, since, label): + with open(log, encoding='utf-8', errors='replace') as f: + f.seek(since) + new = f.read() + for rx, why in BAD_LOG: + m = rx.search(new) + if m: + ln = new[max(0, new.rfind('\n', 0, m.start()) + 1):new.find('\n', m.end())] + raise Fail(f'{label}: {why}: {ln.strip()[:500]}') + return new + + +def check_dub(comfy, name, outputs, label): + proj = os.path.join(comfy, 'output', 'lsdub', name) + renders = glob.glob(os.path.join(proj, 'shots', '*', '*', 'render.mp4')) + glob.glob(os.path.join(proj, 'shots', '*', '*', '*.mp4')) + errors = [] + for d in glob.glob(os.path.join(proj, 'shots', '*', '*', 'done.json')): + j = json.load(open(d)) + if j.get('error'): errors.append(f"{os.path.relpath(d, proj)}: {j['error']}") + if errors: raise Fail(f'{label}: dubbed shots raised: {errors}') + if not renders: raise Fail(f'{label}: no shot was dubbed (no render under {proj}/shots)') + cuts = [f for f in glob.glob(os.path.join(comfy, 'output', '**', '*.mp4'), recursive=True) if name in f and 'final' in f] + if not cuts: raise Fail(f'{label}: Save Cut wrote no file') + rep = texts(outputs, '17') + if 'shot' not in rep.lower(): raise Fail(f'{label}: Verify gave no report: {rep[:300]!r}') + say(f'{label}: {len(renders)} render file(s), cut {os.path.relpath(cuts[0], comfy)}; Verify: {rep.splitlines()[0][:120] if rep else ""}') + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split('\n')[0]) + ap.add_argument('--comfyui', required=True) + ap.add_argument('--pack', default=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + ap.add_argument('--port', type=int, default=8199) + ap.add_argument('--demo-zip') + ap.add_argument('--hf-mirror') + ap.add_argument('--save-mirror') + ap.add_argument('--demo-fallback', help='a demo zip to use if fetch_demo.py fails (the step still fails)') + ap.add_argument('--demo-fallback-sha', default='', help="the fallback zip's sha256") + ap.add_argument('--steps', default='nodes,align,dub,mouth') + ap.add_argument('--timeout', type=int, default=3600, help='seconds per workflow') + a = ap.parse_args() + comfy, pack, steps = os.path.abspath(a.comfyui), os.path.abspath(a.pack), set(a.steps.split(',')) + rev, files = pinned(pack) + say(f'pack {pack}; recogniser {REPO}@{rev[:8]} ({len(files)} files)') + + dst = os.path.join(comfy, 'custom_nodes', 'comfyui-lsdub') + shutil.rmtree(dst, ignore_errors=True) + shutil.copytree(pack, dst, ignore=shutil.ignore_patterns('.git', '__pycache__', '.pytest_cache', '.ruff_cache', 'dist', '*.zip')) + failures = [] + fd = [sys.executable, os.path.join(dst, 'scripts', 'fetch_demo.py'), '--comfyui', comfy] + r = subprocess.run(fd + (['--zip', os.path.abspath(a.demo_zip)] if a.demo_zip else []), capture_output=True, text=True) + if r.returncode: + failures.append(f'demo: scripts/fetch_demo.py fails for a user: {(r.stderr or r.stdout).strip().splitlines()[-1][:400]}') + say(failures[-1]) + if not a.demo_fallback: raise Fail(failures[-1]) + say(f'demo: carrying on with the stand-in {a.demo_fallback} (sha256 {a.demo_fallback_sha[:12]}) so the other steps still report') + stand_in(fd, a.demo_fallback, a.demo_fallback_sha) + else: + say('demo: fetch_demo.py ok') + rec = os.path.join(comfy, 'models', RECOGNISER_DIR) + if os.path.exists(rec): raise Fail(f'{rec} is not empty: the align step needs an empty models folder') + + env = dict(os.environ, PYTHONUNBUFFERED='1') + if a.hf_mirror and all(os.path.isfile(os.path.join(a.hf_mirror, f)) for f in files): + env['HF_ENDPOINT'] = serve_mirror(os.path.abspath(a.hf_mirror), rev, files) + say(f'recogniser files from the cached mirror {a.hf_mirror} on {env["HF_ENDPOINT"]}') + else: + say('recogniser files from the Hugging Face Hub') + log = os.path.join(comfy, 'install-path-comfyui.log') + lf = open(log, 'w') + proc = subprocess.Popen([sys.executable, 'main.py', '--cpu', '--listen', '127.0.0.1', '--port', str(a.port), '--disable-auto-launch'], + cwd=comfy, env=env, stdout=lf, stderr=subprocess.STDOUT) + try: + t0 = time.time() + while True: + if proc.poll() is not None: raise Fail(f'ComfyUI exited ({proc.returncode}); see {log}') + try: + info = api(a.port, '/object_info', timeout=60) + break + except (OSError, urllib.error.URLError): + if time.time() - t0 > 600: raise Fail('ComfyUI did not start in 600 s') from None + time.sleep(2) + want, got = node_ids(pack), {k for k in info if k.startswith('LSDub')} + check_log(log, 0, 'startup') + if want - got: raise Fail(f'nodes not registered: {sorted(want - got)} (see {log})') + say(f'nodes: all {len(want)} LSDub nodes registered') + + def step(label, fn): + try: fn() + except Fail as e: + failures.append(str(e.code)) + say(str(e.code)) + + def align(): + pos = os.path.getsize(log) + out = run_prompt(a.port, workflow('ip-align', aligner='ctc', align_only=True), 'align (ctc, empty models folder)', a.timeout) + check_log(log, pos, 'align') + have = sorted(os.path.relpath(p, rec) for p in glob.glob(os.path.join(rec, '**', '*'), recursive=True) if os.path.isfile(p)) + snap = os.path.join(rec, 'models--' + REPO.replace('/', '--'), 'snapshots', rev) + if any(p.endswith('.safetensors') for p in have): raise Fail(f'align: an unpinned safetensors was downloaded: {have}') + for f, sha in files.items(): + p = os.path.join(snap, f) + if not os.path.isfile(p): raise Fail(f'align: pinned file {f} missing from {snap}; have {have}') + h = hashlib.sha256() + with open(p, 'rb') as fh: + for b in iter(lambda: fh.read(1 << 20), b''): h.update(b) + if h.hexdigest() != sha: raise Fail(f'align: {f} sha256 {h.hexdigest()} != pinned {sha}') + blobs = [p for p in have if '/blobs/' in p or p.startswith('models--') and '/blobs' in p] + if len(blobs) > len(files): raise Fail(f'align: more than the {len(files)} pinned files were downloaded: {blobs}') + rep = texts(out, '13') + if 'aligner ctc' not in rep: raise Fail(f'align: the lyric check does not say aligner ctc: {rep[:400]!r}') + say(f'align: four pinned files at {rev[:8]}, sha256 ok, no safetensors; {rep.splitlines()[2] if len(rep.splitlines()) > 2 else rep[:100]}') + if a.save_mirror and 'HF_ENDPOINT' not in env: + os.makedirs(a.save_mirror, exist_ok=True) + for f in files: shutil.copyfile(os.path.join(snap, f), os.path.join(a.save_mirror, f)) + say(f'align: saved the four files to {a.save_mirror} for the cache') + + def dub(name, label, kw): + pos = os.path.getsize(log) + out = run_prompt(a.port, workflow(name, **kw), label, a.timeout) + check_log(log, pos, label) + check_dub(comfy, name, out, label) + + if 'align' in steps: step('align', align) + if 'dub' in steps: step('dub', lambda: dub('ip-dub-defaults', 'dub at defaults', {})) + if 'mouth' in steps: + step('mouth', lambda: dub('ip-dub-mouth', 'dub with the mouth Combos (' + ', '.join(f'{k} {v}' for k, v in MOUTH.items()) + ')', + dict(aligner='ctc', mouth=True, identity=USER_FACE))) + if failures: raise Fail(f'{len(failures)} step(s) failed:\n ' + '\n '.join(failures)) + say('PASS') + finally: + proc.terminate() + try: proc.wait(30) + except subprocess.TimeoutExpired: proc.kill() + lf.close() + + +if __name__ == '__main__': + main() diff --git a/scripts/release.py b/scripts/release.py new file mode 100644 index 0000000..cb86409 --- /dev/null +++ b/scripts/release.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2026 Humanjava Enterprises. LSDub: https://github.com/HumanjavaEnterprises/comfyui-lsdub +"""release.py: the one release command. Maintainers only (it needs the private engine checkout and gh signed in). + + python scripts/release.py X.Y.Z [--dry-run] [--engine ../lsdub] [--site ../lsdub.web.microsite.src] + +In order, stopping at the first failure with a plain message (nothing remote changes before step 9): + 1. refuses unless the engine and pack trees are clean, on main and equal to origin/main, and engine CI is green on its HEAD; + 2. refuses unless CHANGELOG.md has a `## X.Y.Z` section that passes release_notes.py, the microsite's main has a + matching section in docs/changelog.html, and the demo zip fetch_demo.py points at is published with its checksum; + 3. bumps pyproject (version, Icon/Banner tag URLs) and CITATION.cff; regenerates the README node tables and the bug form; + 4. vendors the engine (a vendor record commit in the engine, pushed only in step 9); + 5. runs the pack checks CI runs (ruff registry rules, pytest, docs and models checks), including the private-methods scrub; + 6. commits "X.Y.Z: ..." on a release/X.Y.Z branch; + 7. pushes release/X.Y.Z and waits for CI and Install path to pass on that commit; + 8. (dry run stops before this: it works in a throwaway worktree and pushes nothing) + 9. fast-forwards main to it, tags vX.Y.Z, pushes main and the tag, pushes the engine record; + 10. watches the publish run, confirms the Comfy Registry lists X.Y.Z, and makes sure the site deploy ran. +""" +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.request + +PACK = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PACK_REPO = 'HumanjavaEnterprises/comfyui-lsdub' +ENGINE_REPO = 'HumanjavaEnterprises/lsdub' +SITE_REPO = 'HumanjavaEnterprises/lsdub.web.microsite.src' +REGISTRY = 'https://api.comfy.org/nodes/lsdub/versions' +EVIDENCE = ('Evidence: release-check source=lsdub/core/lyric_align.py read the vendored tree: vendor_engine scrub CHECK clean, ' + 'pack tests and install-path green on release/{v}') + + +class Stop(SystemExit): + def __init__(self, msg): + pushed = '\n'.join(f' - {p}' for p in PUSHED) + tail = ('\nNothing remote was changed.' if not PUSHED else + f'\nAlready pushed:\n{pushed}\nRecovery: CONTRIBUTING.md, Releases, "If release.py stops part way".') + super().__init__(f'\nrelease: STOPPED. {msg}{tail}') + + +PUSHED = [] +STEP = [0] + + +def step(msg): + STEP[0] += 1 + print(f'\n[{STEP[0]}] {msg}', flush=True) + + +def sh(*cmd, cwd=PACK, check=True, quiet=False, env=None): + if not quiet: print(' $', ' '.join(cmd), flush=True) + r = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) + if check and r.returncode: + raise Stop(f'`{" ".join(cmd)}` failed (exit {r.returncode}) in {cwd}:\n{(r.stdout + r.stderr).strip()[-3000:]}') + return r.stdout.strip() + + +def git(repo, *a, **k): return sh('git', *a, cwd=repo, **k) + + +def gh_json(*a): return json.loads(sh('gh', *a, quiet=True) or 'null') + + +def clean_main(repo, name): + if git(repo, 'status', '--porcelain', quiet=True): raise Stop(f'the {name} tree at {repo} has uncommitted changes. Commit or stash them first.') + br = git(repo, 'rev-parse', '--abbrev-ref', 'HEAD', quiet=True) + if br != 'main': raise Stop(f'the {name} checkout is on {br!r}, not main.') + git(repo, 'fetch', '-q', 'origin', 'main', quiet=True) + if git(repo, 'rev-parse', 'HEAD', quiet=True) != git(repo, 'rev-parse', 'origin/main', quiet=True): + raise Stop(f'the {name} main is not origin/main (pull or push first).') + return git(repo, 'rev-parse', 'HEAD', quiet=True) + + +def runs_green(repo, sha, workflows, wait, timeout=4 * 3600): + """every named workflow has a successful run on sha (waits for them when wait=True).""" + t0 = time.time() + while True: + runs = gh_json('run', 'list', '-R', repo, '--commit', sha, '--json', 'workflowName,status,conclusion,url', '-L', '50') + state = {} + for w in workflows: + mine = [r for r in runs if r['workflowName'] == w] + if any(r['conclusion'] == 'success' for r in mine): state[w] = 'success' + elif any(r['status'] != 'completed' for r in mine): state[w] = 'running' + elif mine: state[w] = f"{mine[0]['conclusion']} ({mine[0]['url']})" + else: state[w] = 'no run yet' + bad = {w: s for w, s in state.items() if s not in ('success', 'running', 'no run yet')} + if bad: raise Stop(f'{repo} @ {sha[:9]}: ' + '; '.join(f'{w} {s}' for w, s in bad.items())) + if all(s == 'success' for s in state.values()): return + if not wait: raise Stop(f'{repo} @ {sha[:9]} is not green yet: {state}') + if time.time() - t0 > timeout: raise Stop(f'{repo} @ {sha[:9]}: gave up waiting: {state}') + print(f' waiting: {state}', flush=True) + time.sleep(60) + + +def read(p): return open(p, encoding='utf-8').read() + + +def write(p, s): open(p, 'w', encoding='utf-8').write(s) + + +def bump(tree, v): + pp = os.path.join(tree, 'pyproject.toml') + s = read(pp) + s = re.sub(r'^version = "[^"]+"', f'version = "{v}"', s, count=1, flags=re.M) + s = re.sub(r'(comfyui-lsdub/)v\d+\.\d+\.\d+(/assets/)', rf'\g<1>v{v}\g<2>', s) + write(pp, s) + cf = os.path.join(tree, 'CITATION.cff') + write(cf, re.sub(r'^version: "[^"]+"', f'version: "{v}"', read(cf), flags=re.M)) + + +def demo_published(tree): + src = read(os.path.join(tree, 'scripts', 'fetch_demo.py')) + ver = re.search(r"^VERSION = '([^']+)'", src, re.M).group(1) + tag = re.search(r"^TAG = (.+)$", src, re.M).group(1).strip() + tag = f'v{ver}' if tag == "f'v{VERSION}'" else tag.strip("'") + sha = re.search(r"^SHA256 = '([0-9a-f]{64})'", src, re.M).group(1) + url = f'https://github.com/{PACK_REPO}/releases/download/{tag}/lsdub-demo-{ver}.zip' + try: + with urllib.request.urlopen(url, timeout=120) as r: got = hashlib.sha256(r.read()).hexdigest() # noqa: S310 (fixed https URL) + except OSError as e: + got = f'unreachable ({e})' + if got != sha: + raise Stop(f'the demo zip fetch_demo.py points at is not published as pinned: {url} -> {got}, pinned {sha}.\n' + f'Attach it first (a new tag for a new zip, never a replaced asset):\n' + f' gh release create {tag} dist/lsdub-demo-{ver}.zip -R {PACK_REPO} --latest=false ' + f'--title "LSDub demo assets {ver}" --notes "Demo assets for the example workflows (scripts/fetch_demo.py)."') + return url + + +def main(): + ap = argparse.ArgumentParser(description='LSDub: the one release command', epilog=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('version') + ap.add_argument('--dry-run', action='store_true', help='every check, in a throwaway worktree; pushes nothing') + ap.add_argument('--engine', default=os.path.join(os.path.dirname(PACK), 'lsdub')) + ap.add_argument('--site', default=os.path.join(os.path.dirname(PACK), 'lsdub.web.microsite.src')) + a = ap.parse_args() + v, dry = a.version, a.dry_run + eng, site = os.path.abspath(a.engine), os.path.abspath(a.site) + print(f'LSDub release {v}{" (DRY RUN: nothing is pushed)" if dry else ""}') + if not re.fullmatch(r'\d+\.\d+\.\d+', v): raise Stop(f'{v!r} is not X.Y.Z') + cur = re.search(r'^version = "([^"]+)"', read(os.path.join(PACK, 'pyproject.toml')), re.M).group(1) + if tuple(map(int, v.split('.'))) <= tuple(map(int, cur.split('.'))): raise Stop(f'{v} is not newer than the current {cur}') + for t in ('git', 'gh'): + if not shutil.which(t): raise Stop(f'{t} is not on PATH') + sh('gh', 'auth', 'status', quiet=True) + + step('engine and pack trees clean, on main, equal to origin/main; engine CI green on its HEAD') + eng_sha = clean_main(eng, 'engine') + pack_sha = clean_main(PACK, 'pack') + runs_green(ENGINE_REPO, eng_sha, ['ci'], wait=False) + if git(PACK, 'ls-remote', '--tags', 'origin', f'refs/tags/v{v}', quiet=True): raise Stop(f'tag v{v} already exists on origin') + if git(PACK, 'ls-remote', '--heads', 'origin', f'release/{v}', quiet=True): raise Stop(f'branch release/{v} already exists on origin') + print(f' engine {eng_sha[:9]} (ci green), pack {pack_sha[:9]}') + + step(f'CHANGELOG section {v} (release_notes.py), the microsite changelog section, the demo zip') + sys.path.insert(0, os.path.join(PACK, 'scripts')) + import release_notes as RN + try: RN.check(read(os.path.join(PACK, 'CHANGELOG.md')), v) + except ValueError as e: raise Stop(f'CHANGELOG.md: {e}. Write the {v} section first (CONTRIBUTING.md, Release notes).') from None + git(site, 'fetch', '-q', 'origin', 'main', quiet=True) + r = subprocess.run(['git', 'show', 'origin/main:docs/changelog.html'], cwd=site, capture_output=True, text=True) + if r.returncode: raise Stop(f'the microsite main ({site}) has no docs/changelog.html. The changelog page has to be on the site\'s main first.') + page = r.stdout + if f'id="v{v}"' not in page: + raise Stop(f'the microsite main has no
in docs/changelog.html. Add it on the site\'s main first.') + print(f' demo zip ok: {demo_published(PACK)}') + + tree, made_worktree = PACK, False + if dry: + tree = tempfile.mkdtemp(prefix=f'lsdub-release-{v}-') + os.rmdir(tree) + git(PACK, 'worktree', 'add', '-q', '--detach', tree, 'HEAD') + made_worktree = True + records = os.path.join(eng, 'scripts', 'vendor_records.tsv') + records_before = read(records) + try: + step(f'bump to {v}: pyproject version + Icon/Banner URLs, CITATION.cff; regenerate README node tables and the bug form') + if not dry: git(PACK, 'checkout', '-q', '-b', f'release/{v}') + bump(tree, v) + sh(sys.executable, 'scripts/make_readme_nodes.py', cwd=tree) + + step('vendor the engine') + sh(sys.executable, os.path.join(eng, 'scripts', 'vendor_engine.py'), '--pack', tree, cwd=eng) + + step('pack checks (what CI runs, plus the private-methods scrub in tests/test_registry_rules.py)') + sh(sys.executable, '-m', 'ruff', 'check', '--select', 'S102,S307,E702,F', '--target-version', 'py310', '.', cwd=tree) + sh(sys.executable, '-m', 'pytest', '-q', 'tests', cwd=tree) + sh(sys.executable, 'scripts/make_readme_nodes.py', '--check', cwd=tree) + sh(sys.executable, 'scripts/check_models_md.py', cwd=tree) + sh(sys.executable, 'scripts/release_notes.py', cwd=tree, quiet=True) + + step(f'commit {v} on release/{v}') + title = RN.top_section(read(os.path.join(tree, 'CHANGELOG.md')))[1] + first = next((ln[2:].strip() for ln in title if ln.startswith('- ')), '') + git(tree, 'add', '-A') + if dry: + print(f' would commit "{v}: {first[:100]}" with: ' + git(tree, 'diff', '--cached', '--stat', quiet=True).replace('\n', '\n ')) + print(' would then run vendor_engine.py --check on the committed record') + print(f' would commit the engine vendor record ({EVIDENCE.format(v=v)[:60]}...)') + print(f'\n[{STEP[0] + 1}..] would push release/{v}, wait for CI + Install path, fast-forward main, tag v{v}, push the tag and the ' + 'engine record, watch publish, confirm the Registry, check the site deploy.\nDRY RUN OK: every check passed.') + return + git(tree, 'commit', '-q', '-m', f'{v}: {first[:120]}') + rel_sha = git(tree, 'rev-parse', 'HEAD', quiet=True) + if read(records) != records_before: + git(eng, 'commit', '-q', '-m', f'{v} vendor record\n\n{EVIDENCE.format(v=v)}', '--', 'scripts/vendor_records.tsv') + sh(sys.executable, os.path.join(eng, 'scripts', 'vendor_engine.py'), '--pack', tree, '--check', cwd=eng) # clean engine again + + step(f'push release/{v} and wait for CI + Install path on {rel_sha[:9]}') + git(PACK, 'push', '-q', 'origin', f'release/{v}') + PUSHED.append(f'pack branch release/{v} ({rel_sha[:9]}); main, tags and the Registry untouched') + print(' (a branch only; main, tags and the Registry are untouched until both pass)') + time.sleep(20) + runs_green(PACK_REPO, rel_sha, ['CI', 'Install path'], wait=True) + + step(f'fast-forward main, tag v{v}, push; push the engine record') + git(PACK, 'checkout', '-q', 'main') + git(PACK, 'merge', '-q', '--ff-only', f'release/{v}') + git(PACK, 'tag', '-a', f'v{v}', '-m', f'LSDub {v}') + git(PACK, 'push', '-q', '--atomic', 'origin', 'main', f'v{v}') # main and the tag land together or not at all + PUSHED.append(f'pack main -> {rel_sha[:9]} and tag v{v} (publish starts from the tag)') + tag_time = time.time() + git(eng, 'push', '-q', 'origin', 'main') + PUSHED.append('engine main: the vendor record commit') + + step('watch publish (it re-runs Install path on the tag first)') + time.sleep(20) + runs_green(PACK_REPO, rel_sha, ['Publish to the Comfy Registry'], wait=True) + + step(f'the Comfy Registry lists {v}') + for _ in range(30): + with urllib.request.urlopen(REGISTRY, timeout=60) as r: vs = {x.get('version'): x.get('status') for x in json.loads(r.read())} # noqa: S310 + if v in vs: + st = {'NodeVersionStatusActive': 'Active', 'NodeVersionStatusPending': 'Pending'}.get(vs[v], vs[v]) + print(f' {v}: {st}' + (' (listed; the Registry still has to review it before it is Active)' if st == 'Pending' else '')) + break + time.sleep(30) + else: + raise Stop(f'publish passed but {REGISTRY} does not list {v} after 15 minutes; check the publisher page.') + + step('site deploy (the publish workflow dispatches it; started here if it did not)') + time.sleep(30) + runs = gh_json('run', 'list', '-R', SITE_REPO, '--workflow', 'deploy.yml', '--json', 'databaseId,createdAt,event,url', '-L', '10') + fresh = [r for r in runs if time.mktime(time.strptime(r['createdAt'], '%Y-%m-%dT%H:%M:%SZ')) - time.timezone >= tag_time - 60] + if not fresh: + sh('gh', 'workflow', 'run', 'deploy.yml', '-R', SITE_REPO, '--ref', 'main') + time.sleep(15) + fresh = gh_json('run', 'list', '-R', SITE_REPO, '--workflow', 'deploy.yml', '--json', 'databaseId,createdAt,event,url', '-L', '1') + if fresh: + sh('gh', 'run', 'watch', str(fresh[0]['databaseId']), '-R', SITE_REPO, '--exit-status', check=False) + print(f' site deploy run: {fresh[0]["url"]} (a skipped deploy means CLOUDFLARE_API_TOKEN is not set there)') + print(f'\nreleased {v}.') + finally: + if made_worktree: + git(PACK, 'worktree', 'remove', '--force', tree, check=False, quiet=True) + if read(records) != records_before: write(records, records_before) # the dry run's vendor record, undone + + +if __name__ == '__main__': + main() diff --git a/tests/engine/test_watermark.py b/tests/engine/test_watermark.py index 92583fc..c744d24 100644 --- a/tests/engine/test_watermark.py +++ b/tests/engine/test_watermark.py @@ -72,7 +72,7 @@ def _top_atoms(path): def test_the_watermarked_file_is_faststart(clip, tmp_path): - """moov ahead of mdat (G10 lane 11: the saved cut, the site's redub.mp4, was the one file with its moov at the end).""" + """moov ahead of mdat (the saved cut, the site's redub.mp4, was the one file with its moov at the end).""" out = str(tmp_path / "wm.mp4") watermark.apply(clip, out) a = _top_atoms(out) diff --git a/tests/test_registry_rules.py b/tests/test_registry_rules.py index 4d2bee4..c7cbb42 100644 --- a/tests/test_registry_rules.py +++ b/tests/test_registry_rules.py @@ -22,7 +22,7 @@ def _files(exts=('.py', '.md', '.json', '.txt', '.toml', '.cff', '.yaml', '.yml' ds[:] = [x for x in ds if x not in SKIP_DIRS] for f in fs: if f in SKIP_DIRS: continue # a worktree's .git is a FILE - if os.path.splitext(f)[1] in exts: yield os.path.join(d, f) + if exts is None or os.path.splitext(f)[1] in exts: yield os.path.join(d, f) def test_no_machine_paths(): @@ -34,6 +34,30 @@ def test_no_machine_paths(): assert not hits, hits +# the pack ships product code and user docs; the methods stay private (the engine's ledger tools, review reports, +# research and plan docs). release.py runs this file, so a tree that carries any of them is never tagged. +METHODS = re.compile('|'.join(['G1' + r'\d\b', 'truth' + '-fix', 'lsdub-tryit' + '-assets', 'review/' + 'G1'])) +METHODS_PATHS = re.compile(r'(^|/)ledger(/|$)|^tests/ledger|^docs/(research|plans)/') + + +def _tracked(): + try: + out = subprocess.run(['git', '-C', ROOT, 'ls-files', '-z'], capture_output=True, check=True).stdout.decode() + return [f for f in out.split('\0') if f] + except (OSError, subprocess.CalledProcessError): + return [os.path.relpath(p, ROOT).replace(os.sep, '/') for p in _files(exts=None)] + + +def test_no_private_methods_in_shipped_tree(): + files = [f for f in _tracked() if os.path.isfile(os.path.join(ROOT, f))] + hits = [f for f in files if METHODS_PATHS.search(f)] + for f in files: + try: txt = open(os.path.join(ROOT, f), encoding='utf-8').read() + except (UnicodeDecodeError, OSError): continue + hits += [f'{f}:{i + 1}' for i, ln in enumerate(txt.splitlines()) if METHODS.search(ln)] + assert not hits, f'private methods in the shipped tree (keep them in the engine repo): {hits}' + + def _ruff(): exe = shutil.which('ruff') if exe: return [exe]