diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index a2674bc..abe5153 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -8,6 +8,18 @@ on: branches: [main] workflow_dispatch: +# Read-only by default for every job; the jobs that publish an image ask for +# `packages: write` themselves. A job that neither declares nor needs more gets no +# more, so a compromised step cannot push to the repository with the workflow token. +permissions: + contents: read + +# Every action below is pinned to a commit, with the release it was when pinned in the +# comment. A tag like `v5` can be moved to point at anything, including code the +# publisher did not write; a commit cannot. To bump one: pick the release, resolve it +# with `gh api repos///git/ref/tags/ --jq .object.sha` (for an +# annotated tag, `gh api repos///git/tags/ --jq .object.sha` gives +# the commit), and update the comment with it. env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} @@ -16,8 +28,8 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - uses: actions/setup-node@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 with: node-version: 22 cache: npm @@ -48,13 +60,13 @@ jobs: digest-amd64: ${{ steps.export.outputs.digest-amd64 }} digest-arm64: ${{ steps.export.outputs.digest-arm64 }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 - - uses: docker/setup-buildx-action@v3 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Log in to GHCR if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -62,7 +74,7 @@ jobs: - name: Build and push by digest id: build - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . platforms: ${{ matrix.platform }} @@ -85,7 +97,7 @@ jobs: mkdir -p /tmp/digests echo -n "${{ steps.build.outputs.digest }}" > "/tmp/digests/${{ matrix.suffix }}" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: github.event_name != 'pull_request' with: name: digest-${{ matrix.suffix }} @@ -102,16 +114,16 @@ jobs: contents: read packages: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: path: /tmp/digests pattern: digest-* merge-multiple: true - - uses: docker/setup-buildx-action@v3 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Log in to GHCR - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -119,7 +131,7 @@ jobs: - name: Work out the tags id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | @@ -174,7 +186,7 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 # The CUDA toolchain and build cache need more room than a runner has free. - name: Free disk space @@ -182,11 +194,11 @@ jobs: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL df -h / - - uses: docker/setup-buildx-action@v3 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Log in to GHCR if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -194,7 +206,7 @@ jobs: - name: Work out the tags id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} flavor: | @@ -207,7 +219,7 @@ jobs: type=sha,format=short - name: Build and push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . file: Dockerfile.cuda diff --git a/.gitignore b/.gitignore index 53c5b8c..cb53f20 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ node_modules/ .devdata/ +# A second dev instance beside the first (another port, another data folder). +.devdata-*/ .dockerdata/ *.log .DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md index dc03f84..0817ea7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,33 @@ changes what your listeners see, it says so. range it had asked for before the row was written, so the log could not show why. The range is kept. +### Security — hardening from the October audit + +- **An oversized image on the share could take the container down.** The shows folder is a + network share other people can write to, and SelfPod decoded whatever image it found there + — a cover, an episode's sidecar, the artwork embedded in an MP3 — at sharp's own ceiling of + roughly 268 megapixels. A 60 KB PNG of one colour declaring 16000×16000 decodes to a + gigabyte, and the scanner decodes sidecars on its own, on every scan: one such file was a + crash loop with no banner to explain it. Every decode now stops at 40 megapixels, at the + header, and the scan log says the image was too large rather than "could not be read". +- **Before sign-in, only the permission banners are shown.** The sign-in page and the + anonymous half of `/api/status` carried every health banner, and those name things: a + show's title when its cut copy goes missing, the host of a feed being followed. Feeds are + private by their token, so their names should not be readable by whoever finds the tunnel + hostname. The banners about `/data` and the shows folder — the ones that have to be visible + before anyone can sign in — still are. +- **Changing the password signs every other session out.** It used to leave them valid for + up to thirty days, which is the opposite of what someone changing their password because + they suspect a stolen session wants. The session that made the change stays signed in. +- **A sign-in always gets a fresh session id**, rather than promoting whatever cookie it was + handed. Nothing was exploitable before — no session exists until the password is proved — + so this is belt and braces. +- **A stranger can no longer keep you locked out.** The account-level backoff reached five + minutes and reset on every wrong guess, so one wrong password every few minutes from + anywhere held the door shut indefinitely. The account now waits at most a minute; the + address doing the guessing waits up to five. A throttled attempt answers 429 with + `Retry-After` instead of 401, so it can be told apart from a wrong password. + ### Changed - Dependencies brought up to patched releases: Fastify 5.12.5 (seven advisories, including an @@ -45,6 +72,204 @@ changes what your listeners see, it says so. (libheif and librsvg), music-metadata 11.16.1 (crafted MP4, EBML and DSF files could crash the scanner — and the scanner reads whatever is dropped on the share). `npm audit` is clean. +### Fixed — things you could do that gave no answer, or the wrong one + +- **Remove or delete from an episode's own page now works.** The response re-targeted the + episode table, which only the show page has; on the episode page htmx refused it and the + page went on showing an episode that was gone. Both actions now take you to the show, with + the confirmation, when they were started on the episode page. +- **Check now, Pause, Resume and Stop following on the Subscription page do something you + can see.** They used to post into a redirect the browser swallowed; the page re-reads + itself now and says what happened. Stop following asks first, like every other action + that cannot be undone. +- **A password change is confirmed with a real notice.** The toast arrived as bare text with + a × that threw. +- **A button that is working says so.** Every decision, teach, check, rescan and settings + button is held while its request is out and shows a ring; a second press while one is in + flight is dropped rather than queued. A request that gets no answer at all — offline, a + tunnel that dropped, an extension in the way — now says so instead of failing in silence, + and a session that has run out no longer flashes "Something went wrong" on its way to the + sign-in page. +- **Check now and Teach no longer hold the page for the whole pass.** The panel comes back at + once with a note, and the work strip follows the pass; the plain-form path still waits. +- **A background refresh no longer wipes what you are typing.** The Adverts panel and the + episode's cuts card re-read themselves after every pass, scheduled ones included, and used + to throw away a half-typed boundary and close the settings fold under you. A refresh now + waits while a field inside it is in use, and is taken once it is not; a fold you opened + stays open; and the second render after every decision — the stream announcing what the + response already carried — is gone. +- **The Show page topbar fits a phone**: the actions take a row of their own under the + breadcrumb instead of pushing it onto four lines and "Follow a feed" off the edge. +- **The access log and the statistics tables are readable on a phone**: each row is a block + with its column names, like the episode table already was, rather than a six-column table + scrolling sideways. The column headers stay for screen readers. +- **The scan-progress strip sent over the live stream could be left behind forever** when + the stream dropped mid-scan: the streamed copy omitted the poll the clicked one has. It is + the same strip now. +- "New show" no longer opens with "New show" already typed into the name, and a refused form + keeps what you typed. The upload queue's success line is no longer painted red. The + dashboard is highlighted in the sidebar when you are on it — and `aria-current` on the + sidebar's current item is now a real attribute rather than escaped text. Settings toggles + and inline saves confirm themselves. The cut pill in the episode table links to the + episode page, where the bar always is, rather than to an anchor the Adverts page may have + paged away. + +### Changed + +- **A date at the start of a filename is the episode's publish date.** `2026-09-01-…`, + `2026_09_01…` or `20260901…` is taken as that day at midnight in `TZ`, unless the file's + modification time already falls on that day, in which case the time of day is kept. Three + dated files copied onto the share together used to land on the one minute they were + copied. Undated files keep the modification time, and a date set on the episode page is + never changed by a rescan. README, *How it finds new episodes*. +- **Colour and size, for people and thumbs.** Primary buttons, accent-coloured text and + selected chips use the darker terracotta (`#B84A1E`, 7.4:1 on white) rather than the one + that fell short of AA for text; the signal colour stays on bars, dots and marks. Muted + grey text that carried information — the "· test" affordance, stat-card hints, timestamps, + label hints — is a shade darker. On a touch screen every control is at least 44px: the + stretch ▶, the toast ×, chips, sign-out, the menu button, the small decision buttons, and + the cut bar gets a taller area to tap without looking any different. +- **Keyboard focus survives a swap.** After saving a setting, sorting the log, restoring an + episode or closing a modal, focus goes back to the control it was on — or to the control + that opened the modal — rather than to the top of the page. Errors are announced as alerts + and stay until dismissed; other notices stay six seconds plus fifty milliseconds a + character, and pausing with the pointer resumes when it leaves. The two Settings switches + have accessible names, card titles are real headings, a refused field says so to assistive + technology, the current breadcrumb is marked, and a refused sign-in puts the cursor on the + password. + +### Fixed — the statistics, the rest of the way + +- **The day boundaries were an hour out on clock-change days in Sydney, Auckland, Lord + Howe and Santiago.** Local midnight was found by probing the zone's offset once, at a + moment ten or eleven hours later than the real one — on the far side of a 02:00 change. + A second probe lands it. London and New York were never affected, and every day of a + year in eight zones is now checked by a test. +- **Deleting an episode no longer deletes its history.** The request log cascaded away + with the episode row, so the show's and the instance's totals shrank. The rows stay, + and the log says "An episode since deleted" where the title was. Removing a show still + removes its log, as it should. +- **"vs the previous 30 days" compares like with like.** The current period runs to + tomorrow and so held twenty-nine days and a morning, measured against thirty whole + days: every card read "down" until the evening. The previous period is now measured + to the same point, and the card says so. +- **Filtering the statistics page to a show filters the whole page.** The link from a + show's page landed on instance-wide cards, chart and tables above a show-only log. Now + every figure follows the show, with a note and a link back to every show. +- **"Failed requests" and "Failures only" agree.** The card counted only audio that + failed; the log showed every failure, a cover that would not serve included. Both now + count every kind. Bytes and "episodes fetched" are still audio only. +- **The CSV holds every row.** It stopped quietly at fifty thousand while the page + promised all of them; it is now streamed, batch by batch, with no ceiling. + +### Changed — the advert pass no longer gets in the way of listeners + +- **Reading an episode no longer stalls everything else.** Decoding an hour-long MP3 to + fingerprint it, or to hand its opening minutes to the recogniser, used to happen on the + same thread that serves downloads and answers the container's health check — about ten + seconds in which a 10 ms timer did not fire once, range requests hung, the live updates + stopped, and on a slow NAS the health check could time out three times and have the + container restarted in the middle of a pass. That work now runs in the worker thread the + corpus searches already used: measured on the same hour of audio, the timer fires one + millisecond late instead of ten seconds late, and the fingerprint comes out byte for byte + the same. A worker that crashes or wedges on a file is reported as a failure of that + episode — a banner, a line in the activity log — and started again for the next one. +- **A pass over a show nothing has happened to costs almost nothing.** Every few minutes, + for every show, the pass used to load every fingerprint into memory, look for the station + jingle in every episode again, parse every transcript and match every known read against + every episode again — to find exactly what it found last time. It now remembers what it + last looked at and skips the detectors while nothing has changed; a new or replaced + episode, a transcript arriving, a decision, a boundary taught, a jingle confirmed or + forgotten, or a changed setting opens them again. The jingle is looked for only in episodes + it has not been looked for in yet, and a fingerprint is read only when it is needed. A + taught boundary that agrees with the jingle now links to it in the same pass rather than + the one after. +- **The pipeline will not read anything but an ordinary file inside the show's folder.** The + show folder is normally a share others can write to, and the pages that serve files have + always refused a link that points outside it. Fingerprinting, listening and cutting read + the same files and used to read whatever the path named — a link to `/dev/zero`, a pipe, a + file of many gigabytes — and hang or run out of memory, every few minutes, with nothing + saying why. All three now refuse a link out of the folder, anything that is not an ordinary + file, and anything over 768 MB, before a byte is read; the banner names the file and says + which, and the pass's activity entry does too. +- **Playing a cut no longer reads the whole episode.** The ▶ on the Adverts page used to read + the entire file off the share and walk every frame of it to play ten seconds. It now reads + only the bytes of those frames, from a table of where they are that is kept per file. + Trimming an episode holds one copy of it in memory rather than two. +- **Pages, stylesheets, scripts and the JSON behind them are compressed over the wire** when + the browser asks — around a quarter of the size over a tunnel. Episode audio and the feed + are exactly as before: audio is never compressed, and the feed compresses itself once. +- Smaller things: the session is no longer re-saved on every request; SQLite's planner + statistics are refreshed once a day and on shutdown, and its journal is kept from growing + without bound; the advert tables' rule and restore columns are indexed; a few statements + the scanner runs for every file are prepared once. + +### Changed — a lighter boot, a smaller image, and nothing taken on trust + +- **A network share is no longer checked twice over.** When live file detection does not + work on a volume — normal for SMB and NFS — SelfPod polls the share instead, and it was + polling every minute while the scheduled rescan also went through every file at the + rescan interval: two full sweeps of the one mount where every look at a file is a round + trip. The poll now runs at the rescan interval, which is what the notice on the + dashboard had been saying all along, and the container log says so when polling starts. +- **The container starts quicker.** The image library is loaded the first time a cover or + an episode's artwork is handled, and the QR encoder the first time a show page is drawn, + rather than before the first page can be served — whole seconds on a small NAS. The + permissions on SelfPod's own code used to be rewritten file by file on every start, + 4,500 files; they are now checked and left alone when already right, and the image + ships them right for the default `PUID`/`PGID`. +- **The image is 20 MB smaller**, by leaving out the SQLite source and the seven other + platforms' binaries that the database library ships with and never reads. An image built + for one box can also leave out the speech model it will not use (`WHISPER_MODELS` in the + README's *Building your own image*); the published image still carries both. +- **`TRUST_PROXY`** says whose `X-Forwarded-*` headers to believe. The default is every + connection, as before; set it to your tunnel's or reverse proxy's address when anything + else can reach port 8080 directly, and a LAN client can no longer claim to have arrived + over HTTPS from somewhere else. A value with a typo in it trusts nobody and says so in + the log, rather than quietly going back to trusting everyone. +- **Everything the build fetches is pinned to exactly what was reviewed:** the base images + by digest, whisper.cpp by commit rather than a tag that could be moved, and every GitHub + Action by commit. The build workflow's token is read-only except where it publishes. + +### Changed — the pages no longer ask for anything an ad blocker would refuse + +- **The Adverts page and its parts live at addresses no filter list matches.** 1.9.1 renamed + the things on the page that an ad blocker hides by name; the addresses the page fetches + were still named after adverts — `/ad-panel`, `/ad-work`, `/adverts`, the clip a ▶ plays + from `/api/ad-segments/…` — and uBlock Origin and AdGuard carry rules that block a request + by its path just as they hide an element by its name. A blocked fetch fails without a word, + so the page would have sat there half-drawn with nothing to say why. The page is now + `/shows//cuts`, everything it fetches is named for cuts, jingles and boundaries, and + the clips play from `/api/cuts/…`. The old page address still arrives, for bookmarks, and + the `/api/ad-…` addresses still answer for scripts. The test that refuses an advert-like + class or id now refuses one in any address the browser fetches too. +- **The dashboard is lighter.** Each card used to load the show's full cover — 1400 pixels or + more, often a few megabytes — into a 250-pixel tile, so a dozen shows over a tunnel was tens + of megabytes of artwork. A card now loads a 400-pixel copy, made once per cover and kept + under `/data/.covers`; a cover the resizer cannot read is served in full rather than as a + broken image. *Rescan all* re-reads the grid once when the sweep finishes, rather than every + card re-reading itself as its show's turn came — one fetch instead of one per show. And the + page asks the access log for every show at once rather than four questions per show. +- **The sidebar says when there are more shows than it lists.** It stops at eight, and used to + stop silently; past eight there is now *All N shows…*, which goes to the dashboard. +- **Typing into the ledger's search no longer fills the Back button.** Each pause while typing + pushed a history entry, so going back stepped through "t", "ta", "tap". Typing now replaces + the address; picking from the two selects still pushes one, as choosing a view should. +- **The first tap on a transcript word picks that word**, not everything from it to the end + ("Removing 4,000 words" until the second tap). Tap the first word, then the last. +- **Names, made consistent.** Following a feed is *Follow a feed* on the page, in its crumb and + in Settings, rather than "Subscription" here and "Follow podcast feeds" there. The episode + page is headed by the episode's title, with "Edit episode" as the small line above it, and + the crumb is the title. The button beside an episode's reach figures, which went to this + show's statistics, now says *Show statistics* rather than "All shows". The cut-settings Save + sits in the same right-aligned row every other form uses, and a few quiet notes that were + styled by a class nothing defined are now actually quiet. One clock formats every position + in an episode, on every page and in the API; the day a remembered cut was decided is now + shown in your time zone like every other date, rather than the server's. +- Under the hood of the stylesheet, the colours that were spelt out by hand in a dozen places + are named once, with the values they always had, and twenty-odd rules nothing used are gone. + Nothing looks different. + ## 1.9.1 — 2026-09-16 ### Fixed diff --git a/Dockerfile b/Dockerfile index 02236af..400b056 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,11 @@ # arm64, so this normally installs without compiling anything; python3/make/g++ # are kept as insurance against a missing prebuild. # --------------------------------------------------------------------------- -FROM node:22-alpine AS build +# Pinned by digest (the multi-arch index, so the same line serves amd64 and arm64) and +# not just by tag, so a rebuild gets the base image that was reviewed and not whatever +# the tag points at that day. To bump: `docker buildx imagetools inspect node:22-alpine +# --format '{{.Manifest.Digest}}'`, then paste the digest here and in the runtime stage. +FROM node:22-alpine@sha256:0a7108bf6c7bf5de370ffb1a3ed6be93d405b43ff159f681a8d18c0e2bc2e402 AS build # cmake and git are for whisper.cpp, below; the rest is insurance for native modules. RUN apk add --no-cache python3 make g++ cmake git @@ -20,6 +24,12 @@ COPY package.json package-lock.json ./ # platform binaries are optional deps, and dropping them breaks it at runtime. RUN npm ci --omit=dev +# Drops the 24 MB of better-sqlite3 that this platform never reads (the SQLite source +# and the seven other platforms' binaries) — before the check below, so the check also +# proves the module loads with only what is kept. The script says what it removed. +COPY docker/trim-node-modules.mjs ./docker/ +RUN node docker/trim-node-modules.mjs node_modules + # Fail the build here rather than at container start if a native module is unusable. RUN node -e "require('better-sqlite3'); require('sharp'); console.log('native modules OK')" @@ -28,7 +38,7 @@ COPY src ./src # --------------------------------------------------------------------------- # whisper.cpp — the speech recogniser behind spoken-advert detection (spec §19.6). # -# Built from a pinned tag rather than installed, because there is no musl build to +# Built from a pinned commit rather than installed, because there is no musl build to # install and because what goes into the image should be what was reviewed. MIT. # # Two builds on amd64, chosen at boot by src/lib/cpu-features.js: `-v3` uses AVX2 @@ -42,14 +52,25 @@ COPY src ./src # The model is fetched here, checked against a pinned digest, and never fetched # again: the running container makes no request this feature adds. # --------------------------------------------------------------------------- -ARG WHISPER_CPP_TAG=b4938 +# The commit tag b4938 pointed at when it was reviewed. A tag can be moved; a commit +# cannot. To bump: pick a release tag, then +# `gh api repos/ggml-org/whisper.cpp/git/ref/tags/ --jq .object.sha` (if that +# object is itself a tag, `gh api repos/ggml-org/whisper.cpp/git/tags/` gives the +# commit) and paste it here and in Dockerfile.cuda. +ARG WHISPER_CPP_COMMIT=371b5a7561823ab2bb32142d2751e35e7534727b # Two models. `base` is the default: 60 MB, quick, good enough for English. `small` is # 190 MB and about three times the work, and markedly better in French — set -# WHISPER_MODEL=small on the container to use it. Both are proved at build time. +# WHISPER_MODEL=small on the container to use it. Every model baked in is proved at +# build time; WHISPER_MODELS chooses which (both, unless an image is built for one +# box that only ever uses one of them — `--build-arg WHISPER_MODELS=base` saves 190 MB). +ARG WHISPER_MODELS=base,small ARG WHISPER_BASE_SHA256=422f1ae452ade6f30a004d7e5c6a43195e4433bc370bf23fac9cc591f01a8898 ARG WHISPER_SMALL_SHA256=ae85e4a935d7a567bd102fe55afc16bb595bdb618e11b2fc7591bc08120411bb -RUN git clone --depth 1 --branch "$WHISPER_CPP_TAG" https://github.com/ggml-org/whisper.cpp /tmp/whisper.cpp +# A shallow fetch of one commit; `git clone --branch` only takes names. +RUN git init -q /tmp/whisper.cpp \ + && git -C /tmp/whisper.cpp fetch -q --depth 1 https://github.com/ggml-org/whisper.cpp "$WHISPER_CPP_COMMIT" \ + && git -C /tmp/whisper.cpp checkout -q FETCH_HEAD ENV WHISPER_CMAKE="-DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DGGML_STATIC=ON -DGGML_NATIVE=OFF -DGGML_OPENMP=OFF \ -DWHISPER_BUILD_EXAMPLES=ON -DWHISPER_BUILD_TESTS=OFF -DWHISPER_BUILD_SERVER=OFF -DWHISPER_SDL2=OFF -DWHISPER_CURL=OFF" @@ -71,15 +92,26 @@ RUN set -e; mkdir -p /app/whisper; \ esac; \ rm -rf /tmp/wb /tmp/wb2 /tmp/wb3 /tmp/whisper.cpp -RUN wget -qO /app/whisper/ggml-base-q5_1.bin https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base-q5_1.bin \ - && echo "$WHISPER_BASE_SHA256 /app/whisper/ggml-base-q5_1.bin" | sha256sum -c - \ - && wget -qO /app/whisper/ggml-small-q5_1.bin https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small-q5_1.bin \ - && echo "$WHISPER_SMALL_SHA256 /app/whisper/ggml-small-q5_1.bin" | sha256sum -c - +# Each model named in WHISPER_MODELS is fetched and checked against its pinned digest; +# a name this file has no digest for fails the build rather than fetching something +# unchecked, and an empty list fails it too. +RUN set -e; \ + for model in $(printf '%s' "$WHISPER_MODELS" | tr ',' ' '); do \ + case "$model" in \ + base) sum="$WHISPER_BASE_SHA256";; \ + small) sum="$WHISPER_SMALL_SHA256";; \ + *) echo "WHISPER_MODELS names \"$model\", which this Dockerfile has no digest for (it knows base and small)." >&2; exit 1;; \ + esac; \ + wget -qO "/app/whisper/ggml-$model-q5_1.bin" "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-$model-q5_1.bin" \ + && echo "$sum /app/whisper/ggml-$model-q5_1.bin" | sha256sum -c -; \ + done; \ + ls /app/whisper/ggml-*.bin >/dev/null 2>&1 || { echo "WHISPER_MODELS=\"$WHISPER_MODELS\" baked no model into the image." >&2; exit 1; } # --------------------------------------------------------------------------- # Runtime stage # --------------------------------------------------------------------------- -FROM node:22-alpine +# The same digest as the build stage; bump both together (see the top of the file). +FROM node:22-alpine@sha256:0a7108bf6c7bf5de370ffb1a3ed6be93d405b43ff159f681a8d18c0e2bc2e402 # su-exec drops privileges without leaving a process between PID 1 and node; # shadow provides usermod/groupmod for the PUID/PGID pattern; tzdata makes the @@ -95,7 +127,9 @@ RUN deluser --remove-home node \ WORKDIR /app -COPY --from=build /app/node_modules ./node_modules +# Owned by the default PUID/PGID from the start, so a container run with the defaults +# never has to chown the 4,500 files in here at boot (docker/entrypoint.sh). +COPY --from=build --chown=1000:1000 /app/node_modules ./node_modules COPY --from=build /app/package.json ./package.json COPY --from=build /app/whisper ./whisper COPY src ./src @@ -108,6 +142,13 @@ COPY scripts ./scripts # A broken build fails here, not on the NAS after the update has been applied. RUN node docker/whisper-smoke.mjs +# The model the container uses unless WHISPER_MODEL says otherwise. It has to be one of +# the models baked in: an image whose default model is missing would start, say the +# recogniser is unavailable, and publish every episode unheard. So the build refuses. +ARG WHISPER_DEFAULT_MODEL=base +RUN test -f "/app/whisper/ggml-${WHISPER_DEFAULT_MODEL}-q5_1.bin" \ + || { echo "WHISPER_DEFAULT_MODEL=${WHISPER_DEFAULT_MODEL} is not among the models baked in ($(ls /app/whisper | grep '^ggml-' | tr '\n' ' ')). Add it to WHISPER_MODELS, or set WHISPER_DEFAULT_MODEL to one of them." >&2; exit 1; } + RUN chmod +x docker/entrypoint.sh \ && mkdir -p /data \ && chown selfpod:selfpod /app @@ -117,7 +158,7 @@ ENV NODE_ENV=production \ DATA_DIR=/data \ PUID=1000 \ PGID=1000 \ - WHISPER_MODEL=base + WHISPER_MODEL=${WHISPER_DEFAULT_MODEL} EXPOSE 8080 VOLUME ["/data"] diff --git a/Dockerfile.cuda b/Dockerfile.cuda index f398389..1a5cee3 100644 --- a/Dockerfile.cuda +++ b/Dockerfile.cuda @@ -15,17 +15,23 @@ # mounts into the container at start. The build proves the CPU builds on real audio, # and that the CUDA build links against everything but that one library. # --------------------------------------------------------------------------- -ARG CUDA_VERSION=12.6.3 -ARG UBUNTU=ubuntu22.04 - -FROM nvidia/cuda:${CUDA_VERSION}-devel-${UBUNTU} AS build +# CUDA 12.6.3 on Ubuntu 22.04, pinned by digest so a rebuild gets the base images that +# were reviewed rather than whatever the tags point at that day. To bump, pick the new +# tags and run `docker buildx imagetools inspect nvidia/cuda: --format +# '{{.Manifest.Digest}}'` for the devel image here and the runtime image below. +FROM nvidia/cuda:12.6.3-devel-ubuntu22.04@sha256:d49bb8a4ff97fb5fe477947a3f02aa8c0a53eae77e11f00ec28618a0bcaa2ad1 AS build ARG NODE_VERSION=22.23.2 ARG NODE_SHA256=d60acfe00a2932254bb0ad20e01b0d74397a0875595de719654b214f4b03f307 # The RTX 3050 is Ampere, compute capability 8.6. Building for every architecture takes # several times as long and is several times the size; override for other cards. ARG CUDA_ARCHITECTURES=86 -ARG WHISPER_CPP_TAG=b4938 +# The commit tag b4938 pointed at when it was reviewed; the Dockerfile says how to bump +# it, and the two must stay the same commit. +ARG WHISPER_CPP_COMMIT=371b5a7561823ab2bb32142d2751e35e7534727b +# Which models to bake in (the Dockerfile explains). Both by default; the GPU makes +# `small` cheap, so an image for one GPU box could carry only that one. +ARG WHISPER_MODELS=base,small ARG WHISPER_BASE_SHA256=422f1ae452ade6f30a004d7e5c6a43195e4433bc370bf23fac9cc591f01a8898 ARG WHISPER_SMALL_SHA256=ae85e4a935d7a567bd102fe55afc16bb595bdb618e11b2fc7591bc08120411bb @@ -44,9 +50,15 @@ WORKDIR /app COPY package.json package-lock.json ./ # glibc prebuilds for better-sqlite3 and sharp; never --omit=optional (sharp's binaries are optional deps). RUN npm ci --omit=dev +# Drops what better-sqlite3 carries for other platforms (the Dockerfile explains); +# before the check, so the check proves the module loads with only what is kept. +COPY docker/trim-node-modules.mjs ./docker/ +RUN node docker/trim-node-modules.mjs node_modules RUN node -e "require('better-sqlite3'); require('sharp'); console.log('native modules OK')" -RUN git clone --depth 1 --branch "$WHISPER_CPP_TAG" https://github.com/ggml-org/whisper.cpp /tmp/whisper.cpp +RUN git init -q /tmp/whisper.cpp \ + && git -C /tmp/whisper.cpp fetch -q --depth 1 https://github.com/ggml-org/whisper.cpp "$WHISPER_CPP_COMMIT" \ + && git -C /tmp/whisper.cpp checkout -q FETCH_HEAD ENV WHISPER_CMAKE="-DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DGGML_NATIVE=OFF -DGGML_OPENMP=OFF \ -DWHISPER_BUILD_EXAMPLES=ON -DWHISPER_BUILD_TESTS=OFF -DWHISPER_BUILD_SERVER=OFF -DWHISPER_SDL2=OFF -DWHISPER_CURL=OFF" @@ -66,15 +78,25 @@ RUN set -e; mkdir -p /app/whisper; \ && cp /tmp/wb2/bin/whisper-cli /app/whisper/whisper-cli-v2; \ rm -rf /tmp/wbc /tmp/wb3 /tmp/wb2 /tmp/whisper.cpp -RUN curl -fsSLo /app/whisper/ggml-base-q5_1.bin https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base-q5_1.bin \ - && echo "$WHISPER_BASE_SHA256 /app/whisper/ggml-base-q5_1.bin" | sha256sum -c - \ - && curl -fsSLo /app/whisper/ggml-small-q5_1.bin https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small-q5_1.bin \ - && echo "$WHISPER_SMALL_SHA256 /app/whisper/ggml-small-q5_1.bin" | sha256sum -c - +# Each model named in WHISPER_MODELS is fetched and checked against its pinned digest; +# a name with no digest here fails the build, and so does an empty list. +RUN set -e; \ + for model in $(printf '%s' "$WHISPER_MODELS" | tr ',' ' '); do \ + case "$model" in \ + base) sum="$WHISPER_BASE_SHA256";; \ + small) sum="$WHISPER_SMALL_SHA256";; \ + *) echo "WHISPER_MODELS names \"$model\", which this Dockerfile has no digest for (it knows base and small)." >&2; exit 1;; \ + esac; \ + curl -fsSLo "/app/whisper/ggml-$model-q5_1.bin" "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-$model-q5_1.bin" \ + && echo "$sum /app/whisper/ggml-$model-q5_1.bin" | sha256sum -c -; \ + done; \ + ls /app/whisper/ggml-*.bin >/dev/null 2>&1 || { echo "WHISPER_MODELS=\"$WHISPER_MODELS\" baked no model into the image." >&2; exit 1; } # --------------------------------------------------------------------------- # Runtime stage: CUDA runtime libraries (cuBLAS, the CUDA runtime), no compilers. # --------------------------------------------------------------------------- -FROM nvidia/cuda:${CUDA_VERSION}-runtime-${UBUNTU} +# The runtime twin of the devel image above; bump both together. +FROM nvidia/cuda:12.6.3-runtime-ubuntu22.04@sha256:63a18dd805367dacfb077aeced8384ab2fb569598ec5f5f5220c3f90a5c23650 ENV DEBIAN_FRONTEND=noninteractive # util-linux: setpriv, which drops privileges as su-exec does on Alpine. @@ -92,7 +114,8 @@ COPY --from=build /opt/node /opt/node ENV PATH=/opt/node/bin:$PATH WORKDIR /app -COPY --from=build /app/node_modules ./node_modules +# Owned by the default PUID/PGID from the start, so a default run never chowns it at boot. +COPY --from=build --chown=1000:1000 /app/node_modules ./node_modules COPY --from=build /app/package.json ./package.json COPY --from=build /app/whisper ./whisper COPY src ./src @@ -107,17 +130,22 @@ RUN WHISPER_SMOKE_SKIP=whisper-cli-cuda node docker/whisper-smoke.mjs \ && if [ -n "$missing" ]; then echo "whisper-cli-cuda is missing libraries: $missing"; exit 1; fi \ && echo "whisper-cli-cuda links (libcuda.so.1 comes from the driver at run time)" +# `small` by default here: the GPU makes it cheap, and it hears French markedly better. +# It has to be one of the models baked in, or the build refuses (the Dockerfile explains). +ARG WHISPER_DEFAULT_MODEL=small +RUN test -f "/app/whisper/ggml-${WHISPER_DEFAULT_MODEL}-q5_1.bin" \ + || { echo "WHISPER_DEFAULT_MODEL=${WHISPER_DEFAULT_MODEL} is not among the models baked in ($(ls /app/whisper | grep '^ggml-' | tr '\n' ' ')). Add it to WHISPER_MODELS, or set WHISPER_DEFAULT_MODEL to one of them." >&2; exit 1; } + RUN chmod +x docker/entrypoint.sh \ && mkdir -p /data \ && chown selfpod:selfpod /app -# `small` by default here: the GPU makes it cheap, and it hears French markedly better. ENV NODE_ENV=production \ PORT=8080 \ DATA_DIR=/data \ PUID=1000 \ PGID=1000 \ - WHISPER_MODEL=small \ + WHISPER_MODEL=${WHISPER_DEFAULT_MODEL} \ NVIDIA_VISIBLE_DEVICES=all \ NVIDIA_DRIVER_CAPABILITIES=compute,utility diff --git a/README.md b/README.md index 353c88b..9e5ea2c 100644 --- a/README.md +++ b/README.md @@ -263,7 +263,7 @@ of it can be changed later in **Settings** without touching the container. |---|---|---| | `PUBLIC_BASE_URL` | — | The address your proxy serves SelfPod on, e.g. `https://podcast.example.com`. Every feed and media URL is built from it. No trailing slash. Can also be set in the setup wizard. | | `PUID` / `PGID` | `1000` | The user and group SelfPod runs as. Match the owner of your audio files. | -| `TZ` | `UTC` | Used for displayed dates and an episode's default publish date. | +| `TZ` | `UTC` | Used for displayed dates and an episode's default publish date — see [How it finds new episodes](#how-it-finds-new-episodes). | | `PORT` | `8080` | Port inside the container. | | `RESCAN_INTERVAL_SECONDS` | `300` | How often the whole library is re-checked as a fallback. 60–21600. | | `MISSING_GRACE_SECONDS` | `86400` | How long an episode whose file vanished stays in the feed, so a brief share outage doesn't drop episodes. | @@ -272,11 +272,12 @@ of it can be changed later in **Settings** without touching the container. | `SUBSCRIPTIONS_ENABLED` | off | Lets SelfPod follow remote feeds. Off by default: it is the only thing that makes SelfPod fetch from the internet. Seeds the setting on first run; after that the Settings page wins. | | `REMOTE_POLL_INTERVAL_SECONDS` | `3600` | How often to check a followed feed. Clamped to 15 minutes – 24 hours — this is someone else's server. | | `ALLOW_PRIVATE_FEED_HOSTS` | *(empty)* | Comma-separated **IP addresses** that may be followed despite being private, e.g. a feed on your own NAS. Exempts only the addresses listed, and only from the address and port rules. Env-only on purpose: it weakens a guarantee, so changing it should mean touching the container. | +| `TRUST_PROXY` | `true` | Whose `X-Forwarded-*` headers to believe: `true` (every connection — what SelfPod always did), `false` (none), or a comma-separated list of the addresses and ranges your proxy connects from, e.g. `172.18.0.0/16` or `loopback`. Narrow it when something other than your proxy can reach port 8080 directly — the default compose file publishes it on every interface — so a LAN client cannot claim to have arrived over HTTPS from somewhere else. Env-only, for the same reason as the row above. | | `ADMIN_USERNAME` | `admin` | First-run only. | | `ADMIN_PASSWORD` | — | First-run only. If unset, a random password is generated and printed once to the logs. | | `SESSION_SECRET` | generated | First-run only; afterwards it lives in the database. | | `LOG_LEVEL` | `info` | `debug`, `info`, `warn`, `error`. | -| `WHISPER_MODEL` | `base` | Which speech model hears the words: `base` or `small` (both ship in the image), or a path to a whisper.cpp model file. `small` is about twice the work and markedly better in French. | +| `WHISPER_MODEL` | `base` | Which speech model hears the words: `base` or `small` (both ship in the published image; see [Building your own image](#building-your-own-image) to ship one), or a path to a whisper.cpp model file. `small` is about twice the work and markedly better in French. | | `WHISPER_CLI` | *(the image's own)* | Path to a `whisper-cli` binary, for a build of your own. The image picks its AVX2 or SSE4.2 build for the CPU at boot. | | `WHISPER_THREADS` | half the logical cores, less one (at least 2) | Threads the recogniser may use. An eight-core, sixteen-thread i7 gets 7. | | `WHISPER_NICE` | `10` | How politely the recogniser yields the processor, 0 (normal) to 19. It only matters when something else wants the processor — on a NAS, usually a listener's download. | @@ -471,6 +472,15 @@ shows the current mode. It deliberately does not sit in a banner across every pa a banner that is always there is a banner nobody reads when something is actually wrong. +**The publish date** of a new episode comes from its filename when the name starts +with a date — `2026-09-01-episode-one.mp3`, `2026_09_01 …` or `20260901 …` — at +midnight in your `TZ`. Otherwise it is the file's modification time, which is right +for a file that was just recorded and wrong for three dated files copied onto the +share together, which would all land on the minute they were copied. A modification +time that already falls on the named day is kept, because it also carries the time +of day. Either way it is only a starting point: the date you set on the episode page +is never changed by a rescan. + ## Security SelfPod is built to be published to the internet, so the design assumes the address @@ -743,6 +753,24 @@ docker build -t selfpod:test . ./test/acceptance/run.sh # 32 end-to-end checks against a real container ``` +### Building your own image + +The published image carries both speech models, `base` (60 MB) and `small` (190 MB), +so one image serves everyone. An image built for one box that will only ever use one +of them can leave the other out: + +```bash +docker build --build-arg WHISPER_MODELS=small --build-arg WHISPER_DEFAULT_MODEL=small -t selfpod:small . +``` + +`WHISPER_MODELS` is the comma-separated list to bake in (default `base,small`), and +`WHISPER_DEFAULT_MODEL` is what the container uses unless `WHISPER_MODEL` says otherwise +(default `base`; `small` in `Dockerfile.cuda`). The default has to be one of the models +baked in, or the build refuses and says so — an image whose default model is missing +would start, report the recogniser unavailable, and publish every episode unheard. +Every model that is baked in is checked against a pinned digest and proved on a +one-second recording during the build. + The acceptance script checks the behaviours that matter most: an `.m4a` appearing with no restart, a filename containing an emoji and curly quotes surviving into a downloadable URL, `cover.png` being found, a rename keeping its GUID, an edited diff --git a/doc/selfpod-spec.md b/doc/selfpod-spec.md index fdba2c0..6d5e406 100644 --- a/doc/selfpod-spec.md +++ b/doc/selfpod-spec.md @@ -246,7 +246,12 @@ Required behavior: used only as a *suggested* title if the user hasn't set one, never silently overwrites a user-edited title on rescan. - Compute/assign a stable GUID (§7.2). - - Insert into the database with `pub_date` defaulting to file mtime. + - Insert into the database with `pub_date` defaulting to the date the filename + starts with (`YYYY-MM-DD`, with `-`, `_`, `.` or nothing between the parts, + at midnight in `TZ`) when there is one and the file mtime falls on another + day; otherwise to the file mtime. First insert only — a rescan never + rewrites `pub_date`, and a user-set date (`pub_date_is_custom`) is never + touched. 3. For each file previously known but no longer present on disk, mark the episode `missing` (soft state, not deleted) rather than removing it from the feed immediately — avoids apps re-downloading/duplicating @@ -519,6 +524,7 @@ the prototype by hand: | `SESSION_SECRET` | No | random, generated on first run and persisted in the DB | Signs admin session cookies | | `LOG_LEVEL` | No | `info` | `debug`/`info`/`warn`/`error` | | `MAX_UPLOAD_SIZE_MB` | No | `1024` | Cap for UI-based file uploads (§11.4) | +| `TRUST_PROXY` | No | `true` | Whose `X-Forwarded-*` headers are believed: `true` (every hop), `false` (none), or a comma-separated list of the proxy's addresses and CIDR ranges (passed to Fastify's `trustProxy`). Narrow it to the tunnel's or reverse proxy's address when anything else can reach the port directly, so a LAN client cannot forge the protocol or source address. Env-only, never a setting in the UI. | Only one required volume: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 793ce1a..37168db 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -77,8 +77,20 @@ else log "running as $APP_USER ($RUN_AS)" fi -# Only ever touches the app's own directory, never anything under /data. -chown -R "$RUN_AS" /app/node_modules 2>/dev/null || true +# Only ever touches the app's own directory, never anything under /data — and only +# when the owner is actually wrong. `chown -R` walks the 4,500 files in node_modules, +# which took seconds on a Celeron on every boot, for a tree that was already right +# after the first one. The image ships it owned by the default ids, and the directory +# and a file deep inside it are both checked, so a chown that was interrupted part-way +# is finished rather than skipped. +node_modules_owner() { + stat -c '%u:%g' /app/node_modules /app/node_modules/.package-lock.json 2>/dev/null | sort -u +} +if [ "$(node_modules_owner)" != "$RUN_AS" ]; then + log "changing the owner of /app/node_modules to $RUN_AS" + chown -R "$RUN_AS" /app/node_modules 2>/dev/null \ + || log "could not change the owner of /app/node_modules; continuing, but the app may be unable to read its own code." +fi # --------------------------------------------------------------------------- # Read + write self-test as the target user. A failure is reported in detail and diff --git a/docker/trim-node-modules.mjs b/docker/trim-node-modules.mjs new file mode 100644 index 0000000..64c3c19 --- /dev/null +++ b/docker/trim-node-modules.mjs @@ -0,0 +1,61 @@ +#!/usr/bin/env node +/** + * Drops what node_modules carries that the image never reads, at build time. + * + * better-sqlite3 ships a prebuilt binary for every platform it supports — eight of + * them, 16 MB — and loads exactly one, chosen by its own `getPrebuildPath()`. The + * image is built for one platform, so the other seven are dead weight; so is `deps`, + * the SQLite source it would only compile if no prebuilt matched. Together they are + * about 24 MB of an image that is mostly whisper models. + * + * Asked of the package rather than guessed from `process.platform`, so a musl build + * keeps the musl binary. Run in the build stage, *before* the native-module check, + * so that check proves the module still loads with only what is kept. A layout this + * script does not recognise fails the build with a sentence saying so, rather than + * trimming blind or quietly doing nothing. + */ +import { createRequire } from 'node:module'; +import { existsSync, readdirSync, rmSync, statSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; + +const root = resolve(process.argv[2] ?? 'node_modules'); +const pkg = join(root, 'better-sqlite3'); +const require = createRequire(import.meta.url); + +function sizeOf(path) { + const stats = statSync(path); + if (!stats.isDirectory()) return stats.size; + return readdirSync(path).reduce((total, name) => total + sizeOf(join(path, name)), 0); +} +const mb = (bytes) => `${(bytes / 1024 / 1024).toFixed(1)} MB`; + +let removed = 0; + +const deps = join(pkg, 'deps'); +if (existsSync(deps)) { + removed += sizeOf(deps); + rmSync(deps, { recursive: true, force: true }); +} + +// The absolute path sidesteps the package's `exports` map, which does not list lib/. +let getPrebuildPath; +try { + ({ getPrebuildPath } = require(join(pkg, 'lib', 'binding.js'))); +} catch (err) { + console.error(`better-sqlite3's loader is not where this script expects it (${err.message}); update docker/trim-node-modules.mjs for this version before trimming.`); + process.exit(1); +} +const keep = getPrebuildPath?.(); +if (!keep) { + console.error('better-sqlite3 reports no prebuilt binary for this platform, so there is nothing safe to trim; it will try to compile instead, which the image does not support.'); + process.exit(1); +} +const prebuilds = dirname(keep); +for (const name of readdirSync(prebuilds)) { + const file = join(prebuilds, name); + if (file === keep) continue; + removed += sizeOf(file); + rmSync(file, { force: true }); +} + +console.log(`trimmed node_modules: kept ${keep.slice(root.length + 1)}, removed ${mb(removed)}`); diff --git a/docker/whisper-smoke.mjs b/docker/whisper-smoke.mjs index e4d2f68..fb45479 100644 --- a/docker/whisper-smoke.mjs +++ b/docker/whisper-smoke.mjs @@ -12,7 +12,7 @@ * it in the shape SelfPod reads, not that it spelled the name. */ import { execFileSync } from 'node:child_process'; -import { existsSync, readFileSync, readdirSync, rmSync } from 'node:fs'; +import { readFileSync, readdirSync, rmSync, statSync } from 'node:fs'; import { join } from 'node:path'; const dir = process.env.WHISPER_DIR ?? '/app/whisper'; @@ -33,18 +33,21 @@ if (!binaries.length) { } /* - * Every binary is proved with the default model. The larger model is proved only with - * the fastest binary: on the SSE4.2-only build a quantised model runs through scalar - * code and takes minutes even on a one-second file, and no box that needs that build - * should be running the larger model — the README says so — so proving the pairing - * would only prove the build host's patience. + * Every binary is proved with the smallest model shipped (`base`, unless the image was + * built without it — WHISPER_MODELS in the Dockerfile). Any larger model is proved only + * with the fastest binary: on the SSE4.2-only build a quantised model runs through + * scalar code and takes minutes even on a one-second file, and no box that needs that + * build should be running the larger model — the README says so — so proving the + * pairing would only prove the build host's patience. */ const fastest = binaries.includes('whisper-cli-v3') ? 'whisper-cli-v3' : binaries[0]; +const smallest = [...models].sort((a, b) => statSync(join(dir, a)).size - statSync(join(dir, b)).size)[0]; const pairs = binaries.flatMap((binary) => models - .filter((model) => model.includes('base') || binary === fastest) + .filter((model) => model === smallest || binary === fastest) .map((model) => [binary, model]), ); +console.log(`models: ${models.join(', ')}; every binary is proved with ${smallest}`); let failed = 0; for (const [name, modelName] of pairs) { diff --git a/package-lock.json b/package-lock.json index 66c5e38..ef19c50 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.9.1", "license": "MIT", "dependencies": { + "@fastify/compress": "9.2.1", "@fastify/cookie": "11.1.2", "@fastify/formbody": "9.0.0", "@fastify/multipart": "10.1.0", @@ -106,6 +107,27 @@ "integrity": "sha512-yXSS27qPExaXeuLvMRMXOLtpipzfQYNjG3FkunDWKGfMYjKuhFXko9CVzqxm8jcF+lmtS9Fd89QNdh9XDjnbNg==", "license": "MIT" }, + "node_modules/@fastify/compress": { + "version": "9.2.1", + "resolved": "https://registry.npmjs.org/@fastify/compress/-/compress-9.2.1.tgz", + "integrity": "sha512-YTLN+YyMCr0lUmzuw+CEh0//Q2pynfzAcZrhfbL6pQyiMTyuGeiskr7LM4armP4tjeIPKzoZZUSLC/XYvGMghg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT", + "dependencies": { + "@fastify/accept-negotiator": "^2.0.0", + "fastify-plugin": "^6.0.0", + "mime-db": "^1.52.0" + } + }, "node_modules/@fastify/cookie": { "version": "11.1.2", "resolved": "https://registry.npmjs.org/@fastify/cookie/-/cookie-11.1.2.tgz", @@ -1941,6 +1963,15 @@ "node": ">=10.0.0" } }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/minimatch": { "version": "10.2.6", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", diff --git a/package.json b/package.json index 0cf365f..93ba6d3 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "reset-password": "node scripts/reset-password.js" }, "dependencies": { + "@fastify/compress": "9.2.1", "@fastify/cookie": "11.1.2", "@fastify/formbody": "9.0.0", "@fastify/multipart": "10.1.0", diff --git a/src/app.js b/src/app.js index 0793de6..a2a24ff 100644 --- a/src/app.js +++ b/src/app.js @@ -28,9 +28,10 @@ export async function buildApp(services) { loggerInstance: logger, // The app always runs behind the user's own reverse proxy or tunnel, so // X-Forwarded-Proto has to be honoured for Secure cookies and correct URLs. - // Note this makes `request.ip` client-influenced — see plugins/auth.js for why - // rate limiting deliberately does not use it. - trustProxy: true, + // Every hop is trusted by default, which makes `request.ip` client-influenced — + // see plugins/auth.js for why rate limiting deliberately does not use it — and + // TRUST_PROXY narrows it to the proxy's own address (config.js). + trustProxy: config.trustProxy, bodyLimit: 2 * 1024 * 1024, // form posts only; uploads stream via multipart // Episode filenames are route parameters, and Fastify rejects any parameter diff --git a/src/config.js b/src/config.js index 7151d40..a42fa90 100644 --- a/src/config.js +++ b/src/config.js @@ -94,6 +94,63 @@ function readAllowedPrivateHosts(raw, { name }) { return { value: Object.freeze(allowed), warnings }; } +/** The range names proxy-addr (what Fastify's `trustProxy` is built on) understands. */ +const TRUST_PROXY_RANGE_NAMES = new Set(['loopback', 'linklocal', 'uniquelocal']); + +/** + * Parses TRUST_PROXY into what Fastify's `trustProxy` option takes: `true` (every hop + * is trusted — the default, and what SelfPod always did), `false` (no hop is), or a + * list of the addresses and CIDR ranges that proxies arrive from. + * + * Why it exists: with every hop trusted, whoever can open a connection to port 8080 + * directly — anyone on the LAN, given compose publishes it on every interface — can + * set X-Forwarded-Proto and X-Forwarded-For themselves, and SelfPod believes them. + * Nothing security-relevant keys off those headers today (plugins/auth.js explains + * why rate limiting never used `request.ip`), so the default stays open; narrowing it + * to the tunnel's or reverse proxy's address closes the door anyway. + * + * A list with an entry that is not an address drops that entry and says so. A list + * with nothing usable in it trusts no hop at all rather than falling back to every + * hop: the operator plainly meant to stop trusting everyone, and trusting nobody + * costs only the Secure flag on a tunnel's cookie, which browsers do not need. + */ +function readTrustProxy(raw, { name }) { + const value = String(raw ?? '').trim(); + if (value === '' || value.toLowerCase() === 'true') return { value: true, warnings: [] }; + if (value.toLowerCase() === 'false') return { value: false, warnings: [] }; + + const warnings = []; + const entries = []; + for (const entry of value.split(',')) { + const item = entry.trim(); + if (!item) continue; + if (TRUST_PROXY_RANGE_NAMES.has(item.toLowerCase())) { + entries.push(item.toLowerCase()); + continue; + } + const [address, prefix, ...rest] = item.split('/'); + const bare = address.startsWith('[') && address.endsWith(']') ? address.slice(1, -1) : address; + const family = isIP(bare); + const prefixOk = + prefix === undefined || (/^\d{1,3}$/.test(prefix) && Number(prefix) <= (family === 4 ? 32 : 128)); + if (family === 0 || rest.length > 0 || !prefixOk) { + warnings.push( + `${name} entry "${item}" is not an IP address or CIDR range and was ignored. List the addresses your proxy connects from, like 172.18.0.5 or 10.0.0.0/8, or use true or false.`, + ); + continue; + } + entries.push(prefix === undefined ? bare : `${bare}/${prefix}`); + } + + if (entries.length === 0) { + warnings.push( + `${name} ("${value}") names no usable address, so no proxy is trusted: forwarded headers are ignored and every request is treated as arriving directly. Fix the value, or set TRUST_PROXY=true to trust every hop as before.`, + ); + return { value: false, warnings }; + } + return { value: Object.freeze(entries), warnings }; +} + const LOG_LEVELS = new Set(['trace', 'debug', 'info', 'warn', 'error', 'fatal', 'silent']); export function loadConfig(env = process.env) { @@ -136,6 +193,7 @@ export function loadConfig(env = process.env) { tempDir: join(dataDir, DIRECTORY_NAMES.TEMP), episodeArtDir: join(dataDir, DIRECTORY_NAMES.EPISODE_ART), fingerprintDir: join(dataDir, DIRECTORY_NAMES.FINGERPRINTS), + coverThumbDir: join(dataDir, DIRECTORY_NAMES.COVER_THUMBS), trimmedDir: join(dataDir, DIRECTORY_NAMES.TRIMMED), transcriptDir: join(dataDir, DIRECTORY_NAMES.TRANSCRIPTS), databasePath: join(dataDir, FILE_NAMES.DATABASE), @@ -275,6 +333,12 @@ export function loadConfig(env = process.env) { warnings.push(...privateHosts.warnings); config.allowedPrivateFeedHosts = privateHosts.value; + // Env-only for the same reason: it decides whose word on "this came over HTTPS from + // that address" is believed, and that should not be changeable from a web page. + const trustProxy = readTrustProxy(env.TRUST_PROXY, { name: 'TRUST_PROXY' }); + warnings.push(...trustProxy.warnings); + config.trustProxy = trustProxy.value; + if (!isAbsolute(config.dataDir)) { throw new Error(`DATA_DIR must be an absolute path (got "${config.dataDir}").`); } diff --git a/src/constants.js b/src/constants.js index 5ecf3da..22642ec 100644 --- a/src/constants.js +++ b/src/constants.js @@ -197,6 +197,15 @@ export const DIRECTORY_NAMES = Object.freeze({ * also serves media, and inside the file the operator backs up. */ FINGERPRINTS: '.fp', + /** + * Small copies of show covers for the dashboard, as `/data/.covers/{show_id}-{width}-{hash}.jpg`. + * + * A cover is 1400 px or more and several hundred kilobytes; the card it sits in is + * 250 px. One copy per cover, named by the cover's content hash so a changed cover + * makes a new one, and a cache like `.art`: lose the folder and it is rebuilt on + * the next request. + */ + COVER_THUMBS: '.covers', /** * Trimmed copies, as `/data/.trimmed/{show_id}/{episode_id}.{ext}`. * @@ -355,6 +364,17 @@ export const REMOTE_IMAGE_MAX_BYTES = 10 * 1024 * 1024; */ export const REMOTE_IMAGE_MAX_PIXELS = 40 * 1000 * 1000; +/** + * The same ceiling for every image sharp decodes, wherever it came from. + * + * The shows folder is a network share that other people can write to, so a cover + * or an episode sidecar dropped there is as much a stranger's choice as a remote + * image is — and an image embedded in an MP3's tag arrives the same way. A 300 KB + * WebP declaring 16000×16000 decodes to a gigabyte; passing this to `sharp()` makes + * it refuse at the header, before a byte is decoded. + */ +export const IMAGE_MAX_INPUT_PIXELS = REMOTE_IMAGE_MAX_PIXELS; + /** * Four timers, not one, because they catch four different failures. * @@ -574,3 +594,14 @@ export const FINGERPRINTABLE_EXTENSIONS = Object.freeze(['.mp3']); * publishes bytes to your subscribers. */ export const TRIMMABLE_EXTENSIONS = Object.freeze(['.mp3']); + +/** + * The largest episode file the advert pipeline will read whole, in bytes. + * + * Fingerprinting, listening and cutting all read a whole file into memory, and the + * show folder is a share others can write to — so a multi-gigabyte file dropped there, + * or a symlink to one, must be refused for the price of a `stat` rather than read + * into the heap on every pass. 768 MB is about five hours at 320 kbit/s, which is + * already past the longest MP3 the frame reader will take in one piece (MAX_FRAMES). + */ +export const MAX_PIPELINE_FILE_BYTES = 768 * 1024 * 1024; diff --git a/src/db/index.js b/src/db/index.js index ceefca8..bd66f08 100644 --- a/src/db/index.js +++ b/src/db/index.js @@ -39,6 +39,10 @@ export function openDatabase(path, { logger, readonly = false } = {}) { } db.pragma('synchronous = NORMAL'); + // The WAL is truncated back to this once a checkpoint has emptied it, so a burst of + // writes — a first scan of a large library — does not leave a file the size of + // that burst beside the database for ever. + db.pragma('journal_size_limit = 67108864'); const migration = runMigrations(db, { logger }); @@ -57,6 +61,7 @@ export function openDatabase(path, { logger, readonly = false } = {}) { */ export function closeDatabase(db, { logger } = {}) { if (!db || !db.open) return; + optimiseDatabase(db, { logger }); try { db.pragma('wal_checkpoint(TRUNCATE)'); } catch (err) { @@ -68,3 +73,18 @@ export function closeDatabase(db, { logger } = {}) { logger?.warn({ err }, 'error closing database'); } } + +/** + * `PRAGMA optimize`: lets SQLite gather the statistics its planner uses to pick + * indexes, for the tables that have changed enough to matter. Cheap — milliseconds — + * and SQLite's own documentation asks for it on close and every so often while + * running; the scheduler calls it once a day, after the access log is trimmed. + */ +export function optimiseDatabase(db, { logger } = {}) { + if (!db || !db.open) return; + try { + db.pragma('optimize'); + } catch (err) { + logger?.debug({ err }, 'PRAGMA optimize failed (harmless)'); + } +} diff --git a/src/db/migrations/012-advert-indexes.sql b/src/db/migrations/012-advert-indexes.sql new file mode 100644 index 0000000..3b40cf5 --- /dev/null +++ b/src/db/migrations/012-advert-indexes.sql @@ -0,0 +1,13 @@ +-- Indexes for the four foreign keys the advert tables are joined and cascaded on. +-- +-- Additive only, and safe to roll back from: an older image neither knows nor +-- needs these, and SQLite ignores an index it did not create. +-- +-- Every one of these columns is used as a join or a lookup on every pass and every +-- page render — the rule behind a cut (marker_id, anchor_id), the restores on a cut +-- (segment_id) — and as the child side of an ON DELETE cascade, which without an +-- index means a full scan of the child table for every parent row deleted. +CREATE INDEX IF NOT EXISTS idx_ad_cut_overrides_segment ON ad_cut_overrides(segment_id); +CREATE INDEX IF NOT EXISTS idx_ad_segments_marker ON ad_segments(marker_id); +CREATE INDEX IF NOT EXISTS idx_ad_segments_anchor ON ad_segments(anchor_id); +CREATE INDEX IF NOT EXISTS idx_ad_anchors_marker ON ad_anchors(marker_id); diff --git a/src/db/migrations/013-media-access-keeps-history.sql b/src/db/migrations/013-media-access-keeps-history.sql new file mode 100644 index 0000000..25f816c --- /dev/null +++ b/src/db/migrations/013-media-access-keeps-history.sql @@ -0,0 +1,39 @@ +-- Request history outlives the episode it was about. +-- +-- `media_access.episode_id` cascaded on delete, so removing an episode — from the +-- page, or by deleting its file — also removed every request ever made for it, and +-- the show's and the instance's totals shrank to match. A download that happened +-- happened; the log is a record of it, not a property of the episode row. The link +-- to `episodes` is dropped (the id is kept, so the log can still say which episode +-- it was and the page can say it is gone), and the link to `shows` keeps cascading: +-- removing a show is the one case where the owner means to forget it. +-- +-- SQLite cannot alter a foreign key in place, so the table is rebuilt. Additive in +-- effect — every row and index comes back as it was — and an older image reading +-- this table sees exactly the columns it expects. +CREATE TABLE media_access_kept ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + episode_id TEXT, + show_id TEXT REFERENCES shows(id) ON DELETE CASCADE, + requested_at TEXT NOT NULL, + kind TEXT NOT NULL, + status_code INTEGER NOT NULL, + bytes_sent INTEGER, + total_bytes INTEGER, + range_header TEXT, + client TEXT, + error TEXT +); + +INSERT INTO media_access_kept + (id, episode_id, show_id, requested_at, kind, status_code, bytes_sent, total_bytes, range_header, client, error) + SELECT id, episode_id, show_id, requested_at, kind, status_code, bytes_sent, total_bytes, range_header, client, error + FROM media_access; + +DROP TABLE media_access; +ALTER TABLE media_access_kept RENAME TO media_access; + +CREATE INDEX idx_media_access_episode ON media_access(episode_id, requested_at DESC); +CREATE INDEX idx_media_access_show ON media_access(show_id, requested_at DESC); +CREATE INDEX idx_media_access_time ON media_access(requested_at DESC); +CREATE INDEX idx_media_access_failures ON media_access(status_code, requested_at DESC); diff --git a/src/index.js b/src/index.js index 43f96b2..447e708 100644 --- a/src/index.js +++ b/src/index.js @@ -7,6 +7,7 @@ import { buildApp, loggerOptions } from './app.js'; import { loadConfig } from './config.js'; import { closeDatabase, openDatabase } from './db/index.js'; import { describeFsError } from './lib/errors.js'; +import { createAudioWorker } from './lib/audio-search.js'; import { createEventBus } from './lib/events.js'; import { createActivity } from './services/activity.js'; import { bootstrap } from './services/bootstrap.js'; @@ -76,6 +77,7 @@ async function main() { await mkdir(config.tempDir, { recursive: true }); await mkdir(config.episodeArtDir, { recursive: true }); await mkdir(config.fingerprintDir, { recursive: true }); + await mkdir(config.coverThumbDir, { recursive: true }); await mkdir(config.trimmedDir, { recursive: true }); await mkdir(config.transcriptDir, { recursive: true }); } catch (err) { @@ -123,8 +125,11 @@ async function main() { }); const stats = createStats({ db, logger }); const subscriptions = createSubscriptions({ db, config, events, logger }); - const transcriber = createTranscriber({ db, config, events, logger, health, shows, episodes }); - const adDetect = createAdDetect({ db, config, events, logger, shows, episodes, transcriber }); + // One worker thread for every piece of audio work — decoding, fingerprinting and + // the corpus searches — so none of it runs on the thread serving listeners. + const audio = createAudioWorker({ logger }); + const transcriber = createTranscriber({ db, config, events, logger, health, shows, episodes, audio }); + const adDetect = createAdDetect({ db, config, events, logger, shows, episodes, transcriber, health, audioSearch: audio }); // Rows an older image wrote after a rollback carry no kind; give them one before any page reads them. adDetect.reconcileKinds(); const remoteFeeds = createRemoteFeeds({ @@ -134,7 +139,7 @@ async function main() { const trimmer = createTrimmer({ config, events, logger, health, shows, episodes, adDetect, metadata, }); - const advertsView = createAdvertsView({ db, adDetect, transcriber, episodes, shows }); + const advertsView = createAdvertsView({ db, config, adDetect, transcriber, episodes, shows }); const adPipeline = createAdPipeline({ db, events, logger, health, shows, episodes, adDetect, trimmer, activity, transcriber, // A new or changed file starts a pass straight away, and one runs after the startup scan. @@ -144,7 +149,7 @@ async function main() { const timeline = createTimeline({ db, logger }); const watcher = createWatcher({ config, settings, events, logger, scanner, shows, health }); const scheduler = createScheduler({ - settings, events, logger, scanner, episodes, watcher, activity, stats, remoteFeeds, adPipeline, + db, settings, events, logger, scanner, episodes, watcher, activity, stats, remoteFeeds, adPipeline, }); const presenters = createPresenters({ settings, shows, episodes, covers, activity, stats, readiness }); @@ -239,7 +244,7 @@ async function startDegradedServer(cause) { logger.error({ err: cause }, `SelfPod cannot open its database. ${detail}`); - const app = Fastify({ loggerInstance: logger, trustProxy: true }); + const app = Fastify({ loggerInstance: logger, trustProxy: config.trustProxy }); app.get('/health', async (request, reply) => { reply.header('access-control-allow-origin', '*').header('cache-control', 'no-store'); diff --git a/src/lib/audio-search.js b/src/lib/audio-search.js index 4cc7aa7..e9a6629 100644 --- a/src/lib/audio-search.js +++ b/src/lib/audio-search.js @@ -1,26 +1,44 @@ import { Worker } from 'node:worker_threads'; import { findHeadAnchors } from './audio-anchor.js'; +import { decodeWindowToWav, fingerprintFile, profileFile } from './audio-tasks.js'; import { findRepeatedAudio } from './repeated-audio.js'; const WORKER_URL = new URL('../workers/audio-search.js', import.meta.url); /** - * Runs the corpus searches in a worker thread, one long-lived worker per instance. + * How long a file task may take before the worker is stopped and the episode + * reported as failed. Generous on purpose: a five-hour episode on a two-core NAS is + * minutes of decoding, and a timeout that fires on honest work would turn a slow box + * into one that never fingerprints anything. It exists for the other case — a decoder + * wedged on a hostile file — so that a pass is reported as failed rather than hung. + */ +export const FILE_TASK_TIMEOUT_MS = 20 * 60 * 1000; + +/** + * Runs the audio work in a worker thread, one long-lived worker per instance. * - * Long-lived rather than one per call because a pass asks twice (the jingle and the - * repeated audio) and starting a worker costs tens of milliseconds and a module load. - * `unref`'d so it never keeps the process alive, and restarted on the next call if it - * died. If a worker cannot be started at all the search runs inline, as it did before — - * slower to live with, never a feature that silently stops. + * Long-lived rather than one per call because a pass asks many times — every + * episode to fingerprint, every window to decode, then the two corpus searches — + * and starting a worker costs tens of milliseconds and a module load. `unref`'d so + * it never keeps the process alive, and restarted on the next call if it died. If a + * worker cannot be started at all the work runs inline, as it did before — slower to + * live with, never a feature that silently stops. + * + * A worker that crashes, or a file task that outlives its timeout, rejects the + * calls waiting on it with an error that says so; the caller reports that episode + * as failed and moves on, and the next call starts a fresh worker. */ -export function createAudioSearch({ logger = null, inline = false } = {}) { +export function createAudioWorker({ logger = null, inline = false, fileTimeoutMs = FILE_TASK_TIMEOUT_MS } = {}) { let worker = null; let nextId = 1; const pending = new Map(); function failAll(error) { - for (const { reject } of pending.values()) reject(error); + for (const { reject, timer } of pending.values()) { + clearTimeout(timer); + reject(error); + } pending.clear(); worker = null; } @@ -28,42 +46,67 @@ export function createAudioSearch({ logger = null, inline = false } = {}) { function ensureWorker() { if (worker) return worker; // No inherited flags: --input-type or a test runner's own flags are refused in a worker. - worker = new Worker(WORKER_URL, { execArgv: [] }); - worker.unref(); - worker.on('message', ({ id, result, error }) => { + const started = new Worker(WORKER_URL, { execArgv: [] }); + worker = started; + started.unref(); + started.on('message', ({ id, result, error }) => { const waiting = pending.get(id); if (!waiting) return; pending.delete(id); - if (error) waiting.reject(Object.assign(new Error(error.message), { stack: error.stack })); - else waiting.resolve(result); + clearTimeout(waiting.timer); + if (error) { + waiting.reject(Object.assign(new Error(error.message), { stack: error.stack, code: error.code, refused: error.refused })); + } else { + waiting.resolve(result); + } }); - worker.on('error', (error) => { - logger?.warn({ err: error }, 'the audio search worker failed; it will be started again on the next pass'); - failAll(error); + started.on('error', (error) => { + logger?.warn({ err: error }, 'the audio worker failed; it will be started again on the next pass'); + if (worker === started) failAll(error); }); - worker.on('exit', (code) => { - if (pending.size) failAll(new Error(`the audio search worker exited with code ${code}`)); + started.on('exit', (code) => { + // Only this worker's own slot: a replacement started after a timeout or a + // crash must not be forgotten when the old one finally reports its exit. + if (worker !== started) return; + if (pending.size) failAll(new Error(`the audio worker exited with code ${code}`)); worker = null; }); - return worker; + return started; } - function run(task, payload, fallback) { - if (inline) return Promise.resolve(fallback()); + function run(task, payload, fallback, { timeoutMs = 0 } = {}) { + if (inline) return Promise.resolve().then(fallback); let target; try { target = ensureWorker(); } catch (error) { - logger?.warn({ err: error }, 'could not start the audio search worker; searching on the main thread'); - return Promise.resolve(fallback()); + logger?.warn({ err: error }, 'could not start the audio worker; working on the main thread'); + return Promise.resolve().then(fallback); } return new Promise((resolve, reject) => { const id = nextId++; - pending.set(id, { resolve, reject }); + let timer = null; + if (timeoutMs > 0) { + timer = setTimeout(() => { + pending.delete(id); + reject(new Error(`the audio work took longer than ${Math.round(timeoutMs / 1000)} seconds and was stopped`)); + // The worker is wedged on this file, so everything else waiting on it is + // told so too, and the next call gets a fresh one. + const stuck = worker; + worker = null; + failAll(new Error('the audio worker was stopped after a task timed out')); + stuck?.terminate().catch(() => {}); + }, timeoutMs); + timer.unref?.(); + } + pending.set(id, { resolve, reject, timer }); target.postMessage({ id, task, payload }); }); } + /** Inline fallbacks unwrap the `{ result }` envelope the worker would have sent. */ + const unwrap = async (work) => (await work()).result; + return { repeatedAudio(episodes, options) { return run('repeatedAudio', { episodes, options }, () => findRepeatedAudio(episodes, options)); @@ -71,6 +114,18 @@ export function createAudioSearch({ logger = null, inline = false } = {}) { headAnchors(episodes, options) { return run('headAnchors', { episodes, options }, () => findHeadAnchors(episodes, options)); }, + /** @see fingerprintFile in audio-tasks.js */ + fingerprintFile(payload) { + return run('fingerprintFile', payload, () => unwrap(() => fingerprintFile(payload)), { timeoutMs: fileTimeoutMs }); + }, + /** @see profileFile in audio-tasks.js */ + profileFile(payload) { + return run('profileFile', payload, () => unwrap(() => profileFile(payload)), { timeoutMs: fileTimeoutMs }); + }, + /** @see decodeWindowToWav in audio-tasks.js */ + decodeWindow(payload) { + return run('decodeWindow', payload, () => unwrap(() => decodeWindowToWav(payload)), { timeoutMs: fileTimeoutMs }); + }, async close() { const current = worker; worker = null; @@ -78,3 +133,6 @@ export function createAudioSearch({ logger = null, inline = false } = {}) { }, }; } + +/** The name this had when it only searched. */ +export const createAudioSearch = createAudioWorker; diff --git a/src/lib/audio-tasks.js b/src/lib/audio-tasks.js new file mode 100644 index 0000000..a225a68 --- /dev/null +++ b/src/lib/audio-tasks.js @@ -0,0 +1,190 @@ +import { createHash } from 'node:crypto'; + +import { createFingerprinter } from './acoustic-fingerprint.js'; +import { decodeToMono } from './decode-audio.js'; +import { frameProfile } from './mp3-frames.js'; +import { openGatedFile } from './pipeline-file.js'; +import { createEnvelopeBuilder } from './snap-edges.js'; +import { openWavWriter } from './wav.js'; + +/** + * The per-file audio work, written once and run in the worker (spec §19.3, §19.6). + * + * Decoding an MP3 is the single most expensive thing SelfPod does with a file, and it + * is pure CPU: an hour of audio was twelve seconds of a fully blocked event loop on a + * desktop, several times that on a NAS. On the main thread that meant every range + * request in flight stalled, the SSE heartbeat stopped, and `/health` did not answer — + * long enough for the container's own health check to give up and have it restarted + * mid-pass. These functions are what the worker runs; `audio-search.js` dispatches to + * them, and falls back to calling them inline if no worker can be started, so the + * result is the same bytes either way — the only difference is which thread was busy. + * + * Every function here reads its file through `openGatedFile`, which checks the open + * descriptor is an ordinary file of a size SelfPod is willing to hold, so a path + * swapped between the pipeline's own check and this read is still refused. + */ + +/** Reads `[start, end)` of an open file into a fresh buffer. */ +async function readRange(handle, start, end) { + const length = Math.max(0, end - start); + const buffer = Buffer.allocUnsafe(length); + let at = 0; + while (at < length) { + const { bytesRead } = await handle.read(buffer, at, length - at, start + at); + if (!bytesRead) break; // shorter than its stat said: a file truncated under us + at += bytesRead; + } + return at === length ? buffer : buffer.subarray(0, at); +} + +/** + * Reads an episode, hashes it, and fingerprints what it sounds like. + * + * `expectSha256` is the digest of the fingerprint already stored: when the bytes + * still match it the decode is skipped and only the digest comes back, so a touched + * but unchanged file costs one read and no CPU. Returned typed arrays are meant to be + * transferred, not copied. + */ +export async function fingerprintFile({ path, maxBytes, expectSha256 = null }) { + const { handle, size } = await openGatedFile(path, { maxBytes }); + let bytes; + try { + bytes = await readRange(handle, 0, size); + } finally { + await handle.close(); + } + + const sha256 = createHash('sha256').update(bytes).digest('hex'); + if (expectSha256 && sha256 === expectSha256) return { result: { sha256, bytes: bytes.length, unchanged: true } }; + + const profile = frameProfile(bytes); + if (!profile) return { result: { sha256, bytes: bytes.length, noFrames: true } }; + if (profile.truncated) return { result: { sha256, bytes: bytes.length, tooLong: true } }; + + const fingerprinter = createFingerprinter(); + const decoded = await decodeToMono(bytes, profile.frames, (samples) => fingerprinter.push(samples)); + const hashes = fingerprinter.finish(); + + return { + result: { + sha256, + bytes: bytes.length, + hashes, + frameCount: profile.frameCount, + durationMs: profile.durationMs, + sampleRate: profile.sampleRate, + samplesPerFrame: profile.frames[0]?.samplesPerFrame ?? 1152, + decodeErrors: decoded.errors, + discontinuities: profile.discontinuities.length, + }, + transfer: [hashes.buffer], + }; +} + +/** + * Reads an episode once and describes its frames, for the transcriber. + * + * The frame table comes back as four typed arrays rather than an array of objects: + * an hour is 137,000 frames, and a structured clone of that many objects costs more + * than the walk that produced them. `framesFromTable` turns it back into the shape + * `mp3-frames.js` works with, on the thread that needs it. + */ +export async function profileFile({ path, maxBytes }) { + const { handle, size } = await openGatedFile(path, { maxBytes }); + let bytes; + try { + bytes = await readRange(handle, 0, size); + } finally { + await handle.close(); + } + const sha256 = createHash('sha256').update(bytes).digest('hex'); + const profile = frameProfile(bytes); + if (!profile) return { result: { sha256, bytes: bytes.length, noFrames: true } }; + + const count = profile.frames.length; + const offsets = new Uint32Array(count); + const lengths = new Uint16Array(count); + const sampleRates = new Uint32Array(count); + const samplesPerFrame = new Uint16Array(count); + for (let i = 0; i < count; i += 1) { + const frame = profile.frames[i]; + offsets[i] = frame.offset; + lengths[i] = frame.length; + sampleRates[i] = frame.sampleRate; + samplesPerFrame[i] = frame.samplesPerFrame; + } + return { + result: { + sha256, + bytes: bytes.length, + frameCount: profile.frameCount, + durationMs: profile.durationMs, + sampleRate: profile.sampleRate, + truncated: profile.truncated, + table: { offsets, lengths, sampleRates, samplesPerFrame }, + }, + transfer: [offsets.buffer, lengths.buffer, sampleRates.buffer, samplesPerFrame.buffer], + }; +} + +/** The frame objects `mp3-frames.js` reasons about, rebuilt from a transferred table. */ +export function framesFromTable({ offsets, lengths, sampleRates, samplesPerFrame }) { + const frames = new Array(offsets.length); + for (let i = 0; i < offsets.length; i += 1) { + frames[i] = { offset: offsets[i], length: lengths[i], sampleRate: sampleRates[i], samplesPerFrame: samplesPerFrame[i] }; + } + return frames; +} + +/** + * Decodes one stretch of an episode to a 16-bit WAV file, for the recogniser. + * + * Only the bytes of the window are read — `start` to `end`, with `frames` relative + * to `start` — so an hour-long episode is not held in memory for the minutes whisper + * takes over its first five. The WAV is written here, a chunk at a time, rather than + * returned: a whole-episode window is over a hundred megabytes of samples, and the + * whole point of the chunked writer was never to hold that at once. What comes back + * is the loudness envelope the edge-snapping reads, and the sample count. + */ +export async function decodeWindowToWav({ path, maxBytes, start, end, frames, skipSamples = 0, wavPath, targetRate }) { + const { handle } = await openGatedFile(path, { maxBytes }); + let bytes; + try { + bytes = await readRange(handle, start, end); + } finally { + await handle.close(); + } + + const envelope = createEnvelopeBuilder(targetRate); + const writer = openWavWriter(wavPath, { sampleRate: targetRate }); + let toSkip = skipSamples; + let closed = false; + try { + const decoded = await decodeToMono( + bytes, + frames, + (samples) => { + // The primer frames' output: decoded so the bit reservoir is primed by the + // time the window starts, then dropped. + if (toSkip >= samples.length) { + toSkip -= samples.length; + return; + } + const kept = toSkip ? samples.subarray(toSkip) : samples; + toSkip = 0; + writer.write(kept); + envelope.push(kept); + }, + { targetRate, resample: 'average' }, + ); + const { samples } = writer.close(); + closed = true; + const values = envelope.finish(); + return { + result: { samples, errors: decoded.errors, envelope: values, hopMs: envelope.hopMs }, + transfer: [values.buffer], + }; + } finally { + if (!closed) writer.close(); + } +} diff --git a/src/lib/dates.js b/src/lib/dates.js index 0b0738a..1880ea0 100644 --- a/src/lib/dates.js +++ b/src/lib/dates.js @@ -32,6 +32,16 @@ export function formatDuration(totalSeconds) { return h > 0 ? `${h}:${pad(m)}:${pad(s)}` : `${m}:${pad(s)}`; } +/** + * The same `m:ss` / `h:mm:ss` from milliseconds, for positions and lengths inside an + * episode — "cut 0:39", "between 2:10 and 2:14 in". One implementation, so a stretch + * reads the same on the Adverts page, the episode page, the ledger and in the API; + * app.js carries a copy of exactly this for what it formats on the client. + */ +export function formatClock(ms) { + return formatDuration((ms ?? 0) / 1000) ?? '0:00'; +} + /** Zero-padded `HH:MM:SS` for the feed, where a leading hour of 00 is harmless. */ export function formatDurationFeed(totalSeconds) { if (totalSeconds === null || totalSeconds === undefined) return null; @@ -113,9 +123,15 @@ export function toLocalInputValue(value, { timeZone } = {}) { /** * Reads a `datetime-local` value back, interpreting it in the given zone. - * Works by probing the offset that zone had at roughly that instant, which - * handles DST correctly for every case except the one ambiguous hour when - * clocks go back — where either interpretation is defensible. + * + * The zone's offset is probed at the wall-clock instant read as UTC, and then once + * more at the instant that first guess produced. One probe was enough for London + * and New York, where the clocks change in the small hours and the first guess + * lands on the right side of the change; it was an hour out at midnight on the + * transition days in Sydney, Auckland and Santiago, where the guess — ten or eleven + * hours later than the real instant — fell on the far side of a 02:00 change. The + * second probe catches that. The one ambiguous hour when clocks go back has two + * defensible answers; the earlier offset wins. */ export function fromLocalInputValue(input, { timeZone } = {}) { if (typeof input !== 'string' || !input.trim()) return null; @@ -127,8 +143,23 @@ export function fromLocalInputValue(input, { timeZone } = {}) { const [, y, mo, d, h, mi, s = '00'] = match; const asUtc = Date.UTC(+y, +mo - 1, +d, +h, +mi, +s); if (!timeZone) return new Date(asUtc).toISOString(); - const offsetMs = zoneOffsetMs(new Date(asUtc), timeZone); - return new Date(asUtc - offsetMs).toISOString(); + const firstOffset = zoneOffsetMs(new Date(asUtc), timeZone); + const firstGuess = asUtc - firstOffset; + const secondOffset = zoneOffsetMs(new Date(firstGuess), timeZone); + if (secondOffset === firstOffset) return new Date(firstGuess).toISOString(); + const secondGuess = asUtc - secondOffset; + // Two different answers: the wall-clock time asked for sits beside a clock change. + // Each guess is judged by what it reads back as in the zone. One that reads back + // exactly wins (the earlier instant on the one night a time occurs twice). In the + // hour the clocks skip no instant reads back exactly, so the first instant on the + // far side of the gap wins — it is the start of the day that was asked for, which + // is what a midnight is used for; the other guess belongs to the day before. + const readback = (instant) => instant + zoneOffsetMs(new Date(instant), timeZone); + const candidates = [firstGuess, secondGuess].sort((a, b) => a - b); + const exact = candidates.filter((instant) => readback(instant) === asUtc); + if (exact.length) return new Date(exact[0]).toISOString(); + const onOrAfter = candidates.filter((instant) => readback(instant) >= asUtc); + return new Date(onOrAfter.length ? onOrAfter[0] : candidates[candidates.length - 1]).toISOString(); } function zoneOffsetMs(date, timeZone) { diff --git a/src/lib/errors.js b/src/lib/errors.js index 1f80958..20948bd 100644 --- a/src/lib/errors.js +++ b/src/lib/errors.js @@ -19,6 +19,9 @@ export const badRequest = (message, code = 'bad_request') => export const unauthorized = (message = 'Please sign in to continue.', code = 'unauthorized') => new AppError(message, { code, status: 401 }); +export const tooManyRequests = (message = 'Too many attempts. Try again shortly.', code = 'rate_limited') => + new AppError(message, { code, status: 429 }); + export const forbidden = (message, code = 'forbidden') => new AppError(message, { code, status: 403 }); diff --git a/src/lib/fingerprint-file.js b/src/lib/fingerprint-file.js index 7da3ad8..d5eb9b7 100644 --- a/src/lib/fingerprint-file.js +++ b/src/lib/fingerprint-file.js @@ -56,13 +56,9 @@ export function encodeFingerprint({ hashes, sampleRate, samplesPerFrame, duratio * into a broken feature. */ export function decodeFingerprint(buffer) { - if (!buffer || buffer.length < HEADER_BYTES) return null; - if (buffer.readUInt32BE(0) !== MAGIC) return null; - - const version = buffer.readUInt32BE(4); - if (version !== FINGERPRINT_VERSION) return null; - - const frameCount = buffer.readUInt32BE(8); + const header = decodeFingerprintHeader(buffer); + if (!header) return null; + const { version, frameCount } = header; if (buffer.length < HEADER_BYTES + frameCount * 4) return null; // One copy and one byte swap, rather than a call per sub-fingerprint: a pass reads @@ -82,6 +78,31 @@ export function decodeFingerprint(buffer) { }; } +/** The bytes a caller needs to read to learn a fingerprint's timing without its hashes. */ +export const FINGERPRINT_HEADER_BYTES = HEADER_BYTES; + +/** + * Reads only the header: the timing, the length, and how many hashes follow. + * + * An anchor already looked for in an episode needs the episode's sample rate and frame + * size to turn a stored hit into a cut, and nothing else — reading half a megabyte of + * hashes to get at twelve bytes of header was most of what an unchanged pass spent. + * Null for anything unrecognised or written by a different version, as above. + */ +export function decodeFingerprintHeader(buffer) { + if (!buffer || buffer.length < HEADER_BYTES) return null; + if (buffer.readUInt32BE(0) !== MAGIC) return null; + const version = buffer.readUInt32BE(4); + if (version !== FINGERPRINT_VERSION) return null; + return { + version, + frameCount: buffer.readUInt32BE(8), + sampleRate: buffer.readUInt32BE(12) || null, + samplesPerFrame: buffer.readUInt32BE(16) || null, + durationMs: buffer.readUInt32BE(20) || null, + }; +} + /** * The frame index at or before a given millisecond offset. * diff --git a/src/lib/lazy-sharp.js b/src/lib/lazy-sharp.js new file mode 100644 index 0000000..fe24c06 --- /dev/null +++ b/src/lib/lazy-sharp.js @@ -0,0 +1,25 @@ +/** + * sharp, loaded the first time an image is actually handled rather than at boot. + * + * Importing it means loading libvips, which is tens of milliseconds on a laptop and + * whole seconds on the Celeron in a small NAS — paid before the first page could be + * served, for a library that is only needed when a cover is inspected or saved or an + * episode's artwork is extracted. Both services share this one loader, so the module + * is loaded once however many of them ask, and a boot with no images never loads it. + * + * The image itself proves at build time that sharp loads (the native-module check in + * the Dockerfile), so in the container this cannot fail. On a development machine + * where it does, the error keeps sharp's own explanation and names the module, so it + * reads as "the image library is broken" rather than "this image is invalid". + */ +let loading = null; + +export function loadSharp() { + loading ??= import('sharp') + .then((module) => module.default) + .catch((err) => { + loading = null; + throw new Error(`The image library (sharp) could not be loaded: ${err?.message ?? err}`, { cause: err }); + }); + return loading; +} diff --git a/src/lib/mp3-cut.js b/src/lib/mp3-cut.js index b2c473c..dc7e6e5 100644 --- a/src/lib/mp3-cut.js +++ b/src/lib/mp3-cut.js @@ -45,6 +45,23 @@ import { id3v2Size, readFrames, readXing } from './mp3-frames.js'; * @returns {{buffer: Buffer, framesKept: number, framesRemoved: number, durationMs: number} | null} */ export function cutFrames(buffer, ranges) { + const cut = cutFrameParts(buffer, ranges); + if (!cut) return null; + const { parts, ...rest } = cut; + return { buffer: Buffer.concat(parts), ...rest }; +} + +/** + * The same cut, as the pieces of the original it is made of, in order. + * + * Every part but a rewritten Xing header is a view of `buffer`, so the result costs + * no second copy of the episode: the trimmer hashes the parts and writes them to the + * file one after another, which is what keeps an hour-long cut at one episode's + * worth of memory rather than two. + * + * @returns {{parts: Buffer[], framesKept: number, framesRemoved: number, durationMs: number} | null} + */ +export function cutFrameParts(buffer, ranges) { const { frames, truncated } = readFrames(buffer); if (!frames.length) return null; // A file too long to have been read whole cannot be cut. Building the result from a @@ -118,7 +135,7 @@ export function cutFrames(buffer, ranges) { // estimating from the bitrate, which is approximate rather than wrong. return { - buffer: Buffer.concat(parts), + parts, framesKept: kept.length, framesRemoved: frames.length - firstAudio - kept.length, durationMs: Math.round(durationMs), diff --git a/src/lib/mp3-range.js b/src/lib/mp3-range.js new file mode 100644 index 0000000..4734eac --- /dev/null +++ b/src/lib/mp3-range.js @@ -0,0 +1,191 @@ +import { MAX_FRAMES, id3v2Size, readFrameHeader, readXing } from './mp3-frames.js'; +import { openGatedFile } from './pipeline-file.js'; + +/** + * Reading part of an MP3 without reading all of it. + * + * The sample a decision is made by listening to is ten seconds of an episode, and + * serving it used to read the whole episode off the share and walk every frame of + * it, on every play. The frames wanted are known by index, and an index is only a + * byte offset once the headers before it have been walked — so this walks them from + * a file handle a chunk at a time, keeps the table it builds per file, and reads + * only the bytes the frames occupy. + * + * The walk here must agree with `readFrames` frame for frame, because the indices + * stored in the catalogue were measured by it. It is the same loop — the same sync + * search, the same bounded resynchronisation, the same cap — over a sliding window + * instead of a whole buffer, and `test/unit/mp3-range.test.js` holds the two to it. + */ + +/** Longer than any valid MPEG audio frame (1,729 bytes), so a frame is never cut by the window. */ +const WINDOW_SLACK = 4096; +const CHUNK_BYTES = 1024 * 1024; +const MAX_SYNC_SEARCH = 512 * 1024; +const RESYNC_BYTES = 8192; + +/** + * Walks a file's frame headers and returns where every audio frame is. + * + * @returns {Promise<{offsets: Uint32Array, lengths: Uint16Array, frameCount: number, sampleRate: number|null, + * samplesPerFrame: number|null, truncated: boolean, size: number, mtimeMs: number} | null>} + * null when the file has no frames at all. + */ +export async function frameTable(path, { maxFrames = MAX_FRAMES } = {}) { + const { handle, size, mtimeMs } = await openGatedFile(path, { maxBytes: Infinity }); + try { + let buffer = Buffer.alloc(0); + let bufferStart = 0; + + /** Makes sure `[at, at + WINDOW_SLACK)` (or to the end of the file) is in `buffer`. */ + async function windowAt(at) { + const wantedEnd = Math.min(size, at + WINDOW_SLACK); + if (at >= bufferStart && wantedEnd <= bufferStart + buffer.length) return; + const chunk = Buffer.allocUnsafe(Math.min(CHUNK_BYTES, size - at)); + let read = 0; + while (read < chunk.length) { + const { bytesRead } = await handle.read(chunk, read, chunk.length - read, at + read); + if (!bytesRead) break; + read += bytesRead; + } + buffer = read === chunk.length ? chunk : chunk.subarray(0, read); + bufferStart = at; + } + + async function headerAt(at) { + if (at + 4 > size) return null; + await windowAt(at); + return readFrameHeader(buffer, at - bufferStart); + } + + await windowAt(0); + let offset = id3v2Size(buffer); + const offsets = []; + const lengths = []; + let first = null; + + // Find the first frame. Anything before it is a tag or junk. + const searchEnd = Math.min(size - 4, offset + MAX_SYNC_SEARCH); + while (offset <= searchEnd && !(await headerAt(offset))) offset += 1; + + while (offset + 4 <= size && offsets.length < maxFrames) { + const frame = await headerAt(offset); + if (!frame) { + // Resynchronise: skip a byte and look again, but only for a bounded distance, + // so a truncated or corrupt tail cannot turn into a linear scan of megabytes. + let probe = offset + 1; + const limit = Math.min(size - 4, offset + RESYNC_BYTES); + while (probe <= limit && !(await headerAt(probe))) probe += 1; + if (probe > limit) break; + offset = probe; + continue; + } + if (!first) first = { ...frame, offset }; + offsets.push(offset); + lengths.push(frame.length); + offset += frame.length; + } + if (!offsets.length) return null; + + // The Xing frame itself is a header, not audio, and the catalogue's frame indices + // count from the first audio frame — exactly as `frameProfile` counts them. + await windowAt(first.offset); + const firstBytes = buffer.subarray(first.offset - bufferStart, first.offset - bufferStart + first.length); + const xing = readXing(firstBytes, { ...first, offset: 0 }); + const firstAudio = xing ? 1 : 0; + const count = offsets.length - firstAudio; + const table = { + offsets: Uint32Array.from(offsets.slice(firstAudio)), + lengths: Uint16Array.from(lengths.slice(firstAudio)), + frameCount: count, + truncated: offsets.length >= maxFrames, + size, + mtimeMs, + sampleRate: null, + samplesPerFrame: null, + }; + if (count) { + const audioFirst = await headerAt(table.offsets[0]); + table.sampleRate = audioFirst?.sampleRate ?? null; + table.samplesPerFrame = audioFirst?.samplesPerFrame ?? null; + } + return table; + } finally { + await handle.close(); + } +} + +/** + * The bytes of audio frames `[fromFrame, toFrame)`, read straight from the file. + * + * Adjacent frames are read in one piece, so a ten-second clip is one or two reads. + * No tag and no Xing header: a player estimates length from the bitrate, which is + * right for a clip that exists only to be listened to once. + */ +export async function readFrameRange(path, table, fromFrame, toFrame) { + const from = Math.max(0, Math.floor(fromFrame)); + const to = Math.min(table.frameCount, Math.ceil(toFrame)); + if (to <= from) return Buffer.alloc(0); + + const runs = []; + let runStart = table.offsets[from]; + let runEnd = runStart + table.lengths[from]; + for (let i = from + 1; i < to; i += 1) { + if (table.offsets[i] === runEnd) { + runEnd += table.lengths[i]; + continue; + } + runs.push([runStart, runEnd]); + runStart = table.offsets[i]; + runEnd = runStart + table.lengths[i]; + } + runs.push([runStart, runEnd]); + + const { handle } = await openGatedFile(path, { maxBytes: Infinity }); + try { + const parts = []; + for (const [start, end] of runs) { + const part = Buffer.allocUnsafe(end - start); + let read = 0; + while (read < part.length) { + const { bytesRead } = await handle.read(part, read, part.length - read, start + read); + if (!bytesRead) break; + read += bytesRead; + } + parts.push(read === part.length ? part : part.subarray(0, read)); + } + return Buffer.concat(parts); + } finally { + await handle.close(); + } +} + +/** + * Frame tables kept per file, keyed on its path, size and modification time. + * + * A table is a few bytes a frame — under a megabyte for an hour — and a handful of + * them is all the review page ever plays from at once. A file that changes gets a + * new key and the old entry ages out. + */ +export function createFrameTableCache({ maxEntries = 8 } = {}) { + const entries = new Map(); + return { + async get(path, { size, mtimeMs }) { + const key = `${path}\0${size}\0${mtimeMs}`; + const hit = entries.get(key); + if (hit) { + // Re-inserted so the most recently used is the last to go. + entries.delete(key); + entries.set(key, hit); + return hit; + } + const table = await frameTable(path); + if (!table) return null; + entries.set(key, table); + while (entries.size > maxEntries) entries.delete(entries.keys().next().value); + return table; + }, + size() { + return entries.size; + }, + }; +} diff --git a/src/lib/pipeline-file.js b/src/lib/pipeline-file.js new file mode 100644 index 0000000..fb59fdf --- /dev/null +++ b/src/lib/pipeline-file.js @@ -0,0 +1,106 @@ +import { constants as fsConstants } from 'node:fs'; +import { open, stat } from 'node:fs/promises'; + +import { MAX_PIPELINE_FILE_BYTES } from '../constants.js'; +import { resolveContained } from './contained-path.js'; + +/** + * Opening an episode for the advert pipeline — fingerprinting, listening, cutting. + * + * Every route that *serves* a file proves the path is inside the show's folder + * before opening it (`resolveContained`), because the folder is normally an SMB + * share that other accounts can write to. The pipeline reads the same files, and + * used to read them by their joined path with no check at all — so a planted + * + * episode.mp3 -> /dev/zero + * + * or a FIFO, or a symlink to a multi-gigabyte file elsewhere on the NAS, would hang + * the unattended pass or exhaust its heap, every few minutes, for ever. This is the + * one place the pipeline's reads go through, and it refuses three things before a + * byte is read: a path that resolves outside the folder, anything that is not an + * ordinary file, and anything larger than SelfPod will hold in memory. + * + * Refusals come back as a plain sentence naming the file, for the health banner and + * the activity log. A file that is simply gone or unreadable is reported the quiet + * way the pipeline always has — that is ordinary on a share — and told apart here so + * the caller can keep doing so. + */ + +/** + * Checks an episode's file and says whether the pipeline may read it. + * + * @returns {Promise<{path: string, size: number, mtimeMs: number} | {refused: string, message: string, quiet: boolean}>} + */ +export async function gateEpisodeFile(directory, filename, { maxBytes = MAX_PIPELINE_FILE_BYTES } = {}) { + const resolved = await resolveContained(directory, filename); + if (!resolved.path) { + if (resolved.reason === 'escapes') { + return { + refused: 'escapes', + quiet: false, + message: `“${filename}” is a link to somewhere outside the show's folder, so SelfPod will not read it.`, + }; + } + return { + refused: 'unreadable', + quiet: true, + message: `“${filename}” could not be read${resolved.code ? ` (${resolved.code})` : ''}.`, + }; + } + + let info; + try { + info = await stat(resolved.path); + } catch (error) { + return { refused: 'unreadable', quiet: true, message: `“${filename}” could not be read (${error.code ?? error.message}).` }; + } + return gateInfo(info, filename, resolved.path, maxBytes); +} + +/** + * Opens a file that has already passed `gateEpisodeFile` and checks it *again* on the + * open descriptor, so a file swapped between the check and the read — a link + * re-pointed at a device, say — is still refused. `O_NONBLOCK` is the difference + * between refusing a FIFO and waiting on it for ever: opening one for reading blocks + * until something writes to it, and nothing ever will. + */ +export async function openGatedFile(path, { maxBytes = MAX_PIPELINE_FILE_BYTES } = {}) { + const handle = await open(path, fsConstants.O_RDONLY | fsConstants.O_NONBLOCK); + try { + const info = await handle.stat(); + const verdict = gateInfo(info, path.slice(path.lastIndexOf('/') + 1), path, maxBytes); + if (verdict.refused) { + const error = new Error(verdict.message); + error.code = 'refused'; + error.refused = verdict.refused; + throw error; + } + return { handle, size: info.size, mtimeMs: Math.trunc(info.mtimeMs) }; + } catch (error) { + await handle.close().catch(() => {}); + throw error; + } +} + +function gateInfo(info, filename, path, maxBytes) { + if (!info.isFile()) { + return { + refused: 'not_a_file', + quiet: false, + message: `“${filename}” is not an ordinary file (a device, a pipe or a folder), so SelfPod will not read it.`, + }; + } + if (info.size > maxBytes) { + return { + refused: 'too_large', + quiet: false, + message: `“${filename}” is ${formatBytes(info.size)}, more than the ${formatBytes(maxBytes)} SelfPod will read in one piece, so it was not read.`, + }; + } + return { path, size: info.size, mtimeMs: Math.trunc(info.mtimeMs) }; +} + +function formatBytes(bytes) { + if (bytes >= 1024 ** 3) return `${(bytes / 1024 ** 3).toFixed(bytes >= 10 * 1024 ** 3 ? 0 : 1)} GB`; + return `${Math.round(bytes / 1024 ** 2)} MB`; +} diff --git a/src/lib/present-segment.js b/src/lib/present-segment.js index f600f74..0aa9534 100644 --- a/src/lib/present-segment.js +++ b/src/lib/present-segment.js @@ -2,6 +2,7 @@ import { HOLD_REASONS, SEGMENT_KINDS, SEGMENT_SOURCES } from '../constants.js'; import { WORD_KINDS, inferKind } from './segment-kind.js'; import { describeVerdict, flattenTranscript, parseCues, presentExcerpt, LOW_CONFIDENCE } from './present-transcript.js'; import { describeCues } from './advert-cues.js'; +import { formatClock } from './dates.js'; import { meanConfidence } from './transcript.js'; /** @@ -73,7 +74,7 @@ export function presentSegment(segment, { episodes, transcripts = null, mode = ' : 'Sounds like a sponsor read, heard once' : `Repeats across ${segment.episode_count} ${segment.episode_count === 1 ? 'episode' : 'episodes'}`, durationSeconds, - durationLabel: formatDuration(segment.duration_ms ?? 0), + durationLabel: formatClock(segment.duration_ms ?? 0), episodeCount: segment.episode_count, occurrenceCount: segment.occurrence_count, autoApproved: Boolean(segment.auto_approved), @@ -114,7 +115,7 @@ export function presentSegment(segment, { episodes, transcripts = null, mode = ' episodeTitle: episodes?.get(row.episode_id)?.title ?? null, startMs: row.start_ms, endMs: row.end_ms, - atLabel: formatDuration(row.start_ms), + atLabel: formatClock(row.start_ms), })), firstSeenAt: segment.first_seen_at, decidedAt: segment.decided_at, @@ -145,19 +146,10 @@ function describePosition(occurrences, episodes) { if (atEnd) return 'At the very end of every episode'; } - if (latest - earliest < 10_000) return `Around ${formatDuration(earliest)} in, every time`; - return `Between ${formatDuration(earliest)} and ${formatDuration(latest)} in`; + if (latest - earliest < 10_000) return `Around ${formatClock(earliest)} in, every time`; + return `Between ${formatClock(earliest)} and ${formatClock(latest)} in`; } -/** `m:ss`, or `h:mm:ss` past an hour. Segments are seconds; positions can be hours. */ -function formatDuration(ms) { - const total = Math.max(0, Math.round((ms ?? 0) / 1000)); - const hours = Math.floor(total / 3600); - const minutes = Math.floor((total % 3600) / 60); - const seconds = total % 60; - const pad = (n) => String(n).padStart(2, '0'); - return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds)}` : `${minutes}:${pad(seconds)}`; -} /** * Whether SelfPod has looked properly and found nothing — which is not "not yet". diff --git a/src/lib/present-transcript.js b/src/lib/present-transcript.js index 785d8be..2db36fb 100644 --- a/src/lib/present-transcript.js +++ b/src/lib/present-transcript.js @@ -1,6 +1,7 @@ import { HOLD_REASONS, SEGMENT_KINDS } from '../constants.js'; import { inferKind } from './segment-kind.js'; import { describeCues } from './advert-cues.js'; +import { formatClock } from './dates.js'; import { normaliseWord } from './text-normalise.js'; /** @@ -18,20 +19,20 @@ const CONTEXT_MS = 4000; /** Below this mean confidence the recogniser was guessing, and so would SelfPod be. */ export const LOW_CONFIDENCE = 0.45; -export function formatClock(ms) { - const total = Math.max(0, Math.round((ms ?? 0) / 1000)); - const hours = Math.floor(total / 3600); - const minutes = Math.floor((total % 3600) / 60); - const seconds = total % 60; - const pad = (n) => String(n).padStart(2, '0'); - return hours > 0 ? `${hours}:${pad(minutes)}:${pad(seconds)}` : `${minutes}:${pad(seconds)}`; -} +/** One clock for every page: lib/dates.js owns it. Re-exported for the callers here. */ +export { formatClock }; -function formatDay(iso) { +/** + * "9 August", in the instance's zone like every other date the pages show — not the + * server's, which inside the container is UTC whatever `TZ` says the owner lives in. + */ +function formatDay(iso, timeZone) { if (!iso) return null; const date = new Date(iso); if (Number.isNaN(date.getTime())) return null; - return date.toLocaleDateString('en-GB', { day: 'numeric', month: 'long' }); + const options = { day: 'numeric', month: 'long' }; + if (timeZone) options.timeZone = timeZone; + return new Intl.DateTimeFormat('en-GB', options).format(date); } /** Every word of a stored transcript in order, each knowing which window it came from. */ @@ -323,7 +324,7 @@ export function parseCues(cues) { * @param {{episode: object, show: object, row: object|null, spoken: Array, markers: Array, pending: boolean, engineMissing: boolean, anchorStatus?: object|null, anchorCut?: object|null}} input * @returns {{stage: string, sentence: string, at?: string, segmentId?: string, reversible?: boolean}|null} */ -export function describeAdvertStage({ episode, show, row, spoken, markers, pending, engineMissing, listenLabel, anchorStatus = null, anchorCut = null }) { +export function describeAdvertStage({ episode, show, row, spoken, markers, pending, engineMissing, listenLabel, anchorStatus = null, anchorCut = null, timeZone = null }) { if (!show || !show.ad_trim_mode || show.ad_trim_mode === 'off') return null; const isMp3 = /\.mp3$/i.test(episode.filename ?? ''); if (!isMp3) { @@ -408,7 +409,7 @@ export function describeAdvertStage({ episode, show, row, spoken, markers, pendi reversible: true, }; } - const decided = formatDay(first.decided_at); + const decided = formatDay(first.decided_at, timeZone); return { stage: 'cut_remembered', sentence: `Cut ${at} automatically, because you removed the same read${decided ? ` on ${decided}` : ' before'}.`, diff --git a/src/lib/time-range.js b/src/lib/time-range.js index 4d94f2b..e7ba25d 100644 --- a/src/lib/time-range.js +++ b/src/lib/time-range.js @@ -40,28 +40,40 @@ export const DEFAULT_RANGE = '30d'; * edge at `from`, so no event falls in both and none falls in the gap between them. * Deriving it by subtracting a duration from `from` instead would drift by an hour * across a transition and quietly move events between the two periods being compared. + * + * `prevTo` is where the comparison stops: the same distance into the previous period + * as `now` is into this one. The current period ends tomorrow, so at nine in the + * morning it holds `days - 1` whole days and nine hours; measured against `days` + * whole days it read "down" every morning and caught up every evening. The same + * elapsed time in both windows is the comparison the card claims to make. */ export function resolveRange(key, { timeZone, now = new Date() } = {}) { const resolvedKey = Object.prototype.hasOwnProperty.call(RANGES, key) ? key : DEFAULT_RANGE; const { days, label, lede } = RANGES[resolvedKey]; if (days === null) { - return { key: resolvedKey, label, lede, days: null, from: null, to: null, prevFrom: null }; + return { key: resolvedKey, label, lede, days: null, from: null, to: null, prevFrom: null, prevTo: null }; } const instant = now instanceof Date ? now : new Date(now); - const today = localDay(Number.isNaN(instant.getTime()) ? new Date() : instant, timeZone); + const moment = Number.isNaN(instant.getTime()) ? new Date() : instant; + const today = localDay(moment, timeZone); + + // `days - 1` back, not `days`: "the last 7 days" is seven calendar days counting + // today, which is what the label promises the reader. + const from = localMidnight(shiftDay(today, -(days - 1)), timeZone); + const prevFrom = localMidnight(shiftDay(today, -(2 * days - 1)), timeZone); + const elapsedMs = moment.getTime() - new Date(from).getTime(); return { key: resolvedKey, label, lede, days, - // `days - 1` back, not `days`: "the last 7 days" is seven calendar days counting - // today, which is what the label promises the reader. - from: localMidnight(shiftDay(today, -(days - 1)), timeZone), + from, to: localMidnight(shiftDay(today, 1), timeZone), - prevFrom: localMidnight(shiftDay(today, -(2 * days - 1)), timeZone), + prevFrom, + prevTo: new Date(new Date(prevFrom).getTime() + Math.max(0, elapsedMs)).toISOString(), }; } diff --git a/src/plugins/auth.js b/src/plugins/auth.js index a2cbed0..4dd7df8 100644 --- a/src/plugins/auth.js +++ b/src/plugins/auth.js @@ -28,10 +28,25 @@ import { SETTING_KEYS } from '../services/settings.js'; */ const WINDOW_MS = 15 * 60 * 1000; -const MAX_ATTEMPTS_PER_WINDOW = 5; -/** Backoff after consecutive failures, in seconds, regardless of source address. */ -const BACKOFF_STEPS = [0, 0, 1, 3, 10, 30, 60, 120, 300]; +/** + * Two backoffs, because they protect against different people. + * + * The account-level one holds *everyone* back after failures from anywhere, so it + * is what stops a slow guess spread across many addresses — and also what lets a + * stranger who holds the public URL keep the owner out: one wrong password every + * few minutes used to reset a five-minute lock for ever. It is therefore capped at + * a minute, which is still one guess a minute for a patient attacker. + * + * The per-source one is allowed to grow to five minutes, because it only ever + * holds back the address that earned it. Behind Cloudflare the source is the + * edge-set client address; on a LAN it is the socket; behind another proxy that + * sets neither header every visitor shares one source, which is no worse than the + * old account-level lock. + */ +const ACCOUNT_BACKOFF_STEPS = [0, 0, 1, 3, 10, 30, 60]; +const SOURCE_BACKOFF_STEPS = [0, 0, 1, 3, 10, 30, 60, 120, 300]; +const MAX_ACCOUNT_LOCK_SECONDS = ACCOUNT_BACKOFF_STEPS[ACCOUNT_BACKOFF_STEPS.length - 1]; async function authPlugin(fastify, { db, settings, config, logger }) { const recordAttempt = db.prepare( @@ -41,6 +56,10 @@ async function authPlugin(fastify, { db, settings, config, logger }) { `SELECT COUNT(*) AS n, MAX(attempted_at) AS last FROM login_attempts WHERE username = ? AND succeeded = 0 AND attempted_at > ?`, ); + const recentFailuresFromSource = db.prepare( + `SELECT COUNT(*) AS n, MAX(attempted_at) AS last FROM login_attempts + WHERE username = ? AND source = ? AND succeeded = 0 AND attempted_at > ?`, + ); const clearAttempts = db.prepare('DELETE FROM login_attempts WHERE username = ?'); const trimAttempts = db.prepare('DELETE FROM login_attempts WHERE attempted_at < ?'); @@ -57,16 +76,27 @@ async function authPlugin(fastify, { db, settings, config, logger }) { fastify.decorate('loginSourceKey', sourceKey); - /** Seconds the account must wait before another attempt, 0 when clear. */ - fastify.decorate('loginBackoffSeconds', (username) => { - const since = new Date(Date.now() - WINDOW_MS).toISOString(); - const row = recentFailures.get(username, since); + /** Seconds still to wait after the failures in `row`, on the given ladder. */ + function waitFor(steps, row) { const failures = row?.n ?? 0; if (!failures || !row.last) return 0; - const step = BACKOFF_STEPS[Math.min(failures, BACKOFF_STEPS.length - 1)]; + const step = steps[Math.min(failures, steps.length - 1)]; if (!step) return 0; const elapsed = (Date.now() - new Date(row.last).getTime()) / 1000; return Math.max(0, Math.ceil(step - elapsed)); + } + + /** + * Seconds this caller must wait before another attempt, 0 when clear: the longer + * of the account's own wait and the wait this source has earned for itself. + */ + fastify.decorate('loginBackoffSeconds', (username, source = null) => { + const since = new Date(Date.now() - WINDOW_MS).toISOString(); + const account = waitFor(ACCOUNT_BACKOFF_STEPS, recentFailures.get(username, since)); + const bySource = source + ? waitFor(SOURCE_BACKOFF_STEPS, recentFailuresFromSource.get(username, source, since)) + : 0; + return Math.max(account, bySource); }); fastify.decorate('recentFailureCount', (username) => { @@ -83,7 +113,7 @@ async function authPlugin(fastify, { db, settings, config, logger }) { const hash = settings.adminPasswordHash(); const source = request ? sourceKey(request) : null; - const backoff = fastify.loginBackoffSeconds(expectedUser); + const backoff = fastify.loginBackoffSeconds(expectedUser, source); if (backoff > 0) { return { ok: false, @@ -110,12 +140,23 @@ async function authPlugin(fastify, { db, settings, config, logger }) { return { ok: true, username: expectedUser }; }); - fastify.decorate('setAdminPassword', async (password) => { + /** + * Sets the password and signs every *other* session out. + * + * Someone changing their password because they suspect a session was stolen + * expects the thief to be gone; until now the thief stayed signed in for up to + * thirty days. `keepSessionId` is the session doing the changing — the setup + * wizard and the Settings modal pass their own — and `null` means sign out + * everyone, which is what the reset script wants. + */ + fastify.decorate('setAdminPassword', async (password, { keepSessionId = null } = {}) => { const hash = await bcrypt.hash(String(password), BCRYPT_ROUNDS); settings.update( { [SETTING_KEYS.ADMIN_PASSWORD_HASH]: hash, [SETTING_KEYS.MUST_CHANGE_PASSWORD]: '0' }, { skipExport: true }, ); + const endedSessions = fastify.sessionStore?.deleteOthers(keepSessionId) ?? 0; + if (endedSessions) logger?.info({ endedSessions }, 'password changed; other sessions signed out'); return true; }); @@ -209,5 +250,5 @@ function formatSeconds(seconds) { return `${minutes} minute${minutes === 1 ? '' : 's'}`; } -export { MAX_ATTEMPTS_PER_WINDOW, WINDOW_MS }; +export { MAX_ACCOUNT_LOCK_SECONDS, WINDOW_MS }; export default fp(authPlugin, { name: 'selfpod-auth', dependencies: ['selfpod-session'] }); diff --git a/src/plugins/session.js b/src/plugins/session.js index 31d7e79..e547f3b 100644 --- a/src/plugins/session.js +++ b/src/plugins/session.js @@ -19,6 +19,8 @@ class SqliteSessionStore { ON CONFLICT(sid) DO UPDATE SET data = excluded.data, expires_at = excluded.expires_at`, ); this.deleteStmt = db.prepare('DELETE FROM sessions WHERE sid = ?'); + this.deleteOthersStmt = db.prepare('DELETE FROM sessions WHERE sid <> ?'); + this.deleteAllStmt = db.prepare('DELETE FROM sessions'); this.cleanupStmt = db.prepare('DELETE FROM sessions WHERE expires_at < ?'); } @@ -50,6 +52,18 @@ class SqliteSessionStore { } } + /** + * Ends every session but one. The one to keep is the session of whoever just + * changed the password: they proved they hold it, and signing them out of the + * page they did it from would be a strange reward. + */ + deleteOthers(keepSid) { + const info = keepSid + ? this.deleteOthersStmt.run(keepSid) + : this.deleteAllStmt.run(); + return info.changes; + } + destroy(sid, callback) { try { this.deleteStmt.run(sid); @@ -81,7 +95,16 @@ async function sessionPlugin(fastify, { db, settings, logger }) { store, cookieName: 'selfpod.sid', saveUninitialized: false, - rolling: true, + /* + * Not rolling. Rolling re-saves the session and re-sends the cookie on every + * request — one UPSERT per page, per fragment, per stylesheet — to push the + * expiry out each time. The session lasts days (see sessionTtlHours), so an admin + * who signs in is not going to notice it expiring from when they signed in rather + * than from their last click. A session that *changed* — signing in, a flash + * message set or read — is still saved: the store writes whenever the data + * differs from what it loaded, rolling or not. + */ + rolling: false, cookie: { path: '/', httpOnly: true, diff --git a/src/routes/api/ad-segments.js b/src/routes/api/ad-segments.js index eacb76b..e457ddf 100644 --- a/src/routes/api/ad-segments.js +++ b/src/routes/api/ad-segments.js @@ -1,11 +1,9 @@ -import { readFile } from 'node:fs/promises'; -import { join } from 'node:path'; +import { stat } from 'node:fs/promises'; import { AD_TRIM_MODES, SEGMENT_STATUS } from '../../constants.js'; import { resolveContained } from '../../lib/contained-path.js'; -import { badRequest, notFound } from '../../lib/errors.js'; -import { cutFrames } from '../../lib/mp3-cut.js'; -import { frameProfile } from '../../lib/mp3-frames.js'; +import { AppError, badRequest, notFound } from '../../lib/errors.js'; +import { createFrameTableCache, readFrameRange } from '../../lib/mp3-range.js'; import { msToFrame } from '../../lib/fingerprint-file.js'; import { presentSegment } from '../../lib/present-segment.js'; import { describeWork } from '../../services/adverts-view.js'; @@ -45,6 +43,56 @@ export default async function adSegmentRoutes(fastify, services) { fastify.addHook('onRequest', fastify.requireAdminApi); + /* + * Where an episode's frames are, per file, for the two routes that play a few + * seconds of one. Walked once from the file and kept; the whole episode is never + * read to serve a clip of it. Shared by the segment and the jingle samples, which + * play from the same handful of exemplar episodes. + */ + const frameTables = createFrameTableCache({ maxEntries: 8 }); + + /** + * Resolves an episode's file inside its show folder and its frame table, or throws + * the 404 every reason here maps to. `/data/shows` is normally a writable SMB + * share: anyone who can drop a file there can drop a symlink there, and these + * routes return part of whatever they open. The cover, artwork and media routes all + * do this; skipping it here would have made one route the way to read the host + * through a share. + */ + async function sampleSource(episode) { + if (!isSafeFilename(episode.filename)) throw notFound('No episode here.', 'not_found'); + const resolved = await resolveContained(shows.dirFor(shows.getOrThrow(episode.show_id)), episode.filename); + if (!resolved.path) throw notFound('That episode is not readable right now.', 'file_missing'); + let table; + try { + const info = await stat(resolved.path); + if (!info.isFile()) throw notFound('That episode is not readable right now.', 'file_missing'); + table = await frameTables.get(resolved.path, { size: info.size, mtimeMs: Math.trunc(info.mtimeMs) }); + } catch (error) { + if (error instanceof AppError) throw error; + throw notFound('That episode is not readable right now.', 'file_missing'); + } + if (!table?.frameCount) throw notFound('That episode could not be played.', 'not_playable'); + return { path: resolved.path, table }; + } + + /** `?context=N` in frames: a few seconds either side, clamped so a query string cannot ask for minutes. */ + function contextFramesFor(request, table) { + const contextSeconds = Math.max(0, Math.min(10, Number(request.query?.context) || 0)); + return table.sampleRate ? Math.round((contextSeconds * 1000 * table.sampleRate) / (table.samplesPerFrame * 1000)) : 0; + } + + async function sendClip(reply, source, fromFrame, toFrame) { + const clip = await readFrameRange(source.path, source.table, fromFrame, toFrame); + if (!clip.length) throw notFound('That segment could not be played.', 'not_playable'); + return reply + .header('content-type', 'audio/mpeg') + .header('content-length', String(clip.length)) + // Derived from files that can change, and only ever a few seconds long. + .header('cache-control', 'private, no-store') + .send(clip); + } + fastify.get('/shows/:id/ad-segments', async (request) => { const show = shows.getOrThrow(request.params.id); const context = await services.advertsView.segmentsContext(show); @@ -291,11 +339,18 @@ export default async function adSegmentRoutes(fastify, services) { /** * The segment on its own, as audio, so a decision can be made by listening. * - * Built by cutting everything *except* the segment out of the exemplar episode, - * which is the trimmer's own operation run the other way round — so what you hear - * is exactly the frames that would be removed, not an approximation of them. + * Exactly the frames the trimmer would remove — the stored frame range of the + * exemplar occurrence — read straight from the file by their offsets, so what you + * hear is what would go, and serving it costs the clip's own bytes rather than the + * whole episode. + * + * Two addresses for the same clip. The pages play it from `/api/cuts/…`, because an + * ad blocker's network rules match a request by its path, and `/ad-segments/` in + * an `