diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml
index 4cdeec1b..f983fe4e 100644
--- a/.github/workflows/auto-release.yml
+++ b/.github/workflows/auto-release.yml
@@ -1,8 +1,21 @@
name: Auto Release
-# When a push to main changes the version, auto-tag and create a draft release.
-# The draft body is pulled from CHANGELOG.md. Desktop builds attach assets.
-# Only step left: review the draft and publish.
+# ── Single-draft release flow (there must be exactly ONE draft per tag) ──────
+# When a push to main changes the version in pyproject.toml:
+# 1. this workflow tags vX.Y.Z and creates/updates the draft release, with
+# notes pulled from CHANGELOG.md;
+# 2. the tag push triggers desktop-release.yml, which builds every platform,
+# signs the update enclosures, refreshes the appcasts, and attaches the
+# built assets to the SAME draft (softprops matches the release by tag).
+# 3. the maintainer reviews that one draft and publishes it;
+# 4. publish-release.yml then ships PyPI + snap.
+#
+# The draft step below is IDEMPOTENT on purpose: `gh release create` would
+# happily make a SECOND draft for a tag that already has one (GitHub does not
+# enforce tag-uniqueness on drafts), which is exactly the duplicate "Zimi
+# vX.Y.Z" draft that showed up next to a hand-curated one after PR #39. So if a
+# release for the tag already exists (a maintainer-curated draft, or a re-run),
+# we EDIT it instead of creating a duplicate.
on:
push:
@@ -78,7 +91,18 @@ jobs:
ENDNOTES
# Trim leading whitespace from heredoc
sed -i 's/^ //' /tmp/release-notes.md
- gh release create "v${VERSION}" \
- --draft \
- --title "Zimi v${VERSION}" \
- --notes-file /tmp/release-notes.md
+ # One draft per tag: update an existing release (curated draft or a
+ # re-run) instead of creating a duplicate. `gh release view` finds
+ # drafts by tag too.
+ if gh release view "v${VERSION}" >/dev/null 2>&1; then
+ # A curated draft already exists (a human wrote the release body).
+ # Preserve it — do NOT overwrite the notes with the changelog
+ # extract. The tag was created above; the draft binds to it on
+ # publish. This is the one-draft-per-tag path.
+ echo "Release v${VERSION} already exists — preserving its curated notes"
+ else
+ gh release create "v${VERSION}" \
+ --draft \
+ --title "Zimi v${VERSION}" \
+ --notes-file /tmp/release-notes.md
+ fi
diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml
index ec2e86e9..cf8ba744 100644
--- a/.github/workflows/desktop-release.yml
+++ b/.github/workflows/desktop-release.yml
@@ -55,7 +55,7 @@ jobs:
gcc libcairo2-dev pkg-config python3-dev
- name: Install dependencies
- run: pip install -r requirements-desktop.txt
+ run: pip install -r desktop/requirements-desktop.txt
- name: Install libtorrent (in-process BT engine)
shell: bash
@@ -139,7 +139,7 @@ jobs:
"
- name: Build with PyInstaller
- run: pyinstaller --noconfirm zimi_desktop.spec
+ run: pyinstaller --noconfirm desktop/zimi_desktop.spec
# ── Post-build smoke tests ────────────────────────────────────────
@@ -459,8 +459,9 @@ jobs:
# Sign one enclosure and write its appcast.
# $1 label $2 file-to-sign $3 release-asset $4 outfile $5 min-sys line
+ # $6 extra enclosure attributes (e.g. sparkle:installerArguments)
generate_appcast() {
- local LABEL="$1" FILE="$2" ASSET="$3" OUTFILE="$4" MINSYS="$5"
+ local LABEL="$1" FILE="$2" ASSET="$3" OUTFILE="$4" MINSYS="$5" ENC_EXTRA="$6"
if [ ! -f "$FILE" ]; then
echo "WARNING: No $LABEL enclosure found at $FILE"
return 1
@@ -493,7 +494,7 @@ jobs:
url="https://github.com/${REPO}/releases/download/${TAG}/${ASSET}"
length="${LENGTH}"
type="application/octet-stream"
- sparkle:edSignature="${ED_SIG}"
+ sparkle:edSignature="${ED_SIG}"${ENC_EXTRA}
/>
@@ -505,11 +506,16 @@ jobs:
}
MAC_MINSYS=$'\n 12.0'
- generate_appcast "Intel" "Zimi-Intel/Zimi-Intel.dmg" "Zimi-Intel.dmg" "appcast-intel.xml" "$MAC_MINSYS"
- generate_appcast "Apple Silicon" "Zimi-AppleSilicon/Zimi-AppleSilicon.dmg" "Zimi-AppleSilicon.dmg" "appcast-arm64.xml" "$MAC_MINSYS"
- # Windows enclosure is the Inno Setup installer; WinSparkle downloads
- # and runs it to apply the update. No minimumSystemVersion line.
- generate_appcast "Windows" "Zimi-windows-x64-Setup/Zimi-windows-x64-Setup.exe" "Zimi-windows-x64-Setup.exe" "appcast-windows.xml" ""
+ generate_appcast "Intel" "Zimi-Intel/Zimi-Intel.dmg" "Zimi-Intel.dmg" "appcast-intel.xml" "$MAC_MINSYS" ""
+ generate_appcast "Apple Silicon" "Zimi-AppleSilicon/Zimi-AppleSilicon.dmg" "Zimi-AppleSilicon.dmg" "appcast-arm64.xml" "$MAC_MINSYS" ""
+ # Windows enclosure is the Inno Setup installer; WinSparkle (0.9.4)
+ # downloads it, runs it with sparkle:installerArguments via
+ # ShellExecuteEx, then quits WITHOUT relaunching — so /SILENT gives a
+ # click-free progress-only update and the installer's [Run] entry does
+ # the relaunch (windows/zimi.iss). /SP- skips the "ready to install"
+ # prompt. Per-user install (no UAC), so silent is safe.
+ WIN_INSTALLER_ARGS=' sparkle:installerArguments="/SILENT /SP-"'
+ generate_appcast "Windows" "Zimi-windows-x64-Setup/Zimi-windows-x64-Setup.exe" "Zimi-windows-x64-Setup.exe" "appcast-windows.xml" "" "$WIN_INSTALLER_ARGS"
# Commit and push to main
git config user.name "github-actions[bot]"
diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml
index ae9ffa71..81996bf7 100644
--- a/.github/workflows/publish-release.yml
+++ b/.github/workflows/publish-release.yml
@@ -34,12 +34,21 @@ jobs:
- name: Download Snap from release
env:
GH_TOKEN: ${{ github.token }}
+ VERSION: ${{ github.event.release.tag_name }}
run: |
- VERSION="${{ github.event.release.tag_name }}"
- gh release download "$VERSION" --repo epheterson/Zimi --pattern "*.snap" || true
- SNAP_FILE=$(ls *.snap 2>/dev/null | head -1)
+ # The snap is attached by the Desktop Release workflow, which is
+ # usually still building when the release-published event fires.
+ # Poll for up to 60 minutes instead of racing it (the 1.8.0 snap
+ # silently missed the store because of this exact race).
+ for i in $(seq 1 30); do
+ gh release download "$VERSION" --repo epheterson/Zimi --pattern "*.snap" || true
+ SNAP_FILE=$(ls *.snap 2>/dev/null | head -1)
+ [ -n "$SNAP_FILE" ] && break
+ echo "Snap not attached yet (attempt $i/30) — waiting 2m"
+ sleep 120
+ done
if [ -z "$SNAP_FILE" ]; then
- echo "No snap found in release $VERSION — skipping publish"
+ echo "No snap found in release $VERSION after 60m — skipping publish"
echo "SNAP_FOUND=false" >> $GITHUB_ENV
else
echo "Found: $SNAP_FILE"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 091801df..8680597f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,142 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/),
and this project adheres to [Semantic Versioning](https://semver.org/).
-## [1.8.0] - Unreleased — the Community Edition
+## [1.8.1] - 2026-07-28
+
+A polish-and-hardening release on top of the Community Edition. The headline is
+access control: a Zimi server can now be open, limited to a chosen set of ZIMs,
+or sign-in-required — with the security model tightened end to end so private
+really means private. "Did you mean?" grows up into the coverage 1.8.0 promised,
+"sections" become "Categories", light mode gets a real contrast pass, and
+downloads gain a nightly window and bandwidth caps.
+
+### Added
+
+- **Public-access modes — open / limited / private.** An anonymous (not
+ signed-in) visitor now sees one of three things, set by the admin: the whole
+ library (**open**, the default and the legacy behaviour), a chosen allowlist
+ of ZIMs (**limited**), or a **sign-in-required** gate (**private**). Anonymous
+ simply gets an allow-set instead of the all-access sentinel, so every existing
+ choke point (`get_zim_files` / `list_zims` / `zim_allowed` / the search-cache
+ key) filters it with no new leak surface. Admin-only `GET`/`POST
+ /manage/public-access`, an `ZIMI_PUBLIC_ACCESS` env override, and a picker in
+ the Users pane. A corrupt policy file fails **closed** to private.
+- **Per-user server-side data.** A signed-in named user can save their
+ bookmarks, history and preferences to their own server account (`/userdata`
+ GET/POST), stored per-user under `ZIMI_DATA_DIR/userdata/`. The blob is gated
+ to the **session** user — a user can only ever touch their own data; an
+ anonymous or admin-without-a-named-user visitor keeps everything in the
+ browser as before. Casefolded keys, size-capped, path-traversal refused.
+- **Backup & export hub, rebuilt as two cards.** "**My data**" (this browser's
+ bookmarks / history / preferences) and "**Server backup**" (the admin's
+ library, collections and layout, plus every user's server-side data) are now
+ clearly separated, each with its own Export and Import. Imports **merge** by
+ default (union by identity, incoming wins) via a two-step preview→apply, with
+ an overwrite escape hatch, and are **scope-validated** so a My-data bundle
+ can't be applied on the Server card or vice-versa. Signed-in users get
+ Save-to-server / Restore-from-server for their own data.
+- **Scheduled downloads.** An optional nightly window (server-local time,
+ overnight-spanning) holds downloads started outside it as `scheduled` and
+ releases them when the window opens, with a per-download **start-now**
+ override. Configurable in the UI or via `ZIMI_DL_WINDOW`.
+- **Download bandwidth caps.** A shared token-bucket throttle paces all
+ concurrent HTTP pulls to a single global **download** cap (N streams sum to
+ the cap, not N × the cap); the cap is the same number as the BitTorrent down
+ limit. An **upload** cap too, plus bulk **Pause / Resume / Delete-all**
+ controls over the download queue.
+- **BitTorrent tunables.** Max concurrent downloads (`active`) and the global
+ peer-connection limit (`conns`) are now adjustable in Settings and via
+ `ZIMI_BT`; the connection limit lands on the session at startup and can be
+ changed live.
+- **"Did you mean?" — the widening 1.8.0 promised.** The vocabulary now uses
+ lossy-counting eviction plus stride sampling, so words spread thin across many
+ titles (mitochondria, photosynthesis) survive the build instead of being
+ dropped; a trigram index adds **distance-2** corrections for long words
+ (`fotosynthesis` → `photosynthesis`); a frequency-ratio guard corrects a
+ common-typo that is itself in the vocabulary (`einstien` → `einstein`) while
+ leaving genuinely common words alone; and the vocabulary is persisted to disk
+ and reloaded (invalidated by index changes or a builder-version bump) so it
+ isn't rescanned every boot. Still fully offline and time-budgeted.
+- **App theme switch — Auto / Dark / Light** for the whole UI, plus
+ **auto-darken** for raw (non-Reader-View) ZIM articles when the app is dark,
+ and a **Reader View Auto** theme that renders sepia in light mode.
+- **zimgit / PDF collections, first-class.** A `zimgit-*` collection now renders
+ as a searchable document list (title, author, size, description) instead of a
+ raw ZIM page.
+- **Video-ZIM polish** — a resume ledger that restores playback position,
+ correct sizing on mobile, and a random-video card.
+- **Almanac** — a real bright-star field in the sky scene, a **Regional /
+ Worldwide** holidays toggle, a wider city set with click-anywhere map
+ selection, its own header identity, and a larger library tap-through link map
+ (every added Q-ID dual-verified against Wikidata).
+
+### Changed
+
+- **libtorrent now installs by default** via `pip` wherever a prebuilt wheel
+ exists — CPython 3.9–3.13 on manylinux **and** musllinux (incl. aarch64),
+ macOS and Windows — marker-gated below Python 3.14 (no wheel there yet, and no
+ sdist) so a plain install never breaks on a new interpreter. Where no wheel
+ exists, Zimi runs HTTP-only and prints the one-line fix; `pip install zimi[bt]`
+ forces the attempt. The Dockerfile drops its separate libtorrent step (the
+ requirement now resolves from `requirements.txt`); Python 3.13 is added to the
+ classifiers.
+- **Reader Define** popover clamps fully to the viewport and dismisses on scroll
+ like a native menu; on touch it clears more room below the selection (away
+ from the iOS system callout) and flips above when near the top of the screen.
+- Server backup bundle bumped to **schema v3** (carries per-user server data).
+- Repo layout: desktop build entry-points moved under `desktop/`, the Playwright
+ config under `tests/`.
+
+### Fixed
+
+- **#38 — same-page `#fragment` links.** An in-page anchor in a single-page doc
+ now scrolls in place instantly; it previously routed through a full
+ `location.replace()` and hung behind the 15-second safety overlay, making
+ every anchor jump look like a 15s page load.
+- **Health report flags broken scrapes.** A ZIM that opens and reports entries
+ can still be broken — every article a 0-byte shell, or its media all empty. A
+ universal text-sanity sampler and a media sampler now catch both and demote
+ the ZIM to a warning; a stray `.zim.torrent` metadata file is surfaced as a
+ distinct info row, never as a broken ZIM.
+- **iFixit snippets.** `extract_snippet` prefers a device page's own summary
+ block over the one repeated featured-guide `` baked into
+ every iFixit page, without regressing ZIMs (wikipedia / gutenberg / ted) whose
+ meta description *is* the right snippet.
+- **Light mode contrast pass (WCAG AA)** — toggles, badges, amber ink, and
+ disabled-field / blank-icon legibility; a redesigned, deterministic tile card;
+ and a fix for the Safari tab-switch theme flash.
+- **Move to…** stays in place when used from Settings, and its submenu clamps to
+ the viewport.
+- Raw-article horizontal overflow on narrow screens.
+- iOS input-zoom guard on the private-mode login field; the inert Cancel button
+ is hidden in the non-dismissible login gate.
+
+### Security
+
+- **Private/limited mode is fail-closed.** In private mode the request gate
+ 401s every read/data endpoint that isn't on a tiny login-surface allowlist —
+ default-deny, so a newly added read endpoint is covered automatically rather
+ than leaking until someone remembers to list it. In limited mode anonymous is
+ filtered by every choke point, including the `zim_allowed` bypass paths
+ (`/languages`, `/article-languages`, almanac-links).
+- **The service worker never caches or serves stale identity endpoints.**
+ `/whoami`, `/login`, `/logout`, `/list`, `/search`, `/suggest` and `/random`
+ are now network-only. This closes two field bugs: a stale cached anonymous
+ `/whoami` that re-showed the login screen after a successful sign-in ("sign in
+ twice"), and a cached full-library `/list` that could be served to a
+ now-anonymous visitor of a private instance on a network blip.
+- **Admin session cookie.** The primary admin authenticates with a password
+ Bearer that only rides `/manage/*`; the data endpoints (`/list`, `/search`)
+ and the `/w/` reader iframe carry no Authorization header, so a private- or
+ limited-mode admin loaded an empty library and blank article frames. Login and
+ `/whoami` now mint an HttpOnly `zimi_session` admin cookie that authenticates
+ both transports. It is unforgeable, never resolves as a named user, and is
+ revoked server-side on logout and on password rotation.
+- The `zimi_session` cookie sets `Secure` only behind an HTTPS proxy (a browser
+ silently drops a Secure cookie over plain http, which is what broke logins on
+ the LAN); `HttpOnly` and `SameSite=Lax` are always present.
+
+## [1.8.0] - 2026-07-28 — the Community Edition
A release shaped by the people using Zimi: the open issues, the long-standing
asks from the wider ZIM ecosystem, and a week of field testing on real phones.
@@ -128,12 +263,13 @@ delta updates on top.
row in Manage) to move it into another category, including brand-new custom
ones, and reorder the home sections — default categories and collections
alike — from a new Reorder panel in Manage preferences (#37).
-- **Windows portable build** — a one-dir `Zimi-windows-x64.zip` (Edge WebView2
- backend, in-process libtorrent when a wheel is available) built by the desktop
- release CI alongside the macOS DMGs and Linux AppImage/snap. The Windows
- desktop app also self-updates via WinSparkle — a per-user Inno Setup installer
- (no UAC) shipped beside the zip, verified with the same signed appcast key as
- the macOS Sparkle path.
+- **Native Windows app** — a per-user Inno Setup installer
+ (`Zimi-windows-x64-Setup.exe`, no UAC) that self-updates via WinSparkle,
+ verified with the same signed appcast key as the macOS Sparkle path; updates
+ install silently and relaunch on their own. A portable `Zimi-windows-x64.zip`
+ (Edge WebView2 backend, in-process libtorrent when a wheel is available) is
+ shipped beside it for no-install use. Both are built by the desktop release CI
+ alongside the macOS DMGs and Linux AppImage/snap.
- **Delta updates over BitTorrent.** When updating a ZIM that has a torrent,
Zimi copies the previous version into staging under the new name first, so
libtorrent's hash check salvages every unchanged piece and only the changed
@@ -395,89 +531,6 @@ looks like the Moon.
### Added
-- **"New" and "Updated" badges** on ZIM cards, so fresh or changed sources
- stand out in a large library instead of being lost in the grid (#34). A
- fresh install gets a solid "New" pill; a source whose file changed on an
- update gets a quieter "Updated" pill. A badge clears the moment you open
- that ZIM, and auto-expires after a week even if you never do — so it never
- lingers.
-
-## [1.7.4] - 2026-07-20
-
-A polish drop that closes the two live issues the post-1.7.3 code audit
-surfaced, and grows the almanac up: an accurate Chinese calendar, a real
-high-res moon, and a solar system you can travel out to. The moon finally
-looks like the Moon.
-
-### Almanac
-
-- **Accurate Chinese calendar.** Replaced the mean-lunation approximation with
- real astronomy — month boundaries are true new moons in China Standard Time,
- leap months placed by the solar-term rule against the winter solstice.
- Verified against the Hong Kong Observatory (New Year dates and leap months
- 2014–2033). It's a browsable calendar system again, with the 闰 leap-month
- marker and the correct zodiac animal.
-- **Holidays land on every calendar.** Worldwide days, regional holidays, the
- Easter cycle, solstices, meteor showers and Hindu/Sikh festivals are now
- projected onto whatever calendar you're viewing (Hebrew, Islamic, Chinese…)
- instead of vanishing when you switch away from Gregorian.
-- **A real high-resolution moon**, reprojected from NASA's seamless lunar
- albedo map — genuine crater detail and subtle true colour. The animated sky
- moon shares the hero's shading now, earthshine dark side and all.
-- **Travel the solar system.** The interstellar probes — Voyager 1 & 2,
- Pioneer 10 & 11, New Horizons — are plotted at their real bearings out past
- Neptune and creep outward as the clock runs, framed by labelled asteroid,
- Kuiper and heliopause markers.
-- **Easier location picking** — the world map cycles through overlapping
- cities on repeated clicks, and search reaches 354 cities.
-
-### Security
-
-### Security
-
-- **`/dl/` no longer serves whole ZIMs to the public internet.** On a
- host-networked deploy behind a containerized reverse proxy, every WAN
- request reached Zimi from the docker bridge gateway — a private IP with no
- real client IP propagated — so the whole internet was classified "private"
- and could pull raw `.zim` files (public content, so not a data breach, but
- unmetered use of the operator's uplink) and got the trusted rate tier.
- Zimi now honours Cloudflare's un-forgeable `CF-Connecting-IP`, trusts a
- forwarded client only from a private proxy hop (optionally an explicit
- `ZIMI_TRUSTED_PROXIES` allowlist), and refuses a forwarded value that claims
- loopback. WAN clients resolve to their real IP again.
-
-### Fixed
-
-- **A libzim segfault under normal use.** Opening an article kicked off a
- background thread that read a shared libzim archive with no lock, while the
- request read the same archive — two threads in a non-thread-safe library, a
- silent crash. It now uses its own handle. Added a real-ZIM concurrency
- stress test so this class of bug can't hide behind mocked archives again.
-- Search results could silently drop a ZIM under concurrent load (an archive
- published before its lock); the setup order is fixed.
-- LAN peer discovery advertised the wrong BitTorrent port when a custom port
- was set; it now advertises the real one.
-- Malformed HTTP Range headers no longer 500 the download endpoint.
-- The almanac topbar showed the underlying ZIM's icon; the breadcrumb is now
- just "Zimi" while the almanac is open (you reach it only from home).
-- Picking a location on the almanac's world map no longer rebuilds the whole
- panel — it refreshes the location-dependent pieces in place, so the page
- stops jumping and flashing on each click.
-
-### Changed
-
-- **The moon is beautiful now.** The hero and Today-card moon are rendered as
- a single physically-shaded sphere — a soft terminator, gentle limb
- darkening, the maria showing through, and a faint earthshine glow on the
- dark side of a crescent. Gone are the hard light/dark edge, the seam down
- the middle at the quarters, and the flat too-bright disc. The hero moon now
- renders at the display's device resolution (up to 512px) so its shading
- stays crisp when you lean in.
-- **Country holidays get their own colour** on the calendar, apart from the
- worldwide observances (#33).
-
-### Added
-
- **"New" and "Updated" badges** on ZIM cards, so fresh or changed sources
stand out in a large library instead of being lost in the grid (#34). A
fresh install gets a solid "New" pill; a source whose file changed on an
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index c4f6c9e4..5b92ce89 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -27,8 +27,8 @@ python3 tests/test_unit.py --perf # Performance benchmarks
Before tagging, verify these on the feature branch:
- [ ] Versions match: `ZIMI_VERSION` in `server.py`, `version` in `pyproject.toml`, `version` in `snap/snapcraft.yaml`
-- [ ] `zimi_desktop.spec` `hiddenimports` includes ALL `zimi/*.py` modules (add any new ones from server split)
-- [ ] Smoke test locally: `python3 zimi_desktop.py --serve --port 0` starts and prints `READY`
+- [ ] `desktop/zimi_desktop.spec` `hiddenimports` includes ALL `zimi/*.py` modules (add any new ones from server split)
+- [ ] Smoke test locally: `python3 desktop/zimi_desktop.py --serve --port 0` starts and prints `READY`
- [ ] `desktop-release.yml` smoke test commands work with current auth model (e.g. `Sec-Fetch-Site` header for manage endpoints)
- [ ] `node -c zimi/static/app.js` passes (no syntax errors)
- [ ] `python3 -m pytest tests/ -q` passes
@@ -74,8 +74,8 @@ When QA passes:
### Local Build
```bash
-pip install -r requirements-desktop.txt
-pyinstaller --noconfirm zimi_desktop.spec # Output: dist/Zimi.app
+pip install -r desktop/requirements-desktop.txt
+pyinstaller --noconfirm desktop/zimi_desktop.spec # Output: dist/Zimi.app
```
### Icons
@@ -94,6 +94,6 @@ CI handles signing and notarization automatically. See `.github/workflows/deskto
- `dist/` and `build/` are gitignored — never commit build artifacts
- PyInstaller copies source at build time — rebuild after code changes
- The `.spec` file includes `zimi/templates/`, `zimi/assets/`, and `zimi/static/` as data
-- **When splitting modules:** any new `zimi/*.py` file must be added to `hiddenimports` in `zimi_desktop.spec` AND the CI smoke test must still pass. PyInstaller can't discover runtime imports.
+- **When splitting modules:** any new `zimi/*.py` file must be added to `hiddenimports` in `desktop/zimi_desktop.spec` AND the CI smoke test must still pass. PyInstaller can't discover runtime imports.
- **Workflow dispatch and refs:** `gh workflow run --ref v1.X.0` runs the workflow YAML from the tag, not main. Fixes to the workflow on main won't apply unless you omit `--ref`.
- Windows: use `pip install zimi` (no desktop build currently)
diff --git a/Dockerfile b/Dockerfile
index edd54885..828eff18 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,12 +1,11 @@
FROM python:3.11-slim
COPY requirements.txt .
-# libtorrent 2.0 is the in-process BT engine for the optional torrent download
-# path (ZIMI_TORRENT=1). Off by default — HTTP downloads work without it. The
-# manylinux cp311 wheel means no apt/dist-packages games; the base is pinned to
-# python:3.11-slim so this wheel resolves.
-RUN pip install --no-cache-dir -r requirements.txt \
- && pip install --no-cache-dir "libtorrent==2.0.*"
+# requirements.txt pins libtorrent 2.0 (the in-process BT engine, ON by
+# default). The manylinux wheel — cp311 for both amd64 and aarch64 (Synology
+# and other ARM NAS) — resolves cleanly on this python:3.11-slim/glibc base, so
+# no apt/dist-packages games and no separate install step.
+RUN pip install --no-cache-dir -r requirements.txt
WORKDIR /app
COPY zimi/ ./zimi/
diff --git a/README.md b/README.md
index 35034352..5ce16b1f 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
# Zimi
[](https://github.com/epheterson/Zimi/actions/workflows/ci.yml)
-[](#)
+[](#)
[](docs/plans/2026-04-26-accessibility.md)
[](docs/plans/2026-04-26-accessibility.md)
[](#languages)
@@ -19,6 +19,7 @@ A modern experience for your ZIM files.
- **Search that hits everything.** One query, every source, 100M+ articles, the right answer on top. Fast.
- **Multilingual.** Switch any article into any language it has. Ten UI languages built in.
- **A real library.** 1,000+ archives one click away, auto-updates, collections, batch downloads, bookmarks and history.
+- **Yours or everyone's.** Serve the whole library openly, limit anonymous visitors to a chosen set of ZIMs, or require sign-in — with named accounts and per-ZIM access lists on top.
- **Your own network.** Your machines find each other and pass ZIMs around at LAN speed, no internet needed.
- **A good citizen.** Downloads arrive over BitTorrent and seed back to the Kiwix network. One switch makes you a full mirror.
- **Fresh daily.** Picture of the Day, On This Day, a word, a quote, a comic, a live almanac sky. All computed locally, forever.
@@ -52,7 +53,7 @@ Something not right? [Open an issue.](https://github.com/epheterson/Zimi/issues)
Three switches in Server Settings control all of it:
-- **BitTorrent** (on by default). Downloads arrive via the Kiwix swarm and seed back, capped at a ratio you choose. `0` means never seed. The Mac and Linux desktop apps and the Docker image ship their own BitTorrent engine (in-process libtorrent); a bare `pip install` uses it if the wheel is present — and everything quietly falls back to plain HTTP without it. UPnP asks your router to open the port, and the settings panel shows whether it worked.
+- **BitTorrent** (on by default). Downloads arrive via the Kiwix swarm and seed back, capped at a ratio you choose. `0` means never seed. The engine is in-process libtorrent: the desktop apps and Docker image bundle it, and `pip install zimi` pulls it automatically wherever a prebuilt wheel exists (CPython 3.9–3.13 on Linux, macOS, and Windows). If there's no wheel for your interpreter — Python 3.14+ has none yet — Zimi quietly falls back to plain HTTP and prints the one-line fix; `pip install zimi[bt]` forces the attempt. UPnP asks your router to open the port, and the settings panel shows whether it worked. Concurrent downloads and the peer-connection limit are tunable in the same panel.
- **Nearby** (off by default). Flip it on and Zimi devices on your network find each other; a green pill on a catalog card means a neighbor already has that ZIM. Transfers stay on your LAN, never the internet.
- **Mirror** (off). Lifts the seeding cap, for people who want to run a long-term Kiwix mirror.
@@ -139,7 +140,8 @@ Most people set nothing: every setting below has a sensible default or lives in
| `ZIM_DIR` | `/zims` | Path to ZIM files (scanned for `*.zim` on startup) |
| `ZIMI_DATA_DIR` | `/config` (Docker) or `$ZIM_DIR/.zimi` | Cache, indexes, and settings. Mount separately in Docker. |
| `ZIMI_MANAGE_PASSWORD` | _(none)_ | Protect library management |
-| `ZIMI_BT` | `on` | BitTorrent: `off`, or `on,port=6881,ratio=2,up=2048,seed=on,mirror=off,upnp=on,dht=on`. `seed`, `upnp`, and `dht` default on. Fields you set are locked in the UI; fields you leave out stay UI-controlled. `ratio=0` means never seed. |
+| `ZIMI_PUBLIC_ACCESS` | `open` | What an anonymous visitor sees: `open` (whole library), `limited` (an admin-chosen allowlist), or `private` (sign-in required). Also a UI setting; the env var wins when set. |
+| `ZIMI_BT` | `on` | BitTorrent: `off`, or `on,port=6881,ratio=2,up=2048,seed=on,mirror=off,upnp=on,dht=on,active=4,conns=200`. `seed`, `upnp`, and `dht` default on. `active` caps concurrent downloads (the rest queue; governs HTTP too — legacy `ZIMI_MAX_CONCURRENT_DOWNLOADS` still works), `conns` is the global peer-connection limit. Fields you set are locked in the UI; fields you leave out stay UI-controlled. `ratio=0` means never seed. |
| `ZIMI_NEARBY` | `off` | LAN sharing: `off`, or `on,name=my-zimi,public=off,ip=192.168.1.20`. Controls serving *and* fetching between your Zimi devices. Set `ip=` to your host's LAN address when running Docker in bridge mode. |
diff --git a/deploy.sh b/deploy.sh
index 854887ae..c1ec7f54 100755
--- a/deploy.sh
+++ b/deploy.sh
@@ -80,7 +80,7 @@ if [ ! -x "$BUILD_VENV/bin/python" ] || \
"$BUILD_PY" -m venv "$BUILD_VENV"
fi
"$BUILD_VENV/bin/pip" install -q --upgrade pip
-"$BUILD_VENV/bin/pip" install -q -r requirements-desktop.txt
+"$BUILD_VENV/bin/pip" install -q -r desktop/requirements-desktop.txt
# Soft dependency, mirroring CI: no wheel for this interpreter → warn and build
# HTTP-only, never block the build.
if "$BUILD_VENV/bin/pip" install -q "libtorrent==2.0.*" 2>/dev/null; then
@@ -89,7 +89,7 @@ else
echo " WARNING: no libtorrent wheel for $($BUILD_PY --version 2>&1) — app will be HTTP-only"
fi
-"$BUILD_VENV/bin/pyinstaller" zimi_desktop.spec --noconfirm 2>&1 | tail -5
+"$BUILD_VENV/bin/pyinstaller" desktop/zimi_desktop.spec --noconfirm 2>&1 | tail -5
echo " App built"
echo ""
diff --git a/requirements-desktop.txt b/desktop/requirements-desktop.txt
similarity index 100%
rename from requirements-desktop.txt
rename to desktop/requirements-desktop.txt
diff --git a/zimi_desktop.py b/desktop/zimi_desktop.py
similarity index 97%
rename from zimi_desktop.py
rename to desktop/zimi_desktop.py
index 665e5968..010a23a0 100644
--- a/zimi_desktop.py
+++ b/desktop/zimi_desktop.py
@@ -14,6 +14,17 @@
import sys
import threading
+# ---------------------------------------------------------------------------
+# Path bootstrap — this script lives in desktop/, the zimi package at the repo
+# root. When run directly in dev mode (e.g. `python desktop/zimi_desktop.py`),
+# sys.path[0] is desktop/, so `import zimi` would miss the package. Prepend the
+# repo root. In a frozen bundle the package is embedded under _MEIPASS, so skip.
+# ---------------------------------------------------------------------------
+if not getattr(sys, "_MEIPASS", None):
+ _REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ if _REPO_ROOT not in sys.path:
+ sys.path.insert(0, _REPO_ROOT)
+
# ---------------------------------------------------------------------------
# Windows: force pythonnet onto the .NET Framework runtime BEFORE any import
# that triggers pywebview → pythonnet (must run before `import webview`).
@@ -396,9 +407,10 @@ def _init_sparkle_updater():
app_bundle_path, "Frameworks", "Sparkle.framework"
)
if not bundle_path or not os.path.exists(bundle_path):
- # Dev mode: framework in repo root
+ # Dev mode: framework in repo root (one level up from desktop/)
bundle_path = os.path.join(
- os.path.dirname(os.path.abspath(__file__)), "Sparkle.framework"
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+ "Sparkle.framework",
)
if not os.path.exists(bundle_path):
return
diff --git a/zimi_desktop.spec b/desktop/zimi_desktop.spec
similarity index 87%
rename from zimi_desktop.spec
rename to desktop/zimi_desktop.spec
index 92239ba7..b7d8b722 100644
--- a/zimi_desktop.spec
+++ b/desktop/zimi_desktop.spec
@@ -2,8 +2,8 @@
"""
PyInstaller spec for Zimi Desktop.
-Build:
- pyinstaller zimi_desktop.spec
+Build (from the repo root):
+ pyinstaller --noconfirm desktop/zimi_desktop.spec
Output:
dist/Zimi/ — one-dir bundle (all platforms)
@@ -24,6 +24,14 @@ from PyInstaller.utils.hooks import (
block_cipher = None
+# This spec lives in desktop/; the entry script and its winsparkle sibling live
+# beside it, while the zimi package + data dirs (and CI-downloaded Sparkle
+# framework / WinSparkle DLL) live at the repo root. Anchor every path to those
+# two roots explicitly so the build is independent of the invocation CWD.
+# SPECPATH is the absolute directory containing this spec file.
+DESKTOP_DIR = SPECPATH
+REPO_ROOT = os.path.dirname(SPECPATH)
+
# zeroconf (LAN peer discovery) loads submodules dynamically, so PyInstaller's
# static analysis misses them unless we collect the whole package.
zeroconf_hiddenimports = collect_submodules("zeroconf")
@@ -78,7 +86,7 @@ lt_hidden = collect_submodules('libtorrent')
# without auto-update, exactly like the Sparkle.framework soft path.
winsparkle_bins = []
if platform.system() == 'Windows':
- _ws_dll = os.path.join(SPECPATH, 'WinSparkle.dll')
+ _ws_dll = os.path.join(REPO_ROOT, 'WinSparkle.dll')
if os.path.isfile(_ws_dll):
winsparkle_bins.append((_ws_dll, '.'))
@@ -113,13 +121,13 @@ if platform.system() == 'Windows':
]
a = Analysis(
- ['zimi_desktop.py'],
- pathex=[],
+ [os.path.join(DESKTOP_DIR, 'zimi_desktop.py')],
+ pathex=[REPO_ROOT, DESKTOP_DIR],
binaries=libzim_bins + lt_bins + winsparkle_bins + pythonnet_bins,
datas=[
- ('zimi/templates', 'zimi/templates'),
- ('zimi/assets', 'zimi/assets'),
- ('zimi/static', 'zimi/static'),
+ (os.path.join(REPO_ROOT, 'zimi/templates'), 'zimi/templates'),
+ (os.path.join(REPO_ROOT, 'zimi/assets'), 'zimi/assets'),
+ (os.path.join(REPO_ROOT, 'zimi/static'), 'zimi/static'),
] + pythonnet_datas,
hiddenimports=[
'zimi',
@@ -176,7 +184,8 @@ exe = EXE(
strip=False,
upx=True,
console=False,
- icon='zimi/assets/icon.icns' if platform.system() == 'Darwin' else 'zimi/assets/icon.ico',
+ icon=(os.path.join(REPO_ROOT, 'zimi/assets/icon.icns') if platform.system() == 'Darwin'
+ else os.path.join(REPO_ROOT, 'zimi/assets/icon.ico')),
)
coll = COLLECT(
@@ -197,7 +206,7 @@ if platform.system() == 'Darwin':
app = BUNDLE(
coll,
name='Zimi.app',
- icon='zimi/assets/icon.icns',
+ icon=os.path.join(REPO_ROOT, 'zimi/assets/icon.icns'),
bundle_identifier='io.zosia.zimi',
info_plist={
'CFBundleShortVersionString': '1.4.0',
diff --git a/zimi_winsparkle.py b/desktop/zimi_winsparkle.py
similarity index 81%
rename from zimi_winsparkle.py
rename to desktop/zimi_winsparkle.py
index d485dca6..d5df9f25 100644
--- a/zimi_winsparkle.py
+++ b/desktop/zimi_winsparkle.py
@@ -40,6 +40,13 @@
"https://raw.githubusercontent.com/epheterson/Zimi/main/appcast-windows.xml"
)
+# Override for update-testing. Set ZIMI_APPCAST_URL to point a build at a
+# throwaway feed (a local file:// or http:// appcast advertising an RC build)
+# so a released app can be driven through a real WinSparkle update WITHOUT
+# touching the production appcast on `main`. Unset in normal use → the default
+# feed above wins. Windows-only lever; ignored everywhere else the module no-ops.
+_APPCAST_URL_ENV = "ZIMI_APPCAST_URL"
+
# WinSparkle stores its check state (last-check time, skipped versions) here.
_REGISTRY_PATH = b"Software\\Zimi\\WinSparkle"
@@ -101,16 +108,28 @@ def _bind_signatures(dll):
# init / cleanup / check_* take no args and return void — ctypes defaults fine.
-def init_updater(
- version, appcast_url=WINDOWS_APPCAST_URL, pubkey=WINSPARKLE_EDDSA_PUBLIC_KEY
-):
+def _resolve_appcast_url(explicit=None):
+ """Feed URL to use: explicit arg > ZIMI_APPCAST_URL env > production default.
+
+ The env override is the test lever documented on :data:`_APPCAST_URL_ENV`;
+ with it unset (the normal case) the production feed is returned unchanged.
+ """
+ if explicit:
+ return explicit
+ return os.environ.get(_APPCAST_URL_ENV, WINDOWS_APPCAST_URL)
+
+
+def init_updater(version, appcast_url=None, pubkey=WINSPARKLE_EDDSA_PUBLIC_KEY):
"""Initialize WinSparkle and kick off a launch-time update check.
- Returns True if the updater started, False (clean no-op) otherwise — on a
- non-Windows host, a missing DLL, or any load/call error. Safe to call from a
- background thread: WinSparkle spawns its own thread and message loop.
+ ``appcast_url`` defaults to :func:`_resolve_appcast_url` (production feed
+ unless ``ZIMI_APPCAST_URL`` is set). Returns True if the updater started,
+ False (clean no-op) otherwise — on a non-Windows host, a missing DLL, or any
+ load/call error. Safe to call from a background thread: WinSparkle spawns its
+ own thread and message loop.
"""
global _dll
+ appcast_url = _resolve_appcast_url(appcast_url)
dll_path = _find_dll()
if not dll_path:
_log_once("DLL not found — running without auto-update")
diff --git a/docs/plans/2026-07-24-release-scoping.md b/docs/plans/2026-07-24-release-scoping.md
index 3068a63d..b356bae9 100644
--- a/docs/plans/2026-07-24-release-scoping.md
+++ b/docs/plans/2026-07-24-release-scoping.md
@@ -127,3 +127,12 @@ Surface this list at the start of every release cycle.
download (zip stays for portable users) — Eric on first launch: the
bare onedir "_internal" folder look is odd; Program Files install
hides it. Pairs with the Authenticode cert decision.
+- Onboarding / feature tour (Eric, 2026-07-28): "In time we may need
+ readme or onboarding to show users all features and how to." A
+ first-run "What can Zimi do?" tour or Help panel — 1.9 candidate.
+- Private-mode content-cache hardening (1.9): the SW keeps article
+ content (/read, /w/) cached for offline reading — after logout on a
+ private instance, previously-viewed articles remain readable on that
+ device. Same-device-only, low severity, deliberate 1.8.1 tradeoff
+ (identity/library endpoints are network-only as of the r3 fix).
+ Candidate: purge content caches on logout when mode=private.
diff --git a/docs/plans/2026-07-28-windows-gold-standard.md b/docs/plans/2026-07-28-windows-gold-standard.md
new file mode 100644
index 00000000..0a5cfc15
--- /dev/null
+++ b/docs/plans/2026-07-28-windows-gold-standard.md
@@ -0,0 +1,159 @@
+# Windows gold-standard update test (1.8.0 → 1.8.1)
+
+**Goal:** prove, on the maintainer's real Windows PC, that a Zimi Windows
+install auto-updates end-to-end via WinSparkle — detect → download → verify
+signature → run installer **silently** → relaunch at the new version — before
+1.8.1 ships. ~10 minutes.
+
+---
+
+## The one thing you must know first: prerelease tags are NOT safe
+
+A prerelease tag like `v1.8.1-rc1` **is unsafe** as a staging mechanism. It is
+not a theoretical risk — here is exactly what it does:
+
+- `desktop-release.yml` triggers on `push: tags: ['v*.*.*']`. The glob matches
+ `v1.8.1-rc1` (`v1` . `8` . `1-rc1`), so the tag **builds all platforms**.
+- Its `release` job then runs unconditionally for any tag and, in the same run:
+ 1. **rewrites `appcast-windows.xml`, `appcast-arm64.xml`, `appcast-intel.xml`
+ on `main` and pushes them** (job "Sign updates and update appcasts",
+ `git push origin main`). Every existing 1.8.0 user's app polls those files
+ — they would immediately be told to "update" to the RC.
+ 2. **pushes a Homebrew cask bump** to `epheterson/homebrew-zimi`.
+ 3. creates a GitHub draft release for the RC.
+
+So a prerelease tag mutates the three **production** appcasts and the Homebrew
+tap. Do not use one for testing. (If we ever want RC tags to be safe, the fix is
+to gate the "update appcasts" + "Homebrew cask" steps on
+`!contains(github.ref_name, '-')` — out of scope here, noted for later.)
+
+**The safe lever instead:** `ZIMI_APPCAST_URL` (added to `zimi_winsparkle.py`
+this release). Setting it points a build at a throwaway feed without touching
+anything on `main`. Caveat that shapes the whole procedure below: **the shipped
+1.8.0 predates this override**, so the real 1.8.0 artifact cannot be redirected
+— it can only ever read the production feed. We therefore split the test.
+
+---
+
+## Part A — sanity-check the real shipped 1.8.0 (2 min)
+
+Confirms the artifact users actually have is wired to the right feed.
+
+1. Download `Zimi-windows-x64-Setup.exe` from the published
+ [1.8.0 release](https://github.com/epheterson/Zimi/releases/tag/v1.8.0) and
+ run it (per-user, no UAC prompt).
+2. Launch Zimi. Leave it open ~30 s.
+3. **Pass:** the app opens, and the log shows WinSparkle initialising and
+ checking against `raw.githubusercontent.com/.../appcast-windows.xml`. Because
+ production still advertises 1.8.0, it reports "up to date" (no prompt). This
+ proves the shipped updater is alive and pointed at the right URL. It cannot
+ be driven to 1.8.1 here (no override in 1.8.0) — that is Part B's job.
+
+---
+
+## Part B — prove the auto-update mechanism, safely (6 min)
+
+Rehearses 1.8.0 → 1.8.1 with the **real** WinSparkle path and a **local** feed.
+Nothing on `main` is touched.
+
+### One-time prep (done the night before, or first thing)
+
+1. **Staged installer (the "1.8.1" the test updates *to*).** Bump
+ `pyproject.toml` to `1.8.1` in a scratch checkout and build the installer —
+ either locally with Inno Setup:
+ ```
+ iscc /DMyAppVersion=1.8.1 windows\zimi.iss # → dist\Zimi-windows-x64-Setup.exe
+ ```
+ or grab the artifact from a **bare** `workflow_dispatch` run of
+ `desktop-release.yml` with **no tag input** — that runs the build matrix only;
+ the `release` job is skipped (`if: startsWith(github.ref,'refs/tags/') ||
+ inputs.tag != ''`), so no appcast/tag/cask is touched. Rename the artifact to
+ `Zimi-windows-x64-Setup.exe`.
+
+2. **Sign it** with the same Ed25519 key the release uses (on the Mac, with
+ `SPARKLE_PRIVATE_KEY` available):
+ ```
+ sign_update Zimi-windows-x64-Setup.exe --ed-key-file <(echo "$SPARKLE_PRIVATE_KEY")
+ ```
+ Note the `sparkle:edSignature="…"` and `length="…"` it prints.
+
+3. **Write a test appcast** `appcast-windows-test.xml` next to the installer.
+ Same shape the CI generates, advertising **1.8.1**, enclosure `url` pointing
+ at the locally-served installer, with the signature/length from step 2 and
+ the silent-install args:
+ ```xml
+
+
+
+ Zimi Updates
+
+ 1.8.1
+ 1.8.1
+
+
+
+
+ ```
+
+4. **Baseline app (the "1.8.0" the test updates *from*).** Build the current
+ `v1.8.1` branch **as-is** (its `pyproject.toml` still reports `1.8.0`) — this
+ is effectively "1.8.0 + the `ZIMI_APPCAST_URL` override", which is precisely
+ what makes the redirect possible. Install it. It reports version 1.8.0, so
+ WinSparkle will see 1.8.1 in the test feed as an upgrade.
+
+### The test (run in the morning)
+
+1. In the folder holding the signed installer + `appcast-windows-test.xml`:
+ ```
+ python -m http.server 8000
+ ```
+2. Point the baseline app at the test feed and launch it:
+ ```
+ set ZIMI_APPCAST_URL=http://localhost:8000/appcast-windows-test.xml
+ "%LOCALAPPDATA%\Programs\Zimi\Zimi.exe"
+ ```
+3. WinSparkle checks on launch, finds 1.8.1, verifies the EdDSA signature
+ against the bundled public key, and offers the update. Accept it.
+4. Watch: the installer runs **silently** (a progress window, no wizard pages,
+ no UAC), the old app closes, and Zimi **relaunches on its own** at 1.8.1.
+
+### Pass criteria
+
+- [ ] Update prompt appears (signature verified — a bad/missing signature is
+ rejected and no prompt shows).
+- [ ] Install is click-free: `/SILENT /SP-` means progress only, no wizard, no
+ UAC (per-user install).
+- [ ] The app **relaunches by itself** after install (this is the risky bit:
+ WinSparkle 0.9.4 quits without relaunching, so the installer's `[Run]`
+ entry — now without `skipifsilent` — must do it).
+- [ ] Relaunched app reports **1.8.1** (About / `/api` version / window title).
+- [ ] Exactly **one** Zimi instance is running afterwards (no double-launch).
+
+If the relaunch check fails, that is the finding — the `skipifsilent` removal in
+`windows/zimi.iss` is the fix under test; capture the behavior and iterate
+before the real ship.
+
+---
+
+## The real ship (the ultimate truth)
+
+Part B proves the mechanism; the release itself is the final 1.8.0 → 1.8.1
+proof. After merging 1.8.1 (version bump → `auto-release.yml` tags + drafts →
+`desktop-release.yml` builds/signs/attaches to the one draft), **publish** it.
+The production `appcast-windows.xml` now advertises 1.8.1. On the Part-A machine
+(real shipped 1.8.0, production feed), relaunch and let WinSparkle check — it
+pulls 1.8.1 for real. Same pass criteria.
+
+## Rollback
+
+- **Part B:** nothing to undo — local only. Stop the `http.server`, `set
+ ZIMI_APPCAST_URL=` (clear it).
+- **Bad production 1.8.1:** revert the "Update appcasts for v1.8.1" commit on
+ `main` so the feed re-advertises 1.8.0, and mark the 1.8.1 GitHub release as a
+ draft / delete it. A user already on a bad 1.8.1 reinstalls 1.8.0 manually
+ (the stable Inno `AppId` upgrades in place).
diff --git a/docs/release-notes-1.8.0.md b/docs/release-notes-1.8.0.md
index a5baf81a..6617af45 100644
--- a/docs/release-notes-1.8.0.md
+++ b/docs/release-notes-1.8.0.md
@@ -44,7 +44,7 @@ This is the release the people using Zimi shaped: every open issue on the tracke
## A native Windows app
-A one-dir `Zimi-windows-x64.zip` (Edge WebView2) that **self-updates via WinSparkle**, signed with the same appcast key as the macOS Sparkle path, with a per-user installer that needs no admin rights.
+A per-user **installer** (`Zimi-windows-x64-Setup.exe`, no admin rights) that **self-updates via WinSparkle**, signed with the same appcast key as the macOS Sparkle path — updates install silently and relaunch on their own. A portable `Zimi-windows-x64.zip` (Edge WebView2) is there too for no-install use.
## Also in the box
diff --git a/docs/release-notes-1.8.1.md b/docs/release-notes-1.8.1.md
new file mode 100644
index 00000000..a1b16143
--- /dev/null
+++ b/docs/release-notes-1.8.1.md
@@ -0,0 +1,62 @@
+# Zimi 1.8.1
+
+A polish-and-hardening release on top of the Community Edition. The big one is **access control** — a Zimi server can now be open to everyone, limited to a chosen set of ZIMs, or sign-in-required — and the whole private-mode path is tightened so private really means private. On top of that: the offline "did you mean?" grows into the coverage 1.8.0 promised, downloads get a nightly window and bandwidth caps, and light mode gets a proper contrast pass.
+
+## Choose who sees what
+
+- **Three access modes for anonymous visitors.** Set your server to **Open** (the whole library, the default), **Limited** (only the ZIMs you pick), or **Private** (sign-in required to see anything). Named user accounts and their per-ZIM allowlists still work exactly as before on top of this — Limited/Private just decide what someone who *isn't* signed in gets. Set it in Manage → Users, or with `ZIMI_PUBLIC_ACCESS`.
+- **Private really means private.** The private-mode gate is default-deny: every read and data endpoint is blocked for an anonymous visitor unless it's on a tiny login allowlist, so nothing leaks by omission. A corrupt policy file fails *closed*, not open.
+- **Your own account, on the server.** Signed-in users can now save their bookmarks, history and preferences to their own server account and restore them on another device — each user can only ever touch their own data.
+
+## Backup, rebuilt
+
+- **Two clearly-separated cards** — "**My data**" (this browser's bookmarks, history and preferences) and "**Server backup**" (the library, collections, layout and every user's server-side data). Each has its own Export and Import.
+- **Imports merge by default** with a preview-then-apply step and an overwrite escape hatch, and they're scope-checked so you can't drop a My-data file on the Server card (or the reverse) by mistake.
+
+## "Did you mean?", delivered
+
+- 1.8.0 shipped offline spelling correction and flagged that its coverage was a work-in-progress. 1.8.1 is the widening: it now finds words spread thin across your titles (mitochondria, photosynthesis), catches longer two-typo misspellings (`fotosynthesis` → `photosynthesis`), and fixes a common typo even when the misspelling itself appears in your library (`einstien` → `einstein`) — all still built entirely from your own ZIMs, with no network, ever.
+- The corrected vocabulary is now cached to disk, so it isn't rebuilt on every start.
+
+## Downloads on your terms
+
+- **Nightly window** — schedule downloads to run only inside a time window (say, overnight); anything you start outside it waits, with a start-now override when you don't want to wait.
+- **Bandwidth caps** — a download speed limit that's shared across all your transfers (so ten downloads still sum to your cap, not ten times it) and an upload limit, plus bulk Pause / Resume / Delete-all over the queue.
+- **BitTorrent** — max concurrent downloads and max peer connections are now adjustable knobs.
+
+## Read better
+
+- **App theme** — an Auto / Dark / Light switch for the whole interface, with an option to auto-darken raw article pages when the app is dark, and a Reader View Auto theme that turns sepia in light mode.
+- **Light mode contrast pass** — toggles, badges, disabled fields and icons all pass WCAG AA now, tiles are redesigned, and the Safari tab-switch flash is gone.
+- **Define** clamps to the screen and dismisses on scroll like a native menu, and no longer collides with the iOS text-selection callout.
+- **PDF collections** (`zimgit-*` ZIMs) render as a searchable document list — title, author, size, description — instead of a raw page.
+- **Video ZIMs** remember where you left off, size correctly on phones, and get a random-video card.
+
+## Fixes
+
+- **#38** — in-page `#fragment` links (single-page docs like devdocs) now scroll instantly instead of hanging for 15 seconds behind a loading spinner.
+- **Health report** now catches a ZIM that opens fine but is a broken scrape — every article an empty shell, or its media all 0-byte — and flags it, instead of reporting it healthy.
+- **Snippets** on iFixit device pages show the device's own description instead of the one repeated boilerplate blurb baked into every page.
+- Raw articles no longer overflow sideways on narrow screens; the **Move to…** menu stays put and stays on-screen.
+
+## Under the hood
+
+- **BitTorrent installs by default** now. `pip install zimi` pulls libtorrent automatically wherever a prebuilt wheel exists — CPython 3.9–3.13 across Linux (glibc *and* Alpine/musl, including ARM NAS), macOS and Windows — closing the 1.8.0 gap where a bare pip install left you on HTTP-only. No wheel for your interpreter (e.g. Python 3.14, which has none yet)? Zimi quietly falls back to HTTP and tells you the one-line fix; `pip install zimi[bt]` forces it.
+- **Security hardening** across the private-mode path: the service worker no longer caches identity or library endpoints (fixing a "sign in twice" bug and a stale-library leak), and a new HttpOnly admin session cookie means a private-mode admin sees the full library and can open articles again — revoked on logout and on any password change.
+
+## Also in the box
+
+- The almanac gains a real bright-star field, a Regional / Worldwide holidays toggle, a wider set of cities with click-anywhere map selection, and more of its stars, planets and people linking straight into your installed encyclopedias.
+
+## Install
+
+```bash
+brew tap epheterson/zimi && brew install --cask zimi # macOS
+sudo snap install zimi # Linux
+docker run --network host -v ./zims:/zims -v ./zimi-config:/config epheterson/zimi
+pip install zimi
+```
+
+Windows: download the installer from the release page.
+
+Full detail in [CHANGELOG.md](https://github.com/epheterson/Zimi/blob/main/CHANGELOG.md).
diff --git a/pyproject.toml b/pyproject.toml
index eb05e980..7cf673b3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "zimi"
-version = "1.8.0"
+version = "1.8.1"
description = "Offline knowledge server for ZIM files — search and read Wikipedia, Stack Overflow and 50+ sources with no internet"
readme = "README.md"
license = {text = "MIT"}
@@ -23,6 +23,7 @@ classifiers = [
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
"Topic :: Internet :: WWW/HTTP :: HTTP Servers",
"Topic :: Text Processing :: Indexing",
]
@@ -30,16 +31,27 @@ dependencies = [
"libzim>=3.1.0",
"certifi>=2024.1.1",
# LAN peer discovery (mDNS/Zeroconf). Pure-Python; the peer ZIM-sharing
- # transport runs over plain HTTP so it needs no extra binary. libtorrent
- # (the optional in-process BitTorrent accelerator) is bundled in the
- # Docker image and desktop builds; a bare pip install uses it if present.
+ # transport runs over plain HTTP so it needs no extra binary.
"zeroconf>=0.130.0",
+ # BitTorrent accelerator (optional at runtime, attempted by default at
+ # install). Pulled automatically wherever a prebuilt wheel exists — CPython
+ # 3.9–3.13 on manylinux + musllinux (incl. aarch64), macOS, and Windows, so
+ # glibc AND Alpine get it for free. Gated below 3.14 because no 3.14 wheel
+ # is published yet and the project ships no sdist: without the marker pip
+ # would find no distribution there and fail the WHOLE install, breaking the
+ # HTTP soft-fallback contract. On 3.14+ Zimi runs HTTP-only and logs the
+ # one-line fix. (A py≤3.13 arch with no wheel at all — rare, e.g. some BSD —
+ # would still fail the install since there's no sdist; use the Docker image
+ # or `pip install zimi` without forcing BT there.) Force it anywhere with
+ # `pip install zimi[bt]`.
+ "libtorrent==2.0.*; python_version < '3.14'",
]
[project.optional-dependencies]
+bt = ["libtorrent==2.0.*"]
pdf = ["PyMuPDF>=1.23.0"]
mcp = ["mcp>=1.0.0"]
-all = ["PyMuPDF>=1.23.0", "mcp>=1.0.0"]
+all = ["PyMuPDF>=1.23.0", "mcp>=1.0.0", "libtorrent==2.0.*"]
[project.scripts]
zimi = "zimi.server:main"
diff --git a/requirements.txt b/requirements.txt
index f806920a..be5a25be 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,3 +3,9 @@ PyMuPDF>=1.23.0
mcp>=1.0.0
certifi>=2024.1.1
zeroconf>=0.130.0
+# BitTorrent accelerator. Marker-gated below 3.14 (no wheel there yet, and no
+# sdist) so a plain `pip install -r requirements.txt` never breaks on new
+# interpreters; missing = HTTP-only. Wheels cover manylinux + musllinux, macOS
+# and Windows for 3.9–3.13; Docker's base is Python 3.11 (glibc), so a wheel
+# always resolves here.
+libtorrent==2.0.*; python_version < "3.14"
diff --git a/tests/conftest_zim.py b/tests/conftest_zim.py
index 8afb77ce..8c5b408c 100644
--- a/tests/conftest_zim.py
+++ b/tests/conftest_zim.py
@@ -121,3 +121,76 @@ def build_wiki_fixture_zim(path: str) -> str:
creator.add_metadata("Description", "tiny wiki fixture")
assert os.path.exists(path)
return path
+
+
+class _MediaItem(Item):
+ """A media entry (video/audio) with arbitrary bytes — including zero, to
+ stand in for a broken/partial scrape's empty placeholder."""
+
+ def __init__(self, path: str, mimetype: str, content: bytes):
+ super().__init__()
+ self._path = path
+ self._mimetype = mimetype
+ self._content = content
+
+ def get_path(self) -> str:
+ return self._path
+
+ def get_title(self) -> str:
+ return ""
+
+ def get_mimetype(self) -> str:
+ return self._mimetype
+
+ def get_contentprovider(self) -> ContentProvider:
+ return _StringProvider(self._content)
+
+ def get_hints(self) -> dict:
+ return {Hint.FRONT_ARTICLE: False}
+
+
+def build_empty_text_fixture_zim(path: str) -> str:
+ """Write a ZIM that opens fine and has entries, but every text/html article
+ is 0-byte — the media-free shape of a broken scrape. The health check's
+ text-sanity sampler must flag it even though the media sampler finds nothing.
+ Returns the path."""
+ articles = [
+ ("A/Alpha", "Alpha", b""),
+ ("A/Bravo", "Bravo", b""),
+ ("A/Charlie", "Charlie", b""),
+ ]
+ with Creator(path).config_indexing(True, "eng") as creator:
+ creator.set_mainpath("A/Alpha")
+ for p, t, h in articles:
+ creator.add_item(_Article(p, t, h))
+ creator.add_metadata("Title", "Empty Scrape")
+ creator.add_metadata("Language", "eng")
+ creator.add_metadata("Description", "empty-article fixture")
+ assert os.path.exists(path)
+ return path
+
+
+def build_media_fixture_zim(path: str) -> str:
+ """Write a ZIM with one article, one real video, and one ZERO-BYTE video.
+
+ Mirrors the partial-scrape shape (e.g. ted_en_technology_2023-09: real
+ .webm alongside 0-byte .mp4 placeholders) so the health check's media
+ sampler has a genuine empty media entry to flag. Returns the path."""
+ with Creator(path).config_indexing(True, "eng") as creator:
+ creator.set_mainpath("A/Talks")
+ creator.add_item(
+ _Article(
+ "A/Talks",
+ "Talks",
+ b"
Talks
",
+ )
+ )
+ creator.add_item(
+ _MediaItem("videos/1/video.webm", "video/webm", b"\x1aE\xdf\xa3real")
+ )
+ creator.add_item(_MediaItem("videos/2/video.webm", "video/webm", b""))
+ creator.add_metadata("Title", "Test Talks")
+ creator.add_metadata("Language", "eng")
+ creator.add_metadata("Description", "media fixture")
+ assert os.path.exists(path)
+ return path
diff --git a/playwright.config.mjs b/tests/playwright.config.mjs
similarity index 66%
rename from playwright.config.mjs
rename to tests/playwright.config.mjs
index c19dd370..636938c0 100644
--- a/playwright.config.mjs
+++ b/tests/playwright.config.mjs
@@ -1,14 +1,15 @@
import { defineConfig } from '@playwright/test';
export default defineConfig({
- testDir: './tests',
- testMatch: ['visual_validation.spec.mjs', 'test_password_flow.mjs', 'test_interlang.mjs', 'screenshots.mjs', 'test_tabs.mjs', 'test_almanac_hero_clock.spec.mjs', 'test_login_navigation.spec.mjs'],
+ // This config lives in tests/; specs sit beside it, artifacts stay at repo root.
+ testDir: '.',
+ testMatch: ['visual_validation.spec.mjs', 'test_password_flow.mjs', 'test_interlang.mjs', 'screenshots.mjs', 'test_tabs.mjs', 'test_almanac_hero_clock.spec.mjs', 'test_login_navigation.spec.mjs', 'test_private_mode_login.spec.mjs', 'test_admin_private_library.spec.mjs', 'test_backup_hub.spec.mjs'],
timeout: 60000,
expect: { timeout: 10000 },
fullyParallel: false, // Run sequentially — some tests depend on server state
retries: 0,
reporter: [
- ['html', { open: 'always', outputFolder: 'test-results/html-report' }],
+ ['html', { open: 'always', outputFolder: '../test-results/html-report' }],
['list'],
],
use: {
@@ -19,5 +20,5 @@ export default defineConfig({
viewport: { width: 1440, height: 900 },
actionTimeout: 10000,
},
- outputDir: 'test-results/artifacts',
+ outputDir: '../test-results/artifacts',
});
diff --git a/tests/test_admin_private_library.spec.mjs b/tests/test_admin_private_library.spec.mjs
new file mode 100644
index 00000000..2325152d
--- /dev/null
+++ b/tests/test_admin_private_library.spec.mjs
@@ -0,0 +1,136 @@
+// SHIP BLOCKER (1.8.1 field): a signed-in ADMIN saw an EMPTY library in private
+// (and limited) public-access mode — home "No knowledge sources found", Library
+// "No ZIMs installed" — even though Settings listed every ZIM. The existing
+// private-mode spec only proved the admin gate CLEARS; it never checked the
+// admin's library was populated or that an article rendered. This spec closes
+// both gaps.
+//
+// Root cause: the admin's credential is a password Bearer that only rides
+// /manage/* calls. The DATA endpoints (/list, /search) are plain fetch() with no
+// header, and the /w/ reader iframe is a browser navigation that CAN'T send one,
+// so the server saw neither user nor admin and returned an empty library. Fix:
+// an HttpOnly zimi_session admin cookie, minted at login, that rides both
+// transports.
+//
+// Run against a FRESH passwordless server on plain http; beforeAll configures it:
+// ZIM_DIR=./zims ZIMI_DATA_DIR=/tmp/zimi-admin ZIMI_MANAGE=1 \
+// python3 -m zimi serve --port 8878
+// BASE_URL=http://localhost:8878 npx playwright test --project=webkit \
+// --config=tests/playwright.config.mjs tests/test_admin_private_library.spec.mjs
+
+import { test, expect } from '@playwright/test';
+
+const BASE = process.env.BASE_URL || 'http://localhost:8878';
+const ADMIN_PW = 'adminpw';
+const USER = 'kid';
+const USER_PW = 'crayons';
+
+// Loopback is a private client: set the password + seed a limited user while the
+// instance is still passwordless/open, then flip to private.
+test.beforeAll(async ({ request }) => {
+ await request.post(`${BASE}/manage/set-password`, {
+ headers: { 'Content-Type': 'application/json' },
+ data: { password: ADMIN_PW },
+ });
+ const auth = { 'Content-Type': 'application/json', Authorization: 'Bearer ' + ADMIN_PW };
+ await request.post(`${BASE}/manage/users`, {
+ headers: auth,
+ // A user whose allowlist is EMPTY — so if isolation ever broke, they'd see
+ // the admin's ZIMs and the test would catch it.
+ data: { action: 'create', name: USER, password: USER_PW, allowlist: [] },
+ });
+ await request.post(`${BASE}/manage/public-access`, {
+ headers: auth,
+ data: { mode: 'private' },
+ });
+});
+
+// The client's live view through the (SW-controlled) fetch path — the security
+// contract that actually matters. Also pulls the /list payload so we can assert
+// the admin's library is POPULATED, not just 200-but-empty.
+async function serverView(page) {
+ return page.evaluate(async () => {
+ const who = await (await fetch('/whoami', { credentials: 'same-origin' })).json();
+ const listRes = await fetch('/list?layout=1', { credentials: 'same-origin' });
+ let zimCount = 0;
+ if (listRes.status === 200) {
+ const data = await listRes.json();
+ const zims = Array.isArray(data) ? data : data.zims || [];
+ zimCount = zims.length;
+ }
+ return { role: who.role, listStatus: listRes.status, zimCount };
+ });
+}
+
+async function adminSignIn(page) {
+ await page.goto(BASE);
+ await page.waitForFunction(() => window._loginRequired === true, { timeout: 8000 });
+ await page.fill('#pw-username', 'admin');
+ await page.fill('#pw-input', ADMIN_PW);
+ await page.evaluate(() => submitPw());
+ // submitPw reloads on a gate sign-in; wait until the gate is gone.
+ await expect.poll(() => page.evaluate(() => window._loginRequired), { timeout: 8000 }).toBeFalsy();
+ await page.waitForTimeout(400);
+}
+
+test('private-mode admin sees a POPULATED library (not the empty state)', async ({ page }) => {
+ await adminSignIn(page);
+ // The authoritative contract: the admin's DATA endpoints return the WHOLE
+ // library, not the empty/limited set the bug produced. Read it through the
+ // client's own (SW-controlled) fetch path — a plain fetch with no Auth header,
+ // exactly like the app's /list call. This is stronger than scraping the home
+ // grid (whose exact DOM varies with how many ZIMs the fixture installs).
+ const view = await serverView(page);
+ expect(view.role).toBe('admin');
+ expect(view.listStatus).toBe(200);
+ expect(view.zimCount, 'admin /list must return the installed ZIMs').toBeGreaterThan(0);
+ // And the login gate must be gone — the admin is inside the app, not stuck at
+ // the sign-in overlay or a blank library.
+ expect(await page.evaluate(() => window._loginRequired)).toBeFalsy();
+ await expect(page.locator('#pw-overlay.open')).toHaveCount(0);
+});
+
+test('private-mode admin can OPEN an article — the /w/ iframe renders, not blank', async ({ page }) => {
+ await adminSignIn(page);
+ // The reader iframe is a browser navigation carrying only the cookie. Load a
+ // real /w/ content URL in an iframe and confirm the server served it (200) AND
+ // the frame has a non-empty document — the exact thing that was blank before.
+ const result = await page.evaluate(async () => {
+ const listRes = await fetch('/list?layout=1', { credentials: 'same-origin' });
+ const data = await listRes.json();
+ const zims = Array.isArray(data) ? data : data.zims || [];
+ const z = zims[0];
+ const url = '/w/' + z.name + '/' + (z.main_path || 'index');
+ const status = (await fetch(url, { credentials: 'same-origin' })).status;
+ // Now the true iframe transport: no Authorization header possible.
+ const bodyLen = await new Promise((resolve) => {
+ const f = document.createElement('iframe');
+ f.style.display = 'none';
+ f.onload = () => {
+ try {
+ resolve((f.contentDocument.body.innerText || '').length);
+ } catch (e) {
+ resolve(-1);
+ }
+ f.remove();
+ };
+ f.src = url;
+ document.body.appendChild(f);
+ });
+ return { status, bodyLen };
+ });
+ expect(result.status, '/w/ content must serve to a cookie-only request').toBe(200);
+ expect(result.bodyLen, 'the reader iframe must render a non-empty article').toBeGreaterThan(0);
+});
+
+test('admin logout in private mode returns to the gate and cuts off access', async ({ page }) => {
+ await adminSignIn(page);
+ expect((await serverView(page)).role).toBe('admin');
+ // manageLogout POSTs /logout (dropping the admin cookie session) then reloads.
+ await page.evaluate(() => manageLogout());
+ await page.waitForFunction(() => window._loginRequired === true, { timeout: 8000 });
+ await expect(page.locator('#pw-overlay.open')).toBeVisible();
+ const view = await serverView(page);
+ expect(view.role).toBe('anonymous');
+ expect(view.listStatus).toBe(401);
+});
diff --git a/tests/test_almanac_hero_clock.spec.mjs b/tests/test_almanac_hero_clock.spec.mjs
index 2ee2eb95..78e37883 100644
--- a/tests/test_almanac_hero_clock.spec.mjs
+++ b/tests/test_almanac_hero_clock.spec.mjs
@@ -10,7 +10,7 @@
// Start a server first (no ZIMs needed):
// ZIM_DIR=/tmp/zimi-empty python3 -m zimi serve --port 8877
// Run:
-// BASE_URL=http://localhost:8877 npx playwright test tests/test_almanac_hero_clock.spec.mjs
+// BASE_URL=http://localhost:8877 npx playwright test --config=tests/playwright.config.mjs tests/test_almanac_hero_clock.spec.mjs
import { test, expect } from '@playwright/test';
diff --git a/tests/test_backup.py b/tests/test_backup.py
new file mode 100644
index 00000000..2d414ddc
--- /dev/null
+++ b/tests/test_backup.py
@@ -0,0 +1,566 @@
+"""Handler-level tests for the backup & export hub (v1.8.1, schema v2).
+
+Covers the server half of a backup bundle: `_build_backup_bundle` (device +
+admin-only server scope), and the `/manage/backup` POST importer — a two-step
+preview→apply that MERGES by default (union by identity, incoming wins) with an
+overwrite escape hatch. Per-browser state (bookmarks/history/preferences) is
+client-only and never reaches the server, so it isn't exercised here.
+"""
+
+import os
+import sys
+from urllib.parse import urlparse
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import zimi.manage as manage # noqa: E402
+import zimi.server as server # noqa: E402
+
+
+class _Handler:
+ def __init__(self, private=True, auth=None):
+ self.status = None
+ self.body = None
+ self._private = private
+ self.headers = {}
+ if auth is not None:
+ self.headers["Authorization"] = auth
+
+ def _json(self, status, body):
+ self.status = status
+ self.body = body
+ return None
+
+ def _is_private_client(self):
+ return self._private
+
+
+def _setup(monkeypatch, tmp_path, *, password_hash="", private=True):
+ data_dir = tmp_path / "data"
+ data_dir.mkdir()
+ monkeypatch.setattr(server, "ZIMI_DATA_DIR", str(data_dir))
+ monkeypatch.setattr(server, "ZIMI_MANAGE", True)
+ monkeypatch.setattr(manage, "_get_manage_password_hash", lambda: password_hash)
+ monkeypatch.setattr(manage, "_get_api_token", lambda: "")
+ # No ZIMs on disk in these tests — keep the library list empty + fast.
+ monkeypatch.setattr(server, "list_zims", lambda: [])
+ monkeypatch.setattr(server, "get_zim_files", lambda: {})
+ return data_dir
+
+
+def _post(handler, body):
+ manage.handle_manage_post(handler, urlparse("/manage/backup"), body)
+ return handler.status, handler.body
+
+
+def _apply(handler, body):
+ """POST an apply (the two-step's confirm leg)."""
+ b = dict(body)
+ b["action"] = "apply"
+ return _post(handler, b)
+
+
+# ── Export shape ──
+
+
+def test_build_bundle_shape(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ bundle = manage._build_backup_bundle()
+ assert bundle["schema"] == "zimi-backup"
+ assert bundle["schema_version"] == manage._BACKUP_SCHEMA_VERSION
+ assert bundle["library"] == []
+ assert bundle["collections"] == {"version": 1, "favorites": [], "collections": {}}
+ assert bundle["library_layout"] == {
+ "overrides": {},
+ "section_order": [],
+ "sections": [],
+ }
+ assert "created" in bundle and "zimi_version" in bundle
+
+
+def test_build_bundle_lists_installed_library(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ monkeypatch.setattr(
+ server,
+ "list_zims",
+ lambda: [
+ {
+ "name": "wikipedia_en",
+ "file": "wikipedia_en_2026-01.zim",
+ "date": "2026-01",
+ "language": "eng",
+ "article_count": 100,
+ "size_bytes": 999,
+ "title": "Wikipedia",
+ }
+ ],
+ )
+ lib = manage._build_backup_bundle()["library"]
+ assert lib == [
+ {
+ "name": "wikipedia_en",
+ "file": "wikipedia_en_2026-01.zim",
+ "date": "2026-01",
+ "language": "eng",
+ "article_count": 100,
+ "size_bytes": 999,
+ "title": "Wikipedia",
+ }
+ ]
+
+
+# ── Import round-trips (apply leg) ──
+
+
+def test_import_restores_collections(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ bundle = {
+ "schema": "zimi-backup",
+ "collections": {
+ "version": 1,
+ "favorites": ["wikipedia_en"],
+ "collections": {"survival": {"label": "Survival", "zims": ["a", "b"]}},
+ },
+ }
+ status, body = _apply(_Handler(), bundle)
+ assert status == 200
+ assert "collections" in body["applied"]
+ saved = server._load_collections()
+ assert saved["favorites"] == ["wikipedia_en"]
+ assert saved["collections"]["survival"]["zims"] == ["a", "b"]
+
+
+def test_import_restores_layout(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ bundle = {
+ "schema": "zimi-backup",
+ "library_layout": {
+ "overrides": {"wikipedia_en": "Books"},
+ "section_order": ["cat:Books"],
+ },
+ }
+ status, body = _apply(_Handler(), bundle)
+ assert status == 200
+ assert "library_layout" in body["applied"]
+ layout = server._load_library_layout()
+ assert layout["overrides"] == {"wikipedia_en": "Books"}
+ assert layout["section_order"] == ["cat:Books"]
+
+
+def test_import_partial_bundle_applies_only_present(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ status, body = _apply(
+ _Handler(),
+ {"schema": "zimi-backup", "library_layout": {"overrides": {"a": "Books"}}},
+ )
+ assert status == 200
+ assert body["applied"] == ["library_layout"]
+
+
+def test_import_empty_bundle_ok_noop(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ status, body = _apply(_Handler(), {"schema": "zimi-backup"})
+ assert status == 200
+ assert body["applied"] == []
+
+
+# ── Merge rules: union / dedupe / newest-wins ──
+
+
+def test_favorites_merge_union_and_dedupe(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ server._save_collections({"version": 1, "favorites": ["old"], "collections": {}})
+ status, body = _apply(
+ _Handler(),
+ {
+ "schema": "zimi-backup",
+ # "old" is a duplicate; "new" is added.
+ "collections": {
+ "version": 1,
+ "favorites": ["old", "new"],
+ "collections": {},
+ },
+ },
+ )
+ assert status == 200
+ # Union preserves the existing entry, appends the new one, drops the dupe.
+ assert server._load_collections()["favorites"] == ["old", "new"]
+ assert body["preview"]["collections"]["fav_dupes"] == 1
+
+
+def test_collections_newest_wins_on_conflict(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ server._save_collections(
+ {
+ "version": 1,
+ "favorites": [],
+ "collections": {"c": {"label": "current", "updated": 100}},
+ }
+ )
+ # Incoming carries a newer timestamp → it wins.
+ _apply(
+ _Handler(),
+ {
+ "schema": "zimi-backup",
+ "collections": {
+ "version": 1,
+ "favorites": [],
+ "collections": {"c": {"label": "incoming", "updated": 200}},
+ },
+ },
+ )
+ assert server._load_collections()["collections"]["c"]["label"] == "incoming"
+
+
+def test_collections_older_incoming_loses(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ server._save_collections(
+ {
+ "version": 1,
+ "favorites": [],
+ "collections": {"c": {"label": "current", "updated": 300}},
+ }
+ )
+ _apply(
+ _Handler(),
+ {
+ "schema": "zimi-backup",
+ "collections": {
+ "version": 1,
+ "favorites": [],
+ "collections": {"c": {"label": "incoming", "updated": 200}},
+ },
+ },
+ )
+ assert server._load_collections()["collections"]["c"]["label"] == "current"
+
+
+def test_layout_overrides_merge_not_replace(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ server._save_library_layout({"overrides": {"a": "Books"}, "section_order": []})
+ _apply(
+ _Handler(),
+ {"schema": "zimi-backup", "library_layout": {"overrides": {"b": "Docs"}}},
+ )
+ # Merge keeps the existing override AND adds the incoming one.
+ assert server._load_library_layout()["overrides"] == {"a": "Books", "b": "Docs"}
+
+
+def test_overwrite_replaces_favorites(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ server._save_collections({"version": 1, "favorites": ["old"], "collections": {}})
+ _post(
+ _Handler(),
+ {
+ "action": "apply",
+ "overwrite": True,
+ "schema": "zimi-backup",
+ "collections": {"version": 1, "favorites": ["new"], "collections": {}},
+ },
+ )
+ assert server._load_collections()["favorites"] == ["new"]
+
+
+def test_import_favorites_capped(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ _apply(
+ _Handler(),
+ {
+ "schema": "zimi-backup",
+ "collections": {
+ "version": 1,
+ "favorites": [str(i) for i in range(200)],
+ "collections": {},
+ },
+ },
+ )
+ assert len(server._load_collections()["favorites"]) == 100
+
+
+# ── Preview-before-apply contract ──
+
+
+def test_preview_is_default_and_applies_nothing(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ server._save_collections({"version": 1, "favorites": ["keep"], "collections": {}})
+ status, body = _post( # no action → preview
+ _Handler(),
+ {
+ "schema": "zimi-backup",
+ "collections": {"version": 1, "favorites": ["new"], "collections": {}},
+ },
+ )
+ assert status == 200
+ assert body["status"] == "preview"
+ assert body["preview"]["collections"]["fav_added"] == 1
+ # Nothing was written — the existing favorites are untouched.
+ assert server._load_collections()["favorites"] == ["keep"]
+
+
+def test_preview_reports_missing_zims(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ monkeypatch.setattr(server, "get_zim_files", lambda: {"have_it": object()})
+ status, body = _post(
+ _Handler(),
+ {
+ "schema": "zimi-backup",
+ "library": [
+ {"name": "have_it"},
+ {"name": "missing_a"},
+ {"name": "missing_b"},
+ ],
+ },
+ )
+ assert status == 200
+ assert body["preview"]["missing_zims"] == 2
+
+
+# ── Validation ──
+
+
+def test_import_rejects_foreign_schema(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ status, body = _apply(_Handler(), {"schema": "something-else"})
+ assert status == 400
+
+
+def test_import_rejects_bad_collections(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ status, _ = _apply(
+ _Handler(), {"schema": "zimi-backup", "collections": {"favorites": "nope"}}
+ )
+ assert status == 400
+
+
+def test_import_rejects_bad_layout(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ status, _ = _apply(
+ _Handler(),
+ {"schema": "zimi-backup", "library_layout": {"section_order": ["bogus:x"]}},
+ )
+ assert status == 400
+
+
+# ── Auth matrix (mirrors library-layout) ──
+
+
+def test_import_passwordless_public_locked(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path, private=False)
+ status, body = _apply(_Handler(private=False), {"schema": "zimi-backup"})
+ assert status == 403
+ assert body["error"] == "public_locked"
+
+
+# ── Full-server scope: build + admin gate ──
+
+
+def _seed_server_state(monkeypatch, tmp_path):
+ """Give the instance some server-owned state so a server bundle is non-empty."""
+ from zimi import library as _lib
+ from zimi import p2p
+ from zimi import users as _users
+
+ monkeypatch.setattr(
+ server,
+ "_DOWNLOAD_SCHEDULE_CONFIG",
+ str(tmp_path / "data" / "download_schedule.json"),
+ raising=False,
+ )
+ p2p.set_prefs_path(str(tmp_path / "data" / "bt_prefs.json"))
+ _users._save_users({"alice": {"name": "alice", "pw": "HASH", "role": "user"}})
+ _users.set_public_access("open")
+ _lib._save_download_schedule(True, "02:00", "05:00", True, 20)
+ p2p.set_pref("seed", False)
+ _lib.record_seed("wikipedia_en_2026.zim")
+
+
+def test_build_server_bundle_includes_hashes_and_policy(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ _seed_server_state(monkeypatch, tmp_path)
+ bundle = manage._build_backup_bundle(scope="server")
+ assert bundle["scope"] == "server"
+ # users.json rides along WITH hashes — it's the admin's own backup.
+ assert bundle["users"]["alice"]["pw"] == "HASH"
+ assert bundle["public_access"]["mode"] == "open"
+ assert bundle["schedule"]["upload_restrict"] is True
+ assert bundle["bt_prefs"]["seed"] is False
+ assert "wikipedia_en_2026.zim" in bundle["seed_intents"]
+ # v3 additions — the rest of a full-restore's coverage.
+ assert isinstance(bundle["hot_zims"], list)
+ assert set(bundle["auto_update"]) == {"enabled", "frequency"}
+ assert isinstance(bundle["history"], list)
+ assert isinstance(bundle["user_data"], dict)
+
+
+def test_device_bundle_has_no_server_state(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ _seed_server_state(monkeypatch, tmp_path)
+ bundle = manage._build_backup_bundle() # device
+ assert bundle["scope"] == "device"
+ for k in (
+ "users",
+ "public_access",
+ "schedule",
+ "bt_prefs",
+ "seed_intents",
+ "hot_zims",
+ "auto_update",
+ "history",
+ "user_data",
+ ):
+ assert k not in bundle
+
+
+def test_server_export_requires_admin(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ monkeypatch.setattr(manage, "admin_kind", lambda h: None) # not an admin
+ h = _Handler()
+ manage.handle_manage_get(
+ h, urlparse("/manage/backup?scope=server"), {"scope": ["server"]}
+ )
+ assert h.status == 403
+
+
+def test_server_restore_refuses_non_admin(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ _seed_server_state(monkeypatch, tmp_path)
+ monkeypatch.setattr(manage, "admin_kind", lambda h: None)
+ # A server-scope bundle from a non-admin session is refused in BOTH legs.
+ for action in ("preview", "apply"):
+ status, body = _post(
+ _Handler(),
+ {
+ "schema": "zimi-backup",
+ "scope": "server",
+ "action": action,
+ "users": {"mallory": {"name": "mallory", "pw": "X", "role": "admin"}},
+ },
+ )
+ assert status == 403
+ # And the malicious user was never written.
+ from zimi import users as _users
+
+ assert "mallory" not in _users._load_users()
+
+
+def test_server_restore_merges_users_for_admin(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ _seed_server_state(monkeypatch, tmp_path) # existing user "alice"
+ monkeypatch.setattr(manage, "admin_kind", lambda h: "primary")
+ status, body = _apply(
+ _Handler(),
+ {
+ "schema": "zimi-backup",
+ "scope": "server",
+ "users": {"bob": {"name": "bob", "pw": "BHASH", "role": "user"}},
+ },
+ )
+ assert status == 200
+ assert "users" in body["applied"]
+ from zimi import users as _users
+
+ merged = _users._load_users()
+ assert "alice" in merged and "bob" in merged # union, not replace
+
+
+def test_device_bundle_ignores_stray_server_keys(monkeypatch, tmp_path):
+ """A device bundle carrying server keys must NOT touch server state — the
+ scope, not the presence of keys, gates the server path."""
+ _setup(monkeypatch, tmp_path)
+ _seed_server_state(monkeypatch, tmp_path)
+ monkeypatch.setattr(manage, "admin_kind", lambda h: "primary")
+ _apply(
+ _Handler(),
+ {
+ "schema": "zimi-backup", # no scope → device
+ "users": {"mallory": {"name": "mallory", "pw": "X", "role": "admin"}},
+ },
+ )
+ from zimi import users as _users
+
+ assert "mallory" not in _users._load_users()
+
+
+# ── v3 server state: hot list, auto-update, history, per-user data round-trip ──
+
+
+def test_server_restore_roundtrips_v3_fields(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ _seed_server_state(monkeypatch, tmp_path)
+ monkeypatch.setattr(manage, "admin_kind", lambda h: "primary")
+ monkeypatch.setattr(
+ server,
+ "_AUTO_UPDATE_CONFIG",
+ str(tmp_path / "data" / "auto_update.json"),
+ raising=False,
+ )
+ monkeypatch.delenv("ZIMI_HOT_ZIMS", raising=False)
+ monkeypatch.delenv("ZIMI_AUTO_UPDATE", raising=False)
+ monkeypatch.setattr(server, "_auto_update_env_locked", False, raising=False)
+ from zimi import library as _lib
+
+ status, body = _apply(
+ _Handler(),
+ {
+ "schema": "zimi-backup",
+ "scope": "server",
+ "hot_zims": ["wikipedia_en", "gutenberg"],
+ "auto_update": {"enabled": True, "frequency": "daily"},
+ "history": [{"event": "download", "name": "x"}],
+ "user_data": {
+ "kid": {"version": 1, "bookmarks": [{"zim": "a", "path": "b"}]}
+ },
+ },
+ )
+ assert status == 200
+ for k in ("hot_zims", "auto_update", "history", "user_data"):
+ assert k in body["applied"]
+ assert server.get_hot_zims() == ["wikipedia_en", "gutenberg"]
+ assert _lib._load_auto_update_config() == (True, "daily")
+ assert server._load_history() == [{"event": "download", "name": "x"}]
+ from zimi import users as _users
+
+ assert _users.load_user_data("kid")["bookmarks"] == [{"zim": "a", "path": "b"}]
+
+
+def test_history_restore_preserves_running_log_without_overwrite(monkeypatch, tmp_path):
+ """Merge-mode restore into a NON-empty history is a no-op (a live server's
+ real event stream is never duplicated/reordered); overwrite replaces it."""
+ _setup(monkeypatch, tmp_path)
+ monkeypatch.setattr(manage, "admin_kind", lambda h: "primary")
+ server._save_history([{"event": "existing"}])
+ _apply(
+ _Handler(),
+ {
+ "schema": "zimi-backup",
+ "scope": "server",
+ "history": [{"event": "incoming"}],
+ },
+ )
+ assert server._load_history() == [{"event": "existing"}] # unchanged
+ _post(
+ _Handler(),
+ {
+ "action": "apply",
+ "overwrite": True,
+ "schema": "zimi-backup",
+ "scope": "server",
+ "history": [{"event": "incoming"}],
+ },
+ )
+ assert server._load_history() == [{"event": "incoming"}]
+
+
+def test_hot_zims_env_lock_wins_over_restore(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ monkeypatch.setattr(manage, "admin_kind", lambda h: "primary")
+ monkeypatch.setenv("ZIMI_HOT_ZIMS", "locked_zim")
+ status, body = _apply(
+ _Handler(),
+ {"schema": "zimi-backup", "scope": "server", "hot_zims": ["from_backup"]},
+ )
+ assert status == 200
+ assert "hot_zims" not in body["applied"] # env-locked → skipped
+ assert server.get_hot_zims() == ["locked_zim"]
diff --git a/tests/test_backup_hub.spec.mjs b/tests/test_backup_hub.spec.mjs
new file mode 100644
index 00000000..2f297284
--- /dev/null
+++ b/tests/test_backup_hub.spec.mjs
@@ -0,0 +1,109 @@
+// Backup hub (v1.8.1): the two clearly-separated cards — "My data" and "Server
+// backup" — each with their own Export + Import, scope-validated imports, and a
+// signed-in user's Save-to / Restore-from-server round-trip.
+//
+// Start a password-protected server with a scratch data dir first:
+// ZIM_DIR=./zims ZIMI_DATA_DIR=/tmp/zimi-backup-hub ZIMI_MANAGE=1 \
+// python3 -m zimi serve --port 8877
+// Run:
+// BASE_URL=http://localhost:8877 npx playwright test \
+// --config=tests/playwright.config.mjs tests/test_backup_hub.spec.mjs
+
+import { test, expect } from '@playwright/test';
+
+const BASE = process.env.BASE_URL || 'http://localhost:8893';
+const PW = 'hunter2';
+
+test.beforeAll(async ({ request }) => {
+ // Loopback client is private, so it may set the admin password.
+ await request.post(`${BASE}/manage/set-password`, {
+ headers: { 'Content-Type': 'application/json' },
+ data: { password: PW },
+ });
+ // A named user for the signed-in My-data flow.
+ await request.post(`${BASE}/manage/users`, {
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${PW}` },
+ data: { action: 'create', name: 'kiddo', password: 'kidpw', role: 'user' },
+ });
+});
+
+async function freshAdminServerPane(page) {
+ await page.goto(BASE);
+ await page.evaluate(() => { localStorage.clear(); sessionStorage.clear(); });
+ await page.goto(BASE);
+ await page.waitForFunction(() => typeof enterManage === 'function');
+ // Seed the admin token + enable manage directly so the test doesn't race the
+ // boot probe; then enter manage and land on the Server pane.
+ await page.evaluate((pw) => { manageEnabled = true; _manageToken = pw; _manageUser = 'admin'; enterManage(); }, PW);
+ await expect.poll(() => page.evaluate(() => mode)).toBe('manage');
+ // renderManage fetches /manage/status then lands on the Library pane; its
+ // async switchMs('library') can clobber a one-shot switch, so re-apply
+ // switchMs('server') each poll until the Server pane (and its cards) sticks.
+ await expect.poll(() =>
+ page.evaluate(() => { switchMs('server'); return !!document.querySelector('#ms-server-file'); })
+ ).toBe(true);
+}
+
+test('admin Server pane renders both cards with separate controls', async ({ page }) => {
+ await freshAdminServerPane(page);
+ // My-data card controls.
+ await expect(page.locator('#ms-mydata-file')).toHaveCount(1);
+ await expect(page.locator('#ms-mydata-overwrite')).toHaveCount(1);
+ await expect(page.locator('#ms-mydata-status')).toHaveCount(1);
+ // Server-backup card controls — distinct ids, its own import/preview slot.
+ await expect(page.locator('#ms-server-file')).toHaveCount(1);
+ await expect(page.locator('#ms-server-overwrite')).toHaveCount(1);
+ await expect(page.locator('#ms-server-import')).toHaveCount(1);
+ // Admin (no named-user session) → the My-data card is file-only, no
+ // Save/Restore-to-server buttons.
+ await expect(page.locator('button', { hasText: 'Save to server' })).toHaveCount(0);
+});
+
+test('scope-mismatch imports fail clearly, pointing at the right card', async ({ page }) => {
+ await freshAdminServerPane(page);
+ // A My-data bundle dropped on the Server-backup card.
+ await page.evaluate(() =>
+ _previewServerBackup(JSON.stringify({ schema: 'zimi-backup', scope: 'my-data', bookmarks: [] }))
+ );
+ await expect.poll(() =>
+ page.evaluate(() => document.getElementById('ms-server-status').textContent)
+ ).toContain('My data card');
+ // A server bundle dropped on the My-data card.
+ await page.evaluate(() =>
+ _applyMyDataFile(JSON.stringify({ schema: 'zimi-backup', scope: 'server' }))
+ );
+ await expect.poll(() =>
+ page.evaluate(() => document.getElementById('ms-mydata-status').textContent)
+ ).toContain('Server backup card');
+});
+
+test('signed-in user saves then restores My data from their server account', async ({ page }) => {
+ await page.goto(BASE);
+ await page.evaluate(() => { localStorage.clear(); sessionStorage.clear(); });
+ await page.goto(BASE);
+ await page.waitForFunction(() => typeof openLoginModal === 'function');
+ await page.evaluate(() => openLoginModal());
+ await page.waitForSelector('#pw-overlay.open');
+ await page.fill('#pw-username', 'kiddo');
+ await page.fill('#pw-input', 'kidpw');
+ await page.evaluate(() => submitPw());
+ await expect.poll(() => page.evaluate(() => !!(_userSession && _userSession.name))).toBe(true);
+ // The user's account view carries their My-data card (with server buttons).
+ await expect.poll(() =>
+ page.evaluate(() => { manageEnabled = true; enterManage(); return !!document.querySelector('#ms-mydata-file'); })
+ ).toBe(true);
+ await expect(page.locator('button', { hasText: 'Save to server' })).toHaveCount(1);
+ // Seed a bookmark, save to server, wipe locally, restore — it comes back.
+ await page.evaluate(() =>
+ _setStorageJSON(SK.BOOKMARKS, [{ zim: 'w', path: '/a', timestamp: 1 }])
+ );
+ await page.evaluate(() => saveMyDataToServer());
+ await expect.poll(() =>
+ page.evaluate(() => document.getElementById('ms-mydata-status').textContent)
+ ).toContain('account');
+ await page.evaluate(() => _setStorageJSON(SK.BOOKMARKS, []));
+ await page.evaluate(() => restoreMyDataFromServer());
+ await expect.poll(() =>
+ page.evaluate(() => _getStorageJSON(SK.BOOKMARKS, []).length)
+ ).toBe(1);
+});
diff --git a/tests/test_define_touch_position.cjs b/tests/test_define_touch_position.cjs
new file mode 100644
index 00000000..322e50cc
--- /dev/null
+++ b/tests/test_define_touch_position.cjs
@@ -0,0 +1,133 @@
+// DOM-free regression test for the Define-chip touch/iOS positioning logic.
+//
+// Guards three properties added for the iOS system-callout collision fix:
+// 1. Touch devices get a bigger clearance below the selection than desktop
+// (_defineRangeRect's margin), so the chip doesn't sit under the OS
+// callout's tail even though both default to "below the selection."
+// 2. Touch devices flip the chip ABOVE the selection when it's near the top
+// of the viewport — that's exactly where iOS itself has no room above
+// and flips its own callout below, so mirroring keeps them apart.
+// 3. Desktop (non-touch) positioning is completely unaffected: no near-top
+// flip, margin stays at the original 4px.
+//
+// Pure-helper approach, same pattern as test_reader_font.cjs: extract the
+// constants + functions straight from app.js by source markers, eval them in
+// a sandboxed vm context with a controllable matchMedia + a fake popover
+// element, and assert on the computed style.left/top.
+//
+// Run: node tests/test_define_touch_position.cjs (exit 0 = pass, non-zero = fail)
+
+const fs = require('fs');
+const path = require('path');
+const vm = require('vm');
+
+const APP_JS = path.join(__dirname, '..', 'zimi', 'static', 'app.js');
+const src = fs.readFileSync(APP_JS, 'utf8');
+
+function extract(re, label) {
+ const m = src.match(re);
+ if (!m) { throw new Error('could not extract ' + label + ' from app.js'); }
+ return m[0];
+}
+
+const cDelay = extract(/var _DEFINE_TOUCH_DELAY = \d+;.*$/m, '_DEFINE_TOUCH_DELAY');
+const cMargin = extract(/var _DEFINE_TOUCH_MARGIN = \d+;.*$/m, '_DEFINE_TOUCH_MARGIN');
+const cNearTop = extract(/var _DEFINE_NEAR_TOP_PX = \d+;.*$/m, '_DEFINE_NEAR_TOP_PX');
+const fIsTouch = extract(/function _defineIsTouch\(\)\s*\{[\s\S]*?\n\}/, '_defineIsTouch');
+const fRangeRect = extract(/function _defineRangeRect\(frame, range\)\s*\{[\s\S]*?\n\}/, '_defineRangeRect');
+const fPosition = extract(/function _definePosition\(rect\)\s*\{[\s\S]*?\n\}/, '_definePosition');
+
+// Controllable "is this a touch/coarse-pointer device" flag, flipped per test.
+let touchMode = false;
+
+function makeFrame(top, left) {
+ return { getBoundingClientRect: () => ({ top: top, left: left, bottom: top, right: left }) };
+}
+function makeRange(top, bottom, left, right) {
+ return { getBoundingClientRect: () => ({ top: top, bottom: bottom, left: left, right: right }) };
+}
+function makePopover(w, h) {
+ const style = {};
+ return {
+ classList: { add: () => {} },
+ offsetWidth: w, offsetHeight: h,
+ style,
+ };
+}
+
+const sandbox = {
+ window: {
+ matchMedia: () => ({ matches: touchMode }),
+ innerWidth: 390,
+ innerHeight: 800,
+ },
+};
+vm.createContext(sandbox);
+vm.runInContext([cDelay, cMargin, cNearTop, fIsTouch, fRangeRect, fPosition].join('\n'), sandbox);
+
+let failures = 0;
+function check(name, cond) {
+ if (cond) { console.log(' ok - ' + name); }
+ else { console.log(' FAIL - ' + name); failures++; }
+}
+
+// 1. Desktop: unchanged 4px margin below the selection.
+{
+ touchMode = false;
+ const frame = makeFrame(100, 20);
+ const range = makeRange(150, 170, 30, 60); // selection well below viewport top
+ const rect = vm.runInContext('_defineRangeRect(frame, range)', Object.assign(sandbox, { frame, range }));
+ check('desktop margin is 4px below the selection bottom', rect.y === 170 + 100 + 4);
+}
+
+// 2. Touch: bigger clearance below the selection (collision margin).
+{
+ touchMode = true;
+ const frame = makeFrame(100, 20);
+ const range = makeRange(150, 170, 30, 60);
+ const rect = vm.runInContext('_defineRangeRect(frame, range)', Object.assign(sandbox, { frame, range }));
+ const margin = vm.runInContext('_DEFINE_TOUCH_MARGIN', sandbox);
+ check('touch margin exceeds desktop margin', margin > 4);
+ check('touch rect.y uses the touch margin', rect.y === 170 + 100 + margin);
+}
+
+// 3. Desktop positioning: no near-top flip even when the selection is at the
+// very top of the viewport — only touch devices mirror the OS callout flip.
+{
+ touchMode = false;
+ const popover = makePopover(200, 60);
+ sandbox.__pop = popover;
+ vm.runInContext('_definePopover = __pop', sandbox);
+ const rect = { x: 20, y: 40, top: 10 }; // near top; would trigger the touch flip
+ vm.runInContext('_definePosition(__rect)', Object.assign(sandbox, { __rect: rect }));
+ check('desktop stays below near the top (no flip)', popover.style.top === '40px');
+}
+
+// 4. Touch positioning away from the top: stays below, per the default.
+{
+ touchMode = true;
+ const popover = makePopover(200, 60);
+ sandbox.__pop = popover;
+ vm.runInContext('_definePopover = __pop', sandbox);
+ const rect = { x: 20, y: 400, top: 380 }; // nowhere near the viewport top
+ vm.runInContext('_definePosition(__rect)', Object.assign(sandbox, { __rect: rect }));
+ check('touch away from the top stays below', popover.style.top === '400px');
+}
+
+// 5. Touch positioning near the top: flips ABOVE the selection, mirroring
+// iOS's own callout-below-when-no-room-above behavior.
+{
+ touchMode = true;
+ const popover = makePopover(200, 60);
+ sandbox.__pop = popover;
+ vm.runInContext('_definePopover = __pop', sandbox);
+ const nearTop = vm.runInContext('_DEFINE_NEAR_TOP_PX', sandbox);
+ const margin = vm.runInContext('_DEFINE_TOUCH_MARGIN', sandbox);
+ const rect = { x: 20, y: 60, top: nearTop - 20 }; // inside the near-top zone
+ vm.runInContext('_definePosition(__rect)', Object.assign(sandbox, { __rect: rect }));
+ const expectedY = Math.max(8, rect.top - 60 - margin);
+ check('touch near the top flips above the selection', popover.style.top === expectedY + 'px');
+}
+
+if (failures) { console.log('\n' + failures + ' check(s) FAILED'); process.exit(1); }
+console.log('\nAll Define touch-position checks passed.');
diff --git a/tests/test_did_you_mean.py b/tests/test_did_you_mean.py
index 41258bfd..528dc1ee 100644
--- a/tests/test_did_you_mean.py
+++ b/tests/test_did_you_mean.py
@@ -284,27 +284,53 @@ def test_eviction_sweep_keeps_frequent_drops_singletons(self):
for w in ("aaa", "bbb", "ccc", "ddd"):
self.assertNotIn(w, vocab)
- def test_saturation_stops_the_scan(self):
- # 19 words pre-seeded to count 2 (via a duplicate within each row's
- # own title), then a 20th, brand-new singleton word hits a word cap
- # of 20. It's the ONLY singleton in the vocab at that moment, so the
- # sweep frees just 1 — under the 10% (of 20 = 2) saturation
- # threshold. That must stop the scan outright: a further row after
- # the trigger must never be read.
- titles = [f"W{i:02d} W{i:02d}" for i in range(19)]
- titles.append("Trigger") # 20th distinct word, count 1 → hits the cap
- titles.append("ShouldNeverAppear ShouldNeverAppear") # must not be read
+ def test_cap_evicts_but_never_stops_the_scan(self):
+ # THE coverage-promise regression guard. Five singleton words fill a
+ # word cap of 5; the 5th admission triggers a tiered sweep. Unlike the
+ # old saturation-stop, the scan MUST continue — so "later", which
+ # appears (three times, to clear the final singleton prune) only in a
+ # row read AFTER the eviction, must land in the vocab. This is exactly
+ # the "mitochondria only shows up once the later indexes are scanned"
+ # case, in miniature.
+ titles = [
+ "Aaa",
+ "Bbb",
+ "Ccc",
+ "Ddd",
+ "Eee", # 5th distinct singleton → hits cap of 5, triggers eviction
+ "Later Later Later", # read only after the sweep — must be admitted
+ ]
+ _make_title_index(self.tmp, "wikipedia", titles)
+ with (
+ mock.patch.object(_search, "_VOCAB_MAX_WORDS", 5),
+ mock.patch.object(_search, "_VOCAB_FETCH_BATCH_SIZE", 1),
+ ):
+ vocab = _search._build_vocab()
+ # The singletons were swept (and pruned) — but the post-eviction word
+ # was still counted, proving the scan did not stop.
+ self.assertEqual(vocab.get("later"), 3)
+
+ def test_admissions_freeze_when_eviction_cannot_free_room(self):
+ # A vocab of nothing but high-count words: the sweep can't free the
+ # required room even at the top tier, so NEW words stop being admitted
+ # — but existing words keep counting and every row is still read.
+ # 19 words pre-seeded to a count above every eviction tier, then a new
+ # word appears after the cap is hit; it must be refused admission,
+ # while an already-known word seen again afterward must still increment.
+ titles = [f"W{i:02d} W{i:02d} W{i:02d} W{i:02d}" for i in range(19)]
+ titles.append("Newword") # 20th distinct → hits cap, can't free → frozen
+ titles.append("W00") # already known → must still be counted (5th time)
_make_title_index(self.tmp, "wikipedia", titles)
with (
mock.patch.object(_search, "_VOCAB_MAX_WORDS", 20),
- mock.patch.object(_search, "_VOCAB_EVICT_MIN_FRACTION", 0.10),
+ mock.patch.object(_search, "_VOCAB_EVICT_MAX_TIER", 3),
mock.patch.object(_search, "_VOCAB_FETCH_BATCH_SIZE", 1),
):
vocab = _search._build_vocab()
- for i in range(19):
- self.assertEqual(vocab.get(f"w{i:02d}"), 2)
- self.assertNotIn("trigger", vocab)
- self.assertNotIn("shouldneverappear", vocab)
+ self.assertNotIn("newword", vocab) # admission frozen
+ self.assertEqual(
+ vocab.get("w00"), 5
+ ) # existing word still counted after freeze
def test_final_prune_drops_remaining_singletons(self):
# No cap pressure here — this isolates the end-of-scan prune from
@@ -377,16 +403,18 @@ def test_builder_version_bump_invalidates_cache(self):
# Back to the real (current) version — same indexes, different sig.
self.assertIsNone(_search._vocab_cache_load())
- def test_round3_cache_on_disk_invalidated_by_round4_version_bump(self):
- # The exact real-world case: a round-3 cache (builder version 2,
- # first-come-eviction vocab, no stride sampling) already sitting on
- # disk when round 4's code (version 3) starts up. It must not be
- # loaded — the algorithm producing "words" changed, even though the
- # indexes on disk did not.
- with mock.patch.object(_search, "_VOCAB_BUILDER_VERSION", 2):
- round3_sig = _search._vocab_signature(self.index_dir)
- _search._vocab_cache_save({"python": 4}, round3_sig)
- self.assertEqual(_search._VOCAB_BUILDER_VERSION, 3)
+ def test_prior_version_cache_on_disk_invalidated_by_version_bump(self):
+ # The exact real-world case behind the 1.8.1 coverage promise: a cache
+ # from the previous builder (the saturating-cap algorithm that dropped
+ # spread-thin words) already sitting on disk when the current code
+ # starts up. It must not be loaded — the algorithm producing "words"
+ # changed, even though the indexes on disk did not — so production
+ # re-scans once and picks up the previously-missing words.
+ with mock.patch.object(
+ _search, "_VOCAB_BUILDER_VERSION", _search._VOCAB_BUILDER_VERSION - 1
+ ):
+ prior_sig = _search._vocab_signature(self.index_dir)
+ _search._vocab_cache_save({"python": 4}, prior_sig)
self.assertIsNone(_search._vocab_cache_load())
def test_round_trip_avoids_rebuild(self):
@@ -678,5 +706,111 @@ def test_absent_suggestion_not_shown(self):
self.assertNotIn("Did you mean", out)
+class EditDistanceTests(unittest.TestCase):
+ """The bounded Levenshtein used to verify trigram candidates."""
+
+ def test_zero_distance(self):
+ self.assertEqual(_search._edit_distance_le2("photo", "photo"), 0)
+
+ def test_one_and_two(self):
+ self.assertEqual(_search._edit_distance_le2("photo", "photto"), 1) # 1 insert
+ self.assertEqual(
+ _search._edit_distance_le2("fotosynthesis", "photosynthesis"), 2
+ )
+
+ def test_caps_at_three(self):
+ # Far-apart strings never report a real distance, just the >2 sentinel.
+ self.assertEqual(_search._edit_distance_le2("cat", "elephant"), 3)
+ self.assertEqual(_search._edit_distance_le2("abcdefg", "hijklmn"), 3)
+
+ def test_length_gap_short_circuits(self):
+ self.assertEqual(_search._edit_distance_le2("ab", "abcde"), 3)
+
+
+class TrigramIndexTests(unittest.TestCase):
+ """The trigram inverted index only covers long words and finds
+ distance-2 corrections an exhaustive edit-2 scan would skip for length."""
+
+ def tearDown(self):
+ _search._trigram_index = None
+
+ def test_index_only_holds_long_words(self):
+ vocab = {"photosynthesis": 10, "cat": 5, "python": 3, "database": 7}
+ idx = _search._build_trigram_index(vocab)
+ # "database" (8) and "photosynthesis" (14) are indexed; "cat"/"python"
+ # (< _TRIGRAM_MIN_LEN) are not.
+ indexed = {w for posting in idx.values() for w in posting}
+ self.assertIn("photosynthesis", indexed)
+ self.assertIn("database", indexed)
+ self.assertNotIn("cat", indexed)
+ self.assertNotIn("python", indexed)
+
+ def test_trigrams_helper(self):
+ self.assertEqual(_search._trigrams("abcd"), {"abc", "bcd"})
+ self.assertEqual(_search._trigrams("ab"), {"ab"}) # short-circuit
+
+ def test_long_word_distance2_correction(self):
+ # THE dist-2 promise case: "fotosynthesis" -> "photosynthesis" (two
+ # edits, 13/14 chars — too long for the exhaustive edit-2 path).
+ vocab = {
+ "photosynthesis": 100,
+ "photosynthetic": 20,
+ "mitochondria": 50,
+ "encyclopedia": 80,
+ }
+ _search._rebuild_trigram_index(vocab)
+ self.assertEqual(
+ _search._best_correction("fotosynthesis", vocab), "photosynthesis"
+ )
+
+ def test_long_word_correction_via_did_you_mean(self):
+ vocab = {"photosynthesis": 100, "chlorophyll": 40}
+ _search._rebuild_trigram_index(vocab)
+ deadline = time.monotonic() + 1.0
+ self.assertEqual(
+ _search._did_you_mean("fotosynthesis", vocab, deadline),
+ "photosynthesis",
+ )
+
+ def test_long_word_no_index_fails_soft(self):
+ # No trigram index (never built) → long-word dist-2 is simply skipped,
+ # never raising, returning None.
+ _search._trigram_index = None
+ vocab = {"photosynthesis": 100}
+ self.assertIsNone(_search._best_correction("fotosynthesis", vocab))
+
+ def test_frequency_breaks_ties_among_near_long_words(self):
+ # Two long vocab words are each EXACTLY distance 2 from the typo (the
+ # two-char suffix differs), and distance 2 from each other, so neither
+ # is reachable by the distance-1 path — the trigram path must choose,
+ # and the far more common one wins.
+ vocab = {"helloworldaa": 900, "helloworldbb": 3}
+ _search._rebuild_trigram_index(vocab)
+ out = _search._best_correction("helloworldxx", vocab)
+ self.assertEqual(out, "helloworldaa") # frequency wins the tie
+
+ def test_index_caps_to_highest_count_long_words(self):
+ # With the per-index word cap lowered, only the highest-count long
+ # words are indexed — a rare long word is excluded, a common one kept —
+ # so query latency stays flat as the vocab's low-count tail grows.
+ vocab = {
+ "photosynthesis": 500, # common → indexed
+ "electroencephalography": 3, # rare long word → dropped by the cap
+ "biodiversity": 400, # common → indexed
+ }
+ with mock.patch.object(_search, "_TRIGRAM_MAX_INDEX_WORDS", 2):
+ idx = _search._build_trigram_index(vocab)
+ indexed = {w for posting in idx.values() for w in posting}
+ self.assertIn("photosynthesis", indexed)
+ self.assertIn("biodiversity", indexed)
+ self.assertNotIn("electroencephalography", indexed)
+
+ def test_reset_clears_trigram_index(self):
+ _search._rebuild_trigram_index({"photosynthesis": 5})
+ self.assertIsNotNone(_search._trigram_index)
+ _search._reset_vocab()
+ self.assertIsNone(_search._trigram_index)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_download_queue.py b/tests/test_download_queue.py
index ef29f65b..37b75f18 100644
--- a/tests/test_download_queue.py
+++ b/tests/test_download_queue.py
@@ -69,11 +69,12 @@ def _kiwix_url(name):
# ────────────────────────────────────────────────────────────────────────────
-def test_default_max_concurrent_is_3(monkeypatch):
+def test_default_max_concurrent(monkeypatch):
monkeypatch.delenv("ZIMI_MAX_CONCURRENT_DOWNLOADS", raising=False)
- from zimi import library
+ monkeypatch.delenv("ZIMI_BT", raising=False)
+ from zimi import library, p2p
- assert library._max_concurrent() == 3
+ assert library._max_concurrent() == p2p.DEFAULT_MAX_ACTIVE_DOWNLOADS
def test_env_var_overrides_max_concurrent(monkeypatch):
@@ -85,9 +86,9 @@ def test_env_var_overrides_max_concurrent(monkeypatch):
def test_env_var_invalid_falls_back_to_default(monkeypatch):
monkeypatch.setenv("ZIMI_MAX_CONCURRENT_DOWNLOADS", "not-a-number")
- from zimi import library
+ from zimi import library, p2p
- assert library._max_concurrent() == library._MAX_CONCURRENT_DEFAULT
+ assert library._max_concurrent() == p2p.DEFAULT_MAX_ACTIVE_DOWNLOADS
def test_env_var_zero_clamps_to_one(monkeypatch):
@@ -126,6 +127,28 @@ def test_at_cap_queues_extras(_no_real_threads, _no_mirrors, monkeypatch):
assert library._download_queue[0]["id"] == qid
+def test_pref_cap_queues_extras(_no_real_threads, _no_mirrors, monkeypatch, tmp_path):
+ """The cap wired through the p2p pref (the UI path, no env) queues past it,
+ and raising it live then drains the queue."""
+ from zimi import library, p2p
+
+ monkeypatch.delenv("ZIMI_MAX_CONCURRENT_DOWNLOADS", raising=False)
+ monkeypatch.delenv("ZIMI_BT", raising=False)
+ monkeypatch.setattr(p2p, "_prefs_path", str(tmp_path / "prefs.json"))
+ assert p2p.set_pref("max_active_downloads", 1)
+
+ library._start_download(_kiwix_url("a"), size_bytes=100)
+ qid = library._start_download(_kiwix_url("b"), size_bytes=200)[0]
+ assert len(library._active_downloads) == 1
+ assert [q["id"] for q in library._download_queue] == [qid]
+
+ # Raise the cap and drain — the queued item promotes into the new slot.
+ assert p2p.set_pref("max_active_downloads", 2)
+ library.drain_download_queue()
+ assert qid in library._active_downloads
+ assert len(library._download_queue) == 0
+
+
def test_queue_orders_smallest_first(_no_real_threads, _no_mirrors, monkeypatch):
"""When the cap is full, queued items are sorted smallest-first."""
monkeypatch.setenv("ZIMI_MAX_CONCURRENT_DOWNLOADS", "1")
@@ -274,9 +297,7 @@ def _read_pending(data_dir):
return json.load(f)["pending"]
-def test_pending_downloads_persist_on_enqueue(
- _no_real_threads, _no_mirrors, _data_dir
-):
+def test_pending_downloads_persist_on_enqueue(_no_real_threads, _no_mirrors, _data_dir):
library._start_download(_kiwix_url("aaa"), size_bytes=100)
library._start_download(_kiwix_url("bbb"), size_bytes=200)
pending = _read_pending(_data_dir)
@@ -376,9 +397,7 @@ def test_download_refused_when_size_exceeds_free_space(
monkeypatch.setattr(
library.shutil, "disk_usage", lambda p: _FakeUsage(free=5 * 1024**3)
)
- dl_id, err = library._start_download(
- _kiwix_url("big"), size_bytes=10 * 1024**3
- )
+ dl_id, err = library._start_download(_kiwix_url("big"), size_bytes=10 * 1024**3)
assert dl_id is None
assert "disk space" in err.lower()
diff --git a/tests/test_download_schedule.py b/tests/test_download_schedule.py
new file mode 100644
index 00000000..d4600a64
--- /dev/null
+++ b/tests/test_download_schedule.py
@@ -0,0 +1,386 @@
+"""Tests for scheduled / night-window downloads.
+
+When scheduling is enabled, downloads started outside the local-time window
+are held in the queue as ``scheduled`` and released when the window opens
+(by the watcher tick or a config change). Window logic is server-local time,
+supports overnight-spanning ranges, and never traps downloads on bad config.
+"""
+
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import zimi.library as library # noqa: E402
+import zimi.server as server # noqa: E402
+
+
+@pytest.fixture(autouse=True)
+def _reset(tmp_path, monkeypatch):
+ monkeypatch.setattr(server, "ZIM_DIR", str(tmp_path))
+ monkeypatch.setattr(
+ server,
+ "_DOWNLOAD_SCHEDULE_CONFIG",
+ str(tmp_path / "download_schedule.json"),
+ raising=False,
+ )
+ monkeypatch.delenv("ZIMI_DL_WINDOW", raising=False)
+ with library._download_lock:
+ library._active_downloads.clear()
+ library._download_queue.clear()
+ library._download_counter = 0
+ yield
+ with library._download_lock:
+ library._active_downloads.clear()
+ library._download_queue.clear()
+
+
+@pytest.fixture
+def _no_real_threads(monkeypatch):
+ class _FakeThread:
+ def __init__(self, target=None, args=(), kwargs=None, daemon=None, name=None):
+ self.target, self.args, self.kwargs = target, args, kwargs or {}
+
+ def start(self):
+ pass
+
+ monkeypatch.setattr(library.threading, "Thread", _FakeThread)
+
+
+def _kiwix_url(name):
+ return f"https://download.kiwix.org/zim/{name}.zim"
+
+
+# ────────────────────────────────────────────────────────────────────────────
+# Pure window math
+# ────────────────────────────────────────────────────────────────────────────
+
+
+def test_parse_hhmm():
+ assert library._parse_hhmm("01:00") == 60
+ assert library._parse_hhmm("07:30") == 450
+ assert library._parse_hhmm("23:59") == 1439
+ assert library._parse_hhmm("00:00") == 0
+
+
+def test_parse_hhmm_rejects_garbage():
+ for bad in ("", "24:00", "1:60", "aa:bb", "7", "07-00", None, "25:10"):
+ assert library._parse_hhmm(bad) is None
+
+
+def test_fmt_hhmm():
+ assert library._fmt_hhmm(60) == "01:00"
+ assert library._fmt_hhmm(450) == "07:30"
+ assert library._fmt_hhmm(1440) == "00:00" # wraps
+
+
+def test_in_window_daytime_range():
+ # 01:00–07:00
+ s, e = 60, 420
+ assert not library._in_window(0, s, e) # 00:00 before
+ assert library._in_window(60, s, e) # 01:00 start inclusive
+ assert library._in_window(300, s, e) # 05:00 inside
+ assert not library._in_window(420, s, e) # 07:00 end exclusive
+ assert not library._in_window(600, s, e) # 10:00 after
+
+
+def test_in_window_spans_midnight():
+ # 22:00–06:00
+ s, e = 1320, 360
+ assert library._in_window(1380, s, e) # 23:00
+ assert library._in_window(0, s, e) # 00:00
+ assert library._in_window(300, s, e) # 05:00
+ assert not library._in_window(360, s, e) # 06:00 end exclusive
+ assert not library._in_window(720, s, e) # 12:00 outside
+ assert library._in_window(1320, s, e) # 22:00 start inclusive
+
+
+def test_in_window_equal_bounds_is_always_open():
+ assert library._in_window(0, 120, 120)
+ assert library._in_window(1000, 120, 120)
+
+
+# ────────────────────────────────────────────────────────────────────────────
+# Config load / save
+# ────────────────────────────────────────────────────────────────────────────
+
+
+def test_default_schedule_disabled():
+ sched = library._load_download_schedule()
+ assert sched["enabled"] is False
+ assert sched["start"] == "01:00"
+ assert sched["end"] == "07:00"
+ assert sched["locked"] is False
+
+
+def test_save_and_reload():
+ assert library._save_download_schedule(True, "23:00", "05:00")
+ sched = library._load_download_schedule()
+ assert sched == {
+ "enabled": True,
+ "start": "23:00",
+ "end": "05:00",
+ "locked": False,
+ "upload_restrict": False,
+ "upload_trickle_kb": library._DEFAULT_UPLOAD_TRICKLE_KB,
+ }
+
+
+# ────────────────────────────────────────────────────────────────────────────
+# Upload-window restrictor (restrict seeding to the window)
+# ────────────────────────────────────────────────────────────────────────────
+
+
+def test_upload_fields_persist():
+ assert library._save_download_schedule(True, "01:00", "07:00", True, 25)
+ sched = library._load_download_schedule()
+ assert sched["upload_restrict"] is True
+ assert sched["upload_trickle_kb"] == 25
+
+
+def test_upload_fields_preserved_on_window_edit():
+ # Set the upload policy, then edit only the window — policy must survive.
+ library._save_download_schedule(True, "01:00", "07:00", True, 40)
+ library._save_download_schedule(True, "02:00", "06:00")
+ sched = library._load_download_schedule()
+ assert sched["start"] == "02:00"
+ assert sched["upload_restrict"] is True
+ assert sched["upload_trickle_kb"] == 40
+
+
+def test_upload_trickle_floored_positive():
+ # 0 would mean "unlimited" downstream — a trickle must stay positive.
+ library._save_download_schedule(True, "01:00", "07:00", True, 0)
+ assert library._load_download_schedule()["upload_trickle_kb"] >= 1
+
+
+def test_in_window_now_ignores_enabled(monkeypatch):
+ # Scheduling OFF but the upload restrictor uses raw window membership.
+ library._save_download_schedule(False, "01:00", "07:00", True, 30)
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 180) # 03:00 inside
+ assert library._in_window_now() is True
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 720) # noon outside
+ assert library._in_window_now() is False
+
+
+def test_upload_window_transition_calls_rate_setter(monkeypatch):
+ from zimi import p2p
+
+ calls = []
+ monkeypatch.setattr(p2p, "set_upload_window_cap", lambda kb: calls.append(kb))
+ library._save_download_schedule(True, "01:00", "07:00", True, 30)
+ library._upload_window_applied = None # reset transition memory
+ # Outside the window → trickle to 30.
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 720)
+ library._apply_upload_window()
+ assert calls[-1] == 30
+ # Inside the window → release (None).
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 180)
+ library._apply_upload_window()
+ assert calls[-1] is None
+
+
+def test_upload_window_idempotent_no_transition(monkeypatch):
+ from zimi import p2p
+
+ calls = []
+ monkeypatch.setattr(p2p, "set_upload_window_cap", lambda kb: calls.append(kb))
+ library._save_download_schedule(True, "01:00", "07:00", True, 30)
+ library._upload_window_applied = None
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 720)
+ library._apply_upload_window()
+ n = len(calls)
+ library._apply_upload_window() # same state — must not touch the session again
+ assert len(calls) == n
+
+
+def test_malformed_persisted_times_fall_back():
+ cfg = server._DOWNLOAD_SCHEDULE_CONFIG
+ with open(cfg, "w") as f:
+ f.write('{"enabled": true, "start": "nope", "end": "99:99"}')
+ sched = library._load_download_schedule()
+ assert sched["start"] == "01:00"
+ assert sched["end"] == "07:00"
+
+
+def test_env_var_locks_window(monkeypatch):
+ monkeypatch.setenv("ZIMI_DL_WINDOW", "02:00-04:00")
+ sched = library._load_download_schedule()
+ assert sched["enabled"] is True
+ assert sched["start"] == "02:00"
+ assert sched["end"] == "04:00"
+ assert sched["locked"] is True
+ # A locked window refuses persistence.
+ assert library._save_download_schedule(False, "10:00", "12:00") is False
+
+
+def test_env_var_malformed_ignored(monkeypatch):
+ monkeypatch.setenv("ZIMI_DL_WINDOW", "notatime")
+ assert library._load_download_schedule()["locked"] is False
+
+
+# ────────────────────────────────────────────────────────────────────────────
+# _within_download_window / _schedule_defers_now
+# ────────────────────────────────────────────────────────────────────────────
+
+
+def test_disabled_schedule_always_in_window():
+ library._save_download_schedule(False, "01:00", "07:00")
+ assert library._within_download_window(now_min=720) is True
+ assert library._schedule_defers_now() is False
+
+
+def test_enabled_outside_window_defers(monkeypatch):
+ library._save_download_schedule(True, "01:00", "07:00")
+ # Pin "now" to noon — outside the window.
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 720)
+ assert library._within_download_window() is False
+ assert library._schedule_defers_now() is True
+
+
+def test_enabled_inside_window_does_not_defer(monkeypatch):
+ library._save_download_schedule(True, "01:00", "07:00")
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 180) # 03:00
+ assert library._within_download_window() is True
+ assert library._schedule_defers_now() is False
+
+
+# ────────────────────────────────────────────────────────────────────────────
+# Queue transitions
+# ────────────────────────────────────────────────────────────────────────────
+
+
+def test_outside_window_queues_as_scheduled(_no_real_threads, monkeypatch):
+ monkeypatch.setattr(library, "_fetch_mirrors", lambda url: [])
+ library._save_download_schedule(True, "01:00", "07:00")
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 720) # noon
+ dl_id, err = library._start_download(_kiwix_url("a"), size_bytes=100)
+ assert err is None
+ assert dl_id not in library._active_downloads
+ assert len(library._download_queue) == 1
+ assert library._download_queue[0]["scheduled"] is True
+
+
+def test_inside_window_starts_immediately(_no_real_threads, monkeypatch):
+ monkeypatch.setattr(library, "_fetch_mirrors", lambda url: [])
+ library._save_download_schedule(True, "01:00", "07:00")
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 180) # 03:00
+ dl_id, err = library._start_download(_kiwix_url("a"), size_bytes=100)
+ assert err is None
+ assert dl_id in library._active_downloads
+ assert len(library._download_queue) == 0
+
+
+def test_scheduled_not_drained_outside_window(_no_real_threads, monkeypatch):
+ monkeypatch.setattr(library, "_fetch_mirrors", lambda url: [])
+ library._save_download_schedule(True, "01:00", "07:00")
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 720)
+ library._start_download(_kiwix_url("a"), size_bytes=100)
+ with library._download_lock:
+ library._drain_queue()
+ # Still parked — window is closed.
+ assert len(library._download_queue) == 1
+ assert len(library._active_downloads) == 0
+
+
+def test_window_open_releases_scheduled(_no_real_threads, monkeypatch):
+ monkeypatch.setattr(library, "_fetch_mirrors", lambda url: [])
+ library._save_download_schedule(True, "01:00", "07:00")
+ # Queue it while the window is closed…
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 720)
+ library._start_download(_kiwix_url("a"), size_bytes=100)
+ assert len(library._download_queue) == 1
+ # …then the window opens and the watcher tick releases it.
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 180)
+ library._download_schedule_tick()
+ assert len(library._download_queue) == 0
+ assert len(library._active_downloads) == 1
+
+
+def test_tick_noop_outside_window(_no_real_threads, monkeypatch):
+ monkeypatch.setattr(library, "_fetch_mirrors", lambda url: [])
+ library._save_download_schedule(True, "01:00", "07:00")
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 720)
+ library._start_download(_kiwix_url("a"), size_bytes=100)
+ library._download_schedule_tick() # still outside window
+ assert len(library._download_queue) == 1
+
+
+def test_scheduled_status_serialization(monkeypatch):
+ library._save_download_schedule(True, "01:00", "07:00")
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 180)
+ from zimi import p2p
+
+ monkeypatch.setattr(p2p, "get_download_limit_kb", lambda: 300)
+ monkeypatch.setattr(p2p, "is_bt_down_env_locked", lambda: False)
+ st = library._download_schedule_status()
+ assert st["enabled"] is True
+ assert st["start"] == "01:00"
+ assert st["in_window"] is True
+ assert st["download_kb"] == 300
+ assert st["download_kb_locked"] is False
+
+
+def test_get_downloads_reports_scheduled(_no_real_threads, monkeypatch):
+ monkeypatch.setattr(library, "_fetch_mirrors", lambda url: [])
+ library._save_download_schedule(True, "01:00", "07:00")
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 720)
+ library._start_download(_kiwix_url("a"), size_bytes=100)
+ rows = library._get_downloads()
+ queued = [r for r in rows if r.get("queued")]
+ assert len(queued) == 1
+ assert queued[0]["scheduled"] is True
+
+
+# ────────────────────────────────────────────────────────────────────────────
+# Start-now override (_start_scheduled_now)
+# ────────────────────────────────────────────────────────────────────────────
+
+
+def test_start_now_launches_scheduled_item(_no_real_threads, monkeypatch):
+ monkeypatch.setattr(library, "_fetch_mirrors", lambda url: [])
+ library._save_download_schedule(True, "01:00", "07:00")
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 720) # outside
+ dl_id, _ = library._start_download(_kiwix_url("a"), size_bytes=100)
+ assert library._download_queue[0]["scheduled"] is True
+ # Override the window for this one — still outside it.
+ status, code = library._start_scheduled_now(dl_id)
+ assert (status, code) == ("started", 200)
+ assert dl_id in library._active_downloads
+ assert len(library._download_queue) == 0
+
+
+def test_start_now_at_cap_promotes_to_normal_queue(_no_real_threads, monkeypatch):
+ monkeypatch.setattr(library, "_fetch_mirrors", lambda url: [])
+ monkeypatch.setenv("ZIMI_MAX_CONCURRENT_DOWNLOADS", "1")
+ library._save_download_schedule(True, "01:00", "07:00")
+ # One active (in-window start), one scheduled (outside window).
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 180) # inside
+ library._start_download(_kiwix_url("active"), size_bytes=100)
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 720) # outside
+ sid, _ = library._start_download(_kiwix_url("later"), size_bytes=200)
+ assert library._download_queue[0]["scheduled"] is True
+ # No free slot: start-now clears the scheduled flag but can't launch yet.
+ status, code = library._start_scheduled_now(sid)
+ assert (status, code) == ("queued", 200)
+ assert "scheduled" not in library._download_queue[0]
+ # It now drains on the next slot regardless of the window (not gated).
+ monkeypatch.setattr(library, "_now_local_minutes", lambda: 720)
+ with library._download_lock:
+ library._active_downloads.clear() # free the slot
+ library._drain_queue()
+ assert sid in library._active_downloads
+
+
+def test_start_now_unknown_id_not_found():
+ assert library._start_scheduled_now("nope") == ("not_found", 404)
+
+
+def test_start_now_on_active_item_reports_already_active(_no_real_threads, monkeypatch):
+ monkeypatch.setattr(library, "_fetch_mirrors", lambda url: [])
+ library._save_download_schedule(False, "01:00", "07:00") # off → starts active
+ dl_id, _ = library._start_download(_kiwix_url("a"), size_bytes=100)
+ assert dl_id in library._active_downloads
+ assert library._start_scheduled_now(dl_id) == ("already_active", 200)
diff --git a/tests/test_download_throttle.py b/tests/test_download_throttle.py
new file mode 100644
index 00000000..50876621
--- /dev/null
+++ b/tests/test_download_throttle.py
@@ -0,0 +1,143 @@
+"""Tests for the global HTTP download-speed throttle.
+
+The token bucket (_DownloadThrottle) is shared across every download thread so
+N concurrent pulls sum to the cap, not N × the cap. The pacing math is pure
+(clock injectable) so it can be verified without real sleeps.
+"""
+
+import os
+import sys
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import zimi.library as library # noqa: E402
+import zimi.p2p as p2p # noqa: E402
+
+
+@pytest.fixture(autouse=True)
+def _hermetic_throttle_state():
+ """Snapshot and restore every piece of module-global state these tests
+ touch, so nothing leaks in or out regardless of suite order:
+ - library._rate_cache (the ~2s rate lookup cache)
+ - library._download_throttle (the shared token-bucket singleton)
+ - p2p._prefs_path (set by the mirrors-bt-down test)
+ A dirty _rate_cache or a mid-flight bucket is exactly what produces the
+ "expected 0.0, got 1.0" pacing failures under interleaved runs.
+ """
+ saved_cache = dict(library._rate_cache)
+ saved_prefs = p2p._prefs_path
+ library._rate_cache["ts"] = 0.0
+ library._rate_cache["bps"] = 0
+ library._download_throttle.reset()
+ yield
+ library._rate_cache.clear()
+ library._rate_cache.update(saved_cache)
+ library._download_throttle.reset()
+ p2p._prefs_path = saved_prefs
+
+
+class _FakeClock:
+ def __init__(self):
+ self.t = 0.0
+
+ def __call__(self):
+ return self.t
+
+ def advance(self, dt):
+ self.t += dt
+
+
+# ────────────────────────────────────────────────────────────────────────────
+# Throttle math
+# ────────────────────────────────────────────────────────────────────────────
+
+
+def test_zero_rate_never_throttles():
+ th = library._DownloadThrottle()
+ assert th.consume(1_000_000, 0) == 0.0
+ assert th.consume(1_000_000, -5) == 0.0
+
+
+def test_first_chunk_within_budget_no_sleep():
+ clk = _FakeClock()
+ th = library._DownloadThrottle(clock=clk)
+ # 100 KB/s = 102400 B/s. A 64 KB chunk fits in the first second's burst.
+ assert th.consume(65536, 102400) == 0.0
+
+
+def test_overspend_returns_proportional_sleep():
+ clk = _FakeClock()
+ th = library._DownloadThrottle(clock=clk)
+ rate = 100_000 # bytes/sec
+ # Burst allowance caps at one second (100k). First consume drains it to 0.
+ assert th.consume(100_000, rate) == 0.0
+ # Next consume with no elapsed time goes 50k into deficit -> 0.5s sleep.
+ assert th.consume(50_000, rate) == pytest.approx(0.5, abs=1e-6)
+
+
+def test_elapsed_time_refills_bucket():
+ clk = _FakeClock()
+ th = library._DownloadThrottle(clock=clk)
+ rate = 100_000
+ th.consume(100_000, rate) # drain burst to 0
+ clk.advance(0.5) # refills 50k
+ # 50k available now -> a 50k chunk is free.
+ assert th.consume(50_000, rate) == pytest.approx(0.0, abs=1e-9)
+
+
+def test_burst_allowance_capped_at_one_second():
+ clk = _FakeClock()
+ th = library._DownloadThrottle(clock=clk)
+ rate = 100_000
+ th.consume(1, rate) # seed _last
+ clk.advance(1000) # huge idle — must NOT bank 1000s of credit
+ # Only one second's worth (100k) is available; a 200k read pays for 100k.
+ assert th.consume(200_000, rate) == pytest.approx(1.0, abs=1e-6)
+
+
+def test_aggregate_rate_across_two_streams():
+ """Two 'threads' sharing one bucket are paced to the aggregate rate."""
+ clk = _FakeClock()
+ th = library._DownloadThrottle(clock=clk)
+ rate = 100_000
+ th.consume(100_000, rate) # stream A drains the burst
+ # Stream B, same instant, gets no free credit -> must wait.
+ assert th.consume(100_000, rate) == pytest.approx(1.0, abs=1e-6)
+
+
+def test_reset_clears_state():
+ clk = _FakeClock()
+ th = library._DownloadThrottle(clock=clk)
+ th.consume(100_000, 100_000)
+ th.reset()
+ # Fresh bucket: a within-budget read is free again.
+ assert th.consume(100_000, 100_000) == 0.0
+
+
+# ────────────────────────────────────────────────────────────────────────────
+# Rate lookup + caching + global-cap wiring
+# ────────────────────────────────────────────────────────────────────────────
+
+
+def test_download_rate_bps_reads_global_cap(monkeypatch):
+ library._rate_cache["ts"] = 0.0 # force refresh
+ monkeypatch.setattr(p2p, "get_download_limit_kb", lambda: 256)
+ assert library._download_rate_bps() == 256 * 1024
+
+
+def test_download_rate_bps_zero_is_unlimited(monkeypatch):
+ library._rate_cache["ts"] = 0.0
+ monkeypatch.setattr(p2p, "get_download_limit_kb", lambda: 0)
+ assert library._download_rate_bps() == 0
+
+
+def test_download_limit_mirrors_bt_down(monkeypatch, tmp_path):
+ """The global download cap and the BT down limit are one number."""
+ prefs = tmp_path / "prefs.json"
+ p2p.set_prefs_path(str(prefs))
+ monkeypatch.delenv("ZIMI_BT_DOWN_KB", raising=False)
+ assert p2p.set_pref("bt_down_kb", 512)
+ assert p2p.get_download_limit_kb() == 512
+ assert p2p.get_bt_down_limit_kb() == 512
diff --git a/tests/test_health.py b/tests/test_health.py
index 01f88889..cd2db19e 100644
--- a/tests/test_health.py
+++ b/tests/test_health.py
@@ -11,7 +11,11 @@
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from conftest_zim import build_fixture_zim # noqa: E402
+from conftest_zim import ( # noqa: E402
+ build_empty_text_fixture_zim,
+ build_fixture_zim,
+ build_media_fixture_zim,
+)
import zimi.server as server # noqa: E402
import zimi.health as health # noqa: E402
@@ -85,6 +89,101 @@ def test_report_flags_broken_and_healthy(tmp_path, monkeypatch):
assert st["summary"]["warnings"] == 1
+def test_zero_byte_media_entry_flagged(tmp_path, monkeypatch):
+ """A ZIM that opens fine and has entries can still ship broken media — a
+ 0-byte video the count and size-vs-catalog checks never see. The media
+ sampler must count it and flag the ZIM (issue #38 follow-up: the shape of
+ ted_en_technology_2023-09)."""
+ zdir = tmp_path / "zims"
+ zdir.mkdir()
+ build_media_fixture_zim(str(zdir / "talks_en_2026-06.zim"))
+ ddir = tmp_path / "data"
+ ddir.mkdir()
+ monkeypatch.setattr(server, "ZIM_DIR", str(zdir))
+ monkeypatch.setattr(server, "ZIMI_DATA_DIR", str(ddir))
+ server._archive_pool.clear()
+ server.load_cache(force=True)
+
+ st = _run_check()
+ row = {r["name"]: r for r in st["report"]}["talks"]
+ assert row["opens"] is True
+ assert row["media_sampled"] == 2 # one real + one empty
+ assert row["media_empty"] == 1
+ assert row["status"] == "warn"
+ assert any("empty" in i and "0-byte" in i for i in row["issues"])
+ assert "videos/2/video.webm" in (row.get("media_examples") or [])
+
+
+def test_healthy_media_not_flagged(tmp_path, monkeypatch):
+ """A ZIM whose media all carry bytes stays healthy — the sampler counts
+ them but raises no issue and never demotes an otherwise-clean ZIM."""
+ zdir = tmp_path / "zims"
+ zdir.mkdir()
+ # Rebuild with both videos non-empty by writing a second real-media ZIM.
+ from conftest_zim import _Article, _MediaItem, _StringProvider # noqa: F401
+ from libzim.writer import Creator
+
+ p = str(zdir / "goodtalks_en_2026-06.zim")
+ with Creator(p).config_indexing(True, "eng") as creator:
+ creator.set_mainpath("A/Talks")
+ creator.add_item(_Article("A/Talks", "Talks", b"ok"))
+ creator.add_item(_MediaItem("videos/1/video.webm", "video/webm", b"realbytes1"))
+ creator.add_item(_MediaItem("videos/2/video.webm", "video/webm", b"realbytes2"))
+ creator.add_metadata("Title", "Good Talks")
+ creator.add_metadata("Language", "eng")
+ creator.add_metadata("Description", "media fixture")
+ ddir = tmp_path / "data"
+ ddir.mkdir()
+ monkeypatch.setattr(server, "ZIM_DIR", str(zdir))
+ monkeypatch.setattr(server, "ZIMI_DATA_DIR", str(ddir))
+ server._archive_pool.clear()
+ server.load_cache(force=True)
+
+ st = _run_check()
+ row = {r["name"]: r for r in st["report"]}["goodtalks"]
+ assert row["media_sampled"] == 2
+ assert row["media_empty"] == 0
+ assert row["status"] == "ok"
+ assert not row["issues"]
+
+
+def test_empty_articles_flagged_without_media(tmp_path, monkeypatch):
+ """A ZIM that opens, has entries, and carries no media can still be a broken
+ scrape — every article a 0-byte shell. The universal text-sanity sampler
+ must catch it (the media sampler never fires here) and demote it to warn."""
+ zdir = tmp_path / "zims"
+ zdir.mkdir()
+ build_empty_text_fixture_zim(str(zdir / "empties_en_2026-06.zim"))
+ ddir = tmp_path / "data"
+ ddir.mkdir()
+ monkeypatch.setattr(server, "ZIM_DIR", str(zdir))
+ monkeypatch.setattr(server, "ZIMI_DATA_DIR", str(ddir))
+ server._archive_pool.clear()
+ server.load_cache(force=True)
+
+ st = _run_check()
+ row = {r["name"]: r for r in st["report"]}["empties"]
+ assert row["opens"] is True
+ assert row["entries"] and row["entries"] > 0
+ assert row["text_sampled"] and row["text_sampled"] > 0
+ assert row["text_empty"] == row["text_sampled"]
+ assert row["media_sampled"] is None # no media entries at all
+ assert row["status"] == "warn"
+ assert any("articles empty" in i for i in row["issues"])
+
+
+def test_healthy_text_not_flagged(tmp_path, monkeypatch):
+ """The text-sanity sampler must never demote a normal content ZIM: the
+ survival fixture's articles all carry bytes, so it stays healthy."""
+ _setup(tmp_path, monkeypatch)
+ st = _run_check()
+ good = {r["name"]: r for r in st["report"]}["survival"]
+ assert good["text_sampled"] and good["text_sampled"] > 0
+ assert good["text_empty"] == 0
+ assert good["status"] == "ok"
+ assert not good["issues"]
+
+
def test_second_start_is_rejected_while_running(tmp_path, monkeypatch):
_setup(tmp_path, monkeypatch)
# Occupy the lock to simulate an in-flight run.
diff --git a/tests/test_iframe_link_chaperon.py b/tests/test_iframe_link_chaperon.py
index 2694eb56..c4c09463 100644
--- a/tests/test_iframe_link_chaperon.py
+++ b/tests/test_iframe_link_chaperon.py
@@ -69,21 +69,27 @@ def test_iframe_click_handler_uses_capture_phase():
)
-def test_iframe_contextmenu_uses_no_rewrite_trick():
- """The right-click contextmenu handler also resolves hrefs from the
- iframe — must use the same `_no_rewrite=true` pattern as the left-
- click handler, otherwise wombat ZIMs would have the same doubling
- bug on right-click "Open article" (incomplete fix for #17)."""
+def test_no_custom_contextmenu_inside_article_frame():
+ """Right-click coherence (1.8.1): right-clicking INSIDE article content
+ must yield ONLY the browser's own context menu, which is load-bearing on
+ a ZIM page (copy, open-link-in-new-tab, translate, image save, look up…).
+
+ An earlier build attached a contextmenu listener to the reader iframe that
+ intercepted right-click on in-archive links and swapped in Zimi's own "open
+ article" menu. That was removed: Zimi's custom link menu is now offered ONLY
+ on CHROME article links (search results / home tiles) in the PARENT document.
+ This guards against a regression that re-adds an in-frame interceptor.
+ """
src = _read_app_js()
- # Both occurrences should share the pattern. Easy invariant: the
- # number of "_no_rewrite = true" assignments matches the number of
- # iframe-document listeners that read a.href (click + contextmenu).
- flag_sets = src.count("_no_rewrite = true")
- assert flag_sets >= 2, (
- f"expected ≥2 `_no_rewrite = true` assignments (click + "
- f"contextmenu handlers); found {flag_sets}. The contextmenu "
- f"handler must also use the trick or right-click on wombat "
- f"ZIMs will double the path."
+ assert "frame.contentDocument.addEventListener('contextmenu'" not in src, (
+ "no contextmenu listener may be attached to the reader iframe that shows "
+ "Zimi's own menu — the system menu is load-bearing inside article content "
+ "(1.8.1 right-click coherence)"
+ )
+ # The exact removed call — a link menu positioned in frame-relative coords.
+ assert "_showLinkCtxMenu(e.clientX + frameRect" not in src, (
+ "the reader frame must not open Zimi's link context menu on right-click; "
+ "that affordance lives on parent-document chrome links only"
)
diff --git a/tests/test_interlang.mjs b/tests/test_interlang.mjs
index 8945c157..babf2761 100644
--- a/tests/test_interlang.mjs
+++ b/tests/test_interlang.mjs
@@ -1,5 +1,5 @@
// Cross-language article navigation tests
-// Run against a remote host: BASE_URL=http://knowledge.example.lan npx playwright test tests/test_interlang.mjs --config=playwright.config.mjs
+// Run against a remote host: BASE_URL=http://knowledge.example.lan npx playwright test tests/test_interlang.mjs --config=tests/playwright.config.mjs
import { test, expect } from '@playwright/test';
diff --git a/tests/test_library_layout.py b/tests/test_library_layout.py
index 687976b3..01b9b94c 100644
--- a/tests/test_library_layout.py
+++ b/tests/test_library_layout.py
@@ -96,6 +96,53 @@ def test_order_replaces_not_merges(monkeypatch, tmp_path):
assert body["section_order"] == ["cat:Wikimedia"]
+def test_other_key_is_orderable(monkeypatch, tmp_path):
+ # The uncategorized catch-all rides section_order via the reserved `other`
+ # key so it can be positioned like any real section.
+ _setup(monkeypatch, tmp_path)
+ order = ["other", "cat:Books"]
+ status, body = _post(_Handler(), {"section_order": order})
+ assert status == 200
+ assert body["section_order"] == order
+
+
+def test_sections_round_trip(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ status, body = _post(_Handler(), {"sections": ["Survival", "Field Guides"]})
+ assert status == 200
+ assert body["sections"] == ["Survival", "Field Guides"]
+ assert server._load_library_layout()["sections"] == ["Survival", "Field Guides"]
+
+
+def test_sections_replace_and_dedupe(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ _post(_Handler(), {"sections": ["A", "B"]})
+ # Fully replaces (not merges) and drops case-insensitive duplicates.
+ _, body = _post(_Handler(), {"sections": ["C", "c", "C "]})
+ assert body["sections"] == ["C"]
+
+
+def test_sections_independent_of_overrides_and_order(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ _post(_Handler(), {"overrides": {"z": "Books"}, "section_order": ["cat:Books"]})
+ _, body = _post(_Handler(), {"sections": ["Empty"]})
+ assert body["overrides"] == {"z": "Books"}
+ assert body["section_order"] == ["cat:Books"]
+ assert body["sections"] == ["Empty"]
+
+
+def test_sections_not_a_list_400(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ status, _ = _post(_Handler(), {"sections": "Books"})
+ assert status == 400
+
+
+def test_sections_empty_string_400(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ status, _ = _post(_Handler(), {"sections": ["ok", ""]})
+ assert status == 400
+
+
def test_override_and_order_are_independent(monkeypatch, tmp_path):
_setup(monkeypatch, tmp_path)
_post(_Handler(), {"overrides": {"z": "Books"}})
@@ -187,15 +234,23 @@ def test_corrupt_layout_reads_as_empty(monkeypatch, tmp_path):
data_dir = _setup(monkeypatch, tmp_path)
(data_dir / "library_layout.json").write_text("{ this is not json")
layout = server._load_library_layout()
- assert layout == {"overrides": {}, "section_order": []}
+ assert layout == {"overrides": {}, "section_order": [], "sections": []}
def test_missing_layout_reads_as_empty(monkeypatch, tmp_path):
_setup(monkeypatch, tmp_path)
- assert server._load_library_layout() == {"overrides": {}, "section_order": []}
+ assert server._load_library_layout() == {
+ "overrides": {},
+ "section_order": [],
+ "sections": [],
+ }
def test_wrong_shape_layout_reads_as_empty(monkeypatch, tmp_path):
data_dir = _setup(monkeypatch, tmp_path)
(data_dir / "library_layout.json").write_text(json.dumps([1, 2, 3]))
- assert server._load_library_layout() == {"overrides": {}, "section_order": []}
+ assert server._load_library_layout() == {
+ "overrides": {},
+ "section_order": [],
+ "sections": [],
+ }
diff --git a/tests/test_libtorrent_backend.py b/tests/test_libtorrent_backend.py
index 220ac3fa..406a9056 100644
--- a/tests/test_libtorrent_backend.py
+++ b/tests/test_libtorrent_backend.py
@@ -281,6 +281,40 @@ def test_zero_means_unlimited(self, backend, tmp_path):
assert backend._ses.settings["download_rate_limit"] == 0
+class TestConnectionLimit:
+ def test_session_start_applies_connections_limit(
+ self, backend, tmp_path, monkeypatch
+ ):
+ # The real, enforced knob: connections_limit lands on the session at
+ # startup (not active_downloads/active_seeds — those are inert for
+ # Zimi's manually-managed, non-auto-managed torrents).
+ monkeypatch.setattr(p2p, "_prefs_path", None)
+ monkeypatch.delenv("ZIMI_BT", raising=False)
+ backend._ensure_session()
+ assert backend._ses.settings["connections_limit"] == p2p.DEFAULT_MAX_CONNECTIONS
+
+ def test_set_connections_limit_applies_live(self, backend, tmp_path):
+ backend.add_torrent(MAGNET, dest_dir=str(tmp_path / "staging"))
+ backend.set_connections_limit(333)
+ assert backend._ses.settings["connections_limit"] == 333
+
+ def test_apply_session_limits_pushes_current_cap(
+ self, backend, tmp_path, monkeypatch
+ ):
+ # apply_session_limits() reads the configured cap and pushes it to the
+ # already-running backend (peek_backend, not get_backend).
+ backend.add_torrent(MAGNET, dest_dir=str(tmp_path / "staging"))
+ monkeypatch.setattr(p2p, "peek_backend", lambda: backend)
+ monkeypatch.setattr(p2p, "_prefs_path", None)
+ monkeypatch.setenv("ZIMI_BT", "on,conns=777")
+ p2p.apply_session_limits()
+ assert backend._ses.settings["connections_limit"] == 777
+
+ def test_connections_limit_floor(self, backend, tmp_path):
+ backend.set_connections_limit(1)
+ assert backend._ses.settings["connections_limit"] == 10
+
+
class TestResume:
def test_resume_round_trip(self, backend, tmp_path):
tid = backend.add_torrent(
diff --git a/tests/test_login_navigation.spec.mjs b/tests/test_login_navigation.spec.mjs
index 32e8591f..5b43db55 100644
--- a/tests/test_login_navigation.spec.mjs
+++ b/tests/test_login_navigation.spec.mjs
@@ -13,7 +13,7 @@
// ZIM_DIR=/tmp/zimi-empty ZIMI_DATA_DIR=/tmp/zimi-auth ZIMI_MANAGE=1 \
// python3 -m zimi serve --port 8878
// Run:
-// BASE_URL=http://localhost:8878 npx playwright test tests/test_login_navigation.spec.mjs
+// BASE_URL=http://localhost:8878 npx playwright test --config=tests/playwright.config.mjs tests/test_login_navigation.spec.mjs
import { test, expect } from '@playwright/test';
diff --git a/tests/test_p2p.py b/tests/test_p2p.py
index 1d249963..904c9942 100644
--- a/tests/test_p2p.py
+++ b/tests/test_p2p.py
@@ -106,6 +106,70 @@ def test_staging_dir_override(monkeypatch):
assert p2p.get_staging_dir("/data") == "/fast-ssd/zimi-tmp"
+# ────────────────────────────────────────────────────────────────────────────
+# Concurrency + connection caps
+# ────────────────────────────────────────────────────────────────────────────
+
+
+@pytest.fixture
+def _clean_caps_env(monkeypatch):
+ for k in ("ZIMI_BT", "ZIMI_MAX_CONCURRENT_DOWNLOADS"):
+ monkeypatch.delenv(k, raising=False)
+ monkeypatch.setattr(p2p, "_prefs_path", None)
+
+
+def test_max_active_downloads_default(_clean_caps_env):
+ assert p2p.get_max_active_downloads() == p2p.DEFAULT_MAX_ACTIVE_DOWNLOADS
+ assert p2p.is_max_active_downloads_env_locked() is False
+
+
+def test_max_active_downloads_legacy_env_wins_and_locks(_clean_caps_env, monkeypatch):
+ monkeypatch.setenv("ZIMI_MAX_CONCURRENT_DOWNLOADS", "7")
+ assert p2p.get_max_active_downloads() == 7
+ assert p2p.is_max_active_downloads_env_locked() is True
+
+
+def test_max_active_downloads_bt_subkey(_clean_caps_env, monkeypatch):
+ monkeypatch.setenv("ZIMI_BT", "on,active=6")
+ assert p2p.get_max_active_downloads() == 6
+ assert p2p.is_max_active_downloads_env_locked() is True
+
+
+@pytest.mark.parametrize("raw,expected", [("0", 1), ("99", 20), ("x", 4)])
+def test_max_active_downloads_clamps(_clean_caps_env, monkeypatch, raw, expected):
+ monkeypatch.setenv("ZIMI_MAX_CONCURRENT_DOWNLOADS", raw)
+ assert p2p.get_max_active_downloads() == expected
+
+
+def test_max_connections_default(_clean_caps_env):
+ assert p2p.get_bt_max_connections() == p2p.DEFAULT_MAX_CONNECTIONS
+ assert p2p.is_bt_max_connections_env_locked() is False
+
+
+def test_max_connections_bt_subkey_wins_and_locks(_clean_caps_env, monkeypatch):
+ monkeypatch.setenv("ZIMI_BT", "on,conns=500")
+ assert p2p.get_bt_max_connections() == 500
+ assert p2p.is_bt_max_connections_env_locked() is True
+
+
+@pytest.mark.parametrize("raw,expected", [("5", 10), ("99999", 2000), ("x", 200)])
+def test_max_connections_clamps(_clean_caps_env, monkeypatch, raw, expected):
+ monkeypatch.setenv("ZIMI_BT", f"on,conns={raw}")
+ assert p2p.get_bt_max_connections() == expected
+
+
+def test_caps_read_from_persisted_prefs(_clean_caps_env, monkeypatch, tmp_path):
+ prefs = tmp_path / "prefs.json"
+ monkeypatch.setattr(p2p, "_prefs_path", str(prefs))
+ assert p2p.set_pref("max_active_downloads", 9)
+ assert p2p.set_pref("bt_max_connections", 321)
+ assert p2p.get_max_active_downloads() == 9
+ assert p2p.get_bt_max_connections() == 321
+ # A pref (no env) leaves the UI control unlocked.
+ assert p2p.is_max_active_downloads_env_locked() is False
+ assert p2p.is_bt_max_connections_env_locked() is False
+
+
# ────────────────────────────────────────────────────────────────────────────
# get_backend() — fail-soft to None, else the libtorrent singleton
# ────────────────────────────────────────────────────────────────────────────
diff --git a/tests/test_password_flow.mjs b/tests/test_password_flow.mjs
index f19e2d2e..ed040fe6 100644
--- a/tests/test_password_flow.mjs
+++ b/tests/test_password_flow.mjs
@@ -1,6 +1,6 @@
// Password flow validation - run against a local server WITHOUT ZIMI_MANAGE_PASSWORD env var
// Start: ZIM_DIR=/tmp/zimi-test-zims ZIMI_DATA_DIR=/tmp/zimi-test-data ZIMI_MANAGE=1 python3 -m zimi serve --port 8877
-// Run: BASE_URL=http://localhost:8877 npx playwright test tests/test_password_flow.mjs
+// Run: BASE_URL=http://localhost:8877 npx playwright test --config=tests/playwright.config.mjs tests/test_password_flow.mjs
import { test, expect } from '@playwright/test';
diff --git a/tests/test_private_mode_login.spec.mjs b/tests/test_private_mode_login.spec.mjs
new file mode 100644
index 00000000..52dbc29c
--- /dev/null
+++ b/tests/test_private_mode_login.spec.mjs
@@ -0,0 +1,161 @@
+// Private (sign-in-required) mode: the boot gate must paint the login form as
+// its FIRST frame (no empty library-chrome flash), and a sign-in must STICK
+// across a reload over plain http — for BOTH a named user (session cookie) and
+// the admin (Bearer token in client storage).
+//
+// Regression (1.8.1 private-mode field test):
+// 1. Empty home rendered before the 401/whoami swapped in the login overlay.
+// 2. Admin login "wouldn't stick": /whoami was probed WITHOUT the admin's
+// Bearer token, so the server saw anonymous+login_required and the client
+// re-gated a just-signed-in admin on reload. (The user/cookie path stuck;
+// the token-admin path did not.)
+//
+// Run against a private-mode server on plain http. Recommended engine: webkit
+// (approximates private Safari, the reported environment).
+// ZIM_DIR=./zims ZIMI_DATA_DIR=/tmp/zimi-private ZIMI_MANAGE=1 \
+// python3 -m zimi serve --port 8877
+// BASE_URL=http://localhost:8877 npx playwright test --project=webkit \
+// --config=tests/playwright.config.mjs tests/test_private_mode_login.spec.mjs
+
+import { test, expect } from '@playwright/test';
+
+const BASE = process.env.BASE_URL || 'http://localhost:8877';
+const ADMIN_PW = 'adminpw';
+const USER = 'alice';
+const USER_PW = 'wonderland';
+
+// Loopback is a private client, so it may set the password + seed a user while
+// the instance is still passwordless/open, then flip to private mode.
+test.beforeAll(async ({ request }) => {
+ await request.post(`${BASE}/manage/set-password`, {
+ headers: { 'Content-Type': 'application/json' },
+ data: { password: ADMIN_PW },
+ });
+ const auth = {
+ 'Content-Type': 'application/json',
+ 'Authorization': 'Bearer ' + ADMIN_PW,
+ };
+ await request.post(`${BASE}/manage/users`, {
+ headers: auth,
+ data: { action: 'create', name: USER, password: USER_PW },
+ });
+ await request.post(`${BASE}/manage/public-access`, {
+ headers: auth,
+ data: { mode: 'private' },
+ });
+});
+
+// Sample every animation frame from first paint, recording whether library
+// chrome (loading spinner / home output) is ever visible WITHOUT the opaque
+// login gate covering the page — i.e. the "empty flash".
+async function armFlashSampler(page) {
+ await page.addInitScript(() => {
+ window.__frames = [];
+ const snap = () => {
+ const out = document.getElementById('output');
+ const gateCovering = !!document.querySelector('#pw-overlay.open') &&
+ document.body.classList.contains('login-gate');
+ const outLen = ((out && out.innerText) || '').trim().length;
+ window.__frames.push({ gateCovering, outLen, hasLoading: !!document.querySelector('.loading') });
+ if (window.__frames.length < 300) requestAnimationFrame(snap);
+ };
+ requestAnimationFrame(snap);
+ });
+}
+
+test('private boot paints the login form first — no empty library flash', async ({ page }) => {
+ await armFlashSampler(page);
+ await page.goto(BASE);
+ await page.waitForFunction(() => window._loginRequired === true, { timeout: 8000 });
+ await expect(page.locator('#pw-overlay.open')).toBeVisible();
+ const frames = await page.evaluate(() => window.__frames || []);
+ const flashes = frames.filter(f => (f.hasLoading || f.outLen > 0) && !f.gateCovering);
+ expect(flashes, 'library chrome must never paint before the gate covers it').toHaveLength(0);
+});
+
+test('named-user sign-in sticks across reload (session cookie, plain http)', async ({ page }) => {
+ await page.goto(BASE);
+ await page.waitForFunction(() => window._loginRequired === true, { timeout: 8000 });
+ await page.fill('#pw-username', USER);
+ await page.fill('#pw-input', USER_PW);
+ await page.evaluate(() => submitPw());
+ // submitPw reloads on a gate sign-in; wait until the gate is gone.
+ await expect.poll(() => page.evaluate(() => window._loginRequired), { timeout: 8000 }).toBeFalsy();
+ await page.goto(BASE); // explicit reopen
+ await page.waitForFunction(() => typeof _loginRequired !== 'undefined', { timeout: 8000 });
+ await page.waitForTimeout(500);
+ expect(await page.evaluate(() => window._loginRequired)).toBeFalsy();
+ await expect(page.locator('#pw-overlay.open')).toHaveCount(0);
+});
+
+test('admin sign-in sticks across reload (Bearer token, not re-gated)', async ({ page }) => {
+ await page.goto(BASE);
+ await page.waitForFunction(() => window._loginRequired === true, { timeout: 8000 });
+ await page.fill('#pw-username', 'admin');
+ await page.fill('#pw-input', ADMIN_PW);
+ await page.evaluate(() => submitPw());
+ await expect.poll(() => page.evaluate(() => window._loginRequired), { timeout: 8000 }).toBeFalsy();
+ await page.goto(BASE); // reload: boot gate must recognise the token-admin
+ await page.waitForFunction(() => typeof _loginRequired !== 'undefined', { timeout: 8000 });
+ await page.waitForTimeout(500);
+ expect(await page.evaluate(() => window._loginRequired)).toBeFalsy();
+ await expect(page.locator('#pw-overlay.open')).toHaveCount(0);
+});
+
+// Read the server's identity + library-access as the CLIENT sees it right now,
+// through the (SW-controlled) fetch path. This is the security contract that
+// matters — not DOM text, which depends on how many ZIMs the fixture installs.
+async function serverView(page) {
+ return page.evaluate(async () => {
+ const who = await (await fetch('/whoami', { credentials: 'same-origin' })).json();
+ const listStatus = (await fetch('/list?layout=1', { credentials: 'same-origin' })).status;
+ return { role: who.role, listStatus };
+ });
+}
+
+// BUG 2 (field): "Private mode I had to sign in twice and saw nothing still."
+// A single sign-in must clear the gate AND leave the client authenticated with
+// live library access. The root cause was the service worker serving a STALE
+// anonymous /whoami after the post-login reload, which re-showed the gate and
+// left the client seeing an anonymous (empty/401) library — see
+// tests/test_sw_route_classification.py for the SW-level guard.
+test('named-user sign-in authenticates on the FIRST try (no re-gate, live access)', async ({ page }) => {
+ await page.goto(BASE);
+ await page.waitForFunction(() => window._loginRequired === true, { timeout: 8000 });
+ await page.fill('#pw-username', USER);
+ await page.fill('#pw-input', USER_PW);
+ await page.evaluate(() => submitPw());
+ // Gate must clear and STAY clear — no second prompt.
+ await expect.poll(() => page.evaluate(() => window._loginRequired), { timeout: 8000 }).toBeFalsy();
+ await page.waitForTimeout(500);
+ expect(await page.evaluate(() => window._loginRequired)).toBeFalsy();
+ await expect(page.locator('#pw-overlay.open')).toHaveCount(0);
+ // The client is authenticated and the library is accessible — NOT the stale
+ // anonymous view the SW used to serve after the reload.
+ const view = await serverView(page);
+ expect(view.role).toBe('user');
+ expect(view.listStatus).toBe(200);
+});
+
+// BUG 1 (field): "I set private access, logged out and saw everything." Logging
+// out must return straight to the non-dismissible gate AND cut off library
+// access. The bug was that logout dropped credentials but never re-ran the boot
+// gate, so the full library the session had already loaded stayed on screen and
+// the client still believed it was authorized.
+test('logout in private mode returns to the gate and cuts off access', async ({ page }) => {
+ await page.goto(BASE);
+ await page.waitForFunction(() => window._loginRequired === true, { timeout: 8000 });
+ await page.fill('#pw-username', USER);
+ await page.fill('#pw-input', USER_PW);
+ await page.evaluate(() => submitPw());
+ await expect.poll(() => page.evaluate(() => window._loginRequired), { timeout: 8000 }).toBeFalsy();
+ expect((await serverView(page)).role).toBe('user');
+ // Log out (userLogout reloads); the boot gate must re-show the login overlay.
+ await page.evaluate(() => userLogout());
+ await page.waitForFunction(() => window._loginRequired === true, { timeout: 8000 });
+ await expect(page.locator('#pw-overlay.open')).toBeVisible();
+ // Anonymous again: the server denies the library (401), and the client agrees.
+ const view = await serverView(page);
+ expect(view.role).toBe('anonymous');
+ expect(view.listStatus).toBe(401);
+});
diff --git a/tests/test_public_access.py b/tests/test_public_access.py
new file mode 100644
index 00000000..852695c3
--- /dev/null
+++ b/tests/test_public_access.py
@@ -0,0 +1,673 @@
+"""Anonymous-access policy (1.8.1): open / limited / private.
+
+The public-access policy decides what an ANONYMOUS (not logged-in) visitor may
+see. It reuses the multi-user allowlist machinery — anonymous simply gets an
+allow-set instead of the all-access sentinel — so every existing choke point
+(get_zim_files / list_zims / zim_allowed / search-cache key) filters it with no
+new leak surface. This suite pins:
+
+- storage: default open, round-trip, env override, fail-closed on corruption
+- request_allow per mode × identity (anonymous / admin / logged-in user)
+- the private-mode request gate (login surface reachable, reads 401)
+- leak checks under `limited`: the same choke points filter anonymous, and the
+ /languages- and almanac-style bypass paths (zim_allowed) stay filtered too
+- admin endpoints: GET/POST /manage/public-access are admin-only and round-trip
+"""
+
+import json
+import os
+import sys
+import tempfile
+import unittest
+from types import SimpleNamespace
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import zimi.server as _srv # noqa: E402
+from zimi import manage, users # noqa: E402
+from zimi.http import SESSION_COOKIE_MAX_AGE, ZimHandler # noqa: E402
+
+
+class _FakeHandler:
+ """Minimal ZimHandler stand-in: headers + client privacy + _json capture.
+
+ Borrows the real cookie builders (they only read ``self.headers``) so paths
+ that Set-Cookie — the admin whoami/login — exercise the true header, and
+ records each emitted cookie in ``set_cookies`` for assertions."""
+
+ _session_cookie = ZimHandler._session_cookie
+ _expire_cookie = ZimHandler._expire_cookie
+
+ def __init__(self, headers=None, private=True):
+ self.headers = headers or {}
+ self._private = private
+ self.responses = []
+ self.set_cookies = []
+
+ def _is_private_client(self):
+ return self._private
+
+ def _json(self, code, data):
+ self.responses.append((code, data))
+ return None
+
+ def _json_cookie(self, code, data, set_cookie):
+ self.set_cookies.append(set_cookie)
+ self.responses.append((code, data))
+ return None
+
+ @property
+ def last(self):
+ return self.responses[-1] if self.responses else None
+
+
+def _bearer(token, user=None):
+ h = {"Authorization": "Bearer " + token}
+ if user is not None:
+ h["X-Zimi-User"] = user
+ return h
+
+
+def _cookie(token):
+ return {"Cookie": "zimi_session=" + token}
+
+
+def _gate(handler, path):
+ """Invoke the real private-mode gate against a fake handler."""
+ return ZimHandler._private_access_block(handler, SimpleNamespace(path=path))
+
+
+class _AccessBase(unittest.TestCase):
+ def setUp(self):
+ self._tmp = tempfile.mkdtemp()
+ self._orig_data_dir = _srv.ZIMI_DATA_DIR
+ _srv.ZIMI_DATA_DIR = self._tmp
+ for var in (
+ "ZIMI_MANAGE_PASSWORD",
+ "ZIMI_MANAGE_USER",
+ "ZIMI_API_TOKEN",
+ "ZIMI_PUBLIC_ACCESS",
+ ):
+ os.environ.pop(var, None)
+ manage._env_pw_hash_cache = None
+ _srv.clear_request_allow()
+
+ def tearDown(self):
+ _srv.ZIMI_DATA_DIR = self._orig_data_dir
+ for var in (
+ "ZIMI_MANAGE_PASSWORD",
+ "ZIMI_MANAGE_USER",
+ "ZIMI_API_TOKEN",
+ "ZIMI_PUBLIC_ACCESS",
+ ):
+ os.environ.pop(var, None)
+ manage._env_pw_hash_cache = None
+ _srv.clear_request_allow()
+ import shutil
+
+ shutil.rmtree(self._tmp, ignore_errors=True)
+
+
+# ── Storage: default, round-trip, env override, fail-closed ────────────────
+
+
+class TestPolicyStorage(_AccessBase):
+ def test_default_is_open_when_no_file(self):
+ # Legacy install: no access.json → open (whole library to anonymous).
+ mode, allow = users.get_public_access()
+ self.assertEqual(mode, "open")
+ self.assertEqual(allow, [])
+
+ def test_set_and_get_limited_round_trip(self):
+ ok, err = users.set_public_access("limited", ["a", "b", "b"]) # de-dup
+ self.assertTrue(ok, err)
+ mode, allow = users.get_public_access()
+ self.assertEqual(mode, "limited")
+ self.assertEqual(sorted(allow), ["a", "b"])
+
+ def test_open_and_private_ignore_allowlist(self):
+ users.set_public_access("private", ["a", "b"])
+ _, allow = users.get_public_access()
+ self.assertEqual(allow, []) # not stored for non-limited modes
+
+ def test_invalid_mode_rejected(self):
+ ok, err = users.set_public_access("wideopen")
+ self.assertFalse(ok)
+ self.assertIn("mode", err)
+
+ def test_env_override_wins_over_file(self):
+ users.set_public_access("open")
+ os.environ["ZIMI_PUBLIC_ACCESS"] = "private"
+ mode, _ = users.get_public_access()
+ self.assertEqual(mode, "private")
+
+ def test_env_override_ignored_when_invalid(self):
+ users.set_public_access("limited", ["a"])
+ os.environ["ZIMI_PUBLIC_ACCESS"] = "bogus"
+ mode, allow = users.get_public_access()
+ self.assertEqual(mode, "limited")
+ self.assertEqual(allow, ["a"])
+
+ def test_corrupt_file_fails_closed_to_private(self):
+ # A present-but-unreadable policy must NOT silently fall open.
+ with open(users._access_path(), "w", encoding="utf-8") as f:
+ f.write("{not valid json")
+ mode, _ = users.get_public_access()
+ self.assertEqual(mode, "private")
+
+ def test_corrupt_file_with_env_open_respects_env(self):
+ # The admin's env escape hatch still works over a corrupt file.
+ with open(users._access_path(), "w", encoding="utf-8") as f:
+ f.write("garbage")
+ os.environ["ZIMI_PUBLIC_ACCESS"] = "open"
+ mode, _ = users.get_public_access()
+ self.assertEqual(mode, "open")
+
+ def test_unknown_mode_in_file_fails_closed(self):
+ with open(users._access_path(), "w", encoding="utf-8") as f:
+ json.dump({"version": 1, "mode": "sideways"}, f)
+ mode, _ = users.get_public_access()
+ self.assertEqual(mode, "private")
+
+ def test_status_surfaces_env_control(self):
+ users.set_public_access("limited", ["a"])
+ os.environ["ZIMI_PUBLIC_ACCESS"] = "private"
+ st = users.public_access_status()
+ self.assertEqual(st["mode"], "private") # effective
+ self.assertEqual(st["stored_mode"], "limited") # what was saved
+ self.assertTrue(st["env_controlled"])
+ self.assertEqual(st["env_mode"], "private")
+
+
+# ── request_allow across mode × identity ───────────────────────────────────
+
+
+class TestRequestAllowByMode(_AccessBase):
+ def test_open_anonymous_all_access(self):
+ users.set_public_access("open")
+ self.assertIsNone(users.request_allow(_FakeHandler(private=False)))
+
+ def test_limited_anonymous_gets_public_allowlist(self):
+ users.set_public_access("limited", ["a", "b"])
+ # A non-private anonymous client (e.g. WAN visitor) is restricted.
+ allow = users.request_allow(_FakeHandler(private=False))
+ self.assertEqual(allow, {"a", "b"})
+
+ def test_private_anonymous_gets_empty_set(self):
+ users.set_public_access("private")
+ allow = users.request_allow(_FakeHandler(private=False))
+ self.assertEqual(allow, set()) # defence in depth; gate 401s first
+
+ def test_admin_all_access_in_limited(self):
+ manage._set_manage_password("adminpw")
+ users.set_public_access("limited", ["a"])
+ h = _FakeHandler(_bearer("adminpw"), private=False)
+ self.assertIsNone(users.request_allow(h))
+
+ def test_admin_all_access_in_private(self):
+ manage._set_manage_password("adminpw")
+ users.set_public_access("private")
+ h = _FakeHandler(_bearer("adminpw"), private=False)
+ self.assertIsNone(users.request_allow(h))
+
+ def test_logged_in_limited_user_unaffected_by_open_policy(self):
+ users.set_public_access("open")
+ users.create_user("Kid", "pw", allowlist=["x"], role="limited")
+ token = users.create_session("Kid")
+ allow = users.request_allow(_FakeHandler(_cookie(token)))
+ self.assertEqual(allow, {"x"})
+
+ def test_logged_in_user_own_allowlist_not_public_one(self):
+ # A logged-in user keeps THEIR allowlist even under limited public mode.
+ users.set_public_access("limited", ["public_only"])
+ users.create_user("Kid", "pw", allowlist=["kid_only"], role="limited")
+ token = users.create_session("Kid")
+ allow = users.request_allow(_FakeHandler(_cookie(token)))
+ self.assertEqual(allow, {"kid_only"})
+
+ def test_all_access_user_stays_all_access_under_limited(self):
+ users.set_public_access("limited", ["a"])
+ users.create_user("Grown", "pw", role="user")
+ token = users.create_session("Grown")
+ self.assertIsNone(users.request_allow(_FakeHandler(_bearer(token))))
+
+
+# ── Private-mode request gate ──────────────────────────────────────────────
+
+
+class TestPrivateGate(_AccessBase):
+ def setUp(self):
+ super().setUp()
+ users.set_public_access("private")
+
+ def test_login_surface_paths_pass(self):
+ for p in (
+ "/",
+ "/whoami",
+ "/health",
+ "/login",
+ "/logout",
+ "/favicon.png",
+ "/static/app.js",
+ "/static/i18n/en.json",
+ "/manage/has-password",
+ ):
+ h = _FakeHandler(private=False)
+ self.assertFalse(_gate(h, p), p)
+ self.assertEqual(h.responses, [], p)
+
+ def test_read_endpoints_blocked_for_anonymous(self):
+ # The full read/data surface, not a cherry-pick: every endpoint that can
+ # reveal library contents must 401 an anonymous visitor in private mode.
+ # A new read endpoint that isn't on the tiny login allowlist lands here
+ # automatically (the gate denies everything it doesn't explicitly permit).
+ for p in (
+ "/search",
+ "/list",
+ "/read",
+ "/suggest",
+ "/w/wiki/Foo",
+ "/random",
+ "/chunks",
+ "/almanac-links",
+ "/languages",
+ "/article-languages",
+ "/snippet",
+ "/openapi.json",
+ ):
+ h = _FakeHandler(private=False)
+ self.assertTrue(_gate(h, p), p)
+ code, body = h.last
+ self.assertEqual(code, 401, p)
+ self.assertTrue(body.get("login_required"), p)
+
+ def test_admin_bypasses_gate(self):
+ manage._set_manage_password("adminpw")
+ h = _FakeHandler(_bearer("adminpw"), private=False)
+ self.assertFalse(_gate(h, "/search"))
+ self.assertEqual(h.responses, [])
+
+ def test_logged_in_user_bypasses_gate(self):
+ users.create_user("Kid", "pw", allowlist=["a"], role="limited")
+ token = users.create_session("Kid")
+ h = _FakeHandler(_cookie(token), private=False)
+ self.assertFalse(_gate(h, "/search"))
+ self.assertEqual(h.responses, [])
+
+ def test_open_mode_never_gates(self):
+ users.set_public_access("open")
+ h = _FakeHandler(private=False)
+ self.assertFalse(_gate(h, "/search"))
+
+ def test_limited_mode_never_gates(self):
+ # limited filters via the allow-set, it does NOT 401 — reads still work.
+ users.set_public_access("limited", ["a"])
+ h = _FakeHandler(private=False)
+ self.assertFalse(_gate(h, "/search"))
+
+
+# ── Leak checks under `limited`: anonymous is filtered by every choke point ─
+
+
+class TestLimitedLeakChecks(_AccessBase):
+ def setUp(self):
+ super().setUp()
+ _srv._zim_files_cache = {"a": "/z/a.zim", "b": "/z/b.zim", "c": "/z/c.zim"}
+ _srv._zim_list_cache = [
+ {"name": "a", "entries": 500, "language": "en", "title": "Alpha"},
+ {"name": "b", "entries": 500, "language": "en", "title": "Beta"},
+ {"name": "c", "entries": 500, "language": "fr", "title": "Gamma"},
+ ]
+ users.set_public_access("limited", ["a"])
+ # Simulate the do_GET preamble: the request-allow context is set from the
+ # anonymous policy for a non-private (WAN) visitor.
+ self._allow = users.request_allow(_FakeHandler(private=False))
+ _srv.set_request_allow(self._allow)
+
+ def tearDown(self):
+ _srv._zim_files_cache = None
+ _srv._zim_list_cache = None
+ super().tearDown()
+
+ def test_allow_set_is_public_allowlist(self):
+ self.assertEqual(self._allow, {"a"})
+
+ def test_get_zim_files_filtered(self):
+ self.assertEqual(set(_srv.get_zim_files()), {"a"})
+
+ def test_list_zims_filtered(self):
+ self.assertEqual({z["name"] for z in _srv.list_zims()}, {"a"})
+
+ def test_zim_allowed_bypass_paths_filtered(self):
+ # /languages, /article-languages and almanac-links go through zim_allowed
+ # rather than get_zim_files — assert they see the same filtered view.
+ self.assertTrue(_srv.zim_allowed("a"))
+ self.assertFalse(_srv.zim_allowed("b"))
+ self.assertFalse(_srv.zim_allowed("c"))
+
+ def test_get_archive_fails_closed_for_forbidden(self):
+ _srv._archive_pool["b"] = object()
+ try:
+ self.assertIsNone(_srv.get_archive("b"))
+ finally:
+ _srv._archive_pool.pop("b", None)
+
+ def test_shared_cache_not_mutated(self):
+ _srv.get_zim_files()
+ _srv.clear_request_allow()
+ self.assertEqual(set(_srv.get_zim_files()), {"a", "b", "c"})
+
+
+# ── Admin endpoints for the policy ─────────────────────────────────────────
+
+
+class TestPublicAccessEndpoints(_AccessBase):
+ def setUp(self):
+ super().setUp()
+ manage._set_manage_password("adminpw")
+ _srv._zim_files_cache = {"a": "/z/a.zim", "b": "/z/b.zim"}
+ _srv._zim_list_cache = [
+ {"name": "a", "language": "en", "title": "Alpha", "article_count": 10},
+ {"name": "b", "language": "fr", "title": "Beta", "article_count": 20},
+ ]
+
+ def tearDown(self):
+ _srv._zim_files_cache = None
+ _srv._zim_list_cache = None
+ super().tearDown()
+
+ def _admin(self):
+ return _FakeHandler(_bearer("adminpw"), private=False)
+
+ def _post(self, data, path="/manage/public-access"):
+ h = self._admin()
+ manage.handle_manage_post(h, SimpleNamespace(path=path), data)
+ return h.last
+
+ def _get(self, path="/manage/public-access"):
+ h = self._admin()
+ manage.handle_manage_get(h, SimpleNamespace(path=path), {})
+ return h.last
+
+ def test_get_returns_status_and_picker_options(self):
+ code, body = self._get()
+ self.assertEqual(code, 200)
+ self.assertEqual(body["public_access"]["mode"], "open")
+ opts = {o["name"]: o for o in body["zim_options"]}
+ self.assertEqual(opts["a"]["title"], "Alpha")
+ self.assertEqual(opts["a"]["language"], "en")
+ self.assertEqual(opts["a"]["article_count"], 10)
+
+ def test_post_sets_limited_and_persists(self):
+ code, body = self._post({"mode": "limited", "allowlist": ["a"]})
+ self.assertEqual(code, 200)
+ self.assertEqual(body["public_access"]["mode"], "limited")
+ self.assertEqual(body["public_access"]["allowlist"], ["a"])
+ # Persisted to disk.
+ self.assertEqual(users.get_public_access()[0], "limited")
+
+ def test_post_invalid_mode_rejected(self):
+ code, _ = self._post({"mode": "everything"})
+ self.assertEqual(code, 400)
+
+ def test_users_get_includes_policy_and_options(self):
+ users.set_public_access("limited", ["b"])
+ code, body = self._get("/manage/users")
+ self.assertEqual(code, 200)
+ self.assertEqual(body["public_access"]["mode"], "limited")
+ self.assertIn("zim_options", body)
+
+ def test_endpoints_reject_non_admin_user(self):
+ users.create_user("Kid", "pw", allowlist=["a"], role="limited")
+ token = users.create_session("Kid")
+ h = _FakeHandler(_bearer(token, "Kid"), private=False)
+ manage.handle_manage_post(
+ h, SimpleNamespace(path="/manage/public-access"), {"mode": "open"}
+ )
+ code, _ = h.last
+ self.assertEqual(code, 401)
+ # The policy never changed.
+ self.assertEqual(users.get_public_access()[0], "open")
+
+ def test_get_rejects_non_admin_user(self):
+ users.create_user("Kid", "pw", allowlist=["a"], role="limited")
+ token = users.create_session("Kid")
+ h = _FakeHandler(_bearer(token, "Kid"), private=False)
+ manage.handle_manage_get(h, SimpleNamespace(path="/manage/public-access"), {})
+ code, _ = h.last
+ self.assertEqual(code, 401)
+
+
+# ── Session-cookie attributes (login persistence over plain http) ──────────
+
+
+class TestSessionCookieAttributes(_AccessBase):
+ """The ``zimi_session`` cookie carries ``Secure`` ONLY behind an HTTPS proxy.
+
+ A LAN instance is plain http, and a browser silently DROPS a ``Secure``
+ cookie set over http:// — so gating Secure on the forwarded proto is what
+ lets a login stick on the LAN. ``HttpOnly`` + ``SameSite=Lax`` are always
+ present; ``Max-Age`` appears only when 'remember' is set (else it is a
+ session cookie the browser clears on close).
+ """
+
+ def _cookie(self, remember, proto=None):
+ headers = {}
+ if proto is not None:
+ headers["X-Forwarded-Proto"] = proto
+ h = _FakeHandler(headers, private=False)
+ return ZimHandler._session_cookie(h, "TOK", remember)
+
+ def test_plain_http_omits_secure(self):
+ parts = self._cookie(remember=True).split("; ")
+ self.assertNotIn("Secure", parts)
+ self.assertIn("HttpOnly", parts)
+ self.assertIn("SameSite=Lax", parts)
+ self.assertEqual(parts[0], "zimi_session=TOK")
+
+ def test_https_proxy_sets_secure(self):
+ parts = self._cookie(remember=True, proto="https").split("; ")
+ self.assertIn("Secure", parts)
+
+ def test_https_proto_case_insensitive(self):
+ self.assertIn("Secure", self._cookie(remember=True, proto="HTTPS").split("; "))
+
+ def test_http_forwarded_proto_no_secure(self):
+ # An explicit ``http`` forwarded proto (proxy that terminates TLS
+ # elsewhere but forwards http to us) must NOT set Secure.
+ self.assertNotIn(
+ "Secure", self._cookie(remember=True, proto="http").split("; ")
+ )
+
+ def test_remember_sets_max_age(self):
+ self.assertIn(
+ "Max-Age=" + str(SESSION_COOKIE_MAX_AGE), self._cookie(remember=True)
+ )
+
+ def test_no_remember_is_session_cookie(self):
+ self.assertNotIn("Max-Age", self._cookie(remember=False))
+
+ def test_expire_cookie_clears_with_max_age_zero(self):
+ c = ZimHandler._expire_cookie(_FakeHandler())
+ self.assertIn("zimi_session=;", c)
+ self.assertIn("Max-Age=0", c)
+
+
+# ── whoami recognises a token-authed admin (client boot-gate contract) ──────
+
+
+class TestWhoamiAdminToken(_AccessBase):
+ """The admin's credential is a Bearer token, not the session cookie the
+ named-user path rides on. In private mode the boot gate sends that token on
+ /whoami; the server MUST answer role=admin (never anonymous+login_required),
+ or a just-signed-in admin gets bounced back to the login screen on reload.
+ """
+
+ def setUp(self):
+ super().setUp()
+ self._orig_manage = _srv.ZIMI_MANAGE
+ _srv.ZIMI_MANAGE = True
+ manage._set_manage_password("adminpw")
+ users.set_public_access("private")
+
+ def tearDown(self):
+ _srv.ZIMI_MANAGE = self._orig_manage
+ super().tearDown()
+
+ def test_admin_bearer_token_resolves_to_admin(self):
+ h = _FakeHandler(_bearer("adminpw"), private=False)
+ ZimHandler._handle_whoami(h)
+ code, body = h.last
+ self.assertEqual(code, 200)
+ self.assertEqual(body.get("role"), "admin")
+ self.assertNotIn("login_required", body)
+
+ def test_anonymous_gets_login_required(self):
+ h = _FakeHandler({}, private=False)
+ ZimHandler._handle_whoami(h)
+ code, body = h.last
+ self.assertEqual(code, 200)
+ self.assertEqual(body.get("role"), "anonymous")
+ self.assertTrue(body.get("login_required"))
+
+
+# ── Admin session cookie: the header-less-transport fix ────────────────────
+
+
+class TestAdminSessionToken(_AccessBase):
+ """The primary admin authenticates with a password Bearer, but the reader
+ iframe (/w/) and the plain-fetch data endpoints (/list, /search) send NO
+ Authorization header — only cookies. Without an admin session cookie a
+ private/limited-mode admin loaded an EMPTY library and blank article iframes
+ (the 1.8.1 ship blocker). create_admin_session mints an HttpOnly cookie that
+ _primary_admin_authorized accepts on both transports."""
+
+ def setUp(self):
+ super().setUp()
+ self._orig_manage = _srv.ZIMI_MANAGE
+ _srv.ZIMI_MANAGE = True
+ manage._set_manage_password("adminpw")
+
+ def tearDown(self):
+ _srv.ZIMI_MANAGE = self._orig_manage
+ super().tearDown()
+
+ # --- session primitives ---
+ def test_admin_session_recognised(self):
+ tok = users.create_admin_session()
+ self.assertTrue(users.is_admin_session(tok))
+
+ def test_admin_session_fails_closed_on_junk(self):
+ self.assertFalse(users.is_admin_session(""))
+ self.assertFalse(users.is_admin_session(None))
+ self.assertFalse(users.is_admin_session("not-a-real-token"))
+
+ def test_admin_session_is_not_a_named_user(self):
+ # An admin session must never resolve as a user (no allowlist confusion).
+ tok = users.create_admin_session()
+ self.assertIsNone(users.resolve_session(tok))
+ self.assertIsNone(users.resolve_request_user(_FakeHandler(_cookie(tok))))
+
+ def test_user_session_is_not_an_admin_session(self):
+ users.create_user("Kid", "pw", allowlist=["x"], role="limited")
+ tok = users.create_session("Kid")
+ self.assertFalse(users.is_admin_session(tok))
+
+ # --- transport: cookie-only requests resolve as admin ---
+ def test_cookie_only_request_is_primary_admin(self):
+ tok = users.create_admin_session()
+ h = _FakeHandler(_cookie(tok), private=False) # NO Authorization header
+ self.assertTrue(manage._primary_admin_authorized(h))
+ self.assertIsNone(manage._check_manage_auth(h))
+ self.assertEqual(manage.admin_kind(h), "primary")
+
+ def test_bearer_admin_session_token_also_works(self):
+ tok = users.create_admin_session()
+ h = _FakeHandler(_bearer(tok), private=False)
+ self.assertTrue(manage._primary_admin_authorized(h))
+
+ # --- the actual bug: admin sees the full library on data endpoints ---
+ def test_private_admin_cookie_sees_all(self):
+ users.set_public_access("private")
+ tok = users.create_admin_session()
+ h = _FakeHandler(_cookie(tok), private=False)
+ self.assertIsNone(users.request_allow(h)) # all-access, not empty set
+
+ def test_limited_admin_cookie_sees_all(self):
+ users.set_public_access("limited", ["a"])
+ tok = users.create_admin_session()
+ h = _FakeHandler(_cookie(tok), private=False)
+ self.assertIsNone(users.request_allow(h)) # not just {"a"}
+
+ # --- the second transport gap: the /w/ iframe gate lets the admin in ---
+ def test_private_gate_passes_admin_cookie_iframe(self):
+ users.set_public_access("private")
+ tok = users.create_admin_session()
+ h = _FakeHandler(_cookie(tok), private=False)
+ # A /w/ content request carries only the cookie; the gate must NOT 401.
+ self.assertFalse(_gate(h, "/w/wikipedia/A/Some_Article"))
+
+ def test_private_gate_still_401s_anonymous_iframe(self):
+ # Regression guard: the fix must not open the gate for anonymous.
+ users.set_public_access("private")
+ h = _FakeHandler({}, private=False)
+ self.assertTrue(_gate(h, "/w/wikipedia/A/Some_Article"))
+ self.assertEqual(h.last[0], 401)
+
+ # --- login/logout wiring ---
+ def test_admin_login_sets_httponly_session_cookie(self):
+ users.set_public_access("private")
+ h = _FakeHandler({}, private=False)
+ ZimHandler._handle_login(
+ h, {"username": "admin", "password": "adminpw", "remember": False}
+ )
+ code, body = h.last
+ self.assertEqual(code, 200)
+ self.assertEqual(body.get("role"), "admin")
+ self.assertTrue(h.set_cookies, "admin login must Set-Cookie")
+ cookie = h.set_cookies[-1]
+ self.assertIn("zimi_session=", cookie)
+ self.assertIn("HttpOnly", cookie)
+ # And that cookie authenticates a subsequent cookie-only request.
+ tok = cookie.split("zimi_session=", 1)[1].split(";", 1)[0]
+ self.assertTrue(users.is_admin_session(tok))
+
+ def test_whoami_mints_cookie_for_bearer_admin_without_one(self):
+ users.set_public_access("private")
+ h = _FakeHandler(_bearer("adminpw"), private=False)
+ ZimHandler._handle_whoami(h)
+ self.assertEqual(h.last[1].get("role"), "admin")
+ self.assertTrue(h.set_cookies, "whoami must mint an admin cookie")
+ tok = h.set_cookies[-1].split("zimi_session=", 1)[1].split(";", 1)[0]
+ self.assertTrue(users.is_admin_session(tok))
+
+ def test_whoami_does_not_remint_when_cookie_present(self):
+ users.set_public_access("private")
+ tok = users.create_admin_session()
+ headers = {"Authorization": "Bearer adminpw", "Cookie": "zimi_session=" + tok}
+ h = _FakeHandler(headers, private=False)
+ ZimHandler._handle_whoami(h)
+ self.assertEqual(h.last[1].get("role"), "admin")
+ self.assertFalse(h.set_cookies, "no re-mint when a live admin cookie exists")
+
+ def test_password_rotation_revokes_admin_sessions(self):
+ # An old admin cookie must NOT outlive a password change — the pre-cookie
+ # model (Bearer == password) revoked instantly; this preserves that.
+ tok = users.create_admin_session()
+ self.assertTrue(users.is_admin_session(tok))
+ manage._set_manage_password("newpw")
+ self.assertFalse(users.is_admin_session(tok))
+
+ def test_logout_drops_admin_session_server_side(self):
+ tok = users.create_admin_session()
+ self.assertTrue(users.is_admin_session(tok))
+ h = _FakeHandler(_cookie(tok), private=False)
+ ZimHandler._handle_logout(h)
+ self.assertFalse(
+ users.is_admin_session(tok)
+ ) # dropped, not just expired client-side
+ self.assertIn("Max-Age=0", h.set_cookies[-1])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_reader3.spec.mjs b/tests/test_reader3.spec.mjs
index 424b9b6f..81bb4a01 100644
--- a/tests/test_reader3.spec.mjs
+++ b/tests/test_reader3.spec.mjs
@@ -7,7 +7,7 @@
// Start a server first (uses the bundled devdocs ZIM):
// ZIM_DIR=./zims python3 -m zimi serve --port 8877
// Run:
-// BASE_URL=http://localhost:8877 npx playwright test tests/test_reader3.spec.mjs
+// BASE_URL=http://localhost:8877 npx playwright test --config=tests/playwright.config.mjs tests/test_reader3.spec.mjs
import { test, expect } from '@playwright/test';
diff --git a/tests/test_snippet_boilerplate.py b/tests/test_snippet_boilerplate.py
new file mode 100644
index 00000000..4f9b5498
--- /dev/null
+++ b/tests/test_snippet_boilerplate.py
@@ -0,0 +1,118 @@
+"""Tests for snippet boilerplate-skip (previews.extract_snippet).
+
+iFixit device pages bake one repeated (a featured-guide
+blurb) into every page. extract_snippet must prefer the page's own summary
+block over that boilerplate, without regressing ZIM types whose meta
+description IS the right snippet (wikipedia/gutenberg/ted-style).
+
+Fixtures are trimmed to the load-bearing structure of the real iFixit ZIM
+markup fetched from the NAS (banner-blurb / itemprop="description" span, with
+the wrong SSD meta description in ).
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from zimi.previews import extract_snippet # noqa: E402
+
+# Real iFixit device-page shape (trimmed). The is the
+# repeated featured-guide boilerplate; the banner-blurb span is the truth.
+IFIXIT_AMERIWATER = """
+AmeriWater Water Purification System
+
+
+
+
AmeriWater Water Purification System
+
+
+ Water Purification System is a Sterilization manufactured by AmeriWater. Model numbers 00HC-2015, 00HC-2045.
+
+"""
+
+IFIXIT_ACER = """
+Acer Aspire 5253
+
+
+
+A general purpose laptop released in 2010 with a 15.6" screen.
+"""
+
+
+def test_ifixit_prefers_device_blurb_over_boilerplate_meta():
+ snip = extract_snippet(IFIXIT_AMERIWATER, "ifixit")
+ assert "AmeriWater" in snip
+ assert "SSD" not in snip
+ assert "Lenovo" not in snip
+
+
+def test_ifixit_second_device_also_correct():
+ snip = extract_snippet(IFIXIT_ACER, "ifixit")
+ assert snip.startswith("A general purpose laptop")
+ assert "SSD" not in snip
+
+
+def test_itemprop_description_used_without_banner_class():
+ html = """
+ A widget that does a specific useful thing."""
+ snip = extract_snippet(html, "ifixit")
+ assert snip == "A widget that does a specific useful thing."
+
+
+# ── Regressions: meta description must still win where it's the right source ──
+
+
+def test_wikipedia_meta_description_still_used():
+ html = """
+
Paris (French pronunciation) is the capital of France.
"""
+ snip = extract_snippet(html, "wikipedia_en_all")
+ assert snip == "Paris is the capital and most populous city of France."
+
+
+def test_og_description_variant_used():
+ html = """"""
+ snip = extract_snippet(html, "ted_en")
+ assert snip == "A talk about the future of biology and life."
+
+
+def test_gutenberg_meta_description_used():
+ html = """
+ """
+ snip = extract_snippet(html, "gutenberg_en")
+ assert "Pride and Prejudice" in snip
+
+
+def test_no_meta_falls_back_to_main_body_skipping_nav():
+ html = """
+
+ The genuine article prose starts here and runs on with real content words.
+ """
+ snip = extract_snippet(html, "generic")
+ assert snip.startswith("The genuine article prose")
+ assert "Home" not in snip
+
+
+def test_toc_boilerplate_stripped_from_body_fallback():
+ html = """
+
Table of contents Introduction Steps Comments
+ Actual body content that should become the snippet text here.
+ """
+ snip = extract_snippet(html, "ifixit")
+ assert snip.startswith("Actual body content")
+ assert "Table of contents" not in snip
+
+
+def test_short_own_summary_ignored():
+ """A too-short blurb (<20 chars) should not pre-empt a good meta desc."""
+ html = """
+
Hi
"""
+ snip = extract_snippet(html, "ifixit")
+ assert snip.startswith("A properly detailed description")
+
+
+def test_empty_html_returns_empty_string():
+ assert extract_snippet("", "x") == ""
diff --git a/tests/test_sw_route_classification.py b/tests/test_sw_route_classification.py
new file mode 100644
index 00000000..fabbc203
--- /dev/null
+++ b/tests/test_sw_route_classification.py
@@ -0,0 +1,143 @@
+"""Service-worker route classification — the SW must NEVER cache or serve
+stale the identity/auth-scoped endpoints.
+
+Field regression (1.8.1 private-mode): the SW routed ``/whoami`` through
+stale-while-revalidate and ``/list``/``/search`` through network-first (which
+writes the cache). After a successful sign-in the boot gate read a STALE cached
+anonymous ``/whoami`` and re-showed the login screen ("sign in twice"); and a
+cached full-library ``/list`` could be served on a network blip to a
+now-anonymous visitor of a private instance (library leak).
+
+The fix makes those endpoints network-only. This test loads the REAL
+``zimi/static/sw.js`` in a stubbed ServiceWorker global via ``node`` and asserts
+(a) ``routeStrategy`` classifies each identity endpoint as ``networkOnly`` and
+(b) dispatching a fetch for them NEVER touches the Cache API. It is a loop over
+the endpoint table, not a cherry-pick, so a new identity endpoint that forgets
+the network-only rule fails here.
+"""
+
+import json
+import os
+import shutil
+import subprocess
+import unittest
+
+_HERE = os.path.dirname(os.path.abspath(__file__))
+_SW = os.path.join(_HERE, "..", "zimi", "static", "sw.js")
+
+# Every endpoint whose response varies by authenticated identity or public-access
+# mode. These MUST be network-only in the SW.
+IDENTITY_ENDPOINTS = [
+ "/whoami",
+ "/login",
+ "/logout",
+ "/list",
+ "/search",
+ "/suggest",
+ "/random",
+]
+
+# Query strings must not change the classification (the SW keys off pathname).
+IDENTITY_WITH_QUERY = ["/search?q=water", "/list?layout=1", "/suggest?q=wa"]
+
+_DRIVER = r"""
+const fs = require('node:fs');
+const vm = require('node:vm');
+const src = fs.readFileSync(process.argv[1], 'utf8');
+
+function makeEnv() {
+ const cacheReads = [], cacheWrites = [];
+ const fakeCache = {
+ match: (r) => { cacheReads.push(r.url); return Promise.resolve(undefined); },
+ put: (r) => { cacheWrites.push(r.url); return Promise.resolve(); },
+ addAll: () => Promise.resolve(),
+ };
+ const caches = {
+ open: () => Promise.resolve(fakeCache),
+ match: (r) => { cacheReads.push(r.url); return Promise.resolve(undefined); },
+ keys: () => Promise.resolve([]), delete: () => Promise.resolve(true),
+ };
+ const listeners = {};
+ const self = {
+ addEventListener: (t, fn) => { listeners[t] = fn; },
+ skipWaiting: () => {}, clients: { claim: () => Promise.resolve() },
+ registration: { unregister: () => Promise.resolve() },
+ };
+ const ctx = {
+ self, caches, URL,
+ fetch: () => Promise.resolve({ ok: true, clone: () => ({}) }),
+ Response: class { constructor(b, o) { this.body = b; Object.assign(this, o); } },
+ setInterval: () => {}, console,
+ };
+ vm.createContext(ctx);
+ vm.runInContext(src, ctx);
+ return { self, listeners, cacheReads, cacheWrites };
+}
+
+async function probe(path, mode) {
+ const env = makeEnv();
+ const strategy = env.self.routeStrategy(new URL('http://x' + path).pathname, mode);
+ let responded = null;
+ env.listeners.fetch({ request: { url: 'http://x' + path, mode: mode || 'cors' },
+ respondWith: (p) => { responded = p; } });
+ try { await responded; } catch (e) {}
+ return { strategy, touchedCache: env.cacheReads.length + env.cacheWrites.length };
+}
+
+(async () => {
+ const paths = JSON.parse(process.argv[2]);
+ const out = {};
+ for (const p of paths) out[p] = await probe(p, 'cors');
+ process.stdout.write(JSON.stringify(out));
+})();
+"""
+
+
+def _run_driver(paths):
+ node = shutil.which("node")
+ if not node:
+ raise unittest.SkipTest("node not available")
+ proc = subprocess.run(
+ [node, "-e", _DRIVER, os.path.abspath(_SW), json.dumps(paths)],
+ capture_output=True,
+ text=True,
+ timeout=30,
+ )
+ if proc.returncode != 0:
+ raise AssertionError("SW driver failed: " + proc.stderr)
+ return json.loads(proc.stdout)
+
+
+class TestSwRouteClassification(unittest.TestCase):
+ def test_identity_endpoints_are_network_only(self):
+ res = _run_driver(IDENTITY_ENDPOINTS)
+ for p in IDENTITY_ENDPOINTS:
+ self.assertEqual(
+ res[p]["strategy"], "networkOnly", f"{p} must be network-only"
+ )
+
+ def test_identity_endpoints_never_touch_cache(self):
+ # Behavioural: even after dispatching the fetch event, no Cache API read
+ # or write happens for an identity endpoint — so no stale response can
+ # ever stand in for the live, correctly authorized one.
+ res = _run_driver(IDENTITY_ENDPOINTS)
+ for p in IDENTITY_ENDPOINTS:
+ self.assertEqual(
+ res[p]["touchedCache"], 0, f"{p} must never touch the SW cache"
+ )
+
+ def test_query_string_does_not_change_classification(self):
+ res = _run_driver(IDENTITY_WITH_QUERY)
+ for p in IDENTITY_WITH_QUERY:
+ self.assertEqual(res[p]["strategy"], "networkOnly", p)
+ self.assertEqual(res[p]["touchedCache"], 0, p)
+
+ def test_static_assets_still_cache(self):
+ # Guard the negative: static assets must remain cacheable (offline PWA),
+ # so the network-only rule didn't accidentally swallow everything.
+ res = _run_driver(["/static/app.js"])
+ self.assertEqual(res["/static/app.js"]["strategy"], "staleWhileRevalidate")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_unit.py b/tests/test_unit.py
index ee1a9b68..59fbf92a 100644
--- a/tests/test_unit.py
+++ b/tests/test_unit.py
@@ -604,6 +604,65 @@ def test_build_fts_for_existing_index(self):
self.zimi._close_title_db = orig_close
+class TestZimgitCatalogParse(unittest.TestCase):
+ """zimgit database.js parsing — Python-style single-quoted dicts.
+
+ zimgit ZIMs ship a database.js whose payload is a Python literal (single
+ quotes, not JSON), so parse_catalog must use ast.literal_eval. These tests
+ pin that quirk: the same bytes json.loads chokes on must parse cleanly.
+ """
+
+ def setUp(self):
+ from zimi import search
+
+ self.search = search
+
+ def _archive_with_db(self, content):
+ """MagicMock archive whose database.js entry yields `content`."""
+ from unittest.mock import MagicMock
+
+ archive = MagicMock()
+ entry = MagicMock()
+ entry.get_item.return_value.content = bytearray(content.encode("utf-8"))
+ archive.get_entry_by_path.return_value = entry
+ return archive
+
+ def test_parses_python_style_single_quoted_dicts(self):
+ # Real zimgit shape: single quotes, trailing semicolon, id/ti/dsc/aut/fp.
+ db = (
+ "var DATABASE = ["
+ "{'_id': '00000', 'ti': 'First Aid', 'dsc': 'Field Manual', "
+ "'aut': 'US Army', 'fp': ['First Aid (1).pdf']}"
+ "];"
+ )
+ items = self.search.parse_catalog(self._archive_with_db(db))
+ self.assertEqual(len(items), 1)
+ self.assertEqual(items[0]["ti"], "First Aid")
+ self.assertEqual(items[0]["fp"], ["First Aid (1).pdf"])
+
+ def test_json_would_reject_same_payload(self):
+ # Guards the reason ast.literal_eval exists: JSON can't read single quotes.
+ import ast
+ import json
+
+ payload = "[{'ti': 'X'}]"
+ with self.assertRaises(json.JSONDecodeError):
+ json.loads(payload)
+ self.assertEqual(ast.literal_eval(payload), [{"ti": "X"}])
+
+ def test_missing_database_js_returns_none(self):
+ from unittest.mock import MagicMock
+
+ archive = MagicMock()
+ archive.get_entry_by_path.side_effect = KeyError("database.js")
+ self.assertIsNone(self.search.parse_catalog(archive))
+
+ def test_malformed_payload_returns_none(self):
+ # A non-literal (function call) must fail closed, not raise.
+ archive = self._archive_with_db("var DATABASE = do_evil();")
+ self.assertIsNone(self.search.parse_catalog(archive))
+
+
class TestQuoteExtraction(unittest.TestCase):
"""Test wikiquote quote and attribution extraction from HTML."""
diff --git a/tests/test_userdata.py b/tests/test_userdata.py
new file mode 100644
index 00000000..b21baff4
--- /dev/null
+++ b/tests/test_userdata.py
@@ -0,0 +1,157 @@
+"""Per-user server-side data storage (v1.8.1): bookmarks/history/preferences
+kept per named user under ZIMI_DATA_DIR/userdata/.json.
+
+Two layers are exercised:
+ • the users.py storage primitives (load/save/delete/all/restore) and their
+ isolation guarantees, and
+ • the /userdata GET/POST endpoints, which must be gated to the SESSION user —
+ a user can only ever touch their OWN blob; anonymous/admin-without-a-user is
+ refused (their data stays in the browser).
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import zimi.http as http # noqa: E402
+import zimi.server as server # noqa: E402
+import zimi.users as users # noqa: E402
+
+
+def _setup(monkeypatch, tmp_path):
+ data_dir = tmp_path / "data"
+ data_dir.mkdir()
+ monkeypatch.setattr(server, "ZIMI_DATA_DIR", str(data_dir))
+ return data_dir
+
+
+# ── Storage primitives ──
+
+
+def test_save_and_load_roundtrip(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ ok, err = users.save_user_data(
+ "Alice", {"bookmarks": [{"zim": "a", "path": "b"}], "preferences": {"x": "1"}}
+ )
+ assert ok and err is None
+ blob = users.load_user_data("Alice")
+ assert blob["bookmarks"] == [{"zim": "a", "path": "b"}]
+ assert blob["preferences"] == {"x": "1"}
+ assert "updated" in blob
+
+
+def test_load_missing_returns_empty(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ blob = users.load_user_data("nobody")
+ assert (
+ blob["bookmarks"] == [] and blob["history"] == [] and blob["preferences"] == {}
+ )
+
+
+def test_key_is_casefolded(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ users.save_user_data("Alice", {"bookmarks": [{"zim": "a", "path": "b"}]})
+ # A differently-cased name resolves to the SAME blob.
+ assert users.load_user_data("ALICE")["bookmarks"] == [{"zim": "a", "path": "b"}]
+
+
+def test_users_are_isolated_on_disk(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ users.save_user_data("alice", {"bookmarks": [{"zim": "A", "path": "1"}]})
+ users.save_user_data("bob", {"bookmarks": [{"zim": "B", "path": "2"}]})
+ assert users.load_user_data("alice")["bookmarks"] == [{"zim": "A", "path": "1"}]
+ assert users.load_user_data("bob")["bookmarks"] == [{"zim": "B", "path": "2"}]
+
+
+def test_oversize_blob_rejected(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ huge = [{"zim": "z", "path": "p" * 1000} for _ in range(6000)]
+ ok, err = users.save_user_data("alice", {"bookmarks": huge})
+ assert not ok and err == "data too large"
+
+
+def test_delete_user_removes_their_data(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ users._save_users({"alice": {"name": "alice", "pw": "H", "role": "user"}})
+ users.save_user_data("alice", {"bookmarks": [{"zim": "a", "path": "b"}]})
+ assert os.path.exists(users._userdata_path("alice"))
+ users.delete_user("alice")
+ assert not os.path.exists(users._userdata_path("alice"))
+
+
+def test_key_traversal_is_refused(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ for bad in ("..", ".", "a/b", ""):
+ assert users._safe_userdata_key(bad) is None
+
+
+def test_all_and_restore_roundtrip(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ users.save_user_data("alice", {"bookmarks": [{"zim": "a", "path": "1"}]})
+ users.save_user_data("bob", {"history": [{"zim": "b", "path": "2"}]})
+ snap = users.all_user_data()
+ assert set(snap) == {"alice", "bob"}
+ # Wipe and restore from the snapshot.
+ users.delete_user_data("alice")
+ users.delete_user_data("bob")
+ assert users.all_user_data() == {}
+ n = users.restore_user_data(snap)
+ assert n == 2
+ assert users.load_user_data("alice")["bookmarks"] == [{"zim": "a", "path": "1"}]
+
+
+def test_restore_overwrite_clears_extras(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ users.save_user_data("stale", {"bookmarks": [{"zim": "x", "path": "y"}]})
+ users.restore_user_data({"fresh": {"bookmarks": []}}, overwrite=True)
+ assert set(users.all_user_data()) == {"fresh"} # "stale" was cleared
+
+
+# ── Endpoint gating (/userdata) ──
+
+
+class _Handler(http.ZimHandler):
+ """Minimal ZimHandler stand-in — no socket, just enough to run the two
+ /userdata handlers and capture their response."""
+
+ def __init__(self, session_user=None):
+ self._session_user = session_user
+ self.status = None
+ self.body = None
+ self.headers = {}
+
+ def _json(self, status, body):
+ self.status = status
+ self.body = body
+ return None
+
+
+def test_userdata_get_requires_signed_in_user(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ monkeypatch.setattr(users, "resolve_request_user", lambda h: None) # anon/admin
+ h = _Handler()
+ h._handle_userdata_get()
+ assert h.status == 401
+
+
+def test_userdata_post_saves_only_session_user(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ # The session identity — NOT anything in the body — decides whose blob is hit.
+ monkeypatch.setattr(users, "resolve_request_user", lambda h: "alice")
+ h = _Handler()
+ h._handle_userdata_post({"bookmarks": [{"zim": "a", "path": "b"}], "name": "bob"})
+ assert h.status == 200
+ # Written under alice; bob is untouched (no cross-user write path).
+ assert users.load_user_data("alice")["bookmarks"] == [{"zim": "a", "path": "b"}]
+ assert users.load_user_data("bob")["bookmarks"] == []
+
+
+def test_userdata_get_returns_own_blob(monkeypatch, tmp_path):
+ _setup(monkeypatch, tmp_path)
+ users.save_user_data("alice", {"bookmarks": [{"zim": "a", "path": "b"}]})
+ monkeypatch.setattr(users, "resolve_request_user", lambda h: "alice")
+ h = _Handler()
+ h._handle_userdata_get()
+ assert h.status == 200
+ assert h.body["bookmarks"] == [{"zim": "a", "path": "b"}]
diff --git a/tests/test_winsparkle.py b/tests/test_winsparkle.py
index 66e89718..ff737906 100644
--- a/tests/test_winsparkle.py
+++ b/tests/test_winsparkle.py
@@ -14,7 +14,8 @@
import pytest
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-sys.path.insert(0, REPO_ROOT)
+DESKTOP_DIR = os.path.join(REPO_ROOT, "desktop")
+sys.path.insert(0, DESKTOP_DIR)
import zimi_winsparkle as ws # noqa: E402
@@ -54,9 +55,38 @@ def test_appcast_url_matches_mac_feed_pattern():
assert url.endswith("appcast-windows.xml")
+# ── Appcast URL override (ZIMI_APPCAST_URL) ─────────────────────────────────
+
+
+def test_resolve_appcast_url_defaults_to_production(monkeypatch):
+ monkeypatch.delenv("ZIMI_APPCAST_URL", raising=False)
+ assert ws._resolve_appcast_url() == ws.WINDOWS_APPCAST_URL
+
+
+def test_resolve_appcast_url_honors_env_override(monkeypatch):
+ monkeypatch.setenv("ZIMI_APPCAST_URL", "http://localhost:8000/appcast-test.xml")
+ assert ws._resolve_appcast_url() == "http://localhost:8000/appcast-test.xml"
+
+
+def test_resolve_appcast_url_explicit_arg_wins_over_env(monkeypatch):
+ # An explicit caller argument beats the env override.
+ monkeypatch.setenv("ZIMI_APPCAST_URL", "http://localhost:8000/appcast-test.xml")
+ assert ws._resolve_appcast_url("https://example.com/x.xml") == (
+ "https://example.com/x.xml"
+ )
+
+
+@pytest.mark.skipif(platform.system() == "Windows", reason="no-op path is off-Windows")
+def test_init_updater_noop_even_with_env_override(monkeypatch):
+ # The override changes the feed URL, never the off-Windows soft-fail contract.
+ monkeypatch.setenv("ZIMI_APPCAST_URL", "http://localhost:8000/appcast-test.xml")
+ assert ws.init_updater("1.8.0") is False
+ assert ws._dll is None
+
+
def test_eddsa_key_matches_sparkle_spec_key():
"""WinSparkle reuses the macOS Sparkle keypair — guard against drift."""
- spec = open(os.path.join(REPO_ROOT, "zimi_desktop.spec")).read()
+ spec = open(os.path.join(DESKTOP_DIR, "zimi_desktop.spec")).read()
m = re.search(r"'SUPublicEDKey':\s*'([^']+)'", spec)
assert m, "SUPublicEDKey not found in zimi_desktop.spec"
assert ws.WINSPARKLE_EDDSA_PUBLIC_KEY == m.group(1)
diff --git a/tests/visual_validation.spec.mjs b/tests/visual_validation.spec.mjs
index f8977d1e..b5af5c28 100644
--- a/tests/visual_validation.spec.mjs
+++ b/tests/visual_validation.spec.mjs
@@ -2,15 +2,15 @@
* v1.6 Visual Validation — Complete Release Checklist
*
* Run against a remote host:
- * BASE_URL=https://knowledge.example.com npx playwright test
+ * BASE_URL=https://knowledge.example.com npx playwright test --config=tests/playwright.config.mjs
*
* Run against local:
- * npx playwright test
+ * npx playwright test --config=tests/playwright.config.mjs
*
* View report:
* npx playwright show-report test-results/html-report
*
- * Every test records video automatically (see playwright.config.mjs).
+ * Every test records video automatically (see tests/playwright.config.mjs).
* Screenshots are captured at key moments and embedded in the HTML report.
*/
import { test, expect } from '@playwright/test';
diff --git a/windows/zimi.iss b/windows/zimi.iss
index b15ed9ce..4fc96f2c 100644
--- a/windows/zimi.iss
+++ b/windows/zimi.iss
@@ -66,4 +66,8 @@ Name: "{group}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}"
Name: "{autodesktop}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}"; Tasks: desktopicon
[Run]
-Filename: "{app}\{#MyAppExeName}"; Description: "{cm:LaunchProgram,{#MyAppName}}"; Flags: nowait postinstall skipifsilent
+; Relaunch Zimi after install. No `skipifsilent`: WinSparkle runs this installer
+; with /SILENT for auto-updates and then quits the old app WITHOUT relaunching
+; it (WinSparkle 0.9.4 ShellExecuteEx's the installer, then RequestShutdown),
+; so the installer itself must bring the updated app back up — in silent mode too.
+Filename: "{app}\{#MyAppExeName}"; Description: "{cm:LaunchProgram,{#MyAppName}}"; Flags: nowait postinstall
diff --git a/zimi/health.py b/zimi/health.py
index 81ec462f..ca15be50 100644
--- a/zimi/health.py
+++ b/zimi/health.py
@@ -3,9 +3,10 @@
Admin clicks "Check", the server walks every installed ZIM sequentially
(opening each under ``_srv._zim_lock`` since libzim is not thread-safe),
and reports: opens OK, has a main page, entry count, title-index and Q-ID
-index status, size vs. the cached catalog, and last-updated age. The known
-broken case (``devdocs_en_react``: 0 entries / tiny file) surfaces as a
-warning row.
+index status, size vs. the cached catalog, last-updated age, and a media
+integrity sample (0-byte video/audio entries — broken/partial scrapes that
+the count and size checks can't see). The known broken case
+(``devdocs_en_react``: 0 entries / tiny file) surfaces as a warning row.
Runs on a daemon worker thread; the client polls ``get_state()`` — the same
progress-dict pattern as mirror sync.
@@ -93,6 +94,125 @@ def _age_days(entry, path):
return None
+# A media entry (video/audio) with zero bytes is always broken — a file that
+# can't play — but neither the entry count nor the size-vs-catalog check sees it,
+# because the count is unchanged and the missing bytes are a rounding error
+# against a multi-GB ZIM. Partial breakage is the common shape: e.g.
+# ted_en_technology_2023-09 ships 1184 real .webm videos alongside 26 zero-byte
+# .mp4 placeholders, so browsing one of those 26 talks looks like an app bug
+# (issue #38 follow-up). Sample media entries by extension and flag any empty.
+_MEDIA_EXTS = (
+ ".webm",
+ ".mp4",
+ ".m4v",
+ ".mov",
+ ".mkv",
+ ".ogv",
+ ".avi", # video
+ ".mp3",
+ ".ogg",
+ ".oga",
+ ".m4a",
+ ".opus",
+ ".aac",
+ ".flac",
+ ".wav",
+ ".weba", # audio
+)
+# ZIMs up to this many entries get a full entry walk (a full walk of a
+# 34k-entry video ZIM is ~0.2s, and video/audio ZIMs are rarely larger). Above
+# it — almost always text corpora with little or no media — fall back to a
+# strided sample so the walk stays bounded under the lock (a full walk of a
+# 2.4M-entry Wikipedia is ~8s, far too slow to hold libzim's global lock).
+_MEDIA_FULL_SCAN_MAX = 120_000
+_MEDIA_STRIDED_PROBES = 2_000
+
+
+def _all_entry_count(archive):
+ """All-entries count (not just articles), the id space for
+ ``_get_entry_by_id``. Prefers ``all_entry_count``, falls back."""
+ for attr in ("all_entry_count", "entry_count"):
+ try:
+ n = getattr(archive, attr)
+ if n:
+ return int(n)
+ except Exception:
+ pass
+ return 0
+
+
+def _sample_media(archive):
+ """Walk (or strided-sample) a ZIM for media files that carry no content.
+
+ Returns ``{"sampled","empty","examples","full"}``: media entries examined,
+ how many are 0-byte, up to 5 example paths, and whether the whole entry
+ space was covered (``full``) or only sampled. ``None`` when the libzim build
+ lacks id iteration or the ZIM is empty. Caller holds ``_zim_lock`` — libzim
+ is single-threaded — so this stays a bounded, lock-friendly pass."""
+ if not hasattr(archive, "_get_entry_by_id"):
+ return None
+ n = _all_entry_count(archive)
+ if n <= 0:
+ return None
+ full = n <= _MEDIA_FULL_SCAN_MAX
+ ids = range(n) if full else range(0, n, max(1, n // _MEDIA_STRIDED_PROBES))
+ sampled = empty = 0
+ examples = []
+ for j in ids:
+ try:
+ e = archive._get_entry_by_id(j)
+ if e.is_redirect or not e.path.lower().endswith(_MEDIA_EXTS):
+ continue
+ item = e.get_item()
+ sampled += 1
+ if item.size == 0 or (item.mimetype or "") == "application/x-empty":
+ empty += 1
+ if len(examples) < 5:
+ examples.append(e.path)
+ except Exception:
+ continue
+ return {"sampled": sampled, "empty": empty, "examples": examples, "full": full}
+
+
+# Universal article sanity: even a ZIM with no media can be a broken scrape
+# (every article a 0-byte shell). A tiny strided probe for a few text/html
+# entries confirms at least some carry content — cheap enough to run on every
+# ZIM, unlike the exhaustive media walk. Metadata-only reads (size / mimetype),
+# never the blob.
+_TEXT_SAMPLE = 3 # html articles to size-check
+_TEXT_PROBE_MAX = 60 # entries to probe to find them (bounded)
+
+
+def _sample_text(archive):
+ """Probe a bounded, strided set of entries for a few text/html articles and
+ confirm they carry content. Returns ``{"sampled","empty"}`` (articles
+ examined / of those, 0-byte); ``None`` when id iteration is unavailable or
+ the ZIM is empty. Caller holds ``_zim_lock``."""
+ if not hasattr(archive, "_get_entry_by_id"):
+ return None
+ n = _all_entry_count(archive)
+ if n <= 0:
+ return None
+ step = max(1, n // _TEXT_PROBE_MAX)
+ sampled = empty = 0
+ for j in range(0, n, step):
+ if sampled >= _TEXT_SAMPLE:
+ break
+ try:
+ e = archive._get_entry_by_id(j)
+ if e.is_redirect:
+ continue
+ item = e.get_item()
+ if not (item.mimetype or "").startswith("text/html"):
+ continue
+ sampled += 1
+ if item.size == 0 or (item.mimetype or "") == "application/x-empty":
+ empty += 1
+ except Exception:
+ continue
+ return {"sampled": sampled, "empty": empty}
+
+
def _check_one(entry, path, catalog_sizes):
"""Build a single ZIM's health row."""
name = entry.get("name", "")
@@ -106,11 +226,17 @@ def _check_one(entry, path, catalog_sizes):
"qid_index": "absent", # absent | present
"size_delta": None, # installed - catalog bytes (None if no catalog)
"age_days": _age_days(entry, path),
+ "media_sampled": None, # media entries examined (None = not checked)
+ "media_empty": None, # of those, how many are 0-byte
+ "text_sampled": None, # html articles examined (None = not checked)
+ "text_empty": None, # of those, how many are 0-byte
"status": "warn",
"issues": [],
}
# libzim open + main page — one ZIM at a time under the global read lock.
+ media = None
+ text = None
try:
with _srv._zim_lock:
archive = _srv.open_archive(path)
@@ -124,6 +250,14 @@ def _check_one(entry, path, catalog_sizes):
except Exception:
row["has_main"] = False
row["opens"] = True
+ try:
+ media = _sample_media(archive)
+ except Exception as e:
+ log.debug("health: media sample failed for %s: %s", name, e)
+ try:
+ text = _sample_text(archive)
+ except Exception as e:
+ log.debug("health: text sample failed for %s: %s", name, e)
finally:
del archive
except Exception as e:
@@ -134,6 +268,29 @@ def _check_one(entry, path, catalog_sizes):
row["issues"].append("no main page")
if row["opens"] and row["entries"] == 0:
row["issues"].append("empty (0 entries)")
+ if media and media["sampled"]:
+ row["media_sampled"] = media["sampled"]
+ row["media_empty"] = media["empty"]
+ if media["examples"]:
+ row["media_examples"] = media["examples"]
+ if media["empty"]:
+ # "sampled" qualifier when we only strided a huge ZIM — the true
+ # empty count may be higher than what the sample saw.
+ scope = "" if media["full"] else " in sample"
+ row["issues"].append(
+ f"{media['empty']} of {media['sampled']} media entries "
+ f"empty / 0-byte{scope}"
+ )
+ if text and text["sampled"]:
+ row["text_sampled"] = text["sampled"]
+ row["text_empty"] = text["empty"]
+ # Every sampled article empty is the fingerprint of a broken scrape even
+ # when the ZIM carries no media (a media-empty flag would never fire).
+ if text["empty"] == text["sampled"]:
+ row["issues"].append(
+ f"all {text['sampled']} sampled articles empty / 0-byte "
+ f"— broken scrape?"
+ )
# Index status (sqlite reads — no libzim lock needed).
try:
diff --git a/zimi/http.py b/zimi/http.py
index 7a1b1367..501a22db 100644
--- a/zimi/http.py
+++ b/zimi/http.py
@@ -171,6 +171,33 @@ def _is_trusted_net(ip):
)
)
+# The ONLY paths an anonymous visitor may reach when the public-access policy is
+# ``private``. Everything else 401s until they log in. The set is deliberately
+# minimal — enough to render the login screen and authenticate, nothing that
+# reveals library contents:
+# / the SPA shell (static HTML, no data)
+# /whoami so the client learns it must show the login screen
+# /health aggregate status; already allow-filtered (→ 0 zims)
+# /login /logout the auth transitions themselves
+# favicons / touch icon chrome the shell references
+# Prefixes: /static/ (app.js, app.css, i18n, sw.js, pdfjs, manifest) and
+# /manage/ (self-gated — its own admin challenge blocks non-admins, while the
+# pre-auth has-password/has-token the login modal needs stay reachable).
+_PRIVATE_LOGIN_SURFACE_EXACT = frozenset(
+ (
+ "/",
+ "/whoami",
+ "/health",
+ "/login",
+ "/logout",
+ "/favicon.ico",
+ "/favicon.png",
+ "/favicon-64.png",
+ "/apple-touch-icon.png",
+ )
+)
+_PRIVATE_LOGIN_SURFACE_PREFIX = ("/static/", "/manage/")
+
def _rate_class(path):
"""(is_rate_limited, uses_content_bucket) for a GET path."""
@@ -586,6 +613,27 @@ def _client_ip(self):
return "proxy-unknown"
return direct_ip
+ def _private_access_block(self, parsed):
+ """Enforce ``private`` public-access mode. Returns True (and sends a 401)
+ when an ANONYMOUS request targets anything outside the login surface;
+ False to proceed. Admins and logged-in users always proceed.
+
+ Checks the cheap static path allowlist BEFORE probing identity, so
+ serving the login shell + assets costs no session/admin file reads.
+ """
+ mode, _ = _users.get_public_access()
+ if mode != "private":
+ return False
+ path = parsed.path
+ if path in _PRIVATE_LOGIN_SURFACE_EXACT or path.startswith(
+ _PRIVATE_LOGIN_SURFACE_PREFIX
+ ):
+ return False
+ if _users.resolve_request_user(self) or _users._request_is_admin(self):
+ return False
+ self._json(401, {"error": "authentication required", "login_required": True})
+ return True
+
def do_GET(self):
parsed = urlparse(self.path)
params = parse_qs(parsed.query)
@@ -598,6 +646,17 @@ def param(key, default=None):
# connection re-sets it per request; cleared in the finally for hygiene.
_srv.set_request_allow(_users.request_allow(self))
+ # Private mode: block anonymous reads before any handler runs. The
+ # finally below still clears the request-allow context.
+ try:
+ if self._private_access_block(parsed):
+ return
+ except Exception:
+ # Fail closed: if the policy can't be evaluated, deny rather than
+ # risk serving content an intended-private instance meant to hide.
+ self._json(401, {"error": "authentication required"})
+ return
+
# Rate limit: API endpoints at RATE_LIMIT (10x for trusted clients),
# /w/ content and /snippet at 20x.
limited, is_w_content = _rate_class(parsed.path)
@@ -849,6 +908,7 @@ def param(key, default=None):
{
"zims": result,
"section_order": layout.get("section_order", []),
+ "sections": layout.get("sections", []),
},
)
return self._json(200, result)
@@ -856,6 +916,9 @@ def param(key, default=None):
elif parsed.path == "/whoami":
return self._handle_whoami()
+ elif parsed.path == "/userdata":
+ return self._handle_userdata_get()
+
elif parsed.path == "/languages":
# Installed language summary with native names and ZIM counts
lang_zims = {} # {lang_code: [zim_name, ...]}
@@ -925,28 +988,10 @@ def param(key, default=None):
# Read first 15KB — enough for meta tags + initial content
raw = bytes(item.content)[:15360]
text = raw.decode("UTF-8", errors="replace")
- # Prefer meta description (skips nav/header boilerplate)
- for desc_pat in [
- r' or body (skip nav boilerplate)
- if not snippet:
- for tag in ["main", "article"]:
- tag_m = re.search(
- r"<" + tag + r"[\s>]", text, re.IGNORECASE
- )
- if tag_m:
- plain = _srv.strip_html(text[tag_m.start() :])
- snippet = plain[:300].strip()
- break
- # Last resort: full page text
- if not snippet:
- snippet = _srv.strip_html(text)[:300].strip()
+ # Prefer the page's own summary, then meta description,
+ # then body prose — skipping boilerplate some ZIMs bake
+ # into every page (iFixit device pages, #snippet QA).
+ snippet = _srv.extract_snippet(text, zim)
# Lightweight thumbnail: og:image / twitter:image from
for img_pat in [
r' _srv.MAX_POST_BODY:
+ # Backup import + per-user data save legitimately run large (a full
+ # server bundle carries users/history/every per-user blob); every
+ # other endpoint stays under the tight default cap.
+ body_cap = (
+ _srv.MAX_BACKUP_BODY
+ if parsed.path in ("/manage/backup", "/userdata")
+ else _srv.MAX_POST_BODY
+ )
+ if content_len > body_cap:
return self._json(
413,
- {
- "error": f"Request body too large (max {_srv.MAX_POST_BODY} bytes)"
- },
+ {"error": f"Request body too large (max {body_cap} bytes)"},
)
body = self.rfile.read(content_len) if content_len > 0 else b"{}"
try:
@@ -1352,6 +1409,9 @@ def do_POST(self):
if parsed.path == "/logout":
return self._handle_logout()
+ if parsed.path == "/userdata":
+ return self._handle_userdata_post(data)
+
if parsed.path == "/resolve":
retry_after = _check_rate_limit(
self._client_ip(), limit=self._rate_limit_for_request()
@@ -2200,17 +2260,29 @@ def _handle_login(self, data):
},
self._session_cookie(token, remember),
)
- # Admin account (the existing password account).
+ # Admin account (the existing password account). Mint an HttpOnly admin
+ # session cookie IN ADDITION to the client's password Bearer, so the
+ # header-less transports (reader iframe, plain-fetch data endpoints) carry
+ # admin identity — otherwise a private/limited-mode admin sees an empty
+ # library and blank article iframes. The client still keeps using the
+ # password as its manage Bearer token (unchanged).
from zimi import manage as _manage
if _manage.verify_admin_credentials(username, password):
- return self._json(200, {"role": "admin"})
+ token = _users.create_admin_session()
+ log.info("Admin login (password account)")
+ return self._json_cookie(
+ 200, {"role": "admin"}, self._session_cookie(token, remember)
+ )
return self._json(401, {"error": "invalid credentials"})
def _handle_logout(self):
- """POST /logout — drop the current session + expire the cookie."""
- token = _users._bearer_token(self) or _users._cookie_token(self)
- _users.drop_session(token)
+ """POST /logout — drop the current session(s) + expire the cookie. Drops
+ BOTH the Bearer and cookie tokens: an admin's Bearer is the password (a
+ no-op drop) while its session rides the cookie, so dropping only the
+ first-present token could leave the admin session alive server-side."""
+ _users.drop_session(_users._bearer_token(self))
+ _users.drop_session(_users._cookie_token(self))
return self._json_cookie(200, {"status": "ok"}, self._expire_cookie())
def _handle_whoami(self):
@@ -2243,18 +2315,58 @@ def _handle_whoami(self):
and _manage._get_manage_password_hash()
and _manage._check_manage_auth(self) is None
):
- return self._json(
- 200,
- {"role": "admin", "name": _manage._get_manage_user() or "admin"},
- )
+ resp = {"role": "admin", "name": _manage._get_manage_user() or "admin"}
+ # Ensure the header-less transports (reader iframe, plain-fetch data
+ # endpoints) carry admin identity. If this admin was recognised by the
+ # password Bearer but has no live admin session cookie yet — first boot
+ # after a remembered login, or the cookie expired — mint one now. Boot
+ # awaits /whoami before the first /list, so the cookie lands in time. A
+ # session-scoped cookie (no Max-Age) keeps the "remember" contract: a
+ # non-remembered admin loses it on tab close (its stored Bearer is gone
+ # too), while a remembered admin re-mints from the Bearer each boot.
+ if not _users.is_admin_session(_users._cookie_token(self)):
+ token = _users.create_admin_session()
+ return self._json_cookie(200, resp, self._session_cookie(token, False))
+ return self._json(200, resp)
# Anonymous. Expose a first-login hint ONLY when the default username
# applies — no custom username AND no named users configured. This is
# not an info leak: "the default username is admin" is in the docs.
resp = {"role": "anonymous"}
if not _manage._get_manage_user() and not _users.list_users():
resp["default_username"] = "admin"
+ # Tell the SPA how the public-access policy shapes its view: ``private``
+ # forces the login screen (login_required); ``limited`` just means the
+ # library it receives is already filtered server-side (no client action
+ # needed, but surfaced for messaging). ``open`` omits the field.
+ mode, _ = _users.get_public_access()
+ if mode != "open":
+ resp["public_access"] = mode
+ if mode == "private":
+ resp["login_required"] = True
return self._json(200, resp)
+ def _handle_userdata_get(self):
+ """GET /userdata — the signed-in user's own server-stored My-data blob
+ (bookmarks/history/preferences). Only a NAMED user resolves here; an
+ admin-without-a-user or an anonymous visitor gets 401 and keeps their
+ data in the browser."""
+ name = _users.resolve_request_user(self)
+ if not name:
+ return self._json(401, {"error": "sign in required"})
+ return self._json(200, _users.load_user_data(name))
+
+ def _handle_userdata_post(self, data):
+ """POST /userdata — save the signed-in user's own My-data blob. A user
+ can only ever touch their OWN data: the target is the session identity,
+ never a name from the body, so there is no cross-user write path."""
+ name = _users.resolve_request_user(self)
+ if not name:
+ return self._json(401, {"error": "sign in required"})
+ ok, err = _users.save_user_data(name, data if isinstance(data, dict) else {})
+ if not ok:
+ return self._json(400, {"error": err})
+ return self._json(200, {"status": "ok"})
+
def log_message(self, format, *args):
# Light logging: errors + slow requests. Suppress 200/304 noise.
if len(args) >= 2 and str(args[1]) in ("200", "304"):
diff --git a/zimi/library.py b/zimi/library.py
index b5dabb5d..f00602fb 100644
--- a/zimi/library.py
+++ b/zimi/library.py
@@ -254,25 +254,374 @@ def _auto_update_loop(initial_delay=0):
_download_counter = 0
_download_lock = threading.Lock()
-# Concurrent-download cap. Default 3; overridable via ZIMI_MAX_CONCURRENT_DOWNLOADS.
-# Items beyond the cap are queued in _download_queue, smallest-first.
-_MAX_CONCURRENT_DEFAULT = 3
+# Concurrent-download cap authority lives in p2p (get_max_active_downloads);
+# _max_concurrent() below delegates. Items beyond the cap queue in
+# _download_queue, smallest-first.
_download_queue = [] # [dl, ...] sorted: known sizes ascending, unknown sizes last
-def _max_concurrent():
- """Concurrent-download cap, read from env each call so tests can flip it.
+# ----------------------------------------------------------------------------
+# Global download-speed throttle
+# ----------------------------------------------------------------------------
+# The BT session enforces its own download_rate_limit; HTTP downloads share
+# the same global cap (p2p.get_download_limit_kb) via a token bucket held
+# across every download thread — so N concurrent HTTP pulls sum to the cap,
+# not N × the cap. 0 = unlimited.
+class _DownloadThrottle:
+ """Shared byte-rate limiter across all HTTP download threads.
+
+ ``consume`` accounts ``nbytes`` against a token bucket refilled at
+ ``rate_bps`` bytes/sec and returns how long the caller should sleep to
+ stay under the rate. Pure arithmetic (clock injectable) so the pacing
+ math is unit-testable without real sleeps. ``rate_bps <= 0`` disables it.
+ """
+
+ def __init__(self, clock=time.monotonic):
+ self._lock = threading.Lock()
+ self._clock = clock
+ self._tokens = 0.0
+ self._last = None
+
+ def reset(self):
+ with self._lock:
+ self._tokens = 0.0
+ self._last = None
+
+ def consume(self, nbytes, rate_bps):
+ if rate_bps <= 0:
+ return 0.0
+ with self._lock:
+ now = self._clock()
+ if self._last is None:
+ # Start with a full one-second burst so a fresh (or reset)
+ # bucket doesn't stall the very first chunk.
+ self._last = now
+ self._tokens = rate_bps
+ # Refill, capping the burst allowance at one second's worth so a
+ # long idle can't bank unlimited credit.
+ self._tokens += (now - self._last) * rate_bps
+ self._last = now
+ if self._tokens > rate_bps:
+ self._tokens = rate_bps
+ self._tokens -= nbytes
+ if self._tokens >= 0:
+ return 0.0
+ return -self._tokens / rate_bps
+
+
+_download_throttle = _DownloadThrottle()
+
+# The download cap lives in a prefs file; re-reading it per 64 KB chunk is
+# needless I/O. Cache it briefly so a live change still lands within ~2s.
+_rate_cache = {"ts": 0.0, "bps": 0}
+_RATE_CACHE_TTL = 2.0
+
+
+def _download_rate_bps():
+ """Current global download cap in bytes/sec (0 = unlimited), cached ~2s."""
+ now = time.monotonic()
+ if now - _rate_cache["ts"] > _RATE_CACHE_TTL:
+ try:
+ from zimi import p2p as _p2p
+
+ _rate_cache["bps"] = max(0, _p2p.get_download_limit_kb()) * 1024
+ except Exception:
+ _rate_cache["bps"] = 0
+ _rate_cache["ts"] = now
+ return _rate_cache["bps"]
+
+
+# ----------------------------------------------------------------------------
+# Scheduled downloads — optional night-window queueing
+# ----------------------------------------------------------------------------
+# When enabled, downloads started OUTSIDE the configured local-time window are
+# held in the queue with a "scheduled" marker instead of starting immediately;
+# a background watcher promotes them once the window opens. Disabled by default
+# (new downloads start right away — the pre-existing behavior). Times are
+# minutes-since-local-midnight; a window may span midnight (start > end).
+_DOWNLOAD_SCHEDULE_CONFIG = os.path.join(_srv.ZIMI_DATA_DIR, "download_schedule.json")
+_DEFAULT_WINDOW_START = "01:00"
+_DEFAULT_WINDOW_END = "07:00"
+# Trickle cap (KB/s) applied to seeding when uploads are restricted to the
+# window and we're outside it. A low positive floor, not 0 — 0 means
+# "unlimited" everywhere else in the rate plumbing, which would be the opposite
+# of a trickle.
+_DEFAULT_UPLOAD_TRICKLE_KB = 50
+_schedule_watcher_thread = None
+# Last (restrict, in_window) tuple pushed to the BT session by the upload
+# restrictor, so a 60s tick only touches libtorrent on an actual transition.
+_upload_window_applied = None
+
+
+def _parse_hhmm(s):
+ """'HH:MM' -> minutes since midnight, or None if malformed."""
+ m = re.match(r"^([01]?\d|2[0-3]):([0-5]\d)$", str(s or "").strip())
+ if not m:
+ return None
+ return int(m.group(1)) * 60 + int(m.group(2))
+
- Invalid values (non-integer, negative, zero) clamp to safe defaults.
+def _fmt_hhmm(minutes):
+ """Minutes since midnight -> 'HH:MM'."""
+ minutes = int(minutes) % 1440
+ return f"{minutes // 60:02d}:{minutes % 60:02d}"
+
+
+def _in_window(now_min, start_min, end_min):
+ """True if now_min falls inside [start, end). Equal bounds = always open
+ (a degenerate 24h window). Spans midnight when start > end."""
+ if start_min == end_min:
+ return True
+ if start_min < end_min:
+ return start_min <= now_min < end_min
+ return now_min >= start_min or now_min < end_min
+
+
+def _load_download_schedule():
+ """Return {'enabled', 'start', 'end'} for the download window.
+
+ ZIMI_DL_WINDOW='HH:MM-HH:MM' locks the window and forces scheduling on;
+ otherwise the persisted config (default: disabled, 01:00-07:00)."""
+ env_win = os.environ.get("ZIMI_DL_WINDOW", "").strip()
+ if env_win and "-" in env_win:
+ a, b = env_win.split("-", 1)
+ if _parse_hhmm(a) is not None and _parse_hhmm(b) is not None:
+ return {
+ "enabled": True,
+ "start": a.strip(),
+ "end": b.strip(),
+ "locked": True,
+ "upload_restrict": False,
+ "upload_trickle_kb": _DEFAULT_UPLOAD_TRICKLE_KB,
+ }
+ cfg_path = getattr(_srv, "_DOWNLOAD_SCHEDULE_CONFIG", _DOWNLOAD_SCHEDULE_CONFIG)
+ try:
+ with open(cfg_path, encoding="utf-8") as f:
+ cfg = json.loads(f.read())
+ start = cfg.get("start", _DEFAULT_WINDOW_START)
+ end = cfg.get("end", _DEFAULT_WINDOW_END)
+ if _parse_hhmm(start) is None:
+ start = _DEFAULT_WINDOW_START
+ if _parse_hhmm(end) is None:
+ end = _DEFAULT_WINDOW_END
+ try:
+ trickle = max(
+ 1, int(cfg.get("upload_trickle_kb", _DEFAULT_UPLOAD_TRICKLE_KB))
+ )
+ except (ValueError, TypeError):
+ trickle = _DEFAULT_UPLOAD_TRICKLE_KB
+ return {
+ "enabled": bool(cfg.get("enabled", False)),
+ "start": start,
+ "end": end,
+ "locked": False,
+ "upload_restrict": bool(cfg.get("upload_restrict", False)),
+ "upload_trickle_kb": trickle,
+ }
+ except (OSError, json.JSONDecodeError, ValueError):
+ return {
+ "enabled": False,
+ "start": _DEFAULT_WINDOW_START,
+ "end": _DEFAULT_WINDOW_END,
+ "locked": False,
+ "upload_restrict": False,
+ "upload_trickle_kb": _DEFAULT_UPLOAD_TRICKLE_KB,
+ }
+
+
+def _save_download_schedule(
+ enabled, start, end, upload_restrict=None, upload_trickle_kb=None
+):
+ """Persist the download window. Returns False if the config is env-locked
+ or the write fails.
+
+ The upload-window fields (restrict seeding to the window + the trickle cap)
+ are UI-only — no env lock — and preserved when the caller passes None, so a
+ window edit doesn't clobber the seeding policy and vice versa.
"""
- raw = os.environ.get("ZIMI_MAX_CONCURRENT_DOWNLOADS")
- if raw is None:
- return _MAX_CONCURRENT_DEFAULT
+ cur = _load_download_schedule()
+ if cur.get("locked"):
+ return False
+ if upload_restrict is None:
+ upload_restrict = cur["upload_restrict"]
+ if upload_trickle_kb is None:
+ upload_trickle_kb = cur["upload_trickle_kb"]
try:
- n = int(raw)
+ trickle = max(1, int(upload_trickle_kb))
except (ValueError, TypeError):
- return _MAX_CONCURRENT_DEFAULT
- return max(1, n)
+ trickle = _DEFAULT_UPLOAD_TRICKLE_KB
+ cfg_path = getattr(_srv, "_DOWNLOAD_SCHEDULE_CONFIG", _DOWNLOAD_SCHEDULE_CONFIG)
+ try:
+ _srv._atomic_write_json(
+ cfg_path,
+ {
+ "enabled": bool(enabled),
+ "start": start,
+ "end": end,
+ "upload_restrict": bool(upload_restrict),
+ "upload_trickle_kb": trickle,
+ },
+ )
+ # A policy change is a transition — push the right cap now instead of
+ # waiting up to a full tick for the watcher to notice.
+ _apply_upload_window(force=True)
+ return True
+ except OSError as e:
+ log.warning("could not persist download schedule: %s", e)
+ return False
+
+
+def _now_local_minutes():
+ lt = time.localtime()
+ return lt.tm_hour * 60 + lt.tm_min
+
+
+def _within_download_window(now_min=None):
+ """True when downloads may start NOW. Always True when scheduling is off,
+ or when the window is malformed (never trap downloads on bad config)."""
+ sched = _load_download_schedule()
+ if not sched["enabled"]:
+ return True
+ start = _parse_hhmm(sched["start"])
+ end = _parse_hhmm(sched["end"])
+ if start is None or end is None:
+ return True
+ if now_min is None:
+ now_min = _now_local_minutes()
+ return _in_window(now_min, start, end)
+
+
+def _schedule_defers_now():
+ """True when a newly-started download should be held for the window
+ instead of starting immediately."""
+ return _load_download_schedule()["enabled"] and not _within_download_window()
+
+
+def _in_window_now(sched=None, now_min=None):
+ """Raw window membership, IGNORING the ``enabled`` flag.
+
+ The upload restrictor has its own toggle (``upload_restrict``) over the same
+ window bounds, so it can't reuse ``_within_download_window`` (which
+ short-circuits to True when download-queueing is off). Malformed bounds → in
+ window (never throttle on bad config)."""
+ sched = sched or _load_download_schedule()
+ start = _parse_hhmm(sched["start"])
+ end = _parse_hhmm(sched["end"])
+ if start is None or end is None:
+ return True
+ if now_min is None:
+ now_min = _now_local_minutes()
+ return _in_window(now_min, start, end)
+
+
+def _apply_upload_window(force=False):
+ """Push the correct upload cap to the BT session for the current window.
+
+ When ``upload_restrict`` is on and we're OUTSIDE the window, seeding is
+ trickled to ``upload_trickle_kb``; inside the window (or when the option is
+ off) the normal up limit is restored. Idempotent — only touches libtorrent
+ on a state change, so a 60s tick is free when nothing moved."""
+ global _upload_window_applied
+ from zimi import p2p as _p2p
+
+ sched = _load_download_schedule()
+ restrict = sched["upload_restrict"]
+ inside = _in_window_now(sched)
+ state = (restrict, inside)
+ if not force and state == _upload_window_applied:
+ return
+ _upload_window_applied = state
+ if restrict and not inside:
+ _p2p.set_upload_window_cap(sched["upload_trickle_kb"])
+ else:
+ _p2p.set_upload_window_cap(None)
+
+
+def _download_schedule_status():
+ """Serialize the schedule config for the /manage/download-schedule endpoint."""
+ from zimi import p2p as _p2p
+
+ sched = _load_download_schedule()
+ return {
+ "enabled": sched["enabled"],
+ "start": sched["start"],
+ "end": sched["end"],
+ "locked": sched["locked"],
+ "in_window": _within_download_window(),
+ # The global download-speed cap lives with the BT down limit — one
+ # number governs every transport (see p2p.get_download_limit_kb).
+ "download_kb": _p2p.get_download_limit_kb(),
+ "download_kb_locked": _p2p.is_bt_down_env_locked(),
+ # Upload-window restrictor: throttle seeding to a trickle outside the
+ # window. Shares the window bounds above but is independently toggled.
+ "upload_restrict": sched["upload_restrict"],
+ "upload_trickle_kb": sched["upload_trickle_kb"],
+ "upload_kb": _p2p.get_bt_up_limit_kb(),
+ "upload_kb_locked": _p2p.is_bt_up_env_locked(),
+ # True right now: uploads are actively trickled (restrict on + outside).
+ "upload_throttled": sched["upload_restrict"] and not _in_window_now(sched),
+ }
+
+
+def _download_schedule_tick():
+ """One watcher pass: release scheduled downloads once the window is open and
+ keep the upload cap in step with the window. Cheap no-op otherwise.
+ Extracted so tests can drive it without the loop."""
+ if _within_download_window():
+ with _download_lock:
+ if _download_queue:
+ _drain_queue()
+ _apply_upload_window()
+
+
+def _download_schedule_loop(interval=60):
+ """Background watcher that opens the gate when the window arrives.
+
+ Ticks every ``interval`` seconds. Laptop-sleep resilient: a missed window
+ start just means scheduled downloads begin at the next tick inside the
+ window, not that they're lost. Runs for the process lifetime (daemon)."""
+ while True:
+ try:
+ _download_schedule_tick()
+ except Exception as e:
+ log.debug("download schedule tick failed: %s", e)
+ time.sleep(interval)
+
+
+def start_download_scheduler():
+ """Start the singleton schedule watcher thread (idempotent)."""
+ global _schedule_watcher_thread
+ if _schedule_watcher_thread and _schedule_watcher_thread.is_alive():
+ return
+ _schedule_watcher_thread = threading.Thread(
+ target=_download_schedule_loop, daemon=True, name="download-scheduler"
+ )
+ _schedule_watcher_thread.start()
+ # Set the upload cap to match wherever the window sits right now, so a
+ # process that boots outside its window starts throttled without waiting a
+ # full tick.
+ try:
+ _apply_upload_window(force=True)
+ except Exception as e:
+ log.debug("initial upload-window apply failed: %s", e)
+
+
+def _max_concurrent():
+ """Concurrent-download cap. Authority lives in p2p (legacy
+ ZIMI_MAX_CONCURRENT_DOWNLOADS > ZIMI_BT active= > persisted UI pref >
+ default) so the BitTorrent settings card and this download queue read one
+ number. Invalid/zero values clamp to a safe minimum there."""
+ from zimi import p2p as _p2p
+
+ return _p2p.get_max_active_downloads()
+
+
+def drain_download_queue():
+ """Promote queued downloads into freshly-available slots — e.g. right
+ after the concurrency cap is raised in the UI, when nothing else would
+ trigger a drain until a download finishes. Safe to call any time."""
+ with _download_lock:
+ _drain_queue()
def _active_count():
@@ -280,17 +629,16 @@ def _active_count():
return sum(1 for d in _active_downloads.values() if not d.get("done"))
-def _enqueue_or_start(dl):
- """Either start the download immediately or place it in the queue.
+def _launch_download(dl):
+ """Move dl into an active slot and spawn its thread. Hold _download_lock."""
+ dl.pop("scheduled", None)
+ _active_downloads[dl["id"]] = dl
+ threading.Thread(target=_download_thread, args=(dl,), daemon=True).start()
- Returns True if queued, False if started. Caller must hold _download_lock.
- """
- if _active_count() < _max_concurrent():
- _active_downloads[dl["id"]] = dl
- threading.Thread(target=_download_thread, args=(dl,), daemon=True).start()
- _persist_pending_downloads()
- return False
- # Queue: known sizes ascending; unknown (None) sizes go to the end.
+
+def _insert_into_queue(dl):
+ """Insert dl into the queue, known sizes ascending, unknown sizes last.
+ Hold _download_lock."""
sz = dl.get("size_bytes")
pos = len(_download_queue)
if sz is not None:
@@ -300,19 +648,45 @@ def _enqueue_or_start(dl):
pos = i
break
_download_queue.insert(pos, dl)
+
+
+def _enqueue_or_start(dl):
+ """Either start the download immediately or place it in the queue.
+
+ Returns True if queued, False if started. Caller must hold _download_lock.
+ When download scheduling is on and we're outside the window, the download
+ is queued as ``scheduled`` regardless of free slots — the watcher (or the
+ window opening) releases it later.
+ """
+ if _schedule_defers_now():
+ dl["scheduled"] = True
+ _insert_into_queue(dl)
+ _persist_pending_downloads()
+ return True
+ if _active_count() < _max_concurrent():
+ _launch_download(dl)
+ _persist_pending_downloads()
+ return False
+ _insert_into_queue(dl)
_persist_pending_downloads()
return True
def _drain_queue():
- """Promote queued downloads into active slots while there's room.
+ """Promote eligible queued downloads into active slots while there's room.
- Caller must hold _download_lock.
+ Caller must hold _download_lock. Items marked ``scheduled`` stay put while
+ we're outside the download window; everything else promotes as before.
"""
- while _download_queue and _active_count() < _max_concurrent():
- dl = _download_queue.pop(0)
- _active_downloads[dl["id"]] = dl
- threading.Thread(target=_download_thread, args=(dl,), daemon=True).start()
+ in_window = _within_download_window()
+ i = 0
+ while _active_count() < _max_concurrent() and i < len(_download_queue):
+ dl = _download_queue[i]
+ if dl.get("scheduled") and not in_window:
+ i += 1
+ continue
+ _download_queue.pop(i)
+ _launch_download(dl)
# Refuse downloads that would obviously fill the disk: the expected size
@@ -616,6 +990,32 @@ def seed_ledger_snapshot():
return {k: dict(v) for k, v in _seed_ledger().items()}
+def restore_seed_intents(intents, overwrite=False):
+ """Restore seed-intent entries from a full-server backup.
+
+ ``overwrite`` replaces the whole ledger; otherwise incoming entries are
+ merged in (union by filename, incoming wins on conflict) so a restore never
+ drops a seed this machine already intends. Entries whose ZIM is absent are
+ harmless — reseed_from_ledger drops them at startup. Returns how many
+ entries were added/updated."""
+ if not isinstance(intents, dict):
+ return 0
+ clean = {
+ k: v for k, v in intents.items() if isinstance(k, str) and isinstance(v, dict)
+ }
+ try:
+ with _seed_ledger_lock:
+ ledger = {} if overwrite else _seed_ledger()
+ before = dict(ledger)
+ ledger.update(clean)
+ os.makedirs(os.path.dirname(_seed_ledger_path()), exist_ok=True)
+ _srv._atomic_write_json(_seed_ledger_path(), ledger)
+ return sum(1 for k, v in clean.items() if before.get(k) != v)
+ except Exception as e:
+ log.debug("seed ledger restore failed: %s", e)
+ return 0
+
+
def unrecord_seed(filename):
"""A deliberate stop: this file should NOT come back at startup."""
try:
@@ -882,6 +1282,30 @@ def _cancel_download(dl_id):
return "cancelling", 200
+def _start_scheduled_now(dl_id):
+ """Override the schedule for one queued item: start it now if a slot is
+ free, else clear its ``scheduled`` marker so it drains like any normal
+ queued download. Returns (status, code).
+
+ status: "started" | "queued" | "already_active" | "not_found"
+ """
+ with _download_lock:
+ for i, q in enumerate(_download_queue):
+ if q["id"] == dl_id:
+ q.pop("scheduled", None)
+ if _active_count() < _max_concurrent():
+ _download_queue.pop(i)
+ _launch_download(q)
+ _persist_pending_downloads()
+ return "started", 200
+ _persist_pending_downloads()
+ return "queued", 200
+ dl = _active_downloads.get(dl_id)
+ if dl and not dl.get("done"):
+ return "already_active", 200
+ return "not_found", 404
+
+
def _switch_to_direct(dl_id):
"""Abandon the BitTorrent transfer for an active download and pull it
over HTTP instead. Returns (status, code).
@@ -2428,6 +2852,10 @@ def _download_from_url(dl, url, tmp_dest):
break
f.write(chunk)
dl["downloaded_bytes"] = dl.get("downloaded_bytes", 0) + len(chunk)
+ # Global download-speed cap (shared across all HTTP pulls).
+ delay = _download_throttle.consume(len(chunk), _download_rate_bps())
+ if delay > 0:
+ time.sleep(delay)
except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as e:
resp.close()
return False, f"Transfer error from {urlparse(url).hostname}: {e}"
@@ -2974,6 +3402,7 @@ def _get_downloads():
"elapsed": round(time.time() - dl["started"], 1),
"is_update": dl.get("is_update", False),
"queued": True,
+ "scheduled": bool(dl.get("scheduled", False)),
}
)
for dl_id in to_remove:
diff --git a/zimi/manage.py b/zimi/manage.py
index a7312fb9..41b0bc94 100644
--- a/zimi/manage.py
+++ b/zimi/manage.py
@@ -129,6 +129,16 @@ def _set_manage_password(pw, username=None):
with open(tmp, "w", encoding="utf-8") as f:
f.write(content)
os.replace(tmp, pf)
+ # A password rotation must revoke old admin session cookies immediately (the
+ # pre-cookie model, where the Bearer WAS the password, did so implicitly). The
+ # rotating admin's own Bearer still authenticates and /whoami re-mints a fresh
+ # cookie, so this never locks out the person making the change.
+ try:
+ from zimi import users as _users
+
+ _users.drop_admin_sessions()
+ except Exception:
+ pass
log.info("Manage password %s", "set" if pw else "cleared")
@@ -223,6 +233,23 @@ def _primary_admin_authorized(handler):
# Passwordless: LAN/loopback clients are the (only) primary admin.
return handler._is_private_client()
+ # A primary-admin SESSION token (users.create_admin_session): minted when the
+ # admin password verified, delivered as the HttpOnly zimi_session cookie so
+ # header-less transports carry admin identity — the /w/ reader iframe (a
+ # browser navigation that can't send Authorization) and the plain-fetch data
+ # endpoints (/list, /search, …). Without this, a private/limited-mode admin
+ # loads an EMPTY library and blank article iframes. Checked FIRST (before the
+ # Bearer-format gate below) so a cookie-only request with no Authorization
+ # header still resolves as admin. Accepted from either the cookie (browsers)
+ # or the Bearer header (an API client may present it). As unforgeable as the
+ # password Bearer: a random token, hashed at rest.
+ from zimi import users as _users
+
+ if _users.is_admin_session(_users._cookie_token(handler)):
+ return True
+ if _users.is_admin_session(_users._bearer_token(handler)):
+ return True
+
auth = handler.headers.get("Authorization", "")
if not auth.startswith("Bearer "):
return False
@@ -424,6 +451,518 @@ def _index_dir_stats(path):
}
+# ============================================================================
+# Library layout — shared validate/apply (used by /manage/library-layout POST
+# and the backup importer so the two never drift).
+# ============================================================================
+
+
+def _validate_library_layout(overrides, order, sections=None):
+ """Return an error string if `overrides`/`section_order`/`sections` are
+ malformed, else None. Any may be None (that half is simply skipped)."""
+ if overrides is not None:
+ if (
+ not isinstance(overrides, dict)
+ or len(overrides) > _srv._LAYOUT_MAX_OVERRIDES
+ ):
+ return "invalid overrides"
+ for k, v in overrides.items():
+ if (
+ not isinstance(k, str)
+ or not isinstance(v, str)
+ or len(k) > _srv._LAYOUT_STR_MAX
+ or len(v) > _srv._LAYOUT_STR_MAX
+ ):
+ return "invalid overrides"
+ if order is not None:
+ if not isinstance(order, list) or len(order) > _srv._LAYOUT_MAX_ORDER:
+ return "invalid section_order"
+ for s in order:
+ if (
+ not isinstance(s, str)
+ or len(s) > _srv._LAYOUT_STR_MAX
+ or not _srv._SECTION_KEY_RE.match(s)
+ ):
+ return "invalid section_order"
+ if sections is not None:
+ if not isinstance(sections, list) or len(sections) > _srv._LAYOUT_MAX_SECTIONS:
+ return "invalid sections"
+ for s in sections:
+ if not isinstance(s, str) or not s or len(s) > _srv._LAYOUT_STR_MAX:
+ return "invalid sections"
+ return None
+
+
+def _apply_library_layout(overrides, order, sections=None):
+ """Merge-patch the persisted layout. `overrides` merges key-by-key (empty
+ value clears an entry); `section_order` and `sections` fully replace when
+ present. Returns the saved layout. Caller must have validated first."""
+ with _srv._library_layout_lock:
+ layout = _srv._load_library_layout()
+ if overrides is not None:
+ merged = dict(layout.get("overrides", {}))
+ for k, v in overrides.items():
+ if v == "":
+ merged.pop(k, None) # empty value = revert to heuristic
+ else:
+ merged[k] = v
+ layout["overrides"] = merged
+ if order is not None:
+ layout["section_order"] = list(order)
+ if sections is not None:
+ # De-dupe case-insensitively, first spelling wins, so a declared
+ # empty section can't accumulate near-duplicate rows.
+ seen, deduped = set(), []
+ for name in sections:
+ key = name.strip().lower()
+ if key and key not in seen:
+ seen.add(key)
+ deduped.append(name)
+ layout["sections"] = deduped
+ _srv._save_library_layout(layout)
+ return layout
+
+
+# ============================================================================
+# Backup bundle — export/import a Zimi setup.
+#
+# SCHEMA (zimi-backup, version 3). Two scopes:
+# • "device" — everyone's backup: the installed library list, collections/
+# favorites, and the home layout. The client folds its own per-browser
+# state (bookmarks/history/preferences) in on top; that never touches the
+# server. This is the v1 shape plus a `scope` field. (The redesigned SPA
+# splits this: the "My data" card round-trips the per-browser half through
+# /userdata for a signed-in user; the "Server backup" card owns everything
+# below. The device bundle stays for API clients and back-compat.)
+# • "server" — ADMIN-ONLY. Everything the server owns: the device fields
+# PLUS users.json (WITH password hashes — it's the admin's own backup),
+# the anonymous-access policy, the download schedule, the BitTorrent/
+# sharing prefs, the seed-intent ledger, and (v3) the hot-cache list, the
+# auto-update config, the server-side event history, and every named
+# user's server-stored My-data blob (user_data).
+#
+# Import MERGES by default (union by identity, incoming wins on conflict) and
+# is a two-step: a "preview" pass returns a diff summary and applies nothing;
+# only an explicit "apply" writes. An `overwrite` flag replaces wholesale where
+# the caller wants that. Keep `_BACKUP_SCHEMA_VERSION` in lockstep with the
+# client's `_BACKUP_SCHEMA_VERSION`.
+# ============================================================================
+
+_BACKUP_SCHEMA = "zimi-backup"
+_BACKUP_SCHEMA_VERSION = 3
+
+
+def _bundle_scope(data):
+ """The scope a bundle declares. Missing (v1 bundles) → 'device'. Anything
+ other than 'server' is treated as 'device' so server-only keys can never be
+ processed off a device bundle."""
+ return (
+ "server"
+ if (isinstance(data, dict) and data.get("scope") == "server")
+ else "device"
+ )
+
+
+def _build_backup_bundle(scope="device"):
+ """Assemble a backup bundle. ``scope='server'`` adds the admin-only server
+ state on top of the device fields (caller enforces the admin gate)."""
+ library = [
+ {
+ "name": z["name"],
+ "file": z.get("file"),
+ "date": z.get("date", ""),
+ "language": z.get("language", ""),
+ "article_count": z.get("article_count"),
+ "size_bytes": z.get("size_bytes"),
+ "title": z.get("title", z["name"]),
+ }
+ for z in _srv.list_zims()
+ ]
+ bundle = {
+ "schema": _BACKUP_SCHEMA,
+ "schema_version": _BACKUP_SCHEMA_VERSION,
+ "scope": "device",
+ "created": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+ "zimi_version": _srv.ZIMI_VERSION,
+ "library": library,
+ "collections": _srv._load_collections(),
+ "library_layout": _srv._load_library_layout(),
+ }
+ if scope == "server":
+ from zimi import library as _lib
+ from zimi import p2p
+ from zimi import users as _users
+
+ sched = _lib._load_download_schedule()
+ mode, allow, _ = _users._load_access()
+ au_enabled, au_freq = _lib._load_auto_update_config()
+ bundle["scope"] = "server"
+ bundle["users"] = _users._load_users() # WITH hashes — admin's own backup
+ bundle["public_access"] = {"mode": mode, "allowlist": allow}
+ bundle["schedule"] = {
+ "enabled": sched["enabled"],
+ "start": sched["start"],
+ "end": sched["end"],
+ "upload_restrict": sched["upload_restrict"],
+ "upload_trickle_kb": sched["upload_trickle_kb"],
+ }
+ bundle["bt_prefs"] = p2p.all_prefs()
+ bundle["seed_intents"] = _lib.seed_ledger_snapshot()
+ # v3 additions — the rest of what a full restore needs to resurrect an
+ # instance: the hot-cache list, the auto-update config, the server-side
+ # event history, and every named user's server-stored My-data blob.
+ bundle["hot_zims"] = _srv.get_hot_zims()
+ bundle["auto_update"] = {"enabled": au_enabled, "frequency": au_freq}
+ bundle["history"] = _srv._load_history()
+ bundle["user_data"] = _users.all_user_data()
+ return bundle
+
+
+# ── Merge primitives (pure — compute the merged value + counts, persist later) ──
+
+
+def _collection_newer(incoming, current):
+ """Conflict winner for a same-named collection. Honors an optional
+ ``updated`` epoch when both carry one; otherwise incoming wins (it's the
+ backup the admin chose to restore)."""
+ try:
+ inc_t = float(incoming.get("updated"))
+ cur_t = float(current.get("updated"))
+ except (TypeError, ValueError):
+ return True
+ return inc_t >= cur_t
+
+
+def _merge_collections(cur, incoming, overwrite):
+ """Union favorites (dedupe) and named collections (by name, newest-wins).
+ Returns (merged, counts)."""
+ inc_fav = [f for f in incoming.get("favorites", []) if isinstance(f, str)]
+ inc_cols = incoming.get("collections", {})
+ if overwrite:
+ fav = list(dict.fromkeys(inc_fav))[:100]
+ merged = {"version": 1, "favorites": fav, "collections": dict(inc_cols)}
+ counts = {
+ "fav_added": len(fav),
+ "fav_dupes": 0,
+ "col_added": len(inc_cols),
+ "col_replaced": 0,
+ }
+ return merged, counts
+ fav = list(cur.get("favorites", []))
+ dupes = 0
+ for f in inc_fav:
+ if f in fav:
+ dupes += 1
+ else:
+ fav.append(f)
+ cols = dict(cur.get("collections", {}))
+ added = replaced = 0
+ for name, obj in inc_cols.items():
+ if name in cols:
+ if _collection_newer(obj, cols[name]):
+ cols[name] = obj
+ replaced += 1
+ else:
+ cols[name] = obj
+ added += 1
+ merged = {"version": 1, "favorites": fav[:100], "collections": cols}
+ counts = {
+ "fav_added": max(0, len(fav[:100]) - len(cur.get("favorites", []))),
+ "fav_dupes": dupes,
+ "col_added": added,
+ "col_replaced": replaced,
+ }
+ return merged, counts
+
+
+def _merge_layout(cur, incoming, overwrite):
+ """Merge the home layout: overrides union (incoming wins per ZIM), sections
+ union, section_order replaced when present (an ordering — merging is
+ meaningless). Returns (overrides, order, sections, counts)."""
+ inc_over = incoming.get("overrides") or {}
+ inc_order = incoming.get("section_order")
+ inc_sections = incoming.get("sections") or []
+ cur_over = cur.get("overrides") or {}
+ cur_sections = cur.get("sections") or []
+ if overwrite:
+ over = dict(inc_over)
+ order = inc_order if isinstance(inc_order, list) else cur.get("section_order")
+ sections = list(inc_sections)
+ counts = {
+ "over_added": len(inc_over),
+ "over_changed": 0,
+ "order_changed": inc_order is not None,
+ "sections_added": len(inc_sections),
+ }
+ return over, order, sections, counts
+ over = dict(cur_over)
+ added = changed = 0
+ for k, v in inc_over.items():
+ if k in over:
+ if over[k] != v:
+ changed += 1
+ over[k] = v
+ else:
+ over[k] = v
+ added += 1
+ sections = list(cur_sections)
+ sec_added = 0
+ seen = {s.strip().lower() for s in cur_sections if isinstance(s, str)}
+ for s in inc_sections:
+ if isinstance(s, str) and s.strip().lower() not in seen:
+ seen.add(s.strip().lower())
+ sections.append(s)
+ sec_added += 1
+ order = inc_order if isinstance(inc_order, list) else cur.get("section_order")
+ counts = {
+ "over_added": added,
+ "over_changed": changed,
+ "order_changed": isinstance(inc_order, list)
+ and inc_order != cur.get("section_order"),
+ "sections_added": sec_added,
+ }
+ return over, order, sections, counts
+
+
+def _merge_users(cur, incoming, overwrite):
+ """Union users by casefold key; incoming wins on conflict (or replace all
+ when overwrite). Returns (merged, counts)."""
+ from zimi import users as _users
+
+ inc = {
+ _users._key(v.get("name", k)): v
+ for k, v in incoming.items()
+ if isinstance(v, dict)
+ }
+ if overwrite:
+ return inc, {"added": len(inc), "replaced": 0}
+ merged = dict(cur)
+ added = replaced = 0
+ for k, v in inc.items():
+ if k in merged:
+ if merged[k] != v:
+ replaced += 1
+ merged[k] = v
+ else:
+ merged[k] = v
+ added += 1
+ return merged, {"added": added, "replaced": replaced}
+
+
+def _compute_backup(data, overwrite):
+ """Diff an incoming bundle against current server state WITHOUT persisting.
+
+ Returns (plan, preview, error). ``plan`` is a list of (label, thunk) pairs
+ the apply pass runs in order; ``preview`` is the diff summary the client
+ shows before the admin confirms. Server-only keys are considered only for a
+ ``server``-scope bundle, so a device bundle can never carry server changes.
+ """
+ if not isinstance(data, dict):
+ return None, None, "invalid backup"
+ schema = data.get("schema")
+ if schema is not None and schema != _BACKUP_SCHEMA:
+ return None, None, "not a Zimi backup"
+ scope = _bundle_scope(data)
+ plan = []
+ preview = {"scope": scope}
+
+ coll = data.get("collections")
+ if coll is not None:
+ if not isinstance(coll, dict):
+ return None, None, "invalid collections"
+ fav = coll.get("favorites", [])
+ cols = coll.get("collections", {})
+ if not isinstance(fav, list) or not isinstance(cols, dict):
+ return None, None, "invalid collections"
+ merged, counts = _merge_collections(_srv._load_collections(), coll, overwrite)
+ preview["collections"] = counts
+ plan.append(("collections", lambda m=merged: _persist_collections(m)))
+
+ layout = data.get("library_layout")
+ if layout is not None:
+ if not isinstance(layout, dict):
+ return None, None, "invalid library_layout"
+ over, order, sections, counts = _merge_layout(
+ _srv._load_library_layout(), layout, overwrite
+ )
+ err = _validate_library_layout(over, order, sections)
+ if err:
+ return None, None, err
+ preview["layout"] = counts
+ plan.append(
+ (
+ "library_layout",
+ lambda o=over, r=order, s=sections: _apply_library_layout(o, r, s),
+ )
+ )
+
+ lib = data.get("library")
+ if isinstance(lib, list):
+ installed = set(_srv.get_zim_files().keys())
+ preview["missing_zims"] = sum(
+ 1 for z in lib if isinstance(z, dict) and z.get("name") not in installed
+ )
+
+ if scope == "server":
+ err = _plan_server_scope(data, overwrite, plan, preview)
+ if err:
+ return None, None, err
+
+ return plan, preview, None
+
+
+def _persist_collections(merged):
+ with _srv._collections_lock:
+ _srv._save_collections(merged)
+
+
+def _plan_server_scope(data, overwrite, plan, preview):
+ """Extend the plan/preview with the admin-only server state. Returns an
+ error string or None."""
+ from zimi import library as _lib
+ from zimi import p2p
+ from zimi import users as _users
+
+ users = data.get("users")
+ if users is not None:
+ if not isinstance(users, dict):
+ return "invalid users"
+ merged, counts = _merge_users(_users._load_users(), users, overwrite)
+ preview["users"] = counts
+ plan.append(("users", lambda m=merged: _users._save_users(m)))
+
+ settings_changed = []
+
+ pa = data.get("public_access")
+ if isinstance(pa, dict):
+ mode = pa.get("mode")
+ allow = pa.get("allowlist", [])
+ cur_mode, cur_allow, _ = _users._load_access()
+ if mode != cur_mode or allow != cur_allow:
+ settings_changed.append("public_access")
+ plan.append(
+ ("public_access", lambda m=mode, a=allow: _users.set_public_access(m, a))
+ )
+
+ sched = data.get("schedule")
+ if isinstance(sched, dict):
+ cur = _lib._load_download_schedule()
+ keys = ("enabled", "start", "end", "upload_restrict", "upload_trickle_kb")
+ if any(sched.get(k) != cur.get(k) for k in keys if k in sched):
+ settings_changed.append("download_schedule")
+ plan.append(("schedule", lambda s=sched: _restore_schedule(s)))
+
+ bt = data.get("bt_prefs")
+ if isinstance(bt, dict):
+ if bt != p2p.all_prefs():
+ settings_changed.append("sharing_prefs")
+ plan.append(("bt_prefs", lambda b=bt: _restore_bt_prefs(b, overwrite)))
+
+ intents = data.get("seed_intents")
+ if isinstance(intents, dict):
+ cur = _lib.seed_ledger_snapshot()
+ preview["seed_intents"] = {
+ "added": sum(1 for k, v in intents.items() if cur.get(k) != v)
+ }
+ plan.append(
+ ("seed_intents", lambda i=intents: _lib.restore_seed_intents(i, overwrite))
+ )
+
+ # v3 server state — hot list, auto-update, history, per-user data.
+ hot = data.get("hot_zims")
+ if isinstance(hot, list):
+ # Env-locked hot list (ZIMI_HOT_ZIMS) is authoritative — never clobber it.
+ if "ZIMI_HOT_ZIMS" not in os.environ:
+ if hot != _srv.get_hot_zims():
+ settings_changed.append("hot_zims")
+ plan.append(
+ (
+ "hot_zims",
+ lambda h=[s for s in hot if isinstance(s, str)]: _srv.set_hot_zims(
+ h
+ ),
+ )
+ )
+
+ au = data.get("auto_update")
+ if isinstance(au, dict) and "enabled" in au and "frequency" in au:
+ # Env-locked auto-update (ZIMI_AUTO_UPDATE) wins — skip the restore.
+ if not getattr(_srv, "_auto_update_env_locked", False):
+ cur_en, cur_fr = _lib._load_auto_update_config()
+ if bool(au["enabled"]) != cur_en or au["frequency"] != cur_fr:
+ settings_changed.append("auto_update")
+ plan.append(("auto_update", lambda a=au: _restore_auto_update(a)))
+
+ history = data.get("history")
+ if isinstance(history, list):
+ preview["history"] = {"events": len(history)}
+ plan.append(("history", lambda h=history: _restore_history(h, overwrite)))
+
+ ud = data.get("user_data")
+ if isinstance(ud, dict):
+ preview["user_data"] = {"users": len(ud)}
+ plan.append(("user_data", lambda u=ud: _users.restore_user_data(u, overwrite)))
+
+ if settings_changed:
+ preview["settings"] = settings_changed
+ return None
+
+
+def _restore_auto_update(au):
+ from zimi import library as _lib
+
+ enabled, freq = bool(au.get("enabled")), au.get("frequency", "weekly")
+ _lib._save_auto_update_config(enabled, freq)
+ # Reflect into the live server namespace so /manage/status reads true without
+ # a restart (the loop reads these via _srv — see library._auto_update_loop).
+ _srv._auto_update_enabled = enabled
+ _srv._auto_update_freq = freq
+
+
+def _restore_history(entries, overwrite):
+ """Restore server-side event history. Replace when overwrite; otherwise fill
+ only when the current log is empty (a resurrected instance) so a normal merge
+ into a running server never duplicates or reorders its real event stream."""
+ if overwrite or not _srv._load_history():
+ _srv._save_history(entries)
+
+
+def _restore_schedule(sched):
+ from zimi import library as _lib
+
+ cur = _lib._load_download_schedule()
+ _lib._save_download_schedule(
+ sched.get("enabled", cur["enabled"]),
+ sched.get("start", cur["start"]),
+ sched.get("end", cur["end"]),
+ sched.get("upload_restrict", cur["upload_restrict"]),
+ sched.get("upload_trickle_kb", cur["upload_trickle_kb"]),
+ )
+
+
+def _restore_bt_prefs(bt, overwrite):
+ from zimi import p2p
+
+ p2p.replace_prefs(bt, overwrite=overwrite)
+ p2p.apply_rate_limits()
+
+
+def _apply_backup_bundle(data, overwrite=False):
+ """Apply a backup bundle (MERGE by default). Returns (result, error).
+
+ Prefer the two-step route contract (preview then apply); this runs the whole
+ plan in one shot for direct callers/tests. ``result`` carries the applied
+ keys plus the preview summary."""
+ plan, preview, err = _compute_backup(data, overwrite)
+ if err:
+ return None, err
+ applied = []
+ for label, thunk in plan:
+ thunk()
+ applied.append(label)
+ return {"status": "ok", "applied": applied, "preview": preview}, None
+
+
# ============================================================================
# Manage GET Routes
# ============================================================================
@@ -529,10 +1068,27 @@ def param(key, default=None):
elif parsed.path == "/manage/usage":
return handler._json(200, _srv._get_usage_stats())
+ elif parsed.path == "/manage/download-schedule":
+ from zimi import library as _lib
+
+ return handler._json(200, _lib._download_schedule_status())
+
+ elif parsed.path == "/manage/backup":
+ # scope=device (default, everyone) or scope=server (admin-only: the full
+ # server state incl. users.json with hashes). The manage gate above is
+ # already admin-only, but the explicit check keeps the server-scope
+ # contract self-evident and independently testable.
+ scope = param("scope", "device")
+ if scope == "server" and admin_kind(handler) is None:
+ return handler._json(403, {"error": "full-server backup requires an admin"})
+ return handler._json(200, _build_backup_bundle(scope=scope))
+
elif parsed.path == "/manage/users":
# Named user accounts (multi-user v1) — admin-only (gated above). Returns
- # the roster (no password hashes) plus the installed ZIM names so the
- # admin UI can build the per-user allowlist multi-select.
+ # the roster (no password hashes), the installed ZIM names (legacy field),
+ # the richer picker options (title + language + count) the redesigned
+ # allowlist picker needs, and the public-access policy so the whole Users
+ # panel renders from a single fetch.
from zimi import users as _users
return handler._json(
@@ -540,6 +1096,11 @@ def param(key, default=None):
{
"users": _users.list_users(),
"zims": sorted(_srv.get_zim_files().keys()),
+ # Rich per-ZIM options for the allowlist picker (used by both the
+ # per-user Limited picker and the public-access Limited picker).
+ "zim_options": _zim_picker_options(),
+ # Anonymous-access policy (Open / Limited / Sign-in required).
+ "public_access": _users.public_access_status(),
# The PRIMARY admin (password-file account) is not stored in
# users.json — surface it as a synthetic, non-deletable row so
# the UI can show "the admin" alongside the named users.
@@ -554,6 +1115,19 @@ def param(key, default=None):
},
)
+ elif parsed.path == "/manage/public-access":
+ # Anonymous-access policy on its own, with the picker options — a
+ # lightweight refetch target after the admin changes it.
+ from zimi import users as _users
+
+ return handler._json(
+ 200,
+ {
+ "public_access": _users.public_access_status(),
+ "zim_options": _zim_picker_options(),
+ },
+ )
+
elif parsed.path == "/manage/catalog":
query = param("q", "")
lang = param("lang", "")
@@ -908,10 +1482,24 @@ def param(key, default=None):
if not enabled:
hint = "BT downloads disabled (ZIMI_BT=off). HTTP is used instead."
elif status == "unavailable":
+ import sys as _sys
+
+ # Give the exact next step. Wheels exist for CPython 3.9–3.13; on
+ # 3.14+ there's no wheel yet, so name that specifically instead of
+ # sending the user to a pip command that will fail.
+ if _sys.version_info >= (3, 14):
+ fix = (
+ f"no libtorrent wheel exists for Python "
+ f"{_sys.version_info.major}.{_sys.version_info.minor} yet — "
+ "run Zimi on Python 3.13 or older (or use the Docker image) "
+ "to torrent."
+ )
+ else:
+ fix = "run `pip install libtorrent` (or `pip install zimi[bt]`) to torrent."
hint = (
- "libtorrent isn't importable on this install — downloads "
- "fall back to HTTP. Install libtorrent to torrent and share "
- "load with the Kiwix mirrors."
+ "libtorrent isn't importable on this install — downloads fall "
+ "back to HTTP, which works fine. To share load with the Kiwix "
+ "mirrors, " + fix
)
from zimi import p2p_nat
@@ -941,6 +1529,51 @@ def param(key, default=None):
return handler._json(404, {"error": "not found"})
+def _zim_picker_options():
+ """``[{name, title, language, article_count}]`` for the allowlist pickers,
+ sorted by title. Admin view = all installed ZIMs. Titles/languages come from
+ the startup metadata cache; a ZIM not yet in that cache still appears (by
+ name) so it is always selectable."""
+ opts = []
+ seen = set()
+ for z in _srv._zim_list_cache or []:
+ name = z.get("name")
+ if not name:
+ continue
+ seen.add(name)
+ opts.append(
+ {
+ "name": name,
+ "title": z.get("title") or name,
+ "language": z.get("language", ""),
+ "article_count": z.get("article_count"),
+ }
+ )
+ for name in sorted(_srv.get_zim_files().keys()):
+ if name not in seen:
+ opts.append(
+ {"name": name, "title": name, "language": "", "article_count": None}
+ )
+ opts.sort(key=lambda o: (o["title"] or "").casefold())
+ return opts
+
+
+def _handle_public_access_post(handler, data):
+ """Admin-only: set the anonymous-access policy. ``mode`` ∈ {open, limited,
+ private}; ``allowlist`` applies only to ``limited``. Echoes the fresh status
+ so the UI re-renders in one round trip. Auth already passed (gated in
+ ``handle_manage_post``)."""
+ from zimi import users as _users
+
+ mode = data.get("mode", "")
+ ok, err = _users.set_public_access(mode, data.get("allowlist"))
+ if not ok:
+ return handler._json(400, {"error": err or "operation failed"})
+ return handler._json(
+ 200, {"status": "ok", "public_access": _users.public_access_status()}
+ )
+
+
def _handle_users_post(handler, data):
"""Admin-only user CRUD (multi-user v1). action ∈ {create, delete,
set-password, set-allowlist, set-role}. Errors are returned generically; on
@@ -1066,6 +1699,9 @@ def handle_manage_post(handler, parsed, data):
if parsed.path == "/manage/users":
return _handle_users_post(handler, data)
+ if parsed.path == "/manage/public-access":
+ return _handle_public_access_post(handler, data)
+
if parsed.path == "/manage/download":
url = data.get("url", "")
size_bytes = data.get("size_bytes")
@@ -1136,6 +1772,17 @@ def handle_manage_post(handler, parsed, data):
return handler._json(400, {"error": "Download already finished"})
return handler._json(code, {"status": status, "id": dl_id})
+ elif parsed.path == "/manage/download-start-now":
+ # Override the nightly window for one scheduled item — start it now
+ # (or promote it to a normal queued item if every slot is busy).
+ dl_id = data.get("id", "")
+ from zimi.library import _start_scheduled_now
+
+ status, code = _start_scheduled_now(dl_id)
+ if status == "not_found":
+ return handler._json(404, {"error": "Download not found"})
+ return handler._json(code, {"status": status, "id": dl_id})
+
elif parsed.path == "/manage/switch-direct":
# Escape hatch for a slow BitTorrent swarm: abandon BT for this
# download and pull it over HTTP instead.
@@ -1387,6 +2034,82 @@ def handle_manage_post(handler, parsed, data):
{"enabled": _srv._auto_update_enabled, "frequency": _srv._auto_update_freq},
)
+ elif parsed.path == "/manage/download-schedule":
+ # Night-window queueing + the global download-speed cap. Same env-lock
+ # contract as the other settings endpoints: ZIMI_DL_WINDOW locks the
+ # window, ZIMI_BT_DOWN_KB (via bt_down_kb) locks the speed cap.
+ from zimi import library as _lib
+ from zimi import p2p
+
+ sched = _lib._load_download_schedule()
+ # Window fields + the upload restrictor (restrict seeding to the window
+ # + its trickle cap). All ride the same config file, so one save covers
+ # any subset the client sent; absent fields are preserved.
+ window_keys = (
+ "enabled",
+ "start",
+ "end",
+ "upload_restrict",
+ "upload_trickle_kb",
+ )
+ if any(k in data for k in window_keys):
+ if sched.get("locked"):
+ return handler._json(
+ 403,
+ {
+ "error": "Download window is controlled by the ZIMI_DL_WINDOW env var"
+ },
+ )
+ enabled = bool(data.get("enabled", sched["enabled"]))
+ start = data.get("start", sched["start"])
+ end = data.get("end", sched["end"])
+ if _lib._parse_hhmm(start) is None or _lib._parse_hhmm(end) is None:
+ return handler._json(
+ 400, {"error": "start/end must be 'HH:MM' (24-hour)"}
+ )
+ upload_restrict = (
+ bool(data["upload_restrict"])
+ if "upload_restrict" in data
+ else sched["upload_restrict"]
+ )
+ upload_trickle_kb = sched["upload_trickle_kb"]
+ if "upload_trickle_kb" in data:
+ try:
+ upload_trickle_kb = max(1, int(data["upload_trickle_kb"]))
+ except (ValueError, TypeError):
+ return handler._json(
+ 400, {"error": "upload_trickle_kb must be a number"}
+ )
+ if not _lib._save_download_schedule(
+ enabled, start, end, upload_restrict, upload_trickle_kb
+ ):
+ return handler._json(
+ 500, {"error": "could not save setting (config dir not writable)"}
+ )
+ # Whether the window just opened or scheduling was turned off,
+ # release anything already waiting now — don't wait for a tick.
+ threading.Thread(target=_lib._download_schedule_tick, daemon=True).start()
+ # Global download-speed cap (KB/s, 0 = unlimited) — shared with the BT
+ # download limit so one number governs every transport.
+ if "download_kb" in data:
+ if p2p.is_bt_down_env_locked():
+ return handler._json(
+ 403,
+ {
+ "error": "Download speed limit is controlled by the ZIMI_BT env var"
+ },
+ )
+ try:
+ kb = max(0, int(data["download_kb"]))
+ except (ValueError, TypeError):
+ return handler._json(400, {"error": "download_kb must be a number"})
+ if not p2p.set_pref("bt_down_kb", kb):
+ return handler._json(
+ 500, {"error": "could not save setting (config dir not writable)"}
+ )
+ p2p.apply_rate_limits()
+ return handler._json(200, _lib._download_schedule_status())
+
elif parsed.path == "/manage/seeding-action":
# Pause / resume / stop one seed, or stop everything — the
# sidecar shouldn't need a terminal to be told to quiet down.
@@ -1652,6 +2375,48 @@ def _spawn_now():
changed[_field] = kb
if any(k in changed for k in ("bt_up_kb", "bt_down_kb")):
p2p.apply_rate_limits()
+ # Max concurrent downloads (governs HTTP + BT via library.py's queue).
+ if "max_active_downloads" in data:
+ if p2p.is_max_active_downloads_env_locked():
+ return handler._json(
+ 403,
+ {"error": "Max concurrent downloads is controlled by an env var"},
+ )
+ try:
+ n = max(1, min(20, int(data["max_active_downloads"])))
+ except (ValueError, TypeError):
+ return handler._json(
+ 400, {"error": "max_active_downloads must be a number"}
+ )
+ if not p2p.set_pref("max_active_downloads", n):
+ return handler._json(
+ 500, {"error": "could not save setting (config dir not writable)"}
+ )
+ changed["max_active_downloads"] = n
+ # Raising the cap frees slots that nothing else would drain until a
+ # download finishes — promote queued items now.
+ from zimi import library as _lib_cc
+
+ threading.Thread(target=_lib_cc.drain_download_queue, daemon=True).start()
+ # Max connections — a real libtorrent session setting, applied live.
+ if "bt_max_connections" in data:
+ if p2p.is_bt_max_connections_env_locked():
+ return handler._json(
+ 403,
+ {"error": "Max connections is controlled by the ZIMI_BT env var"},
+ )
+ try:
+ n = max(10, min(2000, int(data["bt_max_connections"])))
+ except (ValueError, TypeError):
+ return handler._json(
+ 400, {"error": "bt_max_connections must be a number"}
+ )
+ if not p2p.set_pref("bt_max_connections", n):
+ return handler._json(
+ 500, {"error": "could not save setting (config dir not writable)"}
+ )
+ changed["bt_max_connections"] = n
+ p2p.apply_session_limits()
if not changed:
return handler._json(
400,
@@ -1692,53 +2457,43 @@ def _spawn_now():
# may be omitted so "Move to…" and "Reorder" send minimal payloads.
overrides = data.get("overrides")
order = data.get("section_order")
- if overrides is None and order is None:
+ sections = data.get("sections")
+ if overrides is None and order is None and sections is None:
return handler._json(400, {"error": "nothing to update"})
- if overrides is not None:
- if (
- not isinstance(overrides, dict)
- or len(overrides) > _srv._LAYOUT_MAX_OVERRIDES
- ):
- return handler._json(400, {"error": "invalid overrides"})
- for k, v in overrides.items():
- if (
- not isinstance(k, str)
- or not isinstance(v, str)
- or len(k) > _srv._LAYOUT_STR_MAX
- or len(v) > _srv._LAYOUT_STR_MAX
- ):
- return handler._json(400, {"error": "invalid overrides"})
- if order is not None:
- if not isinstance(order, list) or len(order) > _srv._LAYOUT_MAX_ORDER:
- return handler._json(400, {"error": "invalid section_order"})
- for s in order:
- if (
- not isinstance(s, str)
- or len(s) > _srv._LAYOUT_STR_MAX
- or not _srv._SECTION_KEY_RE.match(s)
- ):
- return handler._json(400, {"error": "invalid section_order"})
- with _srv._library_layout_lock:
- layout = _srv._load_library_layout()
- if overrides is not None:
- merged = dict(layout.get("overrides", {}))
- for k, v in overrides.items():
- if v == "":
- merged.pop(k, None) # empty value = revert to heuristic
- else:
- merged[k] = v
- layout["overrides"] = merged
- if order is not None:
- layout["section_order"] = list(order)
- _srv._save_library_layout(layout)
+ err = _validate_library_layout(overrides, order, sections)
+ if err:
+ return handler._json(400, {"error": err})
+ layout = _apply_library_layout(overrides, order, sections)
return handler._json(
200,
{
"status": "ok",
"overrides": layout.get("overrides", {}),
"section_order": layout.get("section_order", []),
+ "sections": layout.get("sections", []),
},
)
+ elif parsed.path == "/manage/backup":
+ # Restore a backup bundle. Two-step so nothing lands before the admin
+ # confirms: action="preview" (default) returns a diff summary and
+ # applies NOTHING; action="apply" writes. MERGE by default; overwrite
+ # replaces wholesale. A server-scope bundle is admin-only in BOTH
+ # directions — a non-admin session can't preview OR apply one.
+ if _bundle_scope(data) == "server" and admin_kind(handler) is None:
+ return handler._json(403, {"error": "full-server backup requires an admin"})
+ action = data.get("action", "preview")
+ overwrite = bool(data.get("overwrite"))
+ if action == "apply":
+ result, err = _apply_backup_bundle(data, overwrite=overwrite)
+ if err:
+ return handler._json(400, {"error": err})
+ return handler._json(200, result)
+ # Preview (default): compute the diff, persist nothing.
+ _plan, preview, err = _compute_backup(data, overwrite)
+ if err:
+ return handler._json(400, {"error": err})
+ return handler._json(200, {"status": "preview", "preview": preview})
+
else:
return handler._json(404, {"error": "not found"})
diff --git a/zimi/p2p.py b/zimi/p2p.py
index 540148e8..95cffe11 100644
--- a/zimi/p2p.py
+++ b/zimi/p2p.py
@@ -48,7 +48,7 @@ def _bool_env(key: str, default: bool = False) -> bool:
# ============================================================================
# Compact config blobs — the documented env surface is just two vars:
-# ZIMI_BT="on,port=6881,ratio=2,up=2048,mirror=off"
+# ZIMI_BT="on,port=6881,ratio=2,up=2048,mirror=off,active=4,conns=200"
# ZIMI_NEARBY="on,name=my-zimi,public=off"
# A bare on/off token drives the master switch; key=value pairs set single
# fields. Any field present in the blob is env-locked in the UI — fields
@@ -134,6 +134,50 @@ def set_pref(key: str, value) -> bool:
return True
+def all_prefs() -> dict:
+ """The whole persisted-prefs blob (seed/mirror/rate/port toggles). Used by
+ the full-server backup so every UI-set sharing pref rides along, including
+ ones added later — no per-key list to keep in sync."""
+ if not _prefs_path:
+ return {}
+ with _prefs_lock:
+ try:
+ with open(_prefs_path, encoding="utf-8") as f:
+ data = json.load(f)
+ return data if isinstance(data, dict) else {}
+ except (OSError, ValueError):
+ return {}
+
+
+def replace_prefs(prefs: dict, overwrite: bool = False) -> bool:
+ """Restore prefs from a backup. ``overwrite`` replaces the whole blob;
+ otherwise incoming keys are merged over the current ones (a setting, so
+ incoming wins per key). Returns False when the config dir isn't writable."""
+ if not _prefs_path or not isinstance(prefs, dict):
+ return False
+ with _prefs_lock:
+ merged = {} if overwrite else {}
+ if not overwrite:
+ try:
+ with open(_prefs_path, encoding="utf-8") as f:
+ cur = json.load(f)
+ if isinstance(cur, dict):
+ merged = cur
+ except (OSError, ValueError):
+ pass
+ merged.update(prefs)
+ try:
+ os.makedirs(os.path.dirname(_prefs_path), exist_ok=True)
+ tmp = _prefs_path + ".tmp"
+ with open(tmp, "w", encoding="utf-8") as f:
+ json.dump(merged, f)
+ os.replace(tmp, _prefs_path)
+ except OSError as e:
+ log.warning("could not restore preferences: %s", e)
+ return False
+ return True
+
+
def _env_explicitly_set(key: str) -> bool:
raw = os.environ.get(key)
return raw is not None and raw.strip() != ""
@@ -252,6 +296,17 @@ def get_bt_down_limit_kb() -> int:
return 0
+def get_download_limit_kb() -> int:
+ """Global download-speed cap in KB/s (0 = unlimited).
+
+ A byte is a byte regardless of transport, so one number governs total
+ download speed: the libtorrent session already applies it as its
+ ``download_rate_limit`` (see get_bt_down_limit_kb, wired through
+ apply_rate_limits) and library.py throttles the HTTP read loop to the
+ same value. Same persisted pref as the BT download limit."""
+ return get_bt_down_limit_kb()
+
+
def is_bt_up_env_locked() -> bool:
return "up" in _bt_conf() or _env_explicitly_set("ZIMI_BT_UP_KB")
@@ -260,16 +315,117 @@ def is_bt_down_env_locked() -> bool:
return "down" in _bt_conf() or _env_explicitly_set("ZIMI_BT_DOWN_KB")
+# ============================================================================
+# Concurrency + connection caps
+#
+# Two different enforcement layers, deliberately:
+#
+# active= (max concurrent downloads) is enforced by library.py's own
+# download queue — the rest wait in line, smallest-first — because Zimi
+# manages every torrent by hand (add_torrent strips auto_managed so the
+# ledger, not libtorrent, decides what runs). libtorrent's active_downloads/
+# active_seeds/active_limit only queue AUTO-managed torrents, of which Zimi
+# has none, so setting them would be an inert knob. The queue also governs
+# HTTP pulls, so one number caps concurrency across both transports.
+#
+# conns= (max connections) IS a real libtorrent session setting
+# (connections_limit) — enforced on every torrent regardless of management —
+# so it lives on the session and applies live.
+#
+# Both follow the ZIMI_BT sub-key + persisted-pref + default pattern; a set
+# field env-locks its UI control.
+# ============================================================================
+
+DEFAULT_MAX_ACTIVE_DOWNLOADS = 4
+DEFAULT_MAX_CONNECTIONS = 200
+
+
+def get_max_active_downloads() -> int:
+ """How many downloads run at once; the rest queue (library.py's queue is
+ the enforcer, HTTP and BT alike). Legacy ZIMI_MAX_CONCURRENT_DOWNLOADS
+ wins and locks; then ZIMI_BT's active= field; then the persisted UI
+ value; default 4. Clamped 1..20."""
+ raw = os.environ.get("ZIMI_MAX_CONCURRENT_DOWNLOADS") or _bt_conf().get("active")
+ if raw in (None, ""):
+ raw = _read_pref("max_active_downloads", DEFAULT_MAX_ACTIVE_DOWNLOADS)
+ try:
+ return max(1, min(20, int(raw)))
+ except (ValueError, TypeError):
+ return DEFAULT_MAX_ACTIVE_DOWNLOADS
+
+
+def is_max_active_downloads_env_locked() -> bool:
+ return bool(
+ _env_explicitly_set("ZIMI_MAX_CONCURRENT_DOWNLOADS") or _bt_conf().get("active")
+ )
+
+
+def get_bt_max_connections() -> int:
+ """Global libtorrent connections_limit — the session's total socket cap,
+ shared across all torrents. ZIMI_BT's conns= field wins and locks; then
+ the persisted UI value; default 200 (libtorrent's own default). Clamped
+ 10..2000 — zero would strangle the session."""
+ raw = _bt_conf().get("conns")
+ if raw in (None, ""):
+ raw = _read_pref("bt_max_connections", DEFAULT_MAX_CONNECTIONS)
+ try:
+ return max(10, min(2000, int(raw)))
+ except (ValueError, TypeError):
+ return DEFAULT_MAX_CONNECTIONS
+
+
+def is_bt_max_connections_env_locked() -> bool:
+ return bool(_bt_conf().get("conns"))
+
+
+# Download-window upload restrictor (library.py drives the transitions). When
+# set, this overrides the configured up limit so seeding trickles outside the
+# window; None means "no override — use the normal up limit". Folding it into
+# apply_rate_limits keeps one setter as the sole authority over the live
+# session's upload rate.
+_upload_window_cap_kb: int | None = None
+
+
+def set_upload_window_cap(kb: int | None) -> None:
+ """Trickle (positive KB/s) or release (None) the seeding upload rate for the
+ download window, applied live. Idempotent at the session layer — safe to
+ call on every schedule tick."""
+ global _upload_window_cap_kb
+ _upload_window_cap_kb = None if kb is None else max(0, int(kb))
+ apply_rate_limits()
+
+
+def _effective_up_kb() -> int:
+ """Upload cap to hand the session: the window trickle when one is active,
+ else the configured BT up limit."""
+ if _upload_window_cap_kb is not None:
+ return _upload_window_cap_kb
+ return get_bt_up_limit_kb()
+
+
def apply_rate_limits() -> None:
- """Push the current up/down caps to the running session (live, no restart)."""
+ """Push the current up/down caps to the running session (live, no restart).
+
+ The upload side honors any active download-window trickle (see
+ set_upload_window_cap); the download side is always the configured cap."""
backend = peek_backend()
if backend is not None and hasattr(backend, "set_global_rate_limits"):
try:
- backend.set_global_rate_limits(get_bt_up_limit_kb(), get_bt_down_limit_kb())
+ backend.set_global_rate_limits(_effective_up_kb(), get_bt_down_limit_kb())
except Exception as e:
log.debug("live rate-limit apply failed: %s", e)
+def apply_session_limits() -> None:
+ """Push the connection cap to the running session (live, no restart)."""
+ backend = peek_backend()
+ if backend is not None and hasattr(backend, "set_connections_limit"):
+ try:
+ backend.set_connections_limit(get_bt_max_connections())
+ except Exception as e:
+ log.debug("live connection-limit apply failed: %s", e)
+
+
# Absolute free-space floor shared by the download gate and the seeding
# pause. Percent-of-drive defaults are wrong at both ends: 5% of a 466 GB
# drive is 23 GB of "missing" space, and seeding existing files writes
@@ -387,6 +543,10 @@ def get_mirror_status() -> dict:
"bt_down_kb": get_bt_down_limit_kb(),
"bt_up_env_locked": is_bt_up_env_locked(),
"bt_down_env_locked": is_bt_down_env_locked(),
+ "max_active_downloads": get_max_active_downloads(),
+ "max_active_downloads_env_locked": is_max_active_downloads_env_locked(),
+ "bt_max_connections": get_bt_max_connections(),
+ "bt_max_connections_env_locked": is_bt_max_connections_env_locked(),
# Docker bridge mode advertises an unreachable container IP —
# Nearby silently doesn't work. The UI warns; ZIMI_NEARBY's ip=
# field (or host networking) fixes it.
@@ -527,7 +687,16 @@ def _lt():
_lt_module = libtorrent
except ImportError:
_lt_import_failed = True
- log.info("libtorrent not importable — BT off, downloads use HTTP")
+ import sys as _sys
+
+ if _sys.version_info >= (3, 14):
+ _fix = (
+ f"no wheel for Python {_sys.version_info.major}."
+ f"{_sys.version_info.minor} yet — use Python 3.13 or the Docker image"
+ )
+ else:
+ _fix = "`pip install libtorrent` (or `pip install zimi[bt]`) to enable it"
+ log.info("libtorrent not importable — BT off, downloads use HTTP; %s", _fix)
return None
return _lt_module
@@ -586,8 +755,14 @@ def _ensure_session(self) -> None:
"enable_dht": is_dht_enabled(),
"enable_upnp": is_upnp_enabled(),
"enable_natpmp": is_upnp_enabled(),
- "upload_rate_limit": get_bt_up_limit_kb() * 1024,
+ "upload_rate_limit": _effective_up_kb() * 1024,
"download_rate_limit": get_bt_down_limit_kb() * 1024,
+ # Global socket cap — real and enforced on every torrent. (The
+ # concurrent-DOWNLOAD cap is not a session setting: libtorrent's
+ # active_* only queue auto-managed torrents, which Zimi has none
+ # of, so library.py's own queue enforces it — see
+ # get_max_active_downloads.)
+ "connections_limit": get_bt_max_connections(),
# port_mapping category surfaces UPnP/NAT-PMP success or
# failure — the single most useful signal for "why is BT slow":
# a node that never maps its port is not connectable and starves
@@ -993,6 +1168,10 @@ def set_global_rate_limits(self, up_kb: int, down_kb: int) -> None:
}
)
+ def set_connections_limit(self, n: int) -> None:
+ self._ensure_session()
+ self._ses.apply_settings({"connections_limit": max(10, int(n))})
+
def purge_stopped(self, keep_errors: bool = True) -> None:
"""No-op: libtorrent has no stopped-results ledger to groom.
Finished downloads keep seeding on their live handle; policy
diff --git a/zimi/previews.py b/zimi/previews.py
index ed57b40f..c269f307 100644
--- a/zimi/previews.py
+++ b/zimi/previews.py
@@ -18,6 +18,84 @@ def strip_html(text):
return text
+# Some ZIMs bake a single repeated into every page — iFixit
+# device pages carry a featured-guide blurb ("How to replace the SSD in your
+# Lenovo Legion…") rather than the device's own description. When a page exposes
+# its own summary block (iFixit's .banner-blurb / itemprop="description"), that
+# wins over the meta tag.
+_SNIPPET_OWN_SUMMARY_RES = [
+ re.compile(
+ r'
',
+ re.IGNORECASE | re.DOTALL,
+ ),
+ re.compile(
+ r'<[a-z]+[^>]*\bitemprop=["\']description["\'][^>]*>(.*?)[a-z]+>',
+ re.IGNORECASE | re.DOTALL,
+ ),
+]
+_SNIPPET_META_DESC_RES = [
+ re.compile(
+ r']*>.*?\1>"
+ r'|<[a-z]+[^>]*\bclass=["\'][^"\']*\b'
+ r"(?:toc|breadcrumb|related-guides|featured-guides|navigation|"
+ r'js-dynamic-toc-section)\b[^"\']*["\'][^>]*>.*?[a-z]+>',
+ re.IGNORECASE | re.DOTALL,
+)
+
+
+def extract_snippet(text, zim_name=""):
+ """Best short text snippet for the /snippet endpoint.
+
+ Order of preference:
+ 1. The page's own summary block (iFixit device blurb / itemprop
+ description) — beats a repeated boilerplate .
+ 2. A tag.
+ 3. / body prose, boilerplate chrome stripped.
+ 4. Full page text, boilerplate chrome stripped.
+
+ ``text`` is the already-decoded HTML lead (the caller reads ~15 KB, enough
+ for meta + the opening content). Returns a plain-text string
+ (possibly empty)."""
+ # 1. Page's own summary block.
+ for rx in _SNIPPET_OWN_SUMMARY_RES:
+ m = rx.search(text)
+ if m:
+ s = strip_html(m.group(1))[:300].strip()
+ if len(s) >= 20:
+ return s
+ # 2. Meta description (near the top of ).
+ for rx in _SNIPPET_META_DESC_RES:
+ m = rx.search(text[:8000])
+ if m:
+ s = strip_html(m.group(1))[:300].strip()
+ if s:
+ return s
+ # 3/4. Body prose, with repeated chrome removed first.
+ cleaned = _SNIPPET_BOILERPLATE_RE.sub(" ", text)
+ for tag in ("main", "article"):
+ tag_m = re.search(r"<" + tag + r"[\s>]", cleaned, re.IGNORECASE)
+ if tag_m:
+ s = strip_html(cleaned[tag_m.start() :])[:300].strip()
+ if s:
+ return s
+ return strip_html(cleaned)[:300].strip()
+
+
def _resolve_img_path(archive, path, src):
"""Resolve a relative image src to a ZIM entry path. Returns URL or None."""
decoded = unquote(unquote(src))
@@ -29,7 +107,8 @@ def _resolve_img_path(archive, path, src):
parts = []
for seg in img_path.replace("\\", "/").split("/"):
if seg == "..":
- if parts: parts.pop()
+ if parts:
+ parts.pop()
elif seg and seg != ".":
parts.append(seg)
img_path = "/".join(parts)
@@ -60,18 +139,24 @@ def _extract_preview_title(html_str, entry_title):
for pattern in [
r']*>([^<]+)',
+ r"]*>([^<]+)",
r'
]*>(.*?)
',
r'
]*>(.*?)
',
- r'
]*>(.*?)
',
+ r"
]*>(.*?)
",
]:
tm = re.search(pattern, html_str, re.IGNORECASE | re.DOTALL)
if tm:
clean_title = strip_html(html.unescape(tm.group(1).strip()))
# Strip site suffixes like " | TED Talk", "— The World Factbook"
- clean_title = re.sub(r'\s*[\|–—]\s*(TED\s*Talk|TED|Wikipedia|The World Factbook).*$', '', clean_title)
+ clean_title = re.sub(
+ r"\s*[\|–—]\s*(TED\s*Talk|TED|Wikipedia|The World Factbook).*$",
+ "",
+ clean_title,
+ )
# Strip Factbook region prefixes like "Africa :: " or "Europe :: "
- clean_title = re.sub(r'^[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\s*::\s*', '', clean_title)
+ clean_title = re.sub(
+ r"^[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\s*::\s*", "", clean_title
+ )
if len(clean_title) > 3 and clean_title != entry_title:
return clean_title[:200]
# No HTML title found — title-case the slug as last resort
@@ -80,13 +165,23 @@ def _extract_preview_title(html_str, entry_title):
def _is_real_quote(text):
"""Filter out non-quote text: credits, citations, references, metadata."""
- if re.match(r'^(Directed|Written|Produced|Edited|Narrated|Adapted|Translated|Music)\s+by\b', text, re.IGNORECASE):
+ if re.match(
+ r"^(Directed|Written|Produced|Edited|Narrated|Adapted|Translated|Music)\s+by\b",
+ text,
+ re.IGNORECASE,
+ ):
return False
- if re.match(r'^(In response to|Based on|See also|Main article)\b', text, re.IGNORECASE):
+ if re.match(
+ r"^(In response to|Based on|See also|Main article)\b", text, re.IGNORECASE
+ ):
return False
- if re.search(r'\bRetrieved\s+(on|from)\b|\bISBN\b|\bAssociated Press\b|^\d{4}\s+film\b', text, re.IGNORECASE):
+ if re.search(
+ r"\bRetrieved\s+(on|from)\b|\bISBN\b|\bAssociated Press\b|^\d{4}\s+film\b",
+ text,
+ re.IGNORECASE,
+ ):
return False
- if re.match(r'^[\w\s,]+\(\s*\d{4}\s*\)', text): # "Author Name (Year)" citation
+ if re.match(r"^[\w\s,]+\(\s*\d{4}\s*\)", text): # "Author Name (Year)" citation
return False
return len(text.split()) > 6
@@ -98,46 +193,70 @@ def _extract_wikiquote_attribution(block, inner_ul_pos, page_title):
"""
author = None
# Use page title as fallback only if it looks like a person name (has a space)
- if ' ' in page_title and re.match(r'^[A-Z][a-z]+ [A-Z]', page_title):
+ if " " in page_title and re.match(r"^[A-Z][a-z]+ [A-Z]", page_title):
author = page_title
inner_block = block[inner_ul_pos:]
attr_raw = strip_html(inner_block).strip()
# Normalize double spaces around punctuation (strip_html replaces tags with spaces)
- attr_raw = re.sub(r'\s+([,;:.!?])', r'\1', attr_raw)
- attr_raw = re.sub(r'^[\u2014\u2013\-~]+\s*', '', attr_raw).strip().split('\n')[0].strip()
+ attr_raw = re.sub(r"\s+([,;:.!?])", r"\1", attr_raw)
+ attr_raw = (
+ re.sub(r"^[\u2014\u2013\-~]+\s*", "", attr_raw).strip().split("\n")[0].strip()
+ )
if attr_raw and 3 < len(attr_raw) < 200:
- if not re.search(r'[\[\]{}]|https?:|www\.|^\d', attr_raw, re.IGNORECASE):
+ if not re.search(r"[\[\]{}]|https?:|www\.|^\d", attr_raw, re.IGNORECASE):
# Detect source citations (not person names):
# - Contains ":" mid-text (e.g. "StoptheWarNow: Third peace convoy")
# - Looks like a title/headline (many capitalized words, >5 words)
# - Contains news agency / publication markers
_is_source = bool(
- re.search(r'\w:\s+\w', attr_raw) # colon in middle
- or re.search(r'(?i)\b(Agency|News|Times|Post|Tribune|Journal|Gazette|Herald|Magazine|Review|Report|Press|Daily)\b', attr_raw)
- or (len(attr_raw.split()) > 6 and not re.match(r'^[A-Z][a-z]+(?:\s+[a-z]+)*\s+[A-Z][a-z]+$', attr_raw.split('(')[0].split(',')[0].strip()))
+ re.search(r"\w:\s+\w", attr_raw) # colon in middle
+ or re.search(
+ r"(?i)\b(Agency|News|Times|Post|Tribune|Journal|Gazette|Herald|Magazine|Review|Report|Press|Daily)\b",
+ attr_raw,
+ )
+ or (
+ len(attr_raw.split()) > 6
+ and not re.match(
+ r"^[A-Z][a-z]+(?:\s+[a-z]+)*\s+[A-Z][a-z]+$",
+ attr_raw.split("(")[0].split(",")[0].strip(),
+ )
+ )
)
if not _is_source:
# Extract name: everything before first comma or opening paren
# e.g. "Henry Adams, Mont Saint Michel and Chartres (1904)" → "Henry Adams"
- name_part = re.split(r'[,(]', attr_raw)[0].strip()
+ name_part = re.split(r"[,(]", attr_raw)[0].strip()
# Handle honorifics with commas: "Adams, Henry" or "King, Jr., Martin Luther"
# If name_part is a single word and next part also looks like a name, rejoin
- if name_part and ',' in attr_raw:
- parts = [p.strip() for p in attr_raw.split(',')]
+ if name_part and "," in attr_raw:
+ parts = [p.strip() for p in attr_raw.split(",")]
# "Last, First" pattern: single capitalized word, then capitalized word(s)
- if (len(parts) >= 2 and re.match(r'^[A-Z][a-z]+$', parts[0])
- and re.match(r'^(Jr\.|Sr\.|[A-Z])', parts[1])):
+ if (
+ len(parts) >= 2
+ and re.match(r"^[A-Z][a-z]+$", parts[0])
+ and re.match(r"^(Jr\.|Sr\.|[A-Z])", parts[1])
+ ):
# Check for Jr./Sr. suffix
- if parts[1] in ('Jr.', 'Sr.', 'III', 'II', 'IV') and len(parts) >= 3:
- name_part = parts[2].strip() + ' ' + parts[0] + ', ' + parts[1]
- elif re.match(r'^[A-Z][a-z]', parts[1]):
+ if (
+ parts[1] in ("Jr.", "Sr.", "III", "II", "IV")
+ and len(parts) >= 3
+ ):
+ name_part = (
+ parts[2].strip() + " " + parts[0] + ", " + parts[1]
+ )
+ elif re.match(r"^[A-Z][a-z]", parts[1]):
# "Last, First ..." — but only if second part is short (a name, not a book title)
if len(parts[1].split()) <= 3:
- name_part = parts[1] + ' ' + parts[0]
+ name_part = parts[1] + " " + parts[0]
# Validate: must start with uppercase letter, reasonable length
- if (name_part and 2 < len(name_part) < 60
- and re.match(r'^[A-Z]', name_part)
- and not re.match(r'^(p\.|ch\.|vol\.|see |ibid)', name_part, re.IGNORECASE)):
+ if (
+ name_part
+ and 2 < len(name_part) < 60
+ and re.match(r"^[A-Z]", name_part)
+ and not re.match(
+ r"^(p\.|ch\.|vol\.|see |ibid)", name_part, re.IGNORECASE
+ )
+ ):
author = name_part
return author
@@ -150,14 +269,14 @@ def _extract_preview_wikiquote(html_str, result, entry_title):
# Wikiquote structure:
Quote text
Attribution
# Strategy: find
blocks that contain nested
(quote + attribution).
# Use a simple stack-based approach to find balanced top-level
blocks.
- for ul_m in re.finditer(r'
', html_str):
+ for ul_m in re.finditer(r"
", html_str):
start = ul_m.start()
# Find the matching
by counting nesting depth
depth = 1
pos = ul_m.end()
while depth > 0 and pos < len(html_str) and pos < start + 5000:
- next_open = html_str.find('
', pos)
+ next_open = html_str.find("
", pos)
if next_close < 0:
break
if next_open >= 0 and next_open < next_close:
@@ -170,29 +289,35 @@ def _extract_preview_wikiquote(html_str, result, entry_title):
continue
block = html_str[start:pos]
# Must have a nested
(attribution) to be a quote block
- if block.count('
as the quote
- inner_ul_pos = block.find('
+ inner_ul_pos = block.find("
if inner_ul_pos < 0:
continue
quote_html = block[4:inner_ul_pos] # between outer
]*>", "", quote_html)
text = strip_html(quote_html).strip()
# Check for inline tilde attribution: "Quote text. ~ Author Name"
- tilde_match = re.search(r'\s*~\s*(.+)$', text)
+ tilde_match = re.search(r"\s*~\s*(.+)$", text)
tilde_author = None
if tilde_match:
- text = text[:tilde_match.start()].rstrip()
+ text = text[: tilde_match.start()].rstrip()
tilde_author = tilde_match.group(1).strip()
if 20 < len(text) < 400 and _is_real_quote(text):
- if text.startswith(("Category:", "See also", "External links", "Retrieved")):
+ if text.startswith(
+ ("Category:", "See also", "External links", "Retrieved")
+ ):
continue
result["blurb"] = "\u201c" + text[:250] + "\u201d"
# Attribution: tilde author takes priority (inline convention),
# then try nested
for author name, fall back to page title.
- if tilde_author and 2 < len(tilde_author) < 60 and re.match(r'^[A-Z]', tilde_author):
+ if (
+ tilde_author
+ and 2 < len(tilde_author) < 60
+ and re.match(r"^[A-Z]", tilde_author)
+ ):
result["attribution"] = tilde_author[:100]
break
page_title = result.get("title") or entry_title
@@ -203,27 +328,36 @@ def _extract_preview_wikiquote(html_str, result, entry_title):
# Fallback: if
parsing found nothing, try
blocks or
after "Quotes" heading
if not result.get("blurb"):
# Try
blocks (definition list format used on some wikiquote pages)
- for dd_m in re.finditer(r'
(.*?)
', html_str, re.DOTALL):
+ for dd_m in re.finditer(r"
(.*?)
", html_str, re.DOTALL):
dd_text = strip_html(dd_m.group(1)).strip()
if 30 < len(dd_text) < 400 and _is_real_quote(dd_text):
- if not dd_text.startswith(("Category:", "See also", "External", "Retrieved", "Source")):
+ if not dd_text.startswith(
+ ("Category:", "See also", "External", "Retrieved", "Source")
+ ):
result["blurb"] = "\u201c" + dd_text[:250] + "\u201d"
_pg = result.get("title") or entry_title
- if ' ' in _pg and re.match(r'^[A-Z][a-z]+ [A-Z]', _pg):
+ if " " in _pg and re.match(r"^[A-Z][a-z]+ [A-Z]", _pg):
result["attribution"] = _pg
break
if not result.get("blurb"):
# Try text after a "Quotes" section heading
- quotes_section = re.search(r']*>(?:<[^>]*>)*\s*Quotes?\s*(?:<[^>]*>)*(.*?)(?:]*>(?:<[^>]*>)*\s*Quotes?\s*(?:<[^>]*>)*(.*?)(?:(.*?)
', quotes_section.group(1), re.DOTALL):
+ for li_m in re.finditer(
+ r"
(.*?)
", quotes_section.group(1), re.DOTALL
+ ):
li_text = strip_html(li_m.group(1)).strip()
if 30 < len(li_text) < 400 and _is_real_quote(li_text):
- if not li_text.startswith(("Category:", "See also", "External", "Retrieved")):
+ if not li_text.startswith(
+ ("Category:", "See also", "External", "Retrieved")
+ ):
result["blurb"] = "\u201c" + li_text[:250] + "\u201d"
_pg = result.get("title") or entry_title
- if ' ' in _pg and re.match(r'^[A-Z][a-z]+ [A-Z]', _pg):
+ if " " in _pg and re.match(r"^[A-Z][a-z]+ [A-Z]", _pg):
result["attribution"] = _pg
break
@@ -237,21 +371,32 @@ def _extract_preview_ted(html_str, archive, zim_name, path, result):
"""
speaker = None
last_name = None
- sp_m = re.search(r'
', html_str, re.DOTALL | re.IGNORECASE
+ )
if sp_m:
- last_name = re.sub(r'\s+', ' ', strip_html(sp_m.group(1))).strip()
- if ' ' in last_name:
+ last_name = re.sub(r"\s+", " ", strip_html(sp_m.group(1))).strip()
+ if " " in last_name:
# Already a full name (some playlist ZIMs have full names)
speaker = last_name
# Find full name in speaker_desc by locating the last name in context
if not speaker and last_name:
- sp_desc = re.search(r'
— skip boring inflected forms
- _boring_def = re.compile(r'^(plural of |third-person |simple past |past participle |present participle |alternative |archaic |obsolete |misspelling |eye dialect |nonstandard )', re.IGNORECASE)
- for def_m in re.finditer(r']*>\s*
]*>(.*?)
', section_html, re.DOTALL):
+ _boring_def = re.compile(
+ r"^(plural of |third-person |simple past |past participle |present participle |alternative |archaic |obsolete |misspelling |eye dialect |nonstandard )",
+ re.IGNORECASE,
+ )
+ for def_m in re.finditer(
+ r"]*>\s*
]*>(.*?)
", section_html, re.DOTALL
+ ):
def_text = strip_html(def_m.group(1)).strip()
- def_text = re.split(r'\n', def_text)[0].strip()
- if len(def_text) > 5 and not def_text.startswith(('Category:', 'See also')):
+ def_text = re.split(r"\n", def_text)[0].strip()
+ if len(def_text) > 5 and not def_text.startswith(("Category:", "See also")):
if _boring_def.match(def_text):
result["boring"] = True # signal to retry
else:
@@ -417,11 +612,13 @@ def _extract_preview_wiktionary(html_str, zim_name, result):
if eng_m:
# Slice from English header to next
(next language section) or end
eng_start = eng_m.start()
- next_h2 = re.search(r'
— could be Simple Wiktionary (monolingual, no language headers)
# or a non-English entry. Check if page has any
definitions.
@@ -430,11 +627,16 @@ def _extract_preview_wiktionary(html_str, zim_name, result):
# Simple Wiktionary: treat entire page as English content
eng_section = html_str[:30000]
# Part of speech: Simple Wiktionary uses
for POS (not nested under language)
- _extract_wiktionary_pos_and_def(eng_section, result, '234')
+ _extract_wiktionary_pos_and_def(eng_section, result, "234")
if not result.get("part_of_speech"):
# Try inline pattern: (noun), (verb), etc.
- pos_inline = re.search(r'\((\w+)\)', eng_section[:3000])
- if pos_inline and pos_inline.group(1).lower() in ('noun', 'verb', 'adjective', 'adverb'):
+ pos_inline = re.search(r"\((\w+)\)", eng_section[:3000])
+ if pos_inline and pos_inline.group(1).lower() in (
+ "noun",
+ "verb",
+ "adjective",
+ "adverb",
+ ):
result["part_of_speech"] = pos_inline.group(1).capitalize()
else:
# Full Wiktionary, no English section — flag for the random endpoint to skip
@@ -456,8 +658,11 @@ def _extract_preview_blurb(html_str):
if m and len(m.group(1).strip()) > 20:
return html.unescape(m.group(1).strip())[:200]
# First substantial
text (skip tiny nav/footer paragraphs and boilerplate)
- _skip_blurb = re.compile(r'(Creative Commons|This work is licensed|free to copy and share|All rights reserved|Copyright \d|DMCA)', re.IGNORECASE)
- for pm in re.finditer(r'
]*>(.*?)
', html_str, re.DOTALL | re.IGNORECASE):
+ _skip_blurb = re.compile(
+ r"(Creative Commons|This work is licensed|free to copy and share|All rights reserved|Copyright \d|DMCA)",
+ re.IGNORECASE,
+ )
+ for pm in re.finditer(r"
]*>(.*?)
", html_str, re.DOTALL | re.IGNORECASE):
text = strip_html(pm.group(1))
if len(text) > 40 and not _skip_blurb.search(text):
return text[:200]
@@ -484,7 +689,11 @@ def _extract_preview_thumbnail(html_str, archive, zim_name, path):
m = re.search(pattern, html_str, re.IGNORECASE)
if m:
src = m.group(1)
- if src.startswith("http://") or src.startswith("https://") or src.startswith("//"):
+ if (
+ src.startswith("http://")
+ or src.startswith("https://")
+ or src.startswith("//")
+ ):
continue # external URL, can't serve from ZIM
if not src.lower().endswith(".svg"):
resolved = _resolve_img_path(archive, path, src)
@@ -494,7 +703,7 @@ def _extract_preview_thumbnail(html_str, archive, zim_name, path):
# Fall back to best content image using scoring heuristics
best_img = None
best_score = 0
- for m in re.finditer(r']*)>', html_str, re.IGNORECASE):
+ for m in re.finditer(r"]*)>", html_str, re.IGNORECASE):
attrs = m.group(1)
src_m = re.search(r'src=["\']([^"\']+)["\']', attrs)
if not src_m:
@@ -506,8 +715,14 @@ def _extract_preview_thumbnail(html_str, archive, zim_name, path):
continue
# Skip generic site chrome images (navigation icons, banners)
src_base = src.rsplit("/", 1)[-1].lower()
- if src_base in ("home_on.png", "home_off.png", "banner_ext2.png",
- "photo_on.gif", "one-page-summary.png", "travel-facts.png"):
+ if src_base in (
+ "home_on.png",
+ "home_off.png",
+ "banner_ext2.png",
+ "photo_on.gif",
+ "one-page-summary.png",
+ "travel-facts.png",
+ ):
continue
w_m = re.search(r'width=["\']?(\d+)', attrs)
h_m = re.search(r'height=["\']?(\d+)', attrs)
@@ -518,8 +733,10 @@ def _extract_preview_thumbnail(html_str, archive, zim_name, path):
continue
# Skip images inside header/nav/footer
ctx_start = max(0, m.start() - 300)
- ctx = html_str[ctx_start:m.start()].lower()
- if re.search(r'<(header|nav|footer)\b', ctx) and not re.search(r'(header|nav|footer)>', ctx):
+ ctx = html_str[ctx_start : m.start()].lower()
+ if re.search(r"<(header|nav|footer)\b", ctx) and not re.search(
+ r"(header|nav|footer)>", ctx
+ ):
continue
# Score: area + bonuses for content signals
area = w * h
@@ -561,7 +778,9 @@ def _extract_preview(archive, zim_name, path):
content = bytes(entry.get_item().content)
html_str = content.decode("utf-8", errors="replace")[:80000]
except Exception as e:
- log.debug("Failed to read entry for preview extract %s/%s: %s", zim_name, path, e)
+ log.debug(
+ "Failed to read entry for preview extract %s/%s: %s", zim_name, path, e
+ )
return result
# -- Title: extract from or og:title if entry.title is a slug --
@@ -597,6 +816,8 @@ def _extract_preview(archive, zim_name, path):
# -- Generic thumbnail fallback --
if not result["thumbnail"]:
- result["thumbnail"] = _extract_preview_thumbnail(html_str, archive, zim_name, path)
+ result["thumbnail"] = _extract_preview_thumbnail(
+ html_str, archive, zim_name, path
+ )
return result
diff --git a/zimi/search.py b/zimi/search.py
index ebfd8a4e..3d5b8cc0 100644
--- a/zimi/search.py
+++ b/zimi/search.py
@@ -1158,7 +1158,33 @@ def _dedup_results_by_title(results):
# is at least this many times more common — Norvig-style confidence that lets
# us fix a typo that itself snuck into the vocab (e.g. a misspelled title)
# without ever touching a genuinely common word.
-_VOCAB_MAX_WORDS = 200_000 # cap on distinct vocabulary words held in memory
+# Peak in-memory ceiling on distinct words held DURING the build (not the
+# size of what's persisted — see _VOCAB_MAX_PERSIST_WORDS). The whole library
+# holds ~6M distinct sampled tokens (measured), the vast majority count==1
+# junk from huge Q&A/dictionary indexes. This cap is a memory guard, not an
+# early-stop: hitting it triggers a non-terminating tiered eviction (see
+# _evict_to_free) that frees room and lets the scan CONTINUE through every
+# remaining index, instead of the old 200k cap that saturated after ~3 of 68
+# files and froze the scan — which is exactly why spread-thin words like
+# "mitochondria" and "photosynthesis" never made it in. 4M keeps peak build
+# RSS around ~600MB, comfortable inside the 4GB container alongside the live
+# server.
+_VOCAB_MAX_WORDS = 4_000_000
+# Safety ceiling on the PERSISTED vocab (and the in-memory dict the corrector
+# uses). After the scan, singletons are pruned; only if MORE than this many
+# count>=2 words remain are the top-K by frequency kept. Sized so it does NOT
+# bite for a normal library — the measured 53-file/16.8GB NAS keeps ~1.26M
+# count>=2 words (a ~19MB cache), all retained — so every word appearing even
+# twice survives, honoring the "coverage grows with your title indexes"
+# promise with room to spare (the old cache was 181k words / 2.4MB). The cap
+# only guards against an unbounded dict on a pathologically large library;
+# because it's frequency-ranked, high-count words are never the ones dropped.
+_VOCAB_MAX_PERSIST_WORDS = 1_500_000
+# Highest count-tier _evict_to_free will sweep before giving up and freezing
+# new admissions (it keeps counting/scanning either way). Singletons (tier 1)
+# dominate the junk, so tier 1 almost always frees plenty; tiers 2-3 are a
+# safety valve for pathological libraries.
+_VOCAB_EVICT_MAX_TIER = 3
_VOCAB_MIN_WORD_LEN = 3 # ignore 1-2 char fragments (noise, not misspellings)
# One-time ceiling on the lazy vocab scan when there's no usable cache on
# disk (see _vocab_cache_load / _vocab_cache_save below). Generous on
@@ -1182,24 +1208,54 @@ def _dedup_results_by_title(results):
# file a real chance to be sampled and survive lossy-counting eviction.
_VOCAB_MAX_ROWS_PER_FILE = 3_000_000
_VOCAB_FETCH_BATCH_SIZE = 5000 # sqlite fetchmany() page size during the scan
-# Lossy-counting admission: when the word cap is hit, count==1 entries (the
-# one-off proper nouns and IDs that dominate large title sets) are swept out
-# to make room, and scanning continues — this is what lets genuinely common
-# words admitted later in the scan still get counted, instead of the cap
-# freezing on whichever words happened to appear first. If a sweep frees
-# less than this fraction of the cap, the vocab has saturated (what's left
-# is mostly count>=2) and the scan stops for good.
+# When the peak word cap is hit, tiered eviction (see _evict_to_free) must
+# free at least this fraction of the cap so the sweep is worth its O(N) scan
+# and the vocab doesn't thrash — evicting a few thousand entries only to
+# refill them a batch later. count==1 junk almost always clears far more than
+# this; the fraction only decides how deep the tier escalation goes.
_VOCAB_EVICT_MIN_FRACTION = 0.10
_VOCAB_CACHE_FILENAME = "dym_vocab.json"
_VOCAB_CACHE_PATH = os.path.join(_srv.ZIMI_DATA_DIR, _VOCAB_CACHE_FILENAME)
# Bump whenever the vocab-building algorithm changes in a way that makes an
# on-disk cache from an older version invalid even though the underlying
-# title indexes haven't changed — lossy-counting admission and stride
-# sampling (see _vocab_stride) each changed which words a scan of the same
-# files produces. Folded into _vocab_signature so a stale-algorithm cache is
-# rebuilt transparently rather than loaded forever.
-_VOCAB_BUILDER_VERSION = 3
-_DIST2_MAX_LEN = 7 # only try edit-distance-2 on words this short or shorter
+# title indexes haven't changed — lossy-counting admission, stride sampling,
+# and (v4) non-terminating tiered eviction + top-K persist each changed which
+# words a scan of the same files produces. Folded into _vocab_signature so a
+# stale-algorithm cache is rebuilt transparently rather than loaded forever.
+# v4 is the release-note "coverage grows with your title indexes" rebuild:
+# every production instance re-scans once on upgrade and picks up the
+# previously-missing spread-thin words.
+_VOCAB_BUILDER_VERSION = 4
+# Words this length or shorter get EXHAUSTIVE edit-distance-2 via _edits2 (the
+# candidate set stays small enough to enumerate within budget). Longer words
+# would blow the 50ms budget that way — a 13-char word generates ~450k
+# distance-2 strings (~700ms) — so they go through the trigram index instead
+# (see _trigram_dist2_candidates), which is how "fotosynthesis" reaches
+# "photosynthesis" (distance 2, 14 chars) without enumerating its neighborhood.
+_DIST2_MAX_LEN = 7
+# Long-word distance-2 correction. A character-trigram inverted index over the
+# vocab turns "which vocab words are within edit distance 2 of this long typo?"
+# into a bounded posting-list intersection instead of an unbounded edit-2
+# enumeration. Only words longer than the exhaustive path handles are indexed
+# and corrected this way, so the two paths partition cleanly by length.
+_TRIGRAM_MIN_LEN = _DIST2_MAX_LEN + 1 # 8
+# Trigrams appearing in more vocab words than this are dropped from a query's
+# candidate gather — they're uninformative (every long word shares "ing",
+# "tion", …) and their posting lists dominate the cost. Measured worst-case
+# query stays ~15ms with this set.
+_TRIGRAM_SKIP_POSTING = 15_000
+_TRIGRAM_MAX_SCAN = 40_000 # hard ceiling on postings walked per query (budget guard)
+_TRIGRAM_MAX_VERIFY = 40 # edit-distance-verify only the N best trigram-overlap words
+# Ceiling on how many long words go INTO the trigram index, decoupling
+# long-word distance-2 latency from the size of the persisted vocab. The vocab
+# can grow to _VOCAB_MAX_PERSIST_WORDS (cheap O(1) dist-1 coverage), but only
+# the highest-count long words are indexed for dist-2 — a rare count==2 long
+# word is almost never the right correction, yet each one lengthens the
+# posting lists every query walks. Measured: ~284k long words on the real NAS
+# library gives a ~31ms worst-case long-word query; leaving the index
+# uncapped, a synthetic 1.3M-word vocab (~984k long words) pushed that to
+# ~44ms. Capping here keeps the worst case flat as the library grows.
+_TRIGRAM_MAX_INDEX_WORDS = 400_000
_DYM_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"
_word_split_re = re.compile(r"[^a-z0-9]+")
# Split a query into alternating [gap, word, gap, word, ...]; odd indices are
@@ -1213,6 +1269,13 @@ def _dedup_results_by_title(results):
_vocab = None # {word: count}; None until built, a dict once built (empty on failure)
_vocab_lock = threading.Lock()
_vocab_builder_thread = None # daemon thread building the vocab; test hook for .join()
+# {trigram: [word, ...]} inverted index over the long (>= _TRIGRAM_MIN_LEN)
+# words of _vocab, built by the same background worker right after the vocab is
+# ready. None until then; long-word distance-2 correction is simply skipped
+# while it's absent (fail-soft, like the vocab itself). Not persisted — it's
+# cheap to rebuild (~2-3s) from the loaded vocab, so the cache format is
+# unchanged.
+_trigram_index = None
def _ascii_fold(s):
@@ -1227,15 +1290,61 @@ def _ascii_fold(s):
def _evict_singleton_words(vocab):
"""Sweep-delete every count==1 entry from `vocab`, in place.
- Returns the number removed. Used both mid-scan (lossy-counting admission
- when the word cap is hit) and once at the end (final prune — singletons
- are noise for correction and just bloat the persisted cache)."""
+ Returns the number removed. Used for the final prune — singletons are
+ noise for correction and just bloat the persisted cache."""
singles = [w for w, c in vocab.items() if c == 1]
for w in singles:
del vocab[w]
return len(singles)
+def _evict_to_free(vocab, min_free):
+ """Free at least `min_free` entries from a full vocab WITHOUT stopping the
+ scan, in place. Returns (removed, top_tier).
+
+ Drops the lowest-count tier first (count==1 singletons — the one-off
+ proper nouns, IDs and typos that dominate huge Q&A/dictionary indexes),
+ escalating to count==2, ==3, … only when a tier doesn't free enough,
+ bounded by _VOCAB_EVICT_MAX_TIER. Unlike the old saturation-stop this is
+ NOT a signal to end the scan: the caller keeps reading every remaining
+ index afterward, so a word evicted here is simply re-admitted when it
+ recurs, and — critically — spread-thin words that only appear in
+ later-scanned indexes still get their chance. Singletons almost always
+ clear far more than min_free in one tier; escalation is a safety valve.
+ If even the top tier can't free enough, the caller freezes NEW admissions
+ but still finishes counting existing words and scanning the library."""
+ removed = 0
+ tier = 0
+ while removed < min_free and tier < _VOCAB_EVICT_MAX_TIER:
+ tier += 1
+ victims = [w for w, c in vocab.items() if c == tier]
+ for w in victims:
+ del vocab[w]
+ removed += len(victims)
+ return removed, tier
+
+
+def _cap_vocab_to_top_k(vocab, k):
+ """Keep only the `k` highest-count words in `vocab`, in place. No-op when
+ the vocab already fits. Returns the number dropped.
+
+ Bounds the persisted cache and load time without touching the words a user
+ is likely to type: correction candidates are generated by edit distance,
+ not looked up by rank, so dropping the long low-count tail costs coverage
+ only for words seen a mere handful of times library-wide. Ties at the
+ cutoff count are broken by the word text so the result is deterministic
+ (same indexes → same cache → stable signature)."""
+ if len(vocab) <= k:
+ return 0
+ # Sort by (count desc, word asc); keep the first k.
+ keep = sorted(vocab.items(), key=lambda kv: (-kv[1], kv[0]))[:k]
+ dropped = len(vocab) - len(keep)
+ kept = dict(keep)
+ vocab.clear()
+ vocab.update(kept)
+ return dropped
+
+
def _vocab_stride(conn, cap):
"""Row stride for near-uniform sampling of one title index.
@@ -1263,25 +1372,30 @@ def _build_vocab():
"""Scan the SQLite title indexes into a {word: count} vocabulary.
Opens a FRESH connection per index (sqlite objects aren't shareable across
- threads). Bounded three ways: a distinct-word cap, a per-file row cap, and
- an overall wall-clock budget — whichever hits first stops that file (row
- cap) or the whole scan (word cap / budget). Files are scanned largest-first
- (by byte size) so the richest indexes (Wikipedia-scale) contribute first;
- each file's rows are then STRIDE-SAMPLED (see _vocab_stride) rather than
- read as a contiguous prefix, so the per-file row cap buys breadth across
- the WHOLE file, not just its first N rows — a word occurring only
- occasionally across a huge index still has a real chance to be sampled.
- The word cap uses lossy-counting admission (see
- _evict_singleton_words): hitting it triggers a singleton sweep rather
- than freezing outright, so words seen later in the scan can still be
- counted — a first-come cap otherwise fills entirely with one-off proper
- nouns from a huge index before anything looks "common". A final sweep
- after the whole scan drops any remaining singletons (noise for
- correction, dead weight in the persisted cache). Returns whatever was
- gathered (possibly empty). Never raises; a broken index is skipped.
- Always logs the outcome at info level, including the empty case, so a
- starved scan is visible in production rather than silently returning
- nothing."""
+ threads). Files are scanned largest-first (by byte size) so the richest
+ indexes contribute first if the wall-clock budget is ever hit; each file's
+ rows are STRIDE-SAMPLED (see _vocab_stride) rather than read as a
+ contiguous prefix, so the per-file row cap buys breadth across the WHOLE
+ file — a word occurring only occasionally across a huge index still has a
+ real chance to be sampled.
+
+ The build is bounded by memory, not by a first-come word cap that stops
+ the scan. Hitting the peak word cap triggers NON-TERMINATING tiered
+ eviction (see _evict_to_free): the lowest count-tiers are swept to free
+ room and the scan keeps going through every remaining index. This is the
+ v4 fix for the coverage promise — the old design saturated a 200k cap
+ after ~3 of 68 files and froze, so spread-thin words ("mitochondria",
+ "photosynthesis") that only accumulate once the dictionary/encyclopedia
+ indexes are reached never made it in. Only if eviction genuinely can't
+ free room (a library of almost all high-count words) are NEW admissions
+ frozen — existing words keep counting and every file is still read.
+
+ After the scan: singletons are pruned (noise for correction, dead weight
+ in the cache), then if more than _VOCAB_MAX_PERSIST_WORDS remain, only the
+ top-K by frequency are kept so the persisted cache and load time stay
+ bounded. Returns whatever was gathered (possibly empty). Never raises; a
+ broken index is skipped. Always logs the outcome at info level, including
+ the empty case, so a starved scan is visible in production."""
deadline = time.monotonic() + _VOCAB_BUILD_BUDGET_S
vocab = {}
index_dir = _TITLE_INDEX_DIR
@@ -1297,9 +1411,15 @@ def _build_vocab():
total_files = len(fnames)
files_scanned = 0
rows_scanned = 0
- at_cap = False
+ evictions = 0
+ admissions_frozen = (
+ False # set only if eviction can't free room; never stops the scan
+ )
+ # At least 1 so a tiny cap (tests, degenerate libraries) still frees room
+ # rather than looping on a no-op sweep.
+ min_free = max(1, int(_VOCAB_MAX_WORDS * _VOCAB_EVICT_MIN_FRACTION))
for fname in fnames:
- if at_cap or time.monotonic() > deadline:
+ if time.monotonic() > deadline:
break
db_path = os.path.join(index_dir, fname)
try:
@@ -1320,8 +1440,7 @@ def _build_vocab():
else:
cur = conn.execute("SELECT title_lower FROM titles")
while (
- not at_cap
- and rows_this_file < _VOCAB_MAX_ROWS_PER_FILE
+ rows_this_file < _VOCAB_MAX_ROWS_PER_FILE
and time.monotonic() <= deadline
):
rows = cur.fetchmany(_VOCAB_FETCH_BATCH_SIZE)
@@ -1338,12 +1457,16 @@ def _build_vocab():
continue
if w in vocab:
vocab[w] += 1
- elif not at_cap:
+ elif not admissions_frozen:
vocab[w] = 1
if len(vocab) >= _VOCAB_MAX_WORDS:
- freed = _evict_singleton_words(vocab)
- if freed < _VOCAB_MAX_WORDS * _VOCAB_EVICT_MIN_FRACTION:
- at_cap = True
+ freed, _tier = _evict_to_free(vocab, min_free)
+ evictions += 1
+ # Couldn't free enough even at the top tier:
+ # stop admitting NEW words, but keep counting
+ # existing ones and scanning every file.
+ if freed < min_free:
+ admissions_frozen = True
except Exception as e:
log.debug("Vocab: scan failed for %s: %s", fname, e)
finally:
@@ -1356,16 +1479,20 @@ def _build_vocab():
pruned = _evict_singleton_words(
vocab
) # final prune: singletons are noise, drop them
+ capped = _cap_vocab_to_top_k(vocab, _VOCAB_MAX_PERSIST_WORDS)
log.info(
- "Did-you-mean vocab: %d words (%d singletons pruned) from %d/%d index "
- "files, %d rows (budget_hit=%s, at_cap=%s)",
+ "Did-you-mean vocab: %d words (%d singletons pruned, %d capped by top-K) "
+ "from %d/%d index files, %d rows (budget_hit=%s, evictions=%d, "
+ "admissions_frozen=%s)",
len(vocab),
pruned,
+ capped,
files_scanned,
total_files,
rows_scanned,
budget_hit,
- at_cap,
+ evictions,
+ admissions_frozen,
)
return vocab
@@ -1455,6 +1582,7 @@ def _vocab_build_worker():
len(cached),
_VOCAB_CACHE_PATH,
)
+ _rebuild_trigram_index(cached)
return
try:
built = _build_vocab()
@@ -1467,6 +1595,7 @@ def _vocab_build_worker():
sig = _vocab_signature(_TITLE_INDEX_DIR)
if sig is not None:
_vocab_cache_save(built, sig)
+ _rebuild_trigram_index(_vocab)
def _ensure_vocab():
@@ -1504,11 +1633,12 @@ def _join_vocab_build(timeout=5.0):
def _reset_vocab():
"""Drop the cached vocabulary so the next call rebuilds. Tests only."""
- global _vocab, _vocab_builder_thread
+ global _vocab, _vocab_builder_thread, _trigram_index
_join_vocab_build() # let any in-flight builder finish before we clear state
with _vocab_lock:
_vocab = None
_vocab_builder_thread = None
+ _trigram_index = None
def _edits1(word):
@@ -1521,11 +1651,131 @@ def _edits1(word):
return set(deletes + transposes + replaces + inserts)
+def _trigrams(word):
+ """Set of character 3-grams in `word` (the whole word, itself for len<3)."""
+ if len(word) < 3:
+ return {word}
+ return {word[i : i + 3] for i in range(len(word) - 2)}
+
+
+def _edit_distance_le2(a, b):
+ """Levenshtein distance between `a` and `b`, capped: returns the true
+ distance when it is <= 2, otherwise 3. Rows are computed with early-exit
+ once every cell exceeds 2, so this stays cheap for the long words the
+ trigram path verifies."""
+ la, lb = len(a), len(b)
+ if abs(la - lb) > 2:
+ return 3
+ prev = list(range(lb + 1))
+ for i in range(1, la + 1):
+ cur = [i] + [0] * lb
+ row_best = i
+ ai = a[i - 1]
+ for j in range(1, lb + 1):
+ cost = 0 if ai == b[j - 1] else 1
+ v = prev[j] + 1
+ if cur[j - 1] + 1 < v:
+ v = cur[j - 1] + 1
+ if prev[j - 1] + cost < v:
+ v = prev[j - 1] + cost
+ cur[j] = v
+ if v < row_best:
+ row_best = v
+ if row_best > 2:
+ return 3
+ prev = cur
+ return prev[lb] if prev[lb] <= 2 else 3
+
+
+def _build_trigram_index(vocab):
+ """Inverted trigram index over the long words of `vocab`: {trigram:[word]}.
+
+ Only words >= _TRIGRAM_MIN_LEN are indexed — shorter words are corrected by
+ exhaustive edit-distance-2 and never need this — and at most
+ _TRIGRAM_MAX_INDEX_WORDS of them, the highest-count ones, so query latency
+ stays flat as the vocab grows (see that constant). Posting lists hold the
+ same string objects that key `vocab` (references, not copies), so the index
+ adds only ~pointer-per-posting overhead. Returns the dict (possibly empty)."""
+ longs = [w for w in vocab if len(w) >= _TRIGRAM_MIN_LEN]
+ if len(longs) > _TRIGRAM_MAX_INDEX_WORDS:
+ # Keep the most frequent long words (count desc, word asc for a
+ # deterministic cutoff).
+ longs = sorted(longs, key=lambda w: (-vocab[w], w))[:_TRIGRAM_MAX_INDEX_WORDS]
+ idx = {}
+ for w in longs:
+ for g in _trigrams(w):
+ idx.setdefault(g, []).append(w)
+ return idx
+
+
+def _rebuild_trigram_index(vocab):
+ """Build the trigram index from `vocab` and publish it. Fail-soft: any
+ error just leaves long-word distance-2 correction disabled (index None),
+ never raising into the background worker."""
+ global _trigram_index
+ try:
+ idx = _build_trigram_index(vocab) if vocab else None
+ except Exception as e:
+ log.info("Did-you-mean trigram index: build failed: %s", e)
+ idx = None
+ with _vocab_lock:
+ _trigram_index = idx
+ if idx is not None:
+ log.info("Did-you-mean trigram index: %d trigrams over long words", len(idx))
+
+
+def _trigram_dist2_candidates(word, deadline=None):
+ """Vocab words within edit distance 2 of a long `word`, via the trigram
+ index. Returns only the CLOSEST such words (all at the minimum distance
+ found, which is <= 2), or [] if none / no index / word too short.
+
+ Bounded for the 50ms budget: only selective trigrams are consulted (the
+ ultra-common ones are skipped), their posting lists are walked
+ shortest-first up to a hard scan ceiling, and edit distance is verified on
+ only the best-overlapping candidates."""
+ idx = _trigram_index
+ if idx is None or len(word) < _TRIGRAM_MIN_LEN:
+ return []
+ grams = [
+ g for g in _trigrams(word) if 0 < len(idx.get(g, ())) <= _TRIGRAM_SKIP_POSTING
+ ]
+ grams.sort(key=lambda g: len(idx[g])) # most selective first
+ counts = {}
+ scanned = 0
+ for g in grams:
+ posting = idx[g]
+ if scanned + len(posting) > _TRIGRAM_MAX_SCAN:
+ break
+ scanned += len(posting)
+ for w in posting:
+ counts[w] = counts.get(w, 0) + 1
+ if deadline is not None and time.monotonic() > deadline:
+ break
+ if not counts or (deadline is not None and time.monotonic() > deadline):
+ return []
+ ranked = sorted(counts, key=lambda w: counts[w], reverse=True)[:_TRIGRAM_MAX_VERIFY]
+ best_d = 3
+ best = []
+ for w in ranked:
+ if w == word:
+ continue
+ d = _edit_distance_le2(word, w)
+ if d < best_d:
+ best_d = d
+ best = [w]
+ elif d == best_d and d <= 2:
+ best.append(w)
+ return best if best_d <= 2 else []
+
+
def _best_correction(word, vocab, deadline=None, freq_ratio=None):
"""Best in-vocab correction for `word`, or None. Frequency breaks ties.
- Distance-1 first; distance-2 only for short words (candidate set stays
- bounded) and only if the time budget allows.
+ Distance-1 first, then distance-2 if the time budget allows: short words
+ (<= _DIST2_MAX_LEN) enumerate their full edit-2 neighborhood; longer words
+ use the trigram index instead (see _trigram_dist2_candidates), which keeps
+ long-word correction like "fotosynthesis" -> "photosynthesis" inside the
+ budget an exhaustive enumeration would blow.
If `word` is itself in `vocab`, it's left alone UNLESS `freq_ratio` is
given: then a same-or-lower-distance candidate must be at least
@@ -1552,15 +1802,17 @@ def _pick(cands):
picked = _pick(cands)
if picked:
return picked
- if len(word) <= _DIST2_MAX_LEN and (
- deadline is None or time.monotonic() <= deadline
- ):
- cands2 = set()
- for e1 in _edits1(word):
- for e2 in _edits1(e1):
- if e2 in vocab and e2 != word:
- cands2.add(e2)
- return _pick(cands2)
+ if deadline is None or time.monotonic() <= deadline:
+ if len(word) <= _DIST2_MAX_LEN:
+ cands2 = set()
+ for e1 in _edits1(word):
+ for e2 in _edits1(e1):
+ if e2 in vocab and e2 != word:
+ cands2.add(e2)
+ return _pick(cands2)
+ # Long word: trigram index returns only the closest (<=2) vocab words,
+ # so _pick's frequency tie-break chooses among equally-near candidates.
+ return _pick(_trigram_dist2_candidates(word, deadline))
return None
@@ -2091,12 +2343,22 @@ def get_catalog(zim_name):
docs = []
for doc in catalog:
fps = doc.get("fp", [])
+ path = f"files/{fps[0]}" if fps else None
+ # Cheap size hint: the PDF entry's item.size is metadata (no decompress).
+ # Runs under _zim_lock (caller holds it) so touching libzim here is safe.
+ size = None
+ if path:
+ try:
+ size = archive.get_entry_by_path(path).get_item().size
+ except Exception:
+ size = None
docs.append(
{
"title": doc.get("ti", "?"),
"description": doc.get("dsc", ""),
"author": doc.get("aut", ""),
- "path": f"files/{fps[0]}" if fps else None,
+ "path": path,
+ "size": size,
}
)
return {"zim": zim_name, "documents": docs, "count": len(docs)}
diff --git a/zimi/server.py b/zimi/server.py
index 36e88ab7..afbdb674 100644
--- a/zimi/server.py
+++ b/zimi/server.py
@@ -87,7 +87,7 @@
# SSL context using certifi CA bundle (PyInstaller bundles lack system certs)
SSL_CTX = ssl.create_default_context(cafile=certifi.where())
-ZIMI_VERSION = "1.8.0"
+ZIMI_VERSION = "1.8.1"
# Standing maintenance cadence: catalog TTL is 24h and UPnP leases are
# 24h — run every 12h so both stay fresh at half-life.
@@ -172,6 +172,9 @@ def _init_p2p_background():
from zimi import library as _lib
_lib.resume_pending_downloads()
+ # Watcher that releases night-window-scheduled downloads when the
+ # window opens (no-op unless the user enabled download scheduling).
+ _lib.start_download_scheduler()
# Mirror mode seeds the whole installed library; either way,
# drop seeds whose file an update has replaced, and bring
# session-resumed seeds under the CURRENT settings (a seed
@@ -453,6 +456,11 @@ def _migrate_stray_torrent_files():
50 * 1024 * 1024
) # 50 MB — refuse to serve entries larger than this (prevents OOM)
MAX_POST_BODY = 65536 # max bytes accepted in POST requests (64KB — handles ~500 URLs for batch resolve)
+# Backup bundles and per-user data blobs (bookmarks/history/preferences) are the
+# one class of POST that legitimately runs large — a full-server backup carries
+# users, history and every per-user blob. They get their own ceiling so the tight
+# 64 KB cap keeps guarding every other endpoint.
+MAX_BACKUP_BODY = 8 * 1024 * 1024 # 8 MB — backup import + /userdata save
_BYTES_PER_GB = 1024**3
@@ -653,6 +661,14 @@ def _load_history():
return []
+def _save_history(entries):
+ """Replace the persistent event history with ``entries`` (capped, newest
+ first). Used by the full-server backup restore. Thread-safe."""
+ with _history_lock:
+ clean = [e for e in entries if isinstance(e, dict)][:_HISTORY_MAX]
+ _atomic_write_json(_history_file_path(), clean)
+
+
def _append_history(event):
"""Append an event dict to persistent history. Thread-safe."""
with _history_lock:
@@ -698,18 +714,25 @@ def _save_collections(data):
# ── Library layout: per-ZIM category overrides + home section order (#37) ──
#
# Storage: ZIMI_DATA_DIR/library_layout.json —
-# {"overrides": {"": ""}, "section_order": ["cat:"|"col:", ...]}
+# {"overrides": {"": ""},
+# "section_order": ["cat:"|"col:"|"other", ...],
+# "sections": ["", ...]}
# Overrides win over the _categorize_zim heuristic; section_order drives the
-# home page ordering (unlisted sections append in default order). Reads are
-# public (ride /list); writes are auth-gated /manage endpoints.
+# home page ordering (unlisted sections append in default order); `sections`
+# holds user-declared empty categories that should be offered as Move-to targets
+# and reorder rows before any ZIM lives in them. Reads are public (ride /list);
+# writes are auth-gated /manage endpoints.
_library_layout_lock = threading.Lock()
#: Section-order keys are namespaced so categories and collections can share one
#: ordered list without colliding (a collection named "Books" != category Books).
-_SECTION_KEY_RE = re.compile(r"^(cat:|col:).+")
+#: The bare `other` key is the reserved slot for the uncategorized catch-all, so
+#: it can be ordered like any real section instead of being pinned last.
+_SECTION_KEY_RE = re.compile(r"^(?:(?:cat:|col:).+|other)$")
#: Defensive caps so a hostile/buggy client can't write an unbounded file.
_LAYOUT_MAX_OVERRIDES = 5000
_LAYOUT_MAX_ORDER = 500
+_LAYOUT_MAX_SECTIONS = 200
_LAYOUT_STR_MAX = 128
@@ -721,11 +744,12 @@ def _library_layout_file_path():
def _load_library_layout():
"""Load library layout from disk. Fail-soft to the empty default.
- A missing or corrupt file must degrade to ``{"overrides": {}, "section_order": []}``
- rather than 500 /list — the whole home page renders from this, so a bad read
- can never be allowed to blank the library.
+ A missing or corrupt file must degrade to the empty default
+ (``{"overrides": {}, "section_order": [], "sections": []}``) rather than 500
+ /list — the whole home page renders from this, so a bad read can never be
+ allowed to blank the library.
"""
- empty = {"overrides": {}, "section_order": []}
+ empty = {"overrides": {}, "section_order": [], "sections": []}
try:
with open(_library_layout_file_path(), encoding="utf-8") as f:
data = json.load(f)
@@ -733,9 +757,11 @@ def _load_library_layout():
return empty
overrides = data.get("overrides")
order = data.get("section_order")
+ sections = data.get("sections")
return {
"overrides": overrides if isinstance(overrides, dict) else {},
"section_order": order if isinstance(order, list) else [],
+ "sections": sections if isinstance(sections, list) else [],
}
except FileNotFoundError:
pass # fresh install — no layout yet
@@ -1006,7 +1032,12 @@ def open_archive(path):
return Archive(path)
-from zimi.previews import strip_html, _extract_preview, _resolve_img_path # noqa: E402
+from zimi.previews import ( # noqa: E402
+ strip_html,
+ _extract_preview,
+ _resolve_img_path,
+ extract_snippet,
+)
# ============================================================================
# ZIM Listing & Metadata Cache
@@ -1628,6 +1659,13 @@ def main():
elif args.command == "desktop" or (args.command == "serve" and args.ui):
try:
+ # The desktop entry-point lives in the repo's desktop/ dir (a sibling
+ # of this package), not on the default import path — add it first.
+ _desktop_dir = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "desktop"
+ )
+ if _desktop_dir not in sys.path:
+ sys.path.insert(0, _desktop_dir)
from zimi_desktop import main as desktop_main
except ImportError:
print(
diff --git a/zimi/static/almanac-links.js b/zimi/static/almanac-links.js
index 54871139..866ded10 100644
--- a/zimi/static/almanac-links.js
+++ b/zimi/static/almanac-links.js
@@ -359,6 +359,12 @@
'russiaday': { q: 'Q1432329', en: 'Russia Day' },
'unityday': { q: 'Q1355116', en: 'Unity Day (Russia)' },
+ // ── Round-2 region-pack matches (1.8.1): pack labels that now resolve. ──
+ 'stbrigidsday': { q: 'Q376333', en: 'Imbolc' }, // IE — enwiki merges St Brigid's Day into Imbolc
+ 'blackconsciousnessday': { q: 'Q3082990', en: 'National Day of Zumbi and Black Consciousness' }, // BR
+ 'youthday_cn': { q: 'Q10881633', en: 'Youth Day (China)' },
+ 'youthday_za': { q: 'Q946446', en: 'Youth Day' }, // ZA
+
// ── Worldwide observances (the base Gregorian set) ──
'epiphany': { q: 'Q61556', en: 'Epiphany' },
'holocaustremembranceday': { q: 'Q152960', en: 'International Holocaust Remembrance Day' },
@@ -392,6 +398,32 @@
'worldchildrensday': { q: 'Q37081', en: "Children's Day" },
'humanrightsday': { q: 'Q206206', en: 'Human Rights Day' },
+ // ── Round-2 base-observance matches (1.8.1): keys are _norm(label). ──
+ 'worldbrailleday': { q: 'Q11880318', en: 'World Braille Day' },
+ 'internationaldayofeducation': { q: 'Q59530956', en: 'International Day of Education' },
+ 'worldcancerday': { q: 'Q551942', en: 'World Cancer Day' },
+ 'intldayofwomeninscience': { q: 'Q22084569', en: 'International Day of Women and Girls in Science' },
+ 'worldwildlifeday': { q: 'Q15813320', en: 'World Wildlife Day' },
+ 'internationaldayofhappiness': { q: 'Q5305947', en: 'International Day of Happiness' },
+ 'worldpoetryday': { q: 'Q857171', en: 'World Poetry Day' },
+ 'worldtheatreday': { q: 'Q2076105', en: 'World Theatre Day' },
+ 'worldartday': { q: 'Q15725894', en: 'World Art Day' },
+ 'internationaldanceday': { q: 'Q855927', en: 'International Dance Day' },
+ 'worldpressfreedomday': { q: 'Q592588', en: 'World Press Freedom Day' },
+ 'internationaldayoffamilies': { q: 'Q246959', en: 'International Day of Families' },
+ 'worldbeeday': { q: 'Q47008694', en: 'World Bee Day' },
+ 'worldrefugeeday': { q: 'Q757285', en: 'World Refugee Day' },
+ 'worldmusicday': { q: 'Q1129327', en: 'F\u00eate de la Musique' },
+ 'internationalfriendshipday': { q: 'Q14551661', en: 'Friendship Day' },
+ 'internationalcatday': { q: 'Q25457495', en: 'International Cat Day' },
+ 'worldhumanitarianday': { q: 'Q2723440', en: 'World Humanitarian Day' },
+ 'internationaldogday': { q: 'Q113627236', en: 'International Dog Day' },
+ 'internationaldayofsignlanguages': { q: 'Q47002373', en: 'International Day of Sign Languages' },
+ 'worldteachersday': { q: 'Q864881', en: "World Teachers' Day" },
+ 'worldtelevisionday': { q: 'Q2495614', en: 'World Television Day' },
+ 'intldayofpersonswithdisabilities': { q: 'Q339544', en: "United Nations' International Day of Persons with Disabilities" },
+ 'internationalvolunteerday': { q: 'Q490486', en: 'International Volunteer Day' },
+
// ── Native-calendar observances beyond the base set (famous ones only) ──
'tubishvat': { q: 'Q748816', en: 'Tu BiShvat' },
'lagbaomer': { q: 'Q748801', en: 'Lag BaOmer' },
@@ -589,8 +621,44 @@
'rosetta:georgia-guidestones': { q: 'Q958391', en: 'Georgia Guidestones' }
};
+ // World-clock cities (1.8.1) — the fixed cities of the timezone grid
+ // (_TZ_CITIES in almanac.js), keyed 'city:'. Q-IDs verified both
+ // directions against Wikidata (exact enwiki sitelink). Registered for the
+ // closed set now; like the drawn-only constellations they become live links
+ // once the clock render calls wrap('city:'+key, ...).
+ var CITIES = {
+ 'city:honolulu': { q: 'Q18094', en: 'Honolulu' },
+ 'city:anchorage': { q: 'Q39450', en: 'Anchorage, Alaska' },
+ 'city:los_angeles': { q: 'Q65', en: 'Los Angeles' },
+ 'city:denver': { q: 'Q16554', en: 'Denver' },
+ 'city:mexico_city': { q: 'Q1489', en: 'Mexico City' },
+ 'city:chicago': { q: 'Q1297', en: 'Chicago' },
+ 'city:new_york': { q: 'Q60', en: 'New York City' },
+ 'city:buenos_aires': { q: 'Q1486', en: 'Buenos Aires' },
+ 'city:sao_paulo': { q: 'Q174', en: 'S\u00e3o Paulo' },
+ 'city:london': { q: 'Q84', en: 'London' },
+ 'city:paris': { q: 'Q90', en: 'Paris' },
+ 'city:lagos': { q: 'Q8673', en: 'Lagos' },
+ 'city:cairo': { q: 'Q85', en: 'Cairo' },
+ 'city:johannesburg': { q: 'Q34647', en: 'Johannesburg' },
+ 'city:moscow': { q: 'Q649', en: 'Moscow' },
+ 'city:tehran': { q: 'Q3616', en: 'Tehran' },
+ 'city:dubai': { q: 'Q612', en: 'Dubai' },
+ 'city:karachi': { q: 'Q8660', en: 'Karachi' },
+ 'city:mumbai': { q: 'Q1156', en: 'Mumbai' },
+ 'city:kathmandu': { q: 'Q3037', en: 'Kathmandu' },
+ 'city:dhaka': { q: 'Q1354', en: 'Dhaka' },
+ 'city:bangkok': { q: 'Q1861', en: 'Bangkok' },
+ 'city:singapore': { q: 'Q334', en: 'Singapore' },
+ 'city:shanghai': { q: 'Q8686', en: 'Shanghai' },
+ 'city:tokyo': { q: 'Q1490', en: 'Tokyo' },
+ 'city:adelaide': { q: 'Q5112', en: 'Adelaide' },
+ 'city:sydney': { q: 'Q3130', en: 'Sydney' },
+ 'city:auckland': { q: 'Q37100', en: 'Auckland' }
+ };
+
var MAP = {};
- [PLANETS, PROBES, CONSTELLATIONS, SHOWERS, ECLIPSES, CALENDARS, ZODIAC, STARS, HOLIDAYS, TERMS, SEASONS, BELTS, EVENTS, MESSAGES]
+ [PLANETS, PROBES, CONSTELLATIONS, SHOWERS, ECLIPSES, CALENDARS, ZODIAC, STARS, HOLIDAYS, TERMS, SEASONS, BELTS, EVENTS, MESSAGES, CITIES]
.forEach(function (group) { for (var k in group) if (group.hasOwnProperty(k)) MAP[k] = group[k]; });
// Register curated holidays under a "holiday:" key so wrapHoliday() can
diff --git a/zimi/static/almanac-sky.js b/zimi/static/almanac-sky.js
index b498c5d1..2d3eb112 100644
--- a/zimi/static/almanac-sky.js
+++ b/zimi/static/almanac-sky.js
@@ -75,6 +75,7 @@ function _skyFrame(now, lat, lon, cw, ch) {
var moonPos0 = _moonPosition(now, lat, lon);
var moonM0 = _moonPhase(now);
var projStars = _projectStars(now, lat, lon, cw, ch);
+ var projField = _projectFieldStars(now, lat, lon, cw, ch);
var altStr = sunPos.altitude.toFixed(1);
var labelText = sunPos.altitude > 0
? t('alm_sun') + ' ' + altStr + '°'
@@ -84,7 +85,7 @@ function _skyFrame(now, lat, lon, cw, ch) {
} else {
labelText += ' · ' + t('alm_moon') + ' ' + t('alm_below_horizon');
}
- return { sunPos: sunPos, moonData: { pos: moonPos0, phase: moonM0 }, projStars: projStars, labelText: labelText };
+ return { sunPos: sunPos, moonData: { pos: moonPos0, phase: moonM0 }, projStars: projStars, projField: projField, labelText: labelText };
}
// `animateMoon` -- true only for a repaint that reinitializes this same canvas
@@ -116,7 +117,7 @@ function _initSkyScene(now, lat, lon, animateMoon) {
_skyStartTime = ts;
_skyState = {
canvas: canvas, dpr: dpr, now: now, nowTime: now.getTime(), lat: lat, lon: lon,
- sunPos: f.sunPos, moonData: f.moonData, projStars: f.projStars, labelText: f.labelText,
+ sunPos: f.sunPos, moonData: f.moonData, projStars: f.projStars, projField: f.projField, labelText: f.labelText,
moonAnim: null
};
if (priorMoon) _skyState.moonAnim = { from: priorMoon, to: f.moonData, start: ts, fromTime: priorTime, toTime: now.getTime() };
@@ -127,7 +128,7 @@ function _initSkyScene(now, lat, lon, animateMoon) {
if (!s) return;
var elapsed = (ts - _skyStartTime) / 1000;
var moonNow = _skyMoonAt(s, ts);
- _drawSkyScene(s.canvas, s.dpr, s.sunPos, s.now, s.lat, s.lon, elapsed, s.labelText, s.projStars, moonNow);
+ _drawSkyScene(s.canvas, s.dpr, s.sunPos, s.now, s.lat, s.lon, elapsed, s.labelText, s.projStars, moonNow, s.projField);
// Drive the hero moon's time-travel sweep from this same loop (no second
// rAF). Defined in almanac.js, which loads after this file.
if (typeof _heroMoonTick === 'function') _heroMoonTick(ts);
@@ -152,6 +153,7 @@ function _skySetInstant(now) {
s.sunPos = f.sunPos;
_skyMoonRetarget(s, f.moonData, performance.now(), fromTime, now.getTime());
s.projStars = f.projStars;
+ s.projField = f.projField;
s.labelText = f.labelText;
}
@@ -212,6 +214,132 @@ var _STARS = [
[22.96,-29.62,1.16],[2.53,89.26,1.98],[2.12,23.46,2.00],[9.76,23.77,2.98]
];
+// Real background field — the naked-eye sky beyond the named catalog above.
+// Bright-star catalogue (HYG v41), every star to magnitude 4.0, culled of the
+// ~58 already in _STARS; each row is [RA hours, Dec degrees, magnitude, colour
+// index]. Projected by the SAME horizon geometry as the named stars
+// (_projectFieldStars) so the whole scene is astronomically real and drifts
+// with time/location — this REPLACES the old procedural _ensureSkyBgStars
+// filler, which had no real position and did not move with the sky. Colour
+// index tints each star warm (high B–V) to blue-white (low B–V).
+var _SKY_FIELD_STARS = [
+ [1.63,-57.24,0.45,-0.16],[14.06,-60.37,0.61,-0.23],[9.22,-69.72,1.67,0.07],[22.14,-46.96,1.73,-0.07],
+ [8.16,-47.34,1.75,-0.14],[3.41,49.86,1.79,0.48],[18.40,-34.38,1.79,-0.03],[8.38,-59.51,1.86,1.20],
+ [5.99,44.95,1.90,0.08],[16.81,-69.03,1.91,1.45],[6.63,16.40,1.93,0.00],[8.75,-54.71,1.93,0.04],
+ [20.43,-56.74,1.94,-0.12],[9.46,-8.66,1.99,1.44],[0.73,-17.99,2.04,1.02],[18.92,-26.30,2.05,-0.13],
+ [14.11,-36.37,2.06,1.01],[0.14,29.09,2.07,-0.04],[1.16,35.62,2.07,1.58],[14.85,74.16,2.07,1.47],
+ [22.71,-46.88,2.07,1.61],[17.58,12.56,2.08,0.15],[3.14,40.96,2.09,-0.00],[2.06,42.33,2.10,1.37],
+ [12.69,-48.96,2.20,-0.02],[8.06,-40.00,2.21,-0.27],[9.28,-59.28,2.21,0.19],[15.58,26.71,2.22,0.03],
+ [9.13,-43.43,2.23,1.67],[17.94,51.49,2.24,1.52],[13.66,-53.47,2.29,-0.17],[14.70,-47.39,2.30,-0.15],
+ [14.59,-42.16,2.33,-0.16],[14.75,27.07,2.35,0.97],[21.74,9.88,2.38,1.52],[17.71,-39.03,2.39,-0.17],
+ [0.44,-42.31,2.40,1.08],[17.17,-15.72,2.43,0.06],[23.06,28.08,2.44,1.66],[7.40,-29.30,2.45,-0.08],
+ [21.31,62.59,2.45,0.26],[9.37,-55.01,2.47,-0.14],[23.08,15.21,2.49,-0.00],[3.04,4.09,2.54,1.63],
+ [16.62,-10.57,2.54,0.04],[13.93,-47.29,2.55,-0.18],[5.55,-17.82,2.58,0.21],[12.14,-50.72,2.58,-0.13],
+ [12.26,-17.54,2.58,-0.11],[19.04,-29.88,2.60,0.06],[15.28,-9.38,2.61,-0.07],[15.74,6.43,2.63,1.17],
+ [1.91,20.81,2.64,0.17],[5.66,-34.07,2.65,-0.12],[6.00,37.21,2.65,-0.08],[12.57,-23.40,2.65,0.89],
+ [13.91,18.40,2.68,0.58],[14.98,-43.13,2.68,-0.18],[4.95,33.17,2.69,1.49],[10.78,-49.42,2.69,0.90],
+ [12.62,-69.14,2.69,-0.18],[17.51,-37.30,2.70,-0.18],[7.29,-37.10,2.71,1.62],[18.35,-29.83,2.72,1.38],
+ [19.77,10.61,2.72,1.51],[16.24,-3.69,2.73,1.58],[16.40,61.51,2.73,0.91],[10.72,-64.39,2.74,-0.22],
+ [12.69,-1.45,2.74,0.37],[5.59,-5.91,2.75,-0.21],[13.34,-36.71,2.75,0.07],[14.85,-16.04,2.75,0.15],
+ [17.72,4.57,2.76,1.17],[5.13,-5.09,2.78,0.16],[16.50,21.49,2.78,0.95],[17.24,14.39,2.78,1.16],
+ [17.51,52.30,2.79,0.95],[15.59,-41.17,2.80,-0.22],[5.47,-20.76,2.81,0.81],[16.69,31.60,2.81,0.65],
+ [0.43,-77.25,2.82,0.62],[16.60,-28.22,2.82,-0.21],[18.47,-25.42,2.82,1.02],[0.22,15.18,2.83,-0.19],
+ [8.13,-24.30,2.83,0.46],[15.92,-63.43,2.83,0.32],[3.90,31.88,2.84,0.27],[17.42,-55.53,2.84,1.48],
+ [17.53,-49.88,2.84,-0.14],[3.79,24.11,2.85,-0.09],[13.04,10.96,2.85,0.93],[21.78,-16.13,2.85,0.18],
+ [1.98,-61.57,2.86,0.29],[6.38,22.51,2.87,1.62],[15.32,-68.68,2.87,0.01],[22.31,-60.26,2.87,1.39],
+ [2.97,-40.30,2.88,0.13],[19.16,-21.02,2.88,0.38],[7.45,8.29,2.89,-0.10],[12.93,38.32,2.89,-0.12],
+ [3.96,40.01,2.90,-0.20],[16.35,-25.59,2.90,0.30],[21.53,-5.57,2.90,0.83],[3.08,53.51,2.91,0.72],
+ [9.79,-65.07,2.92,0.27],[22.72,30.22,2.93,0.85],[6.83,-50.61,2.94,1.21],[12.50,-16.52,2.94,-0.01],
+ [22.10,-0.32,2.95,0.97],[3.97,-13.51,2.97,1.59],[5.63,21.14,2.97,-0.15],[18.10,-30.42,2.98,0.98],
+ [13.32,-23.17,2.99,0.92],[17.79,-40.13,2.99,0.51],[19.09,13.86,2.99,0.01],[2.16,34.99,3.00,0.14],
+ [11.16,44.50,3.00,1.14],[15.35,71.83,3.00,0.06],[16.86,-38.05,3.00,-0.20],[21.90,-37.36,3.00,-0.08],
+ [3.72,47.79,3.01,-0.12],[6.34,-30.06,3.02,-0.16],[7.05,-23.83,3.02,-0.08],[12.17,-22.62,3.02,1.33],
+ [5.03,43.82,3.03,0.54],[12.77,-68.11,3.04,-0.18],[14.53,38.31,3.04,0.19],[20.35,-14.78,3.05,0.79],
+ [6.73,25.13,3.06,1.38],[10.37,41.50,3.06,1.60],[19.21,67.66,3.07,0.99],[18.29,-36.76,3.10,1.58],
+ [8.92,5.95,3.11,0.98],[10.83,-16.19,3.11,1.23],[11.60,-63.02,3.11,-0.04],[20.63,-47.29,3.11,1.00],
+ [5.85,-35.77,3.12,1.15],[8.99,48.04,3.12,0.22],[16.98,-55.99,3.12,1.55],[17.25,24.84,3.12,0.08],
+ [14.99,-42.10,3.13,-0.21],[9.35,34.39,3.14,1.55],[9.52,-57.03,3.16,1.54],[17.25,36.81,3.16,1.44],
+ [6.63,-43.20,3.17,-0.10],[9.55,51.68,3.17,0.47],[17.15,65.71,3.17,-0.12],[18.76,-26.99,3.17,-0.11],
+ [5.11,41.23,3.18,-0.15],[14.71,-64.98,3.18,0.26],[4.83,6.96,3.19,0.48],[5.09,-22.37,3.19,1.46],
+ [16.96,9.38,3.19,1.16],[17.83,-37.04,3.19,1.19],[21.22,30.23,3.21,0.99],[23.66,77.63,3.21,1.03],
+ [15.36,-40.65,3.22,-0.23],[16.31,-4.69,3.23,0.97],[18.36,-2.90,3.23,0.94],[21.48,70.56,3.23,-0.20],
+ [6.80,-61.94,3.24,0.23],[20.19,-0.82,3.24,-0.07],[7.49,-43.30,3.25,1.51],[14.11,-26.68,3.25,1.09],
+ [15.07,-25.28,3.25,1.67],[18.98,32.69,3.25,-0.05],[3.79,-74.24,3.26,1.59],[0.66,30.86,3.27,1.27],
+ [17.37,-25.00,3.27,-0.19],[22.91,-15.82,3.27,0.07],[5.22,-16.21,3.29,-0.11],[10.23,-70.04,3.29,-0.07],
+ [15.42,58.97,3.29,1.17],[4.57,-55.04,3.30,-0.08],[10.53,-61.69,3.30,-0.09],[6.25,22.51,3.31,1.60],
+ [17.42,-56.38,3.31,-0.15],[1.10,-46.72,3.32,0.89],[3.09,38.84,3.32,1.53],[17.20,-43.24,3.32,0.44],
+ [17.98,-9.77,3.32,0.99],[19.12,-27.67,3.32,1.17],[4.24,-62.47,3.33,0.92],[11.24,15.43,3.33,-0.00],
+ [7.82,-24.86,3.34,1.22],[5.41,-2.40,3.35,-0.24],[6.75,12.90,3.35,0.44],[8.50,60.72,3.35,0.86],
+ [19.42,3.11,3.36,0.32],[15.38,-44.69,3.37,-0.19],[8.78,6.42,3.38,0.69],[13.58,-0.60,3.38,0.11],
+ [5.59,9.93,3.39,-0.16],[10.28,-61.33,3.39,1.54],[12.93,3.40,3.39,1.57],[22.18,58.20,3.39,1.56],
+ [4.48,15.87,3.40,0.18],[17.17,-15.73,3.40,0.60],[1.47,-43.32,3.41,1.54],[4.01,12.49,3.41,-0.10],
+ [13.83,-41.69,3.41,-0.23],[15.20,-52.10,3.41,0.92],[20.75,61.84,3.41,0.91],[22.69,10.83,3.41,-0.09],
+ [1.88,29.58,3.42,0.49],[16.00,-38.40,3.42,-0.21],[17.77,27.72,3.42,0.75],[20.75,-66.20,3.42,0.16],
+ [9.18,-58.97,3.43,-0.19],[10.28,23.42,3.43,0.31],[19.10,-4.88,3.43,-0.10],[10.28,42.91,3.45,0.03],
+ [0.82,57.82,3.46,0.59],[1.14,-10.18,3.46,1.16],[7.95,-52.98,3.46,-0.18],[15.26,33.31,3.46,0.96],
+ [2.72,3.24,3.47,0.09],[13.83,-42.47,3.47,-0.17],[10.12,16.76,3.48,-0.03],[16.71,38.92,3.48,0.92],
+ [1.73,-15.94,3.49,0.73],[7.03,-27.93,3.49,1.73],[11.31,33.09,3.49,1.40],[15.03,40.39,3.49,0.96],
+ [18.45,-45.97,3.49,-0.18],[22.81,-51.32,3.49,0.08],[6.83,-32.51,3.50,-0.12],[7.34,21.98,3.50,0.37],
+ [22.83,66.20,3.50,1.05],[19.98,19.49,3.51,1.57],[22.83,24.60,3.51,0.93],[3.72,-9.76,3.52,0.92],
+ [9.69,9.89,3.52,0.52],[9.95,-54.57,3.52,-0.07],[18.83,33.36,3.52,0.00],[18.96,-21.11,3.52,1.15],
+ [22.17,6.20,3.52,0.09],[12.69,-1.45,3.52,0.60],[4.48,19.18,3.53,1.01],[8.28,9.19,3.53,1.48],
+ [11.55,-31.86,3.54,0.95],[15.83,-3.43,3.54,-0.04],[17.63,-15.40,3.54,0.26],[4.30,-33.80,3.55,-0.11],
+ [5.78,-14.82,3.55,0.10],[14.32,-46.06,3.55,-0.18],[18.35,72.73,3.55,0.49],[20.15,-66.18,3.55,0.75],
+ [0.32,-8.82,3.56,1.21],[2.28,-51.51,3.56,-0.12],[11.32,-14.78,3.56,1.11],[16.87,-38.02,3.56,-0.21],
+ [7.74,24.40,3.57,0.93],[9.06,47.16,3.57,0.01],[14.53,30.37,3.57,1.30],[15.36,-36.26,3.57,1.53],
+ [7.30,16.54,3.58,0.11],[20.30,-12.54,3.58,0.88],[1.63,48.63,3.59,1.27],[5.29,-6.84,3.59,-0.12],
+ [5.74,-22.45,3.59,0.48],[11.84,1.76,3.59,0.52],[12.36,-60.40,3.59,1.39],[1.40,-8.18,3.60,1.06],
+ [6.88,33.96,3.60,0.10],[8.67,-52.92,3.60,-0.17],[9.51,-40.47,3.60,0.37],[15.62,-28.14,3.60,1.36],
+ [17.52,-60.68,3.60,-0.10],[2.83,27.26,3.61,-0.10],[3.41,9.03,3.61,0.89],[10.18,-12.35,3.61,1.01],
+ [13.04,-71.55,3.61,1.19],[17.76,-64.72,3.61,1.16],[1.52,15.35,3.62,0.97],[3.82,24.05,3.62,-0.07],
+ [7.75,-37.97,3.62,1.71],[16.91,-42.36,3.62,1.39],[23.03,42.33,3.62,-0.10],[11.76,-66.73,3.63,0.16],
+ [20.63,14.60,3.64,0.42],[4.33,15.63,3.65,0.98],[9.53,63.06,3.65,0.36],[15.77,15.42,3.65,0.07],
+ [18.11,-50.09,3.65,-0.10],[22.48,-0.02,3.65,0.41],[15.46,29.11,3.66,0.32],[15.64,-29.78,3.66,-0.18],
+ [14.07,64.38,3.67,-0.05],[20.91,-58.45,3.67,1.25],[4.85,5.61,3.68,-0.16],[8.73,-33.19,3.68,-0.18],
+ [19.79,18.53,3.68,1.31],[23.16,-21.17,3.68,1.20],[0.62,53.90,3.69,-0.20],[1.93,-51.61,3.69,0.84],
+ [5.04,41.08,3.69,1.15],[9.75,-62.51,3.69,1.01],[11.77,47.78,3.69,1.18],[21.67,-16.66,3.69,0.32],
+ [3.33,-21.76,3.70,1.61],[17.96,29.25,3.70,0.94],[23.29,3.28,3.70,0.92],[4.90,2.44,3.71,-0.18],
+ [5.94,-14.17,3.71,0.34],[7.87,-40.58,3.71,1.01],[15.85,4.48,3.71,0.15],[18.12,9.56,3.71,0.16],
+ [19.92,6.41,3.71,0.85],[3.55,-9.46,3.72,0.88],[3.75,24.11,3.72,-0.10],[5.99,54.28,3.72,1.01],
+ [21.08,43.93,3.72,1.61],[3.45,9.73,3.73,-0.08],[14.77,1.89,3.73,-0.01],[17.89,56.87,3.73,1.18],
+ [21.69,-77.39,3.73,1.01],[22.88,-7.58,3.73,1.63],[1.86,-10.34,3.74,1.14],[16.37,19.15,3.74,0.30],
+ [21.25,38.05,3.74,0.39],[9.07,-47.10,3.75,1.17],[17.80,2.71,3.75,0.04],[5.56,-62.49,3.76,0.64],
+ [5.86,-20.88,3.76,0.98],[6.48,-7.03,3.76,-0.11],[19.08,-21.74,3.76,1.01],[19.50,51.73,3.76,0.15],
+ [22.52,50.28,3.76,0.03],[2.84,55.90,3.77,1.69],[3.75,42.58,3.77,0.42],[4.38,17.54,3.77,0.98],
+ [5.65,-2.60,3.77,-0.19],[8.43,-66.14,3.77,1.13],[8.68,-46.65,3.77,0.67],[16.83,-59.04,3.77,1.56],
+ [20.66,15.91,3.77,-0.06],[21.44,-22.41,3.77,1.00],[22.12,25.35,3.77,0.43],[7.15,-70.50,3.78,1.01],
+ [7.43,27.80,3.78,1.02],[9.85,59.04,3.78,0.29],[10.89,-58.85,3.78,0.94],[14.69,13.73,3.78,0.04],
+ [20.79,-9.50,3.78,0.00],[3.16,44.86,3.79,0.98],[10.89,34.21,3.79,1.04],[3.20,-28.99,3.80,0.54],
+ [7.65,-26.80,3.80,-0.16],[15.58,10.54,3.80,0.27],[19.29,53.37,3.80,0.95],[20.23,46.74,3.80,1.27],
+ [4.59,-30.56,3.81,0.96],[10.46,-58.74,3.81,0.32],[15.71,26.30,3.81,0.02],[23.63,46.46,3.81,0.98],
+ [2.03,2.76,3.82,0.02],[9.31,36.80,3.82,0.07],[11.52,69.33,3.82,1.61],[16.52,1.98,3.82,0.02],
+ [17.66,46.01,3.82,-0.18],[10.43,-16.84,3.83,1.46],[13.97,-42.10,3.83,-0.22],[14.80,-79.04,3.83,1.43],
+ [3.74,-64.81,3.84,1.13],[3.74,32.29,3.84,0.02],[4.48,15.96,3.84,0.95],[8.92,-60.64,3.84,-0.10],
+ [10.55,9.31,3.84,-0.15],[10.62,-48.23,3.84,0.30],[12.54,-72.13,3.84,-0.16],[18.13,28.76,3.84,-0.02],
+ [18.23,-21.06,3.84,0.20],[19.80,70.27,3.84,0.89],[4.23,-42.29,3.85,1.08],[5.79,-51.07,3.85,0.17],
+ [6.37,-33.44,3.85,0.86],[10.25,-42.12,3.85,0.05],[12.56,69.79,3.85,-0.12],[12.63,-48.54,3.85,0.05],
+ [15.94,15.66,3.85,0.48],[18.39,21.77,3.85,1.17],[18.59,-8.24,3.85,1.32],[0.95,38.50,3.86,0.13],
+ [4.64,-14.30,3.86,1.08],[5.52,-35.47,3.86,1.13],[16.26,-63.69,3.86,1.10],[16.56,-78.90,3.86,0.92],
+ [17.94,37.25,3.86,1.35],[22.36,-1.39,3.86,-0.06],[3.76,24.37,3.87,-0.06],[8.77,-46.04,3.87,0.01],
+ [13.98,-44.80,3.87,-0.21],[14.72,-5.66,3.87,0.39],[15.95,-29.21,3.87,-0.20],[19.87,1.01,3.87,0.63],
+ [0.16,-45.75,3.88,1.01],[1.89,19.29,3.88,-0.05],[9.88,26.01,3.88,1.22],[15.20,-48.74,3.88,-0.03],
+ [23.17,-45.25,3.88,1.00],[2.94,-8.90,3.89,1.09],[6.90,-24.18,3.89,1.74],[9.24,2.31,3.89,-0.06],
+ [12.33,-0.67,3.89,0.03],[19.94,35.08,3.89,1.02],[9.66,-1.14,3.90,1.31],[11.35,-54.49,3.90,-0.16],
+ [13.52,-39.41,3.90,1.19],[4.05,5.99,3.91,0.03],[8.43,-3.91,3.91,-0.01],[12.47,-50.23,3.91,-0.19],
+ [15.09,-47.05,3.91,-0.14],[15.59,-14.79,3.91,1.01],[16.33,46.31,3.91,-0.15],[17.00,30.93,3.92,-0.02],
+ [19.36,-17.85,3.92,0.23],[21.26,5.25,3.92,0.55],[0.44,-43.68,3.93,0.17],[1.52,-49.07,3.93,0.97],
+ [2.90,52.76,3.93,0.76],[4.61,-3.35,3.93,-0.21],[7.70,-72.61,3.93,1.03],[11.14,-58.98,3.93,1.23],
+ [16.11,-20.67,3.93,-0.05],[18.01,2.93,3.93,0.03],[1.14,-55.25,3.94,-0.12],[7.69,-9.55,3.94,1.02],
+ [7.73,-28.95,3.94,0.16],[8.74,18.15,3.94,1.08],[20.95,41.17,3.94,0.03],[2.06,72.42,3.95,-0.00],
+ [6.61,-19.26,3.95,1.04],[4.14,47.71,3.96,-0.03],[5.99,-42.82,3.96,1.15],[9.01,41.78,3.96,0.46],
+ [9.19,-62.32,3.96,-0.18],[19.38,-44.46,3.96,-0.09],[19.40,-40.62,3.96,-0.10],[20.26,47.71,3.96,1.45],
+ [23.38,-20.10,3.96,1.08],[4.40,-34.02,3.97,1.47],[5.86,39.15,3.97,1.13],[7.28,-67.96,3.97,0.76],
+ [8.67,-35.31,3.97,0.94],[12.19,-52.37,3.97,-0.16],[15.85,-33.63,3.97,-0.04],[20.01,-72.91,3.97,-0.03],
+ [22.49,-43.50,3.97,1.02],[22.78,23.57,3.97,1.07],[3.98,35.79,3.98,0.02],[21.57,45.59,3.98,0.89],
+ [2.00,-21.08,3.99,1.55],[6.25,-6.27,3.99,1.32],[10.41,-74.03,3.99,0.37],[23.29,-58.24,3.99,0.41],
+ [9.04,-66.40,4.00,0.14],[11.40,10.53,4.00,0.42],[16.20,-19.46,4.00,0.08],
+];
+
// Constellation connecting lines — pairs of _STARS indices
var _CONST_LINES = [
[0,2],[0,5],[2,3],[3,4],[4,5],[3,1],[5,6], // Orion
@@ -253,35 +381,73 @@ function _starLinkKey(idx) {
return 'star:' + nm.toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/_+$/, '');
}
-// Project catalog stars to canvas coordinates for current time/location
-function _projectStars(now, lat, lon, W, H) {
+// Local sidereal time (radians) for an instant + longitude — the one value
+// every star projection in the scene shares.
+function _skyLST(now, lon) {
var JD = _dateToJD(now.getTime());
var GMST = (280.46061837 + 360.98564736629 * (JD - JD_J2000)) % 360;
- var LST = (GMST + lon) * DEG_TO_RAD;
- var latR = lat * DEG_TO_RAD;
+ return (GMST + lon) * DEG_TO_RAD;
+}
+
+// Project one star (RA hours, Dec degrees) onto the horizon-scene canvas for a
+// precomputed LST and observer latitude (sin/cos passed in so a whole catalogue
+// pass computes them once). Returns {x, y, alt} or null when the star falls
+// below the scene's horizon band or outside its 60°–300° azimuth window. Shared
+// by the named catalog (_projectStars) and the real field (_projectFieldStars)
+// so both use byte-for-byte identical geometry.
+function _projectStarXY(raHours, decDeg, LST, sinLat, cosLat, W, H) {
+ var ra = raHours * 15 * DEG_TO_RAD;
+ var dec = decDeg * DEG_TO_RAD;
+ var sinDec = Math.sin(dec), cosDec = Math.cos(dec);
+ var HA = LST - ra;
+ HA = ((HA % (2 * Math.PI)) + 3 * Math.PI) % (2 * Math.PI) - Math.PI;
+ var sinAlt = sinLat * sinDec + cosLat * cosDec * Math.cos(HA);
+ var altitude = Math.asin(sinAlt) * 180 / Math.PI;
+ if (altitude < -2) return null;
+ var cosAz = (sinDec - sinLat * sinAlt) / (cosLat * Math.cos(Math.asin(sinAlt)));
+ cosAz = Math.max(-1, Math.min(1, cosAz));
+ if (isNaN(cosAz)) cosAz = 0;
+ var azimuth = Math.acos(cosAz) * 180 / Math.PI;
+ if (HA > 0) azimuth = 360 - azimuth;
+ var xFrac = (azimuth - 60) / 240;
+ if (xFrac < -0.05 || xFrac > 1.05) return null;
+ xFrac = Math.max(0, Math.min(1, xFrac));
+ return { x: xFrac * W, y: Math.max(0, Math.min(H * 0.66, H * 0.66 - (altitude / 90) * H * 0.56)), alt: altitude };
+}
+
+// Project the named catalog stars to canvas coordinates for current
+// time/location. `alt` rides along so the a11y description can count how many
+// are truly above the horizon.
+function _projectStars(now, lat, lon, W, H) {
+ var LST = _skyLST(now, lon);
+ var latR = lat * DEG_TO_RAD, sinLat = Math.sin(latR), cosLat = Math.cos(latR);
var result = [];
for (var i = 0; i < _STARS.length; i++) {
var s = _STARS[i];
- var ra = s[0] * 15 * DEG_TO_RAD;
- var dec = s[1] * DEG_TO_RAD;
- var HA = LST - ra;
- HA = ((HA % (2 * Math.PI)) + 3 * Math.PI) % (2 * Math.PI) - Math.PI;
- var sinAlt = Math.sin(latR) * Math.sin(dec) + Math.cos(latR) * Math.cos(dec) * Math.cos(HA);
- var altitude = Math.asin(sinAlt) * 180 / Math.PI;
- if (altitude < -2) continue;
- var cosAz = (Math.sin(dec) - Math.sin(latR) * sinAlt) / (Math.cos(latR) * Math.cos(Math.asin(sinAlt)));
- cosAz = Math.max(-1, Math.min(1, cosAz));
- if (isNaN(cosAz)) cosAz = 0;
- var azimuth = Math.acos(cosAz) * 180 / Math.PI;
- if (HA > 0) azimuth = 360 - azimuth;
- var xFrac = (azimuth - 60) / 240;
- if (xFrac < -0.05 || xFrac > 1.05) continue;
- xFrac = Math.max(0, Math.min(1, xFrac));
- result.push({ x: xFrac * W, y: Math.max(0, Math.min(H * 0.66, H * 0.66 - (altitude / 90) * H * 0.56)), mag: s[2], idx: i });
+ var p = _projectStarXY(s[0], s[1], LST, sinLat, cosLat, W, H);
+ if (p) result.push({ x: p.x, y: p.y, alt: p.alt, mag: s[2], idx: i });
}
return result;
}
+// Project the real background field (_SKY_FIELD_STARS) the same way. Only stars
+// genuinely above the horizon are kept; each survivor carries a warm/cool tint
+// from its colour index and a deterministic twinkle phase seeded from its RA, so
+// the shimmer is stable across rebuilds. Rebuilt only when _skyFrame recomputes
+// (init / time jump / drift cadence), never per animation frame.
+function _projectFieldStars(now, lat, lon, W, H) {
+ var LST = _skyLST(now, lon);
+ var latR = lat * DEG_TO_RAD, sinLat = Math.sin(latR), cosLat = Math.cos(latR);
+ var out = [];
+ for (var i = 0; i < _SKY_FIELD_STARS.length; i++) {
+ var fs = _SKY_FIELD_STARS[i];
+ var p = _projectStarXY(fs[0], fs[1], LST, sinLat, cosLat, W, H);
+ if (!p || p.alt < 0) continue;
+ out.push({ x: p.x, y: p.y, mag: fs[2], ci: fs[3], phase: (fs[0] * 137.508) % 6.2832 });
+ }
+ return out;
+}
+
function _drawConstellations(ctx, alpha, t, projStars) {
var byIdx = {};
for (var i = 0; i < projStars.length; i++) byIdx[projStars[i].idx] = projStars[i];
@@ -300,33 +466,19 @@ function _drawConstellations(ctx, alpha, t, projStars) {
ctx.restore();
}
-// Decorative dim background starfield for the horizon scene — generated once
-// per canvas size (cached, like the orrery's own background stars) rather than
-// re-rolled every animation frame. These are pure ambiance (no real RA/Dec, no
-// link), filling out the naked-eye-dense look a ~60-catalog-star sky can't on
-// its own; _lcgRand (almanac-orrery.js) keeps them deterministic.
-var _skyBgStars = null;
-
-function _ensureSkyBgStars(W, H, dpr) {
- if (_skyBgStars && _skyBgStars.W === W && _skyBgStars.H === H) return _skyBgStars.stars;
- var sr = _lcgRand(42);
- var count = 220;
- var stars = [];
- for (var i = 0; i < count; i++) {
- stars.push({
- x: sr() * W,
- y: sr() * H * 0.55,
- r: (0.25 + sr() * 0.35) * dpr,
- freq: 0.8 + sr() * 2.0,
- phase: sr() * 6.28,
- base: 0.03 + sr() * 0.12
- });
- }
- _skyBgStars = { W: W, H: H, stars: stars };
- return stars;
+// "r,g,b" tint for a star from its B–V colour index: hot blue-white stars have
+// a low (even negative) index, cool amber stars a high one. Buckets, not a
+// gradient — plenty at this scale, and cheap. Shared by the field draw.
+function _starTint(ci) {
+ if (ci == null) return '220,230,255';
+ if (ci < 0.0) return '202,222,255'; // blue-white (O/B)
+ if (ci < 0.3) return '226,236,255'; // white (A)
+ if (ci < 0.6) return '248,248,235'; // yellow-white (F)
+ if (ci < 1.0) return '255,240,208'; // yellow (G/K)
+ return '255,214,170'; // orange-red (K/M)
}
-function _drawSkyScene(canvas, dpr, sunPos, now, lat, lon, elapsed, labelText, projStars, moonData) {
+function _drawSkyScene(canvas, dpr, sunPos, now, lat, lon, elapsed, labelText, projStars, moonData, projField) {
var t = elapsed || 0; // 't' is animation time in seconds — not the i18n t() function
var ctx = canvas.getContext('2d');
var W = canvas.width, H = canvas.height;
@@ -384,19 +536,24 @@ function _drawSkyScene(canvas, dpr, sunPos, now, lat, lon, elapsed, labelText, p
if (alt < 8) {
var starOpacity = alt < -14 ? 1 : alt < -2 ? (-2 - alt) / 12 : Math.max(0, (8 - alt) / 20);
- // Dim background stars — a cached deterministic field (only the twinkle,
- // a sine over each star's cached phase/frequency, is recomputed per frame;
- // the layout itself is generated once per canvas size, not re-rolled every
- // RAF tick — see _ensureSkyBgStars).
- var bgStars = _ensureSkyBgStars(W, H, dpr);
- for (var si = 0; si < bgStars.length; si++) {
- var bs = bgStars[si];
- var twinkle = Math.sin(t * bs.freq + bs.phase) * 0.15;
- var bsa = starOpacity * Math.max(0.03, bs.base + twinkle);
- ctx.beginPath();
- ctx.arc(bs.x, bs.y, bs.r, 0, Math.PI * 2);
- ctx.fillStyle = 'rgba(200,210,230,' + bsa.toFixed(3) + ')';
- ctx.fill();
+ // Real background field — the naked-eye sky at astronomically correct
+ // positions (_projectFieldStars, mag ≤ 4.0), replacing the old procedural
+ // filler that never moved with the sky. Positions are cached (rebuilt only
+ // on a time/location change, never per frame); only the twinkle recomputes
+ // each RAF tick, and each star's colour-index tint makes hot stars blue and
+ // cool stars amber, as the real sky is.
+ if (projField) {
+ for (var si = 0; si < projField.length; si++) {
+ var fp = projField[si];
+ var fr = Math.max(0.35, (4.5 - fp.mag) * 0.28) * dpr;
+ var ftw = Math.sin(t * (1.0 + (si % 7) * 0.3) + fp.phase) * 0.12;
+ var fbase = 0.1 + (4.5 - fp.mag) / 4.5 * 0.5;
+ var fsa = starOpacity * Math.max(0.05, Math.min(0.85, fbase + ftw));
+ ctx.beginPath();
+ ctx.arc(fp.x, fp.y, fr, 0, Math.PI * 2);
+ ctx.fillStyle = 'rgba(' + _starTint(fp.ci) + ',' + fsa.toFixed(3) + ')';
+ ctx.fill();
+ }
}
// Catalog stars at astronomically correct positions
@@ -1027,10 +1184,12 @@ function _renderStarChart(baseNow) {
_drawStarChart(baseNow);
}
-// Decorative dim background starfield for the planisphere disc — same idea as
-// _ensureSkyBgStars for the horizon scene: a cached, deterministic field (not
-// astronomically real, not linkable) so the disc reads as a real night sky
-// instead of the ~25-35 catalog stars typically above the horizon at once.
+// Decorative dim background starfield for the planisphere disc — a cached,
+// deterministic field (not astronomically real, not linkable) so the schematic
+// chart disc reads as a dense night sky rather than the ~25-35 catalog stars
+// typically above the horizon at once. (The horizon SCENE uses real positions —
+// see _projectFieldStars — but the planisphere is a labelled diagram, where an
+// even ambient fill behind the named stars reads better than 460 mag dots.)
// Cached by disc size only (not lat/lon/time), so panning/scrubbing is free.
var _starChartBgStars = null;
diff --git a/zimi/static/almanac.css b/zimi/static/almanac.css
index 80bf8285..b9204c3c 100644
--- a/zimi/static/almanac.css
+++ b/zimi/static/almanac.css
@@ -63,7 +63,20 @@
keeps the cursor on it and hover repaints the amber border grey. */
.alm-tz-city-card.alm-tz-city-active,
.alm-tz-city-card.alm-tz-city-active:hover { border: 2px solid var(--amber); background: var(--amber-glow); box-shadow: 0 0 10px var(--amber-glow); padding: 5px 3px; }
-.alm-cal-region { text-align: center; font-size: 11px; color: var(--text3); margin-top: 8px; }
+/* Holiday scope pill — centered directly under the calendar grid it filters.
+ A recessed segmented pill (Regional/Worldwide) sets the scope explicitly,
+ amber-filled when active — same control language as the app-theme picker
+ (app.css .app-theme-seg) so the two families read as one system. The
+ location affordance lives on the sun-map, so there is no caption. */
+.alm-hol-row { display: flex; justify-content: center; align-items: center; margin-top: 14px; }
+.alm-scope-seg { display: flex; gap: 3px; padding: 3px; border-radius: 999px; background: var(--surface2); border: 1px solid var(--border); }
+.alm-scope-btn {
+ border: none; border-radius: 999px; padding: 3px 12px; font: inherit; font-size: 11px;
+ background: none; color: var(--text3); cursor: pointer; line-height: 1.4;
+ transition: color 0.15s, background 0.15s;
+}
+.alm-scope-btn:hover { color: var(--amber); }
+.alm-scope-btn.active { background: var(--amber); color: #000; font-weight: 600; }
.alm-tz-city-active .alm-tz-city-name { color: var(--amber); font-weight: 600; }
.alm-tz-city-active .alm-tz-city-time { color: var(--amber); }
.almanac-hero { display: flex; flex-direction: column; align-items: center; margin-bottom: 32px; position: relative; }
diff --git a/zimi/static/almanac.js b/zimi/static/almanac.js
index e74168fd..5bebc5e4 100644
--- a/zimi/static/almanac.js
+++ b/zimi/static/almanac.js
@@ -65,6 +65,21 @@ function _saveLocation(lat, lon, name) {
_almSelectedTz = _almTzForLocation(lat, lon);
}
+// Holiday scope — 'region' (default: the one national pack for the chosen or
+// detected location) or 'worldwide' (all national packs layered at once, each
+// entry tagged with its country). Session-scoped like the location itself: the
+// almanac is ephemeral, so a refresh returns to the location-following default.
+var _ALM_HOLIDAY_SCOPE_KEY = 'zimi_almanac_holiday_scope';
+function _almHolidayScope() {
+ try { return sessionStorage.getItem(_ALM_HOLIDAY_SCOPE_KEY) === 'worldwide' ? 'worldwide' : 'region'; }
+ catch (e) { return 'region'; }
+}
+function _almSetHolidayScope(scope) {
+ if (scope === _almHolidayScope()) return;
+ try { sessionStorage.setItem(_ALM_HOLIDAY_SCOPE_KEY, scope); } catch (e) {}
+ if (typeof _drawAlmanacGrid === 'function') _drawAlmanacGrid();
+}
+
function _signalDelay(au) {
var sec = au * 499;
return { h: Math.floor(sec / 3600), m: Math.floor((sec % 3600) / 60) };
@@ -2843,6 +2858,7 @@ var _MAP_CITIES = [
{ name: 'Melbourne, Victoria, Australia', lat: -37.81, lon: 144.96 },
{ name: 'Brisbane, Queensland, Australia', lat: -27.47, lon: 153.03 },
{ name: 'Perth, Western Australia, Australia', lat: -31.95, lon: 115.86 },
+ { name: 'Adelaide, South Australia, Australia', lat: -34.93, lon: 138.60 },
{ name: 'Auckland, New Zealand', lat: -36.85, lon: 174.76 },
{ name: 'Wellington, New Zealand', lat: -41.29, lon: 174.78 },
{ name: 'Suva, Fiji', lat: -17.77, lon: 177.97 }
@@ -3082,7 +3098,158 @@ var _SEARCH_CITIES = [
{ name: 'Wellington, New Zealand', lat: -41.29, lon: 174.78 },
{ name: 'Christchurch, New Zealand', lat: -43.53, lon: 172.64 },
{ name: 'Suva, Fiji', lat: -18.14, lon: 178.44 },
- { name: 'Port Moresby, Papua New Guinea', lat: -9.44, lon: 147.18 }
+ { name: 'Port Moresby, Papua New Guinea', lat: -9.44, lon: 147.18 },
+ // ── Wider world coverage (1.8.1): major cities by population, coordinates
+ // from the GeoNames cities15000 dataset; deduped against the lists above. ──
+ { name: 'Aden, Yemen', lat: 12.78, lon: 45.04 },
+ { name: 'Al Basrah al Qadimah, Iraq', lat: 30.50, lon: 47.82 },
+ { name: 'Al Mawsil al Jadidah, Iraq', lat: 36.33, lon: 43.11 },
+ { name: 'Aleppo, Syria', lat: 36.20, lon: 37.16 },
+ { name: 'Andijon, Uzbekistan', lat: 40.78, lon: 72.35 },
+ { name: 'Ankara, Turkey', lat: 39.92, lon: 32.85 },
+ { name: 'Antananarivo, Madagascar', lat: -18.91, lon: 47.54 },
+ { name: 'Antsirabe, Madagascar', lat: -19.87, lon: 47.03 },
+ { name: 'Arequipa, Peru', lat: -16.40, lon: -71.54 },
+ { name: 'Arhus, Denmark', lat: 56.16, lon: 10.21 },
+ { name: 'Ashgabat, Turkmenistan', lat: 37.95, lon: 58.38 },
+ { name: 'Asmara, Eritrea', lat: 15.34, lon: 38.93 },
+ { name: 'Bamako, Mali', lat: 12.61, lon: -7.98 },
+ { name: 'Bamenda, Cameroon', lat: 5.96, lon: 10.15 },
+ { name: 'Bangui, Central African Republic', lat: 4.36, lon: 18.55 },
+ { name: 'Banja Luka, Bosnia and Herzegovina', lat: 44.78, lon: 17.21 },
+ { name: 'Benghazi, Libya', lat: 32.11, lon: 20.07 },
+ { name: 'Bharatpur, Nepal', lat: 27.68, lon: 84.44 },
+ { name: 'Bissau, Guinea-Bissau', lat: 11.86, lon: -15.60 },
+ { name: 'Blantyre, Malawi', lat: -15.78, lon: 35.01 },
+ { name: 'Bo, Sierra Leone', lat: 7.96, lon: -11.74 },
+ { name: 'Bobo-Dioulasso, Burkina Faso', lat: 11.18, lon: -4.29 },
+ { name: 'Borama, Somalia', lat: 9.94, lon: 43.18 },
+ { name: 'Bouake, Ivory Coast', lat: 7.69, lon: -5.03 },
+ { name: 'Bujumbura, Burundi', lat: -3.38, lon: 29.36 },
+ { name: 'Bulawayo, Zimbabwe', lat: -20.15, lon: 28.58 },
+ { name: 'Bursa, Turkey', lat: 40.20, lon: 29.06 },
+ { name: 'Camagueey, Cuba', lat: 21.38, lon: -77.92 },
+ { name: 'Chisinau, Moldova', lat: 47.01, lon: 28.86 },
+ { name: 'Ciudad del Este, Paraguay', lat: -25.50, lon: -54.65 },
+ { name: 'Cochabamba, Bolivia', lat: -17.38, lon: -66.16 },
+ { name: 'Conakry, Guinea', lat: 9.54, lon: -13.68 },
+ { name: 'Constantine, Algeria', lat: 36.37, lon: 6.61 },
+ { name: 'Cork, Ireland', lat: 51.90, lon: -8.47 },
+ { name: 'Cotonou, Benin', lat: 6.37, lon: 2.42 },
+ { name: 'Cuenca, Ecuador', lat: -2.90, lon: -79.00 },
+ { name: 'Damascus, Syria', lat: 33.51, lon: 36.29 },
+ { name: 'Danli, Honduras', lat: 14.03, lon: -86.58 },
+ { name: 'Dasoguz, Turkmenistan', lat: 41.84, lon: 59.97 },
+ { name: 'Djibouti, Djibouti', lat: 11.59, lon: 43.15 },
+ { name: 'Dodoma, Tanzania', lat: -6.17, lon: 35.74 },
+ { name: 'Douala, Cameroon', lat: 4.05, lon: 9.70 },
+ { name: 'Dushanbe, Tajikistan', lat: 38.54, lon: 68.78 },
+ { name: 'Fes, Morocco', lat: 34.03, lon: -5.00 },
+ { name: 'Freetown, Sierra Leone', lat: 8.49, lon: -13.24 },
+ { name: 'Gaborone, Botswana', lat: -24.65, lon: 25.91 },
+ { name: 'Ganja, Azerbaijan', lat: 40.68, lon: 46.36 },
+ { name: 'Gaza, Palestinian Territory', lat: 31.50, lon: 34.47 },
+ { name: 'Georgetown, Guyana', lat: 6.80, lon: -58.16 },
+ { name: 'Gonder, Ethiopia', lat: 12.60, lon: 37.47 },
+ { name: 'Graz, Austria', lat: 47.07, lon: 15.44 },
+ { name: 'Haiphong, Vietnam', lat: 20.86, lon: 106.68 },
+ { name: 'Hamhung, North Korea', lat: 39.92, lon: 127.54 },
+ { name: 'Hargeysa, Somalia', lat: 9.56, lon: 44.06 },
+ { name: 'Herat, Afghanistan', lat: 34.35, lon: 62.20 },
+ { name: 'Homs, Syria', lat: 34.72, lon: 36.73 },
+ { name: 'Homyel\', Belarus', lat: 52.43, lon: 30.98 },
+ { name: 'Hrodna, Belarus', lat: 53.68, lon: 23.83 },
+ { name: 'Iasi, Romania', lat: 47.17, lon: 27.60 },
+ { name: 'Ibadan, Nigeria', lat: 7.38, lon: 3.91 },
+ { name: 'Irbid, Jordan', lat: 32.56, lon: 35.85 },
+ { name: 'Isfara, Tajikistan', lat: 40.13, lon: 70.63 },
+ { name: 'Istaravshan, Tajikistan', lat: 39.91, lon: 69.00 },
+ { name: 'Jijiga, Ethiopia', lat: 9.35, lon: 42.80 },
+ { name: 'Juba, South Sudan', lat: 4.85, lon: 31.58 },
+ { name: 'Kakamega, Kenya', lat: 0.28, lon: 34.75 },
+ { name: 'Kano, Nigeria', lat: 12.00, lon: 8.52 },
+ { name: 'Kaunas, Lithuania', lat: 54.90, lon: 23.91 },
+ { name: 'Kenema, Sierra Leone', lat: 7.88, lon: -11.19 },
+ { name: 'Kigali, Rwanda', lat: -1.95, lon: 30.06 },
+ { name: 'Kingston, Jamaica', lat: 18.00, lon: -76.79 },
+ { name: 'Kitwe, Zambia', lat: -12.80, lon: 28.21 },
+ { name: 'Kosice, Slovakia', lat: 48.71, lon: 21.26 },
+ { name: 'Koutiala, Mali', lat: 12.39, lon: -5.47 },
+ { name: 'Kumasi, Ghana', lat: 6.69, lon: -1.62 },
+ { name: 'Libreville, Gabon', lat: 0.39, lon: 9.45 },
+ { name: 'Lilongwe, Malawi', lat: -13.97, lon: 33.79 },
+ { name: 'Linz, Austria', lat: 48.31, lon: 14.29 },
+ { name: 'Lome, Togo', lat: 6.13, lon: 1.22 },
+ { name: 'Lubumbashi, Democratic Republic of the Congo', lat: -11.66, lon: 27.48 },
+ { name: 'Macau, Macao', lat: 22.20, lon: 113.55 },
+ { name: 'Managua, Nicaragua', lat: 12.13, lon: -86.25 },
+ { name: 'Mandalay, Myanmar', lat: 21.97, lon: 96.08 },
+ { name: 'Maracaibo, Venezuela', lat: 10.64, lon: -71.61 },
+ { name: 'Maradi, Niger', lat: 13.50, lon: 7.10 },
+ { name: 'Maseru, Lesotho', lat: -29.32, lon: 27.48 },
+ { name: 'Mazar-e Sharif, Afghanistan', lat: 36.71, lon: 67.11 },
+ { name: 'Mbuji-Mayi, Democratic Republic of the Congo', lat: -6.14, lon: 23.59 },
+ { name: 'Minsk, Belarus', lat: 53.90, lon: 27.57 },
+ { name: 'Misratah, Libya', lat: 32.38, lon: 15.09 },
+ { name: 'Mogadishu, Somalia', lat: 2.04, lon: 45.34 },
+ { name: 'Monrovia, Liberia', lat: 6.30, lon: -10.80 },
+ { name: 'Mwanza, Tanzania', lat: -2.52, lon: 32.90 },
+ { name: 'Mzuzu, Malawi', lat: -11.47, lon: 34.02 },
+ { name: 'N\'Djamena, Chad', lat: 12.11, lon: 15.04 },
+ { name: 'Namangan, Uzbekistan', lat: 41.00, lon: 71.67 },
+ { name: 'Nampula, Mozambique', lat: -15.12, lon: 39.27 },
+ { name: 'Nassau, Bahamas', lat: 25.06, lon: -77.34 },
+ { name: 'Nay Pyi Taw, Myanmar', lat: 19.75, lon: 96.13 },
+ { name: 'Ndola, Zambia', lat: -12.96, lon: 28.64 },
+ { name: 'Niamey, Niger', lat: 13.51, lon: 2.11 },
+ { name: 'Nicosia, Cyprus', lat: 35.17, lon: 33.35 },
+ { name: 'Nis, Serbia', lat: 43.32, lon: 21.90 },
+ { name: 'Nouakchott, Mauritania', lat: 18.09, lon: -15.98 },
+ { name: 'Novi Sad, Serbia', lat: 45.25, lon: 19.84 },
+ { name: 'Nzerekore, Guinea', lat: 7.76, lon: -8.82 },
+ { name: 'Oran, Algeria', lat: 35.70, lon: -0.64 },
+ { name: 'Osh, Kyrgyzstan', lat: 40.53, lon: 72.80 },
+ { name: 'Ostrava, Czechia', lat: 49.83, lon: 18.28 },
+ { name: 'Ouagadougou, Burkina Faso', lat: 12.37, lon: -1.53 },
+ { name: 'Paramaribo, Suriname', lat: 5.87, lon: -55.17 },
+ { name: 'Plovdiv, Bulgaria', lat: 42.15, lon: 24.75 },
+ { name: 'Podgorica, Montenegro', lat: 42.44, lon: 19.26 },
+ { name: 'Pointe-Noire, Republic of the Congo', lat: -4.78, lon: 11.86 },
+ { name: 'Pokhara, Nepal', lat: 28.27, lon: 83.97 },
+ { name: 'Port-au-Prince, Haiti', lat: 18.54, lon: -72.34 },
+ { name: 'Pristina, Kosovo', lat: 42.67, lon: 21.17 },
+ { name: 'Rabat, Morocco', lat: 34.01, lon: -6.83 },
+ { name: 'San Miguel, El Salvador', lat: 13.48, lon: -88.18 },
+ { name: 'San Pedro Sula, Honduras', lat: 15.51, lon: -88.03 },
+ { name: 'San Salvador, El Salvador', lat: 13.69, lon: -89.19 },
+ { name: 'Sanaa, Yemen', lat: 15.35, lon: 44.21 },
+ { name: 'Santa Cruz de la Sierra, Bolivia', lat: -17.79, lon: -63.18 },
+ { name: 'Santiago de Cuba, Cuba', lat: 20.02, lon: -75.82 },
+ { name: 'Santiago de los Caballeros, Dominican Republic', lat: 19.45, lon: -70.69 },
+ { name: 'Sarajevo, Bosnia and Herzegovina', lat: 43.85, lon: 18.36 },
+ { name: 'Serekunda, Gambia', lat: 13.44, lon: -16.68 },
+ { name: 'Sfax, Tunisia', lat: 34.74, lon: 10.76 },
+ { name: 'Shymkent, Kazakhstan', lat: 42.31, lon: 69.60 },
+ { name: 'Sikasso, Mali', lat: 11.32, lon: -5.67 },
+ { name: 'Skopje, North Macedonia', lat: 42.00, lon: 21.43 },
+ { name: 'Sousse, Tunisia', lat: 35.83, lon: 10.64 },
+ { name: 'Taichung, Taiwan', lat: 24.15, lon: 120.68 },
+ { name: 'Taiz, Yemen', lat: 13.58, lon: 44.02 },
+ { name: 'Takeo, Cambodia', lat: 10.99, lon: 104.78 },
+ { name: 'Tamale, Ghana', lat: 9.40, lon: -0.84 },
+ { name: 'Tampere, Finland', lat: 61.50, lon: 23.79 },
+ { name: 'Tegucigalpa, Honduras', lat: 14.08, lon: -87.21 },
+ { name: 'Tirana, Albania', lat: 41.33, lon: 19.82 },
+ { name: 'Toamasina, Madagascar', lat: -18.15, lon: 49.40 },
+ { name: 'Touba, Senegal', lat: 14.86, lon: -15.88 },
+ { name: 'Trondheim, Norway', lat: 63.43, lon: 10.40 },
+ { name: 'Tuerkmenabat, Turkmenistan', lat: 39.07, lon: 63.58 },
+ { name: 'Varna, Bulgaria', lat: 43.22, lon: 27.91 },
+ { name: 'Winejok, South Sudan', lat: 9.01, lon: 27.57 },
+ { name: 'Wroclaw, Poland', lat: 51.10, lon: 17.03 },
+ { name: 'Yaounde, Cameroon', lat: 3.87, lon: 11.52 },
+ { name: 'Yei, South Sudan', lat: 4.09, lon: 30.68 },
+ { name: 'Zinder, Niger', lat: 13.81, lon: 8.99 },
];
// Coastline data removed — using Natural Earth SVG map (/static/world-map.svg)
@@ -4015,10 +4182,13 @@ function _almRegion() {
return '';
}
-function _applyRegionHolidays(region, year, month, add) {
+function _applyRegionHolidays(region, year, month, add, worldwide) {
var pack = _REGION_HOLIDAYS[region];
if (!pack) return;
- var src = _almRegionName(region);
+ // Region-scoped: the caption already names the country, so each entry's tag is
+ // just its colour. Worldwide: 18 packs at once, so every entry carries a
+ // compact country code ("Bastille Day · FR") to say whose day it is.
+ var src = worldwide ? region : _almRegionName(region);
var i;
for (i = 0; i < (pack.fixed || []).length; i++) {
var fx = pack.fixed[i];
@@ -4030,6 +4200,9 @@ function _applyRegionHolidays(region, year, month, add) {
var day = nh[2] === -1 ? _lastWeekday(year, month, nh[1]) : _nthWeekday(year, month, nh[1], nh[2]);
add(day, nh[3], 'holiday', '', src, region);
}
+ // Clock changes are location-specific; layering 18 countries' worth would be
+ // pure noise, so only the single region-scoped pack contributes them.
+ if (worldwide) return;
// Clock changes: labels hold both hemispheres (October IS spring in AU)
var dst = pack.dst;
if (dst === 'us') {
@@ -4044,6 +4217,16 @@ function _applyRegionHolidays(region, year, month, add) {
}
}
+// Worldwide scope: layer every national pack (skipping the EU pseudo-region,
+// which carries no national days of its own — only a DST rule). Each entry is
+// tagged with its ISO country code via the worldwide path of _applyRegionHolidays.
+function _applyAllRegionHolidays(year, month, add) {
+ for (var iso in _REGION_HOLIDAYS) {
+ if (iso === 'EU') continue;
+ _applyRegionHolidays(iso, year, month, add, true);
+ }
+}
+
// ── Equinoxes & solstices: computed, not hardcoded (Meeus ch. 27) ──────
// JDE0 mean-instant polynomials (valid 1000-3000 CE) plus the 24-term
// periodic correction — accurate to minutes. The old code pinned fixed
@@ -4167,8 +4350,10 @@ function _gregorianBaseEvents(year, month, add) {
// (US, CA, AU, DE, IT, BR, IN, CN, JP and others)
if (month === 5) { add(_nthWeekday(year, 5, 0, 2), t('hol_mothers_day'), 'holiday'); }
if (month === 6) { add(_nthWeekday(year, 6, 0, 3), t('hol_fathers_day'), 'holiday'); }
- // National days + clock changes for the detected region
- _applyRegionHolidays(_almRegion(), year, month, add);
+ // National days: every pack when Worldwide is chosen, else the detected
+ // region's (which also contributes its clock changes).
+ if (_almHolidayScope() === 'worldwide') _applyAllRegionHolidays(year, month, add);
+ else _applyRegionHolidays(_almRegion(), year, month, add);
// Easter and related
var easter = _computeEaster(year);
if (easter.month === month) { add(easter.day, 'Easter', 'holiday'); }
@@ -4385,7 +4570,10 @@ function _drawAlmanacGrid() {
for (var ei = 0; ei < shown; ei++) {
var ev = dayEvents[ei];
var escapedLabel = _th(ev.label).replace(/&/g,'&').replace(//g,'>');
- var srcTitle = ev.src ? ' title="' + _tLookup('alm_region_holiday', '{c} holiday').replace('{c}', ev.src).replace(/"/g, '"') + '"' : '';
+ // Worldwide entries carry a 2-letter ISO code as src; expand it to the full
+ // country name for the tooltip ("France holiday", not "FR holiday").
+ var srcName = (ev.src && ev.src.length === 2) ? (_almRegionName(ev.src) || ev.src) : ev.src;
+ var srcTitle = ev.src ? ' title="' + _tLookup('alm_region_holiday', '{c} holiday').replace('{c}', srcName).replace(/"/g, '"') + '"' : '';
// Country-specific holidays (those with a region src) get their own
// colour so they read apart from the worldwide observances (#33).
var evCls = 'alm-ev alm-ev-' + ev.type + (ev.src ? ' alm-ev-country' : '');
@@ -4405,6 +4593,23 @@ function _drawAlmanacGrid() {
}
html += '';
+ // Holiday scope pill — centered directly under the calendar grid it filters.
+ // A two-segment pill (Regional/Worldwide) sets the scope; the location
+ // affordance lives on the sun-map, so no caption here. Gregorian only — the
+ // national packs are keyed to Gregorian month/day, not other systems.
+ if (_almSystem === 'gregorian') {
+ var scope = _almHolidayScope();
+ html += '
' +
+ '
' +
+ '' +
+ '' +
+ '
' +
+ '
';
+ }
+
// Selected day detail — full event list for the selected day
var selCal = _jdnToCalendar(_almSystem, _almSelectedJDN);
if (selCal.year === _almYear && selCal.month === _almMonth) {
@@ -4426,19 +4631,6 @@ function _drawAlmanacGrid() {
}
}
- // Quiet caption saying whose national days are shown. Always present on
- // the Gregorian calendar — no pack means the worldwide set, and saying
- // so beats an unexplained absence of holidays.
- if (_almSystem === 'gregorian') {
- var regionName = _almRegionName(_almRegion());
- var capText = regionName
- ? _tLookup('alm_showing_holidays', 'Showing {c} holidays').replace('{c}', regionName.replace(/' + capText + '';
- }
-
// Cross-reference — selected date in all calendar systems (replaces pills)
html += _almRenderCrossRef(_almSelectedJDN);
diff --git a/zimi/static/app.css b/zimi/static/app.css
index d472997e..22b80f32 100644
--- a/zimi/static/app.css
+++ b/zimi/static/app.css
@@ -3,21 +3,117 @@
:root {
--bg: #0a0a0b; --surface: #141416; --surface2: #1c1c20;
--border: #27272b; --text: #e8e8ed; --text2: #8a8a94;
+ /* Border for interactive controls whose outline is their only affordance
+ (form fields, toggle tracks). On dark it equals --border; the light block
+ deepens it to clear 3:1 on paper. */
+ --border-strong: var(--border);
--accent: #f5f5f7; --amber: #f59e0b; --amber2: #d97706;
+ /* Ink for text/glyphs sitting ON an amber fill (buttons, active pills,
+ badges, toggle knobs). Dark theme's amber is bright, so black reads best;
+ the light block flips it to white because light's amber is deep. One
+ token keeps ~14 call sites from hardcoding a color that's only legible
+ in one theme. */
+ --on-amber: #000;
--amber-glow: rgba(245, 158, 11, 0.08); --amber-border: rgba(245, 158, 11, 0.25);
/* text3 is reserved for large-text (≥18px or ≥14px bold) only —
it's WCAG-AA compliant against --bg at large sizes (3:1) but
not at normal body text. Use --text2 for everything else. */
--text3: #6e6e7a; --text-dim: #6e6e7a; --hover: rgba(245, 158, 11, 0.08);
--error: #ef4444;
+ /* Translucent bar/menu surface (topbar frosted glass). Tokenised so the
+ light theme can flip it without every backdrop rule carrying a literal. */
+ --topbar-bg: rgba(10, 10, 11, 0.85);
+ /* Raised popover/menu fill (context menus, drawers). One token so the
+ three call sites can't drift, and the light theme flips it in one place. */
+ --card-bg: #1a1a1a;
+ /* Positive/"sharing active" semantics (seed + peer badges). --success is the
+ text/icon green; the faint translucent fills stay inline (they read on both
+ themes). Light theme darkens the ink for contrast on paper. */
+ --success: #6ec56e; --success-strong: #84d684;
--radius: 10px; --shadow-popup: 0 8px 32px rgba(0,0,0,0.5);
color-scheme: dark;
font-family: -apple-system, BlinkMacSystemFont, "Inter", "Segoe UI", sans-serif;
}
+ /* ── Light theme ──
+ Warm-paper, soft-contrast counterpart to the default dark palette — NOT
+ stark white. The amber accent deepens to a burnt amber (#b45309) so it keeps
+ AA contrast as text/link on paper AND still reads as a warm fill with black
+ label text; every `var(--amber*)` reference adapts at once (no per-selector
+ churn). Applied by [data-theme] on , stamped synchronously in the head
+ bootstrap so there's no dark flash. Auto mode resolves to dark or light and
+ sets the same attribute, so the CSS has exactly one code path.
+ Deliberate seam: the almanac overlay + the home "Today"/space cards keep
+ their dark cosmic identity in both themes (a night sky is a night sky). */
+ :root[data-theme="light"] {
+ --bg: #f2efe8; --surface: #fbfaf6; --surface2: #ffffff;
+ --border: #e4dfd4; --text: #26241f; --text2: #6a675e;
+ /* Amber deepened one more step (#b45309 → #9d4a08) so it clears WCAG-AA as
+ text/link on paper (5.35:1 on --bg) AND as small amber ink on the faint
+ amber-glow badge fills (updated / installed / multilingual: 4.70:1). The
+ fills flip their ink to white via --on-amber, so a deeper amber only
+ helps them too. */
+ --accent: #1a1917; --amber: #9d4a08; --amber2: #92400e;
+ --amber-glow: rgba(180, 83, 9, 0.10); --amber-border: rgba(180, 83, 9, 0.28);
+ /* On paper the amber fill is deep, so ink on it flips to white (6.15:1). */
+ --on-amber: #fff;
+ /* text3 deepened (#7c786e → #6f6b60) so the muted 10–12px labels that use it
+ (download sizes, field notes, dropdown sublabels) clear AA at 4.63:1 —
+ the dark-theme "large text only" caveat doesn't survive on paper. */
+ --text3: #6f6b60; --text-dim: #6f6b60; --hover: rgba(180, 83, 9, 0.10);
+ --error: #dc2626;
+ /* Interactive-control borders that carry meaning (form fields, toggle
+ tracks). The decorative --border (#e4dfd4) is only ~1.2:1 on paper — fine
+ for card separators that also have a fill change, too weak for a control
+ whose outline is its only affordance. This deeper grey hits 3:1 on --bg
+ (3.30) and on --surface (3.63). */
+ --border-strong: #8c826f;
+ /* Card definition on paper: a firmer edge than the decorative --border plus
+ a soft warm shadow, so tiles/detail cards read as cards, not loose blocks. */
+ --card-border: #ddd6c8;
+ --card-shadow: 0 1px 2px rgba(66, 54, 30, 0.06), 0 3px 8px rgba(66, 54, 30, 0.05);
+ --topbar-bg: rgba(250, 249, 246, 0.85);
+ --card-bg: #ffffff;
+ --success: #2f7d32; --success-strong: #256428;
+ --shadow-popup: 0 8px 32px rgba(60, 50, 30, 0.16);
+ color-scheme: light;
+ }
+ /* Two single-use status inks that are legible on dark but wash out on paper:
+ the "reused download" teal and the "stale" ochre. Darkened here rather than
+ tokenised — one call site each, so a token would be ceremony. */
+ :root[data-theme="light"] .dl-source-reuse { color: #0f766e; }
+ :root[data-theme="light"] .ci-hier-freshness { color: #a16207; }
+ /* Almanac seam (intentional): the almanac is a canvas-driven night-sky app —
+ its orrery/sky/star-field paint hardcoded cosmic colors that only read on
+ dark. Rather than ship a half-lit almanac, we keep its whole subtree dark
+ under the light app. Custom properties inherit, so redefining the token set
+ on the overlay container re-establishes the dark palette for every child
+ (cards, text, pills) without touching almanac.css. The result is a light app
+ chrome over a deliberately dark cosmic panel — a seam by design, not a bug. */
+ :root[data-theme="light"] .almanac-view {
+ --bg: #0a0a0b; --surface: #141416; --surface2: #1c1c20;
+ --border: #27272b; --text: #e8e8ed; --text2: #8a8a94;
+ --accent: #f5f5f7; --amber: #f59e0b; --amber2: #d97706;
+ --amber-glow: rgba(245, 158, 11, 0.08); --amber-border: rgba(245, 158, 11, 0.25);
+ --text3: #6e6e7a; --text-dim: #6e6e7a; --hover: rgba(245, 158, 11, 0.08);
+ --error: #ef4444; --card-bg: #1a1a1a;
+ --success: #6ec56e; --success-strong: #84d684;
+ color-scheme: dark;
+ }
+
html { height: 100%; overflow-y: auto; overscroll-behavior-y: none; }
body { background: var(--bg); color: var(--text); min-height: 100%; padding-top: 56px; overflow: visible; }
+ /* Anti-flash guard. The head bootstrap stamps .theme-boot on while it
+ (re)applies the theme, then removes it on the next committed paint. While it
+ is present, every element's transitions are killed so a bfcache restore /
+ tab re-activation can't animate a dark->light token flip on pills, cards,
+ and toggles (the residual Safari flash). Animations are left alone — only
+ the colour transitions cause the flash. */
+ html.theme-boot *,
+ html.theme-boot *::before,
+ html.theme-boot *::after { transition: none !important; }
+
/* ── Accessibility ── */
/* Visually-hide content but keep it readable for screen readers. */
.sr-only {
@@ -31,7 +127,7 @@
First tab-stop on the page so keyboard users can jump past the topbar. */
.skip-link {
position: absolute; top: -100px; left: 12px;
- background: var(--amber); color: #000;
+ background: var(--amber); color: var(--on-amber);
padding: 8px 16px; border-radius: 6px;
font-weight: 600; font-size: 14px;
z-index: 10000; text-decoration: none;
@@ -59,7 +155,7 @@
.topbar {
position: fixed; top: 0; left: 0; right: 0; z-index: 200;
height: 56px; display: flex; align-items: center; gap: 10px;
- padding: 0 16px; background: rgba(10, 10, 11, 0.85); backdrop-filter: blur(20px) saturate(180%);
+ padding: 0 16px; background: var(--topbar-bg); backdrop-filter: blur(20px) saturate(180%);
-webkit-backdrop-filter: blur(20px) saturate(180%);
border-bottom: 1px solid var(--border);
}
@@ -96,7 +192,7 @@
/* One-shot coachmark floated beside the book button on first Reader View. */
.reader-coach {
position: fixed; z-index: 9999; max-width: 220px;
- background: var(--amber); color: #000; font-size: 12.5px; font-weight: 600;
+ background: var(--amber); color: var(--on-amber); font-size: 12.5px; font-weight: 600;
line-height: 1.3; padding: 8px 11px; border-radius: 10px;
box-shadow: var(--shadow-popup); cursor: pointer;
opacity: 0; transform: translateY(-4px); transition: opacity .2s, transform .2s;
@@ -108,6 +204,30 @@
border-bottom: 6px solid var(--amber);
}
+ /* One-shot tip floated bottom-centre on first article open when a wiktionary
+ is installed — teaches the select-a-word Define gesture. Dismissible. */
+ .define-hint {
+ position: fixed; z-index: 9999; left: 50%; bottom: 24px;
+ transform: translateX(-50%) translateY(6px);
+ display: flex; align-items: center; gap: 9px;
+ max-width: min(360px, calc(100vw - 32px));
+ min-height: 44px; box-sizing: border-box;
+ background: var(--surface); color: var(--text);
+ border: 1px solid var(--amber-border);
+ font-size: 13px; line-height: 1.35; font-weight: 500;
+ padding: 10px 14px; border-radius: 12px;
+ box-shadow: var(--shadow-popup); cursor: pointer;
+ opacity: 0; transition: opacity .2s, transform .2s;
+ }
+ .define-hint.visible { opacity: 1; transform: translateX(-50%); }
+ .define-hint .define-hint-icon { color: var(--amber); flex-shrink: 0; display: flex; }
+ .define-hint .define-hint-icon svg { width: 18px; height: 18px; }
+
+ @media (prefers-reduced-motion: reduce) {
+ .define-hint { transition: opacity .18s; }
+ .define-hint.visible { transform: translateX(-50%); }
+ }
+
/* ── History trail dropdown ── */
#history-trail {
position: fixed; z-index: 9999; min-width: 220px; max-width: 360px;
@@ -249,13 +369,13 @@
}
.pill:hover, .lang-pill:hover,
.ci-variant-btn:hover, .flavor-pill:hover { border-color: var(--amber-border); color: var(--text); background: var(--amber-glow); }
- .pill.active, .lang-pill.active { background: linear-gradient(135deg, var(--amber), var(--amber2)); color: #000; border-color: transparent; font-weight: 600; }
+ .pill.active, .lang-pill.active { background: linear-gradient(135deg, var(--amber), var(--amber2)); color: var(--on-amber); border-color: transparent; font-weight: 600; }
/* Count chip: a subtle padded pill so the number reads as a distinct count and
never crowds the label or the pill's edge. The parent gap (5px) separates it
from the label; the padding gives the digits breathing room inside the chip. */
.pill .pill-count { opacity: 0.7; font-size: 10px; font-weight: 600; margin-left: 1px; padding: 1px 6px; background: rgba(255, 255, 255, 0.07); border-radius: 8px; }
/* Dark chip on the amber active pill — amber-on-amber is unreadable */
- .pill.active .pill-count { opacity: 1; background: rgba(0, 0, 0, 0.22); color: #000; padding: 1px 6px; border-radius: 8px; }
+ .pill.active .pill-count { opacity: 1; background: rgba(0, 0, 0, 0.22); color: var(--on-amber); padding: 1px 6px; border-radius: 8px; }
.pill.dimmed, .lang-pill.dimmed { opacity: 0.35; pointer-events: none; }
.pills-divider { width: 60px; height: 1px; background: var(--border); margin: 0 auto 4px; opacity: 0.5; }
/* Collection pills carry the layers glyph so they read as groupings, not
@@ -291,7 +411,7 @@
background: var(--surface2); color: var(--text); font-size: 12px; font-weight: 500;
cursor: pointer; white-space: nowrap; transition: all 0.15s;
}
- .lang-banner-btn:hover { background: var(--amber); color: #000; border-color: transparent; }
+ .lang-banner-btn:hover { background: var(--amber); color: var(--on-amber); border-color: transparent; }
.lang-banner-dismiss { margin-inline-start: auto; background: none; border: none; color: var(--text2); cursor: pointer; font-size: 14px; padding: 2px 6px; }
.lang-banner-dismiss:hover { color: var(--text); }
@@ -366,9 +486,50 @@
.ms-user-self-actions { flex-shrink: 0; display: flex; gap: 8px; }
/* Role badges in the users list. Muted by default; admin gets amber. */
.ms-role-badge { display: inline-block; font-size: 10px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.04em; padding: 1px 7px; border-radius: 10px; color: var(--text2); background: var(--surface2); border: 1px solid var(--border); vertical-align: middle; }
- .ms-role-badge.ms-role-admin { color: #000; background: linear-gradient(135deg, var(--amber), var(--amber2)); border-color: transparent; }
+ .ms-role-badge.ms-role-admin { color: var(--on-amber); background: linear-gradient(135deg, var(--amber), var(--amber2)); border-color: transparent; }
/* Role picker reuses .pill; keep it wrapping rather than scrolling in the narrow users pane. */
#new-user-role, #edit-user-role { flex-wrap: wrap; overflow: visible; margin-bottom: 4px; }
+ /* Limited scope on a user row → clickable shortcut into the allowlist editor. */
+ .ms-user-scope-link { color: var(--amber); cursor: pointer; text-decoration: none; border-bottom: 1px dotted var(--amber-border); }
+ .ms-user-scope-link:hover { border-bottom-color: var(--amber); }
+
+ /* ── Allowlist picker (per-user Limited + public-access Limited) ──
+ Searchable, legible checklist: real titles, language badge, article count,
+ select-all/none, live count summary. Comfortable rows, mobile-friendly. */
+ .ms-allowlist-picker { border: 1px solid var(--border); border-radius: 10px; background: var(--surface2); overflow: hidden; max-width: 480px; }
+ .ms-allow-tools { display: flex; align-items: center; gap: 6px; padding: 8px; border-bottom: 1px solid var(--border); flex-wrap: wrap; }
+ .ms-allow-search { flex: 1; min-width: 120px; padding: 7px 10px; font-size: 13px; font-family: inherit; color: var(--text); background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); box-sizing: border-box; }
+ .ms-allow-search:focus { outline: none; border-color: var(--amber-border); box-shadow: 0 0 0 3px var(--amber-glow); }
+ .ms-allow-link { padding: 5px 9px; font-size: 12px; font-family: inherit; color: var(--text2); background: none; border: 1px solid var(--border); border-radius: var(--radius); cursor: pointer; transition: all 0.15s; white-space: nowrap; }
+ .ms-allow-link:hover { color: var(--amber); border-color: var(--amber-border); background: var(--amber-glow); }
+ .ms-allow-summary { padding: 6px 10px; font-size: 12px; color: var(--text2); border-bottom: 1px solid var(--border); }
+ .ms-allow-list { max-height: 260px; overflow-y: auto; padding: 4px; }
+ .ms-allow-row { display: flex; align-items: center; gap: 10px; padding: 8px 8px; font-size: 13px; cursor: pointer; border-radius: 6px; }
+ .ms-allow-row:hover { background: var(--amber-glow); }
+ .ms-allow-row input { accent-color: var(--amber); flex-shrink: 0; width: 15px; height: 15px; }
+ .ms-allow-name { flex: 1; min-width: 0; display: flex; align-items: center; gap: 6px; }
+ .ms-allow-title { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
+ .ms-allow-lang { flex-shrink: 0; font-size: 9.5px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.03em; padding: 1px 5px; border-radius: 8px; color: var(--text2); background: var(--surface); border: 1px solid var(--border); }
+ .ms-allow-count { flex-shrink: 0; font-size: 11px; color: var(--text2); font-variant-numeric: tabular-nums; }
+ .ms-allow-empty { color: var(--text2); font-size: 12px; padding: 4px 0; }
+
+ /* ── Public access card ── governs the anonymous view; sits atop the pane. */
+ .ms-pa-card { margin-bottom: 24px; padding-bottom: 20px; border-bottom: 1px solid var(--border); }
+ .ms-pa-sub { color: var(--text2); font-size: 13px; margin: 2px 0 12px; }
+ .ms-pa-env { font-size: 12px; color: var(--text2); background: var(--amber-glow); border: 1px solid var(--amber-border); border-radius: var(--radius); padding: 6px 10px; margin-bottom: 12px; }
+ .ms-pa-env code { font-size: 11px; color: var(--amber); }
+ .ms-pa-choices { display: flex; flex-direction: column; gap: 8px; max-width: 480px; }
+ .ms-pa-choice { display: flex; align-items: flex-start; gap: 10px; padding: 10px 12px; border: 1px solid var(--border); border-radius: 10px; cursor: pointer; transition: all 0.15s; }
+ .ms-pa-choice:hover { border-color: var(--amber-border); background: var(--amber-glow); }
+ .ms-pa-choice.active { border-color: var(--amber-border); background: var(--amber-glow); }
+ .ms-pa-choice.disabled { opacity: 0.55; cursor: not-allowed; }
+ .ms-pa-choice input { accent-color: var(--amber); margin-top: 2px; flex-shrink: 0; width: 15px; height: 15px; }
+ .ms-pa-choice-body { display: flex; flex-direction: column; gap: 2px; min-width: 0; }
+ .ms-pa-choice-title { font-size: 13px; font-weight: 600; color: var(--text); }
+ .ms-pa-choice-desc { font-size: 12px; color: var(--text2); }
+ .ms-pa-allowlist { margin-top: 12px; max-width: 480px; }
+ .ms-pa-allowlist[hidden] { display: none; }
+ .ms-pa-save { margin-top: 12px; }
/* ── Reader View settings palette ── */
/* Popover from the book button (desktop) / ⋯ Reader View row (mobile). Anchored
@@ -399,6 +560,9 @@
.rv-swatch:hover { border-color: var(--amber-border); color: var(--text); }
.rv-swatch.active { border-color: var(--amber); color: var(--text); }
.rv-sw-dot { width: 22px; height: 22px; border-radius: 50%; border: 1px solid rgba(255,255,255,0.14); }
+ /* Auto = follows the app theme: a split dark/light disc, same language as the
+ app-theme Auto icon. Half-and-half via a hard-stop gradient. */
+ .rv-sw-auto .rv-sw-dot { background: linear-gradient(90deg, #0a0a0b 0 50%, #fbfbf9 50% 100%); border-color: rgba(127,127,127,0.4); }
.rv-sw-dark .rv-sw-dot { background: #0a0a0b; }
.rv-sw-light .rv-sw-dot { background: #fbfbf9; border-color: rgba(0,0,0,0.18); }
.rv-sw-sepia .rv-sw-dot { background: #f4ecd8; border-color: rgba(0,0,0,0.14); }
@@ -461,6 +625,13 @@
border-radius: 50%; background: #fff; transition: inset-inline-start 0.15s;
}
.rv-switch.on .rv-knob { inset-inline-start: 18px; }
+ /* Light theme: same fix as the .switch iOS toggle — the OFF track needs a
+ warm-grey fill + AA outline, and its white knob (fine on the dark ON amber)
+ must darken to --text2 when off so it doesn't vanish on the grey track. */
+ :root[data-theme="light"] .rv-switch {
+ background: #e0dacd; border-color: var(--border-strong);
+ }
+ :root[data-theme="light"] .rv-switch:not(.on) .rv-knob { background: var(--text2); }
.rv-pal-divider { height: 1px; margin: 6px 6px; background: var(--border); }
.rv-exit-row, .rv-action-row {
display: flex; align-items: center; gap: 10px; width: 100%; text-align: start;
@@ -553,6 +724,11 @@
.cat-heading:first-child { margin-top: 0; }
.cat-heading.clickable { cursor: pointer; }
.cat-heading.clickable:hover { opacity: 0.8; }
+ /* The Other catch-all heading reads quieter than a named section (it's the
+ leftover bucket) — flat muted colour instead of the amber gradient. */
+ .cat-heading-other {
+ background: none; -webkit-text-fill-color: initial; color: var(--text2); opacity: 0.7;
+ }
/* The global view toggle rides on the first section header's line, pinned to
its trailing edge. Absolute (not flex) so the gradient-clipped heading text
is untouched; text-fill-color reset so the toggle glyphs aren't transparent. */
@@ -572,15 +748,46 @@
.sh-desc { font-size: 13px; color: var(--text2); margin-top: 4px; line-height: 1.5; }
.sh-actions { margin-top: 10px; display: flex; gap: 8px; flex-wrap: wrap; }
.dl-pill { text-decoration: none; display: inline-block; }
+ /* zimgit "document collection" flag in the source header meta line */
+ .sh-chip {
+ display: inline-block; font-size: 11px; font-weight: 600; letter-spacing: 0.03em;
+ color: var(--amber); background: var(--amber-glow); border: 1px solid var(--amber-border);
+ border-radius: 999px; padding: 1px 8px; vertical-align: middle;
+ }
+ /* ── zimgit / PDF collection document list ── */
+ .zg-filter {
+ width: 100%; box-sizing: border-box; margin: 0 0 12px; padding: 9px 12px;
+ background: var(--surface); color: var(--text); font-size: 14px;
+ border: 1px solid var(--border); border-radius: var(--radius); outline: none;
+ transition: border-color 0.2s, box-shadow 0.2s;
+ }
+ .zg-filter:focus { border-color: var(--amber-border); box-shadow: 0 0 0 1px var(--amber-glow); }
+ .zg-doc { display: flex; gap: 12px; align-items: flex-start; }
+ .zg-icon {
+ flex-shrink: 0; width: 38px; height: 46px; margin-top: 2px; border-radius: 4px;
+ background: var(--surface2); border: 1px solid var(--border);
+ display: flex; align-items: center; justify-content: center;
+ font-size: 9px; font-weight: 700; letter-spacing: 0.05em; color: var(--amber);
+ }
+ .zg-meta { font-size: 11px; color: var(--text2); margin-top: 4px; }
+ .zg-empty { padding: 28px 8px; text-align: center; color: var(--text2); font-size: 13px; }
/* ── Source cards ── */
.stats-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(320px, 1fr)); gap: 8px; margin-top: 8px; }
.stat-card {
background: var(--surface); border-radius: var(--radius); padding: 14px;
- border: 1px solid var(--border);
+ border: 1px solid var(--card-border, var(--border));
+ /* On the light cream the 1px border is ~1.2:1 and cards read as loose
+ icon+text. A soft shadow (--card-shadow, light-only) lifts them into
+ defined cards; dark keeps the flat surface-contrast look. Hover overrides
+ both border-color and box-shadow at higher specificity. */
+ box-shadow: var(--card-shadow, none);
transition: background 0.25s, border-color 0.25s, box-shadow 0.25s, transform 0.25s;
transition-timing-function: cubic-bezier(0.4, 0, 0.2, 1);
display: flex; gap: 14px; align-items: stretch; text-decoration: none; color: inherit; cursor: pointer;
+ /* Long-press opens the context menu (#37) — suppress iOS's own text-selection
+ callout / magnifier so the press reads as a clean right-click. */
+ -webkit-touch-callout: none;
}
.stat-card:hover, .stat-card:focus-visible {
background: var(--surface2); border-color: var(--amber-border);
@@ -599,6 +806,11 @@
}
.card-icon { width: 48px; height: 48px; flex-shrink: 0; border-radius: 10px; overflow: hidden; background: var(--surface2); display: flex; align-items: center; justify-content: center; align-self: center; }
.card-icon img { width: 42px; height: 42px; object-fit: cover; background: #fff; border-radius: 8px; padding: 2px; }
+ /* On paper, a source whose favicon is white/transparent (e.g. Army Publishing
+ Directorate) rendered as an invisible white block on a white card. An inset
+ ring on the icon block gives it a defined edge regardless of the artwork. */
+ :root[data-theme="light"] .card-icon { box-shadow: inset 0 0 0 1px rgba(20, 15, 5, 0.10); }
+ :root[data-theme="light"] .card-icon img { box-shadow: inset 0 0 0 1px rgba(20, 15, 5, 0.06); }
.icon-letter { font-size: 20px; font-weight: 700; background: linear-gradient(135deg, var(--amber), #f97316); -webkit-background-clip: text; -webkit-text-fill-color: transparent; background-clip: text; }
.card-info { flex: 1; min-width: 0; display: flex; flex-direction: column; padding-right: 28px; }
.stat-card .name { font-size: 14px; font-weight: 600; color: var(--accent); margin-bottom: 2px; }
@@ -619,21 +831,49 @@
.lib-view-btn:hover { color: var(--text); background: var(--surface2); }
.lib-view-btn.active { color: var(--amber); background: var(--amber-glow); }
- /* Compact tiles: square-ish cells, icon on top, name under it, several per row.
- Reuses .stat-card markup — only the layout changes. Wider min column than
- before so titles have room to breathe (mobile stays ~3 across, desktop gets
- roomier tiles via the min-width override below). */
+ /* Compact tiles: square-ish cells sharing .stat-card markup (same surface,
+ radius, border, hover as the detail cards — one family, only the layout
+ differs). A fixed vertical rhythm on a 4px grid keeps every tile reading
+ the same regardless of title length:
+ 28px top band (reserved for the corner chips) → icon → two-line-tall
+ title (anchors the baseline so short titles don't float) → language chip
+ in normal flow.
+ The old layout floated the title and pinned the language badge bottom-left,
+ where it collided with any title that wrapped to two lines. */
.stats-grid.tiles { grid-template-columns: repeat(auto-fill, minmax(112px, 1fr)); gap: 10px; }
- .stats-grid.tiles .stat-card { flex-direction: column; align-items: center; gap: 8px; padding: 14px 10px; text-align: center; }
+ .stats-grid.tiles .stat-card {
+ flex-direction: column; align-items: center; justify-content: flex-start;
+ gap: 8px; padding: 20px 12px 12px; text-align: center;
+ }
.stats-grid.tiles .card-icon { align-self: center; }
- .stats-grid.tiles .card-info { align-items: center; padding-right: 0; }
+ .stats-grid.tiles .card-info { align-items: center; padding-right: 0; gap: 6px; flex: 0 0 auto; }
.stats-grid.tiles .desc, .stats-grid.tiles .detail, .stats-grid.tiles .card-cat, .stats-grid.tiles .qid-badge { display: none; }
- .stats-grid.tiles .name { font-size: 12px; margin-bottom: 0; line-height: 1.35; display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical; overflow: hidden; word-break: break-word; }
- /* Badges become corner chips on the tile (icon is centred, so corners are
- clear): NEW top-left, language bottom-left. The inline margins from the list
- layout are dropped since they're absolutely positioned here. */
- .stats-grid.tiles .new-badge { position: absolute; top: 6px; left: 6px; margin: 0; }
- .stats-grid.tiles .lang-badge { display: inline-block; position: absolute; bottom: 6px; left: 6px; margin: 0; }
+ /* The name row becomes a centred column: the title (.zt) on top, the language
+ chip on its own line below. */
+ .stats-grid.tiles .name {
+ display: flex; flex-direction: column; align-items: center; gap: 6px; margin: 0;
+ }
+ /* Reserve two lines (2 x 1.35 line-height) so a one-line title lands at the
+ same Y as a two-line one — the anchor that stops titles from floating. */
+ .stats-grid.tiles .zt {
+ font-size: 12px; line-height: 1.35; min-height: 2.7em;
+ display: -webkit-box; -webkit-line-clamp: 2; -webkit-box-orient: vertical;
+ overflow: hidden; word-break: break-word;
+ }
+ /* Deterministic badge layout: status chip top-left, star top-right — opposite
+ corners, so they can never collide at any title length. The language chip
+ flows on its own line under the title (always visible, never clipped by the
+ title clamp, never overlapping). Consistent tone with the list's inline chip.
+ The top band is tight (20px pad): the chips sit high enough to slightly
+ overlap the icon's top corners, which reads as intentional and reclaims a
+ few px of tile height. */
+ .stats-grid.tiles .new-badge { position: absolute; top: 6px; left: 8px; margin: 0; z-index: 1; }
+ .stats-grid.tiles .star-btn { top: 5px; right: 6px; z-index: 1; }
+ /* The chip owns its own line under the title and spells the language out in
+ full ("Français") — no letter-spacing tracking (that's for the terse code
+ form in list rows), and allow it to wrap rather than overflow the tile. */
+ .stats-grid.tiles .lang-badge { margin: 0; }
+ .stats-grid.tiles .lang-badge-full { letter-spacing: 0; white-space: normal; max-width: 100%; }
/* Compact view hides the empty (unfavorited) star — only a filled star or a
hover (pointer devices) reveals it, so tiles aren't peppered with grey stars. */
.stats-grid.tiles .star-btn:not(.starred) { opacity: 0; pointer-events: none; }
@@ -715,7 +955,10 @@
position: fixed; z-index: 10001; display: none;
background: var(--card-bg, #1a1a1a); border: 1px solid var(--border);
border-radius: 10px; box-shadow: 0 8px 24px rgba(0,0,0,0.5);
- color: var(--text); font-size: 13px; max-width: 320px;
+ color: var(--text); font-size: 13px;
+ /* Never wider than the viewport minus the 8px edge margins the JS clamps to,
+ so a card near the right edge still fits after _definePosition runs. */
+ max-width: min(320px, calc(100vw - 16px));
}
.define-popover.open { display: block; }
/* Stage 1: the compact "Define" trigger button */
@@ -840,7 +1083,7 @@
.cache-top-label { color: var(--text); font-weight: 600; }
.manage-btn-action,
.desktop-card .btn-primary,
- .coll-form button { background: linear-gradient(135deg, var(--amber), var(--amber2)); color: #000; border: none; border-radius: var(--radius); cursor: pointer; }
+ .coll-form button { background: linear-gradient(135deg, var(--amber), var(--amber2)); color: var(--on-amber); border: none; border-radius: var(--radius); cursor: pointer; }
.manage-btn-action:hover,
.desktop-card .btn-primary:hover { opacity: 0.85; }
.manage-btn-action:disabled,
@@ -867,6 +1110,25 @@
.ms-pane .ms-field input:focus { outline: none; border-color: var(--amber-border); }
.ms-pane .ms-field input[readonly] { opacity: 0.6; }
.ms-pane .ms-hint { font-size: 12px; color: var(--text2); margin-top: 8px; }
+ /* App-theme segmented control (Auto / Dark / Light). Three equal pills in a
+ recessed track, amber-filled when active — mirrors the reader palette's
+ control language so the two theme pickers feel like one family. */
+ .ms-theme-label { font-size: 13px; color: var(--text); margin: 4px 0 8px; }
+ .app-theme-seg {
+ display: flex; gap: 4px; padding: 4px; border-radius: var(--radius);
+ background: var(--surface2); border: 1px solid var(--border); max-width: 340px;
+ }
+ .app-theme-btn {
+ flex: 1; display: inline-flex; align-items: center; justify-content: center; gap: 6px;
+ padding: 7px 8px; border: 1px solid transparent; border-radius: 7px;
+ background: none; color: var(--text2); font-size: 12.5px; font-weight: 500;
+ cursor: pointer; transition: color 0.15s, background 0.15s, border-color 0.15s;
+ }
+ .app-theme-btn svg { flex-shrink: 0; }
+ .app-theme-btn:hover { color: var(--amber); border-color: var(--amber-border); background: var(--amber-glow); }
+ .app-theme-btn.active {
+ background: var(--amber); color: var(--on-amber); border-color: transparent; font-weight: 600;
+ }
.ms-section-label, .ci-section-label, .hp-group-label { font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.06em; color: var(--text2); }
.ms-section-label { margin-bottom: 8px; }
/* Auto-fit grid for catalog + downloads. 320px min → 2 columns in the
@@ -922,6 +1184,12 @@
border: 2px solid var(--text2); border-radius: 6px; background: var(--surface2);
transition: background 0.15s, border-color 0.15s;
}
+ /* Light theme: --surface2 is pure white, so the unchecked box vanished on the
+ near-white card. Match the fixed-toggle family — a stronger --border-strong
+ edge over a warm tinted fill — so the empty box clears the 3:1 contrast bar. */
+ :root[data-theme="light"] .catalog-grid .catalog-item .ci-select::before {
+ border-color: var(--border-strong); background: #e0dacd;
+ }
.catalog-grid .catalog-item .ci-select:checked::before {
border-color: var(--amber);
background: var(--amber) url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='%23000' stroke-width='3.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpolyline points='20 6 9 17 4 12'/%3E%3C/svg%3E") center/14px no-repeat;
@@ -946,6 +1214,7 @@
background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius);
margin-bottom: 6px; transition: border-color 0.2s, opacity 0.3s;
max-height: 200px; overflow: hidden;
+ -webkit-touch-callout: none; /* long-press → context menu, not iOS callout (#37) */
}
/* Grid-cell catalog-items don't need a margin — the grid gap handles spacing */
.catalog-grid .catalog-item { margin-bottom: 0; }
@@ -1042,6 +1311,34 @@
.dl-item .dl-size { color: var(--text2); flex-shrink: 0; }
.dl-item .dl-done { color: var(--amber); font-weight: 600; }
.dl-item .dl-error { color: var(--error); font-weight: 600; }
+ .dl-item .dl-scheduled { color: var(--text2); flex-shrink: 0; font-weight: 500; }
+ /* Download-scheduling card */
+ .ms-toggle-row { display: flex; align-items: center; gap: 8px; font-size: 13px; color: var(--text); cursor: pointer; }
+ /* Match the app's styled checkbox (amber accent) instead of the bare native
+ square — consistent with .ms-check / .share-upnp elsewhere in the pane. */
+ .ms-toggle-row input[type="checkbox"] { accent-color: var(--amber); width: 15px; height: 15px; flex-shrink: 0; }
+ .ms-toggle-row input[disabled] { cursor: not-allowed; }
+ .ms-dl-window { display: flex; flex-wrap: wrap; align-items: center; gap: 14px; margin-top: 10px; }
+ .ms-dl-window label { display: flex; align-items: center; gap: 6px; font-size: 12px; color: var(--text2); }
+ .ms-dl-window input[type="time"] {
+ background: var(--surface2); color: var(--text); border: 1px solid var(--border);
+ border-radius: 6px; padding: 4px 6px; font-size: 13px;
+ }
+ .ms-dl-window input[type="time"]:disabled { opacity: 0.6; }
+ .ms-dl-speed { margin-top: 12px; }
+ .dl-window-chip { font-size: 11px; font-weight: 600; padding: 2px 8px; border-radius: 999px; white-space: nowrap; }
+ .dl-window-chip.dl-window-in { color: var(--amber); background: var(--amber-glow); border: 1px solid var(--amber-border); }
+ .dl-window-chip.dl-window-out { color: var(--text2); background: var(--surface2); border: 1px solid var(--border); }
+ /* Backup & export hub */
+ .ms-backup-actions { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; margin-top: 10px; }
+ .ms-backup-import { margin-top: 10px; display: flex; flex-direction: column; gap: 8px; align-items: flex-start; }
+ /* Import preview: a bordered diff summary the admin confirms before anything applies. */
+ .ms-backup-preview {
+ align-self: stretch; padding: 12px; border: 1px solid var(--border);
+ border-radius: 8px; background: var(--surface2); display: flex; flex-direction: column; gap: 4px;
+ }
+ .ms-backup-pv-line { font-size: 13px; color: var(--text); }
+ .ms-dl-trickle { margin-top: 8px; }
.dl-progress { height: 4px; background: var(--surface2); border-radius: 2px; margin-top: 8px; overflow: hidden; }
.dl-progress-bar { height: 100%; background: var(--amber); border-radius: 2px; transition: width 0.5s; }
.dl-actions { display: flex; gap: 8px; align-items: center; margin-top: 6px; }
@@ -1057,10 +1354,18 @@
.dl-filter-pill {
display: inline-flex; align-items: center; gap: 4px;
}
+ /* Top-level bulk controls (Pause all / Resume all / Delete all). Shares the
+ download-row button shape; the danger variant flips to the error tone. */
+ .dl-bulk-bar { display: flex; gap: 6px; flex-wrap: wrap; margin: 0 0 12px 0; }
+ .dl-bulk-btn { background: none; border: 1px solid var(--border); color: var(--text2);
+ padding: 4px 12px; border-radius: 12px; font-size: 11px; font-family: inherit; cursor: pointer;
+ transition: all 0.15s; min-height: 24px; }
+ .dl-bulk-btn:hover { border-color: var(--amber); color: var(--amber); }
+ .dl-bulk-btn.dl-bulk-danger:hover { border-color: var(--error); color: var(--error); }
/* background:none beats .pill.active's amber gradient (same specificity,
later in source) — amber label on amber fill was unreadable. */
.dl-filter-pill.active { color: var(--amber); border-color: var(--amber); background: none; font-weight: 600; }
- .dl-filter-pill.active .pill-count { opacity: 1; background: var(--amber); color: #000; padding: 1px 6px; border-radius: 8px; }
+ .dl-filter-pill.active .pill-count { opacity: 1; background: var(--amber); color: var(--on-amber); padding: 1px 6px; border-radius: 8px; }
/* One shape for the whole download-row button family */
.dl-pause-btn, .dl-cancel-btn, .dl-retry-btn { background: none; border: 1px solid var(--border); color: var(--text2);
padding: 4px 12px; border-radius: 12px; font-size: 11px; cursor: pointer; transition: all 0.15s;
@@ -1081,8 +1386,21 @@
.dl-retry-btn:hover { border-color: var(--amber); color: var(--amber); }
.dl-mirror { font-size: 11px; color: var(--text2); flex: 1; }
/* BT card controls: always rendered (fixed height); dimmed + non-interactive
- when BT is off so a toggle never adds/removes rows and nothing jumps. */
- .share-bt-controls { margin-top: 8px; }
+ when BT is off so a toggle never adds/removes rows and nothing jumps.
+ A strict two-column grid (label | control) so every textbox lines up in a
+ single column, however wide the label. Each .share-field goes
+ display:contents so its