From 65355bc0aaba9f6fa15df68ab270c94be3bd2bfc Mon Sep 17 00:00:00 2001 From: Eric Pheterson Date: Tue, 28 Jul 2026 00:41:06 -0700 Subject: [PATCH 01/58] feat(1.8.1): right-click coherence + Define discoverability Right-click coherence: the reader iframe no longer attaches a contextmenu listener that intercepts right-click on in-archive links and swaps in Zimi's own 'open article' menu. The browser's system context menu is load-bearing on article content (copy, open-link-in-new-tab, translate, image save, look up), so right-click inside a ZIM page now yields ONLY the system menu. Zimi's custom link menu stays on chrome article links (search results / home tiles) in the parent document. The 1.8.0 Define right-click suppression is unchanged and still fires (no preventDefault) so a right-click never pops the Define trigger. Rewrote the obsolete #17 contextmenu guard into a coherence guard. Define discoverability, two teaching anchors on top of the select-a-word gesture (which stays the fast path): - One-shot tip: first article open with a wiktionary installed shows a dismissible bottom toast ('select any word for its definition'), once per browser (localStorage flag), auto-dismiss 6s, and retired the moment Define is ever used or lookup mode is entered. - Reader menu entry 'Look up a word': arms a tap-to-define mode with a subtle 'tap any word' banner; the next tap selects that word and shows the same Define popover, then disarms. Escape or re-tapping the entry exits; closing the reader disarms it. Only shown when a wiktionary is installed (else dormant). i18n x10 (real translations): define_hint, define_lookup, define_lookup_prompt. Extracted _defineRangeRect shared by the selection path and tap-to-define. Co-Authored-By: Claude --- tests/test_iframe_link_chaperon.py | 34 ++--- zimi/static/app.css | 43 +++++++ zimi/static/app.js | 191 +++++++++++++++++++++++------ zimi/static/i18n/ar.json | 3 + zimi/static/i18n/de.json | 3 + zimi/static/i18n/en.json | 3 + zimi/static/i18n/es.json | 3 + zimi/static/i18n/fr.json | 3 + zimi/static/i18n/he.json | 3 + zimi/static/i18n/hi.json | 3 + zimi/static/i18n/pt.json | 3 + zimi/static/i18n/ru.json | 3 + zimi/static/i18n/zh.json | 3 + 13 files changed, 249 insertions(+), 49 deletions(-) diff --git a/tests/test_iframe_link_chaperon.py b/tests/test_iframe_link_chaperon.py index 2694eb56..c4c09463 100644 --- a/tests/test_iframe_link_chaperon.py +++ b/tests/test_iframe_link_chaperon.py @@ -69,21 +69,27 @@ def test_iframe_click_handler_uses_capture_phase(): ) -def test_iframe_contextmenu_uses_no_rewrite_trick(): - """The right-click contextmenu handler also resolves hrefs from the - iframe — must use the same `_no_rewrite=true` pattern as the left- - click handler, otherwise wombat ZIMs would have the same doubling - bug on right-click "Open article" (incomplete fix for #17).""" +def test_no_custom_contextmenu_inside_article_frame(): + """Right-click coherence (1.8.1): right-clicking INSIDE article content + must yield ONLY the browser's own context menu, which is load-bearing on + a ZIM page (copy, open-link-in-new-tab, translate, image save, look up…). + + An earlier build attached a contextmenu listener to the reader iframe that + intercepted right-click on in-archive links and swapped in Zimi's own "open + article" menu. That was removed: Zimi's custom link menu is now offered ONLY + on CHROME article links (search results / home tiles) in the PARENT document. + This guards against a regression that re-adds an in-frame interceptor. + """ src = _read_app_js() - # Both occurrences should share the pattern. Easy invariant: the - # number of "_no_rewrite = true" assignments matches the number of - # iframe-document listeners that read a.href (click + contextmenu). - flag_sets = src.count("_no_rewrite = true") - assert flag_sets >= 2, ( - f"expected ≥2 `_no_rewrite = true` assignments (click + " - f"contextmenu handlers); found {flag_sets}. The contextmenu " - f"handler must also use the trick or right-click on wombat " - f"ZIMs will double the path." + assert "frame.contentDocument.addEventListener('contextmenu'" not in src, ( + "no contextmenu listener may be attached to the reader iframe that shows " + "Zimi's own menu — the system menu is load-bearing inside article content " + "(1.8.1 right-click coherence)" + ) + # The exact removed call — a link menu positioned in frame-relative coords. + assert "_showLinkCtxMenu(e.clientX + frameRect" not in src, ( + "the reader frame must not open Zimi's link context menu on right-click; " + "that affordance lives on parent-document chrome links only" ) diff --git a/zimi/static/app.css b/zimi/static/app.css index d472997e..26ff8c06 100644 --- a/zimi/static/app.css +++ b/zimi/static/app.css @@ -108,6 +108,49 @@ border-bottom: 6px solid var(--amber); } + /* One-shot tip floated bottom-centre on first article open when a wiktionary + is installed — teaches the select-a-word Define gesture. Dismissible. */ + .define-hint { + position: fixed; z-index: 9999; left: 50%; bottom: 24px; + transform: translateX(-50%) translateY(6px); + display: flex; align-items: center; gap: 9px; + max-width: min(360px, calc(100vw - 32px)); + min-height: 44px; box-sizing: border-box; + background: var(--surface); color: var(--text); + border: 1px solid var(--amber-border); + font-size: 13px; line-height: 1.35; font-weight: 500; + padding: 10px 14px; border-radius: 12px; + box-shadow: var(--shadow-popup); cursor: pointer; + opacity: 0; transition: opacity .2s, transform .2s; + } + .define-hint.visible { opacity: 1; transform: translateX(-50%); } + .define-hint .define-hint-icon { color: var(--amber); flex-shrink: 0; display: flex; } + .define-hint .define-hint-icon svg { width: 18px; height: 18px; } + + /* Lookup mode: a subtle top banner while the reader is armed to define the + next tapped word (⋯ › Look up a word). Tap the banner or Escape to exit. */ + .define-lookup-banner { + position: fixed; z-index: 9998; left: 50%; top: 12px; + transform: translateX(-50%) translateY(-6px); + display: flex; align-items: center; gap: 8px; + min-height: 40px; box-sizing: border-box; + background: var(--amber); color: #000; + font-size: 12.5px; font-weight: 600; line-height: 1.3; + padding: 8px 14px; border-radius: 999px; + box-shadow: var(--shadow-popup); cursor: pointer; + opacity: 0; transition: opacity .18s, transform .18s; + } + .define-lookup-banner.visible { opacity: 1; transform: translateX(-50%); } + .define-lookup-banner svg { width: 16px; height: 16px; flex-shrink: 0; } + /* While armed, hint the target: a crosshair over article content. */ + body.define-lookup-armed #reader-frame { cursor: crosshair; } + + @media (prefers-reduced-motion: reduce) { + .define-hint, .define-lookup-banner { transition: opacity .18s; } + .define-hint.visible { transform: translateX(-50%); } + .define-lookup-banner.visible { transform: translateX(-50%); } + } + /* ── History trail dropdown ── */ #history-trail { position: fixed; z-index: 9999; min-width: 220px; max-width: 360px; diff --git a/zimi/static/app.js b/zimi/static/app.js index 6c9712f9..306e0784 100644 --- a/zimi/static/app.js +++ b/zimi/static/app.js @@ -39,6 +39,9 @@ var SK = { // One-shot: set once the "tap again for reading settings" coachmark has been // shown, so the hint never nags a returning reader. READER_COACH: 'zimi_reader_settings_coach', + // One-shot: set once the "select any word to define it" tip has been shown (or + // the moment the reader first uses Define), so the tip appears at most once. + DEFINE_HINT: 'zimi_define_hint_seen', // Last-rendered SHARING rows (Server pane) — restored synchronously on // pane open so the section doesn't pop in after the status fetches. SHARE_ROWS: 'zimi_share_rows', @@ -10574,38 +10577,12 @@ function openReader(url) { frame.contentDocument.addEventListener('auxclick', function(e) { if (e.button === 1) _handleFrameLink(e); }); - // Right-click context menu for links inside iframe - frame.contentDocument.addEventListener('contextmenu', function(e) { - var a = e.target.closest('a[href]'); - if (!a) return; - var href = a.getAttribute('href') || ''; - if (href.startsWith('#')) return; - // Same _no_rewrite trick as the click handler — wombat-rewritten - // anchors return the original URL via .href, but with the flag set - // they return the actual in-archive URL (issue #17). - var fullUrl; - try { - var _prevNoRewrite = a._no_rewrite; - a._no_rewrite = true; - fullUrl = a.href; - a._no_rewrite = _prevNoRewrite; - } catch (ex) { - var frameLoc = frame.contentWindow.location; - try { fullUrl = new URL(href, frameLoc.href).href; } catch(ex2) { fullUrl = a.href; } - } - var wMatch = fullUrl.startsWith(location.origin) && fullUrl.match(/\/w\/([^\/]+)\/(.+)/); - if (wMatch) { - e.preventDefault(); - e.stopPropagation(); - var linkZim = decodeURIComponent(wMatch[1]); - var linkPath = decodeURIComponent(wMatch[2]).replace(/\?.*$/, ''); - // Calculate position relative to parent window - var frameRect = frame.getBoundingClientRect(); - _showLinkCtxMenu(e.clientX + frameRect.left, e.clientY + frameRect.top, { - zim: linkZim, path: linkPath, title: a.textContent.trim(), url: fullUrl - }); - } - }); + // NB: deliberately NO contextmenu listener inside the article frame. The + // system context menu is load-bearing on article content (copy, open link + // in new tab, look up, translate, image save…), so right-click inside a ZIM + // page must yield ONLY the browser's own menu. Zimi's custom link menu is + // offered on chrome article links (search results / tiles) in the parent + // document instead — see the delegated contextmenu handler below. } catch(e) { /* cross-origin — ZIM content loaded from a different origin */ } // Classify links: batch-resolve external URLs to find which ones are available locally // Only highlight links that actually resolve to a different installed ZIM @@ -11145,6 +11122,7 @@ function openArticle(zim, path, title, opts) { function closeReader() { if (!readerOpen) return; _ttsStop(); // stop read-aloud when leaving the reader + _defineExitLookupMode(); // never leave the reader armed for a word tap // Sync the address bar back to the view the reader was covering — an // explicit close otherwise strands the article URL (a reload would // reopen the closed article). On popstate-driven closes the history @@ -11447,6 +11425,13 @@ function _buildTopbarMenuHtml() { '" onclick="event.stopPropagation();_ttsToggle()">' + _TBM_TTS_ICON + ' ' + tH(_ttsSpeaking ? 'tts_stop' : 'tts_speak') + ''; } + // 3b. Look up a word — teaching affordance for Define. Only when a wiktionary + // is installed (else Define is dormant). Arms tap-to-define; a re-tap exits. + if (_defineFindWiktionary(_ttsLang(_readerFrameDoc()))) { + readerGroup += ''; + } // 4. Open in browser — LAST, and only where it's meaningful: the desktop app // or an installed/standalone PWA. In a plain browser tab you're already in a // browser, so it's hidden. Opens the ?a= deep link (full Zimi chrome). @@ -11835,14 +11820,18 @@ function _defineIsWord(s) { return /[\p{L}]/u.test(s); // must contain a letter } -// Selection rect in PARENT-window coords (iframe rect + selection rect). -function _defineSelRect(frame, sel) { +// Range rect in PARENT-window coords (iframe offset + range rect). Shared by the +// selection path and tap-to-define, which each have a Range in hand. +function _defineRangeRect(frame, range) { try { - var r = sel.getRangeAt(0).getBoundingClientRect(); + var r = range.getBoundingClientRect(); var fr = frame.getBoundingClientRect(); return { x: r.left + fr.left, y: r.bottom + fr.top + 4, top: r.top + fr.top }; } catch (e) { return null; } } +function _defineSelRect(frame, sel) { + try { return _defineRangeRect(frame, sel.getRangeAt(0)); } catch (e) { return null; } +} function _definePosition(rect) { if (!rect) return; @@ -11877,6 +11866,8 @@ function _defineShowTrigger(frame, word, wikt, rect) { function _defineRun() { var st = _defineState; if (!st) return; + // The reader has now used Define — retire the one-shot discovery tip for good. + try { localStorage.setItem(SK.DEFINE_HINT, '1'); } catch (e) {} _definePopover.innerHTML = '
' + esc(st.word) + '
' + tH('define_loading') + '
'; @@ -11999,6 +11990,136 @@ function _defineAttachToDoc(frame) { if (e.button === 0) _defineSuppressChip = false; if (_defineState && _defineState.path !== null) _defineHide(); }, true); + // First article open with a wiktionary installed → one-shot discovery tip. + _maybeShowDefineHint(doc); +} + +// ── Define discoverability ── +// Two teaching affordances layered on top of the select-a-word gesture, which +// stays the fast path: (a) a one-shot bottom tip the first time an article opens +// with a wiktionary installed, and (b) a "Look up a word" ⋯-menu entry that arms +// a tap-to-define mode for people who never think to select text. + +// (a) One-shot tip — shows at most once per browser. Skipped entirely when no +// wiktionary is installed (feature dormant) or the reader has already met Define. +function _maybeShowDefineHint(doc) { + if (_getStorageFlag(SK.DEFINE_HINT)) return; + if (!readerOpen) return; + if (!_defineFindWiktionary(_ttsLang(doc))) return; // dormant: nothing to teach + try { localStorage.setItem(SK.DEFINE_HINT, '1'); } catch (e) {} + var tip = document.createElement('div'); + tip.className = 'define-hint'; + tip.setAttribute('role', 'status'); + tip.innerHTML = '' + _DEFINE_BOOK_ICON + '' + + '' + tH('define_hint') + ''; + document.body.appendChild(tip); + requestAnimationFrame(function() { tip.classList.add('visible'); }); + var kill = function() { + tip.classList.remove('visible'); + setTimeout(function() { if (tip.parentNode) tip.remove(); }, 220); + }; + var timer = setTimeout(kill, 6000); + tip.addEventListener('click', function() { clearTimeout(timer); kill(); }); +} + +// (b) Tap-to-define lookup mode. Armed from the ⋯ menu; the next tap on a word in +// the article selects that word and runs the same Define path, then disarms. A +// subtle banner explains it; Escape or re-tapping the menu entry exits. +var _defineLookupMode = false; +var _defineLookupBanner = null; +var _defineLookupDetach = null; + +// The word under a point inside the reader document → { word, rect } (rect in +// PARENT coords), or null. Uses caretRange/PositionFromPoint then expands to word +// boundaries in the text node (letters, digits, hyphen, apostrophe). Reads the +// word and rect straight off the Range so it works even when a programmatic +// iframe selection doesn't "stick" (unfocused iframe); it still applies the +// selection as visual feedback, best-effort. +function _defineWordAt(frame, doc, win, x, y) { + var range = null; + if (doc.caretRangeFromPoint) { + range = doc.caretRangeFromPoint(x, y); + } else if (doc.caretPositionFromPoint) { + var pos = doc.caretPositionFromPoint(x, y); + if (pos) { range = doc.createRange(); range.setStart(pos.offsetNode, pos.offset); range.collapse(true); } + } + if (!range || range.startContainer.nodeType !== 3) return null; // need a text node + var node = range.startContainer, text = node.textContent || '', off = range.startOffset; + var isW = function(c) { return !!c && /[\p{L}\p{N}'’-]/u.test(c); }; + var start = off, end = off; + while (start > 0 && isW(text[start - 1])) start--; + while (end < text.length && isW(text[end])) end++; + if (end <= start) return null; + var wr = doc.createRange(); + wr.setStart(node, start); wr.setEnd(node, end); + var word = (wr.toString() || '').trim(); + if (!word) return null; + var rect = _defineRangeRect(frame, wr); + try { var sel = win.getSelection(); sel.removeAllRanges(); sel.addRange(wr); } catch (e) {} + return { word: word, rect: rect }; +} + +function _defineToggleLookupMode() { + if (_defineLookupMode) { _defineExitLookupMode(); return; } + _defineEnterLookupMode(); +} + +function _defineEnterLookupMode() { + if (_defineLookupMode || !readerOpen) return; + var frame = document.getElementById('reader-frame'); + var doc, win; + try { doc = frame.contentDocument; win = frame.contentWindow; } catch (e) { return; } + if (!doc || !win) return; + if (!_defineFindWiktionary(_ttsLang(doc))) return; // dormant + // Using lookup mode also counts as discovering Define — retire the tip. + try { localStorage.setItem(SK.DEFINE_HINT, '1'); } catch (e) {} + _defineLookupMode = true; + document.body.classList.add('define-lookup-armed'); + _defineHide(); + + var banner = document.createElement('div'); + banner.className = 'define-lookup-banner'; + banner.setAttribute('role', 'status'); + banner.innerHTML = _DEFINE_BOOK_ICON + '' + tH('define_lookup_prompt') + ''; + banner.addEventListener('click', function() { _defineExitLookupMode(); }); + document.body.appendChild(banner); + _defineLookupBanner = banner; + requestAnimationFrame(function() { banner.classList.add('visible'); }); + + // The next tap in the article defines that word (capture, so it runs before the + // frame's own link-click handler and never navigates). Escape exits. + var onClick = function(e) { + e.preventDefault(); e.stopPropagation(); + var hit = null; + try { hit = _defineWordAt(frame, doc, win, e.clientX, e.clientY); } catch (ex) {} + _defineExitLookupMode(); + if (hit && hit.rect && _defineIsWord(hit.word)) { + var wikt = _defineFindWiktionary(_ttsLang(doc)); + if (wikt) { _defineSuppressChip = false; _defineShowTrigger(frame, hit.word, wikt, hit.rect); } + } + }; + var onKey = function(e) { if (e.key === 'Escape') _defineExitLookupMode(); }; + doc.addEventListener('click', onClick, true); + doc.addEventListener('keydown', onKey, true); + document.addEventListener('keydown', onKey, true); + _defineLookupDetach = function() { + try { doc.removeEventListener('click', onClick, true); } catch (ex) {} + try { doc.removeEventListener('keydown', onKey, true); } catch (ex) {} + document.removeEventListener('keydown', onKey, true); + }; +} + +function _defineExitLookupMode() { + if (!_defineLookupMode) return; + _defineLookupMode = false; + document.body.classList.remove('define-lookup-armed'); + if (_defineLookupDetach) { _defineLookupDetach(); _defineLookupDetach = null; } + var banner = _defineLookupBanner; + _defineLookupBanner = null; + if (banner) { + banner.classList.remove('visible'); + setTimeout(function() { if (banner.parentNode) banner.remove(); }, 200); + } } // Close context menu on click anywhere diff --git a/zimi/static/i18n/ar.json b/zimi/static/i18n/ar.json index 023f39e1..f04bd661 100644 --- a/zimi/static/i18n/ar.json +++ b/zimi/static/i18n/ar.json @@ -1003,6 +1003,9 @@ "define_loading": "جارٍ البحث…", "define_no_results": "لم يُعثر على تعريف", "define_open_full": "فتح المدخل الكامل ←", + "define_hint": "نصيحة: حدد أي كلمة لعرض تعريفها", + "define_lookup": "البحث عن كلمة", + "define_lookup_prompt": "انقر على أي كلمة", "library_health": "التحقق من سلامة المكتبة", "library_health_running": "جارٍ فحص المكتبة…", "library_health_failed": "فشل فحص الصحة", diff --git a/zimi/static/i18n/de.json b/zimi/static/i18n/de.json index 4b268c2d..81521d94 100644 --- a/zimi/static/i18n/de.json +++ b/zimi/static/i18n/de.json @@ -1003,6 +1003,9 @@ "define_loading": "Nachschlagen…", "define_no_results": "Keine Definition gefunden", "define_open_full": "Vollständigen Eintrag öffnen →", + "define_hint": "Tipp: Wähle ein Wort aus, um seine Definition zu sehen", + "define_lookup": "Wort nachschlagen", + "define_lookup_prompt": "Auf ein Wort tippen", "library_health": "Bibliothek-Integrität prüfen", "library_health_running": "Bibliothek wird geprüft…", "library_health_failed": "Zustandsprüfung fehlgeschlagen", diff --git a/zimi/static/i18n/en.json b/zimi/static/i18n/en.json index cc2c21ee..576d3d47 100644 --- a/zimi/static/i18n/en.json +++ b/zimi/static/i18n/en.json @@ -1003,6 +1003,9 @@ "define_loading": "Looking up…", "define_no_results": "No definition found", "define_open_full": "Open full entry →", + "define_hint": "Tip: select any word for its definition", + "define_lookup": "Look up a word", + "define_lookup_prompt": "Tap any word", "library_health": "Check library integrity", "library_health_running": "Checking library…", "library_health_failed": "Health check failed", diff --git a/zimi/static/i18n/es.json b/zimi/static/i18n/es.json index 026971a4..bc4dddea 100644 --- a/zimi/static/i18n/es.json +++ b/zimi/static/i18n/es.json @@ -1003,6 +1003,9 @@ "define_loading": "Buscando…", "define_no_results": "No se encontró definición", "define_open_full": "Abrir entrada completa →", + "define_hint": "Consejo: selecciona cualquier palabra para ver su definición", + "define_lookup": "Buscar una palabra", + "define_lookup_prompt": "Toca cualquier palabra", "library_health": "Verificar integridad de la biblioteca", "library_health_running": "Comprobando la biblioteca…", "library_health_failed": "Error en la comprobación", diff --git a/zimi/static/i18n/fr.json b/zimi/static/i18n/fr.json index 2b86b749..bdc70260 100644 --- a/zimi/static/i18n/fr.json +++ b/zimi/static/i18n/fr.json @@ -1003,6 +1003,9 @@ "define_loading": "Recherche…", "define_no_results": "Aucune définition trouvée", "define_open_full": "Ouvrir l'entrée complète →", + "define_hint": "Astuce : sélectionnez un mot pour voir sa définition", + "define_lookup": "Rechercher un mot", + "define_lookup_prompt": "Touchez un mot", "library_health": "Vérifier l'intégrité de la bibliothèque", "library_health_running": "Vérification de la bibliothèque…", "library_health_failed": "Échec de la vérification", diff --git a/zimi/static/i18n/he.json b/zimi/static/i18n/he.json index 38a64810..22379823 100644 --- a/zimi/static/i18n/he.json +++ b/zimi/static/i18n/he.json @@ -1003,6 +1003,9 @@ "define_loading": "מחפש…", "define_no_results": "לא נמצאה הגדרה", "define_open_full": "פתח ערך מלא ←", + "define_hint": "טיפ: בחר מילה כלשהי כדי לראות את הגדרתה", + "define_lookup": "חיפוש מילה", + "define_lookup_prompt": "הקש על מילה כלשהי", "library_health": "בדיקת תקינות הספרייה", "library_health_running": "בודק את הספרייה…", "library_health_failed": "בדיקת התקינות נכשלה", diff --git a/zimi/static/i18n/hi.json b/zimi/static/i18n/hi.json index 21541b3a..a92d070d 100644 --- a/zimi/static/i18n/hi.json +++ b/zimi/static/i18n/hi.json @@ -1003,6 +1003,9 @@ "define_loading": "खोज रहे हैं…", "define_no_results": "कोई परिभाषा नहीं मिली", "define_open_full": "पूरी प्रविष्टि खोलें →", + "define_hint": "सुझाव: किसी भी शब्द का अर्थ देखने के लिए उसे चुनें", + "define_lookup": "शब्द खोजें", + "define_lookup_prompt": "किसी भी शब्द पर टैप करें", "library_health": "लाइब्रेरी अखंडता जांचें", "library_health_running": "लाइब्रेरी जाँच रहे हैं…", "library_health_failed": "स्वास्थ्य जाँच विफल", diff --git a/zimi/static/i18n/pt.json b/zimi/static/i18n/pt.json index 38ba4ba9..959b8b3b 100644 --- a/zimi/static/i18n/pt.json +++ b/zimi/static/i18n/pt.json @@ -1003,6 +1003,9 @@ "define_loading": "Procurando…", "define_no_results": "Nenhuma definição encontrada", "define_open_full": "Abrir entrada completa →", + "define_hint": "Dica: selecione qualquer palavra para ver sua definição", + "define_lookup": "Procurar uma palavra", + "define_lookup_prompt": "Toque em qualquer palavra", "library_health": "Verificar integridade da biblioteca", "library_health_running": "Verificando a biblioteca…", "library_health_failed": "Falha na verificação", diff --git a/zimi/static/i18n/ru.json b/zimi/static/i18n/ru.json index c4e43fc5..14629ebb 100644 --- a/zimi/static/i18n/ru.json +++ b/zimi/static/i18n/ru.json @@ -1003,6 +1003,9 @@ "define_loading": "Поиск…", "define_no_results": "Определение не найдено", "define_open_full": "Открыть полную статью →", + "define_hint": "Совет: выделите любое слово, чтобы увидеть его определение", + "define_lookup": "Найти слово", + "define_lookup_prompt": "Коснитесь любого слова", "library_health": "Проверить целостность библиотеки", "library_health_running": "Проверка библиотеки…", "library_health_failed": "Ошибка проверки", diff --git a/zimi/static/i18n/zh.json b/zimi/static/i18n/zh.json index bd589e0e..62e6c7ea 100644 --- a/zimi/static/i18n/zh.json +++ b/zimi/static/i18n/zh.json @@ -1003,6 +1003,9 @@ "define_loading": "查询中…", "define_no_results": "未找到释义", "define_open_full": "打开完整词条 →", + "define_hint": "提示:选择任意单词即可查看释义", + "define_lookup": "查词", + "define_lookup_prompt": "点按任意单词", "library_health": "检查库完整性", "library_health_running": "正在检查库…", "library_health_failed": "健康检查失败", From ee41a285c32fe3bca485b02818ac8b29da774b45 Mon Sep 17 00:00:00 2001 From: Eric Pheterson Date: Tue, 28 Jul 2026 01:02:09 -0700 Subject: [PATCH 02/58] docs: date the 1.8.0 entry, drop a merge-duplicated 1.7.4 section Co-Authored-By: Claude --- CHANGELOG.md | 98 +++++----------------------------------------------- 1 file changed, 8 insertions(+), 90 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 091801df..8f392d6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/), and this project adheres to [Semantic Versioning](https://semver.org/). -## [1.8.0] - Unreleased — the Community Edition +## [1.8.0] - 2026-07-28 — the Community Edition A release shaped by the people using Zimi: the open issues, the long-standing asks from the wider ZIM ecosystem, and a week of field testing on real phones. @@ -128,12 +128,13 @@ delta updates on top. row in Manage) to move it into another category, including brand-new custom ones, and reorder the home sections — default categories and collections alike — from a new Reorder panel in Manage preferences (#37). -- **Windows portable build** — a one-dir `Zimi-windows-x64.zip` (Edge WebView2 - backend, in-process libtorrent when a wheel is available) built by the desktop - release CI alongside the macOS DMGs and Linux AppImage/snap. The Windows - desktop app also self-updates via WinSparkle — a per-user Inno Setup installer - (no UAC) shipped beside the zip, verified with the same signed appcast key as - the macOS Sparkle path. +- **Native Windows app** — a per-user Inno Setup installer + (`Zimi-windows-x64-Setup.exe`, no UAC) that self-updates via WinSparkle, + verified with the same signed appcast key as the macOS Sparkle path; updates + install silently and relaunch on their own. A portable `Zimi-windows-x64.zip` + (Edge WebView2 backend, in-process libtorrent when a wheel is available) is + shipped beside it for no-install use. Both are built by the desktop release CI + alongside the macOS DMGs and Linux AppImage/snap. - **Delta updates over BitTorrent.** When updating a ZIM that has a torrent, Zimi copies the previous version into staging under the new name first, so libtorrent's hash check salvages every unchanged piece and only the changed @@ -395,89 +396,6 @@ looks like the Moon. ### Added -- **"New" and "Updated" badges** on ZIM cards, so fresh or changed sources - stand out in a large library instead of being lost in the grid (#34). A - fresh install gets a solid "New" pill; a source whose file changed on an - update gets a quieter "Updated" pill. A badge clears the moment you open - that ZIM, and auto-expires after a week even if you never do — so it never - lingers. - -## [1.7.4] - 2026-07-20 - -A polish drop that closes the two live issues the post-1.7.3 code audit -surfaced, and grows the almanac up: an accurate Chinese calendar, a real -high-res moon, and a solar system you can travel out to. The moon finally -looks like the Moon. - -### Almanac - -- **Accurate Chinese calendar.** Replaced the mean-lunation approximation with - real astronomy — month boundaries are true new moons in China Standard Time, - leap months placed by the solar-term rule against the winter solstice. - Verified against the Hong Kong Observatory (New Year dates and leap months - 2014–2033). It's a browsable calendar system again, with the 闰 leap-month - marker and the correct zodiac animal. -- **Holidays land on every calendar.** Worldwide days, regional holidays, the - Easter cycle, solstices, meteor showers and Hindu/Sikh festivals are now - projected onto whatever calendar you're viewing (Hebrew, Islamic, Chinese…) - instead of vanishing when you switch away from Gregorian. -- **A real high-resolution moon**, reprojected from NASA's seamless lunar - albedo map — genuine crater detail and subtle true colour. The animated sky - moon shares the hero's shading now, earthshine dark side and all. -- **Travel the solar system.** The interstellar probes — Voyager 1 & 2, - Pioneer 10 & 11, New Horizons — are plotted at their real bearings out past - Neptune and creep outward as the clock runs, framed by labelled asteroid, - Kuiper and heliopause markers. -- **Easier location picking** — the world map cycles through overlapping - cities on repeated clicks, and search reaches 354 cities. - -### Security - -### Security - -- **`/dl/` no longer serves whole ZIMs to the public internet.** On a - host-networked deploy behind a containerized reverse proxy, every WAN - request reached Zimi from the docker bridge gateway — a private IP with no - real client IP propagated — so the whole internet was classified "private" - and could pull raw `.zim` files (public content, so not a data breach, but - unmetered use of the operator's uplink) and got the trusted rate tier. - Zimi now honours Cloudflare's un-forgeable `CF-Connecting-IP`, trusts a - forwarded client only from a private proxy hop (optionally an explicit - `ZIMI_TRUSTED_PROXIES` allowlist), and refuses a forwarded value that claims - loopback. WAN clients resolve to their real IP again. - -### Fixed - -- **A libzim segfault under normal use.** Opening an article kicked off a - background thread that read a shared libzim archive with no lock, while the - request read the same archive — two threads in a non-thread-safe library, a - silent crash. It now uses its own handle. Added a real-ZIM concurrency - stress test so this class of bug can't hide behind mocked archives again. -- Search results could silently drop a ZIM under concurrent load (an archive - published before its lock); the setup order is fixed. -- LAN peer discovery advertised the wrong BitTorrent port when a custom port - was set; it now advertises the real one. -- Malformed HTTP Range headers no longer 500 the download endpoint. -- The almanac topbar showed the underlying ZIM's icon; the breadcrumb is now - just "Zimi" while the almanac is open (you reach it only from home). -- Picking a location on the almanac's world map no longer rebuilds the whole - panel — it refreshes the location-dependent pieces in place, so the page - stops jumping and flashing on each click. - -### Changed - -- **The moon is beautiful now.** The hero and Today-card moon are rendered as - a single physically-shaded sphere — a soft terminator, gentle limb - darkening, the maria showing through, and a faint earthshine glow on the - dark side of a crescent. Gone are the hard light/dark edge, the seam down - the middle at the quarters, and the flat too-bright disc. The hero moon now - renders at the display's device resolution (up to 512px) so its shading - stays crisp when you lean in. -- **Country holidays get their own colour** on the calendar, apart from the - worldwide observances (#33). - -### Added - - **"New" and "Updated" badges** on ZIM cards, so fresh or changed sources stand out in a large library instead of being lost in the grid (#34). A fresh install gets a solid "New" pill; a source whose file changed on an From 87301adac0baaf2a6c0cd9d883f85afebb4bde1b Mon Sep 17 00:00:00 2001 From: Eric Pheterson Date: Tue, 28 Jul 2026 01:06:20 -0700 Subject: [PATCH 03/58] feat(almanac): real bright-star field in the horizon scene Replace the procedural _ensureSkyBgStars filler (220 random dots that never moved with the sky) with 459 real HYG-v41 stars to magnitude 4.0, projected by the same horizon geometry as the named catalog and cached with the same discipline (rebuilt only on a time/location change, not per frame). Each field star is tinted by its B-V colour index and twinkles on a deterministic, RA-seeded phase. Extract the shared per-star projection into _projectStarXY and _skyLST; _projectStars now also carries alt so the a11y star count is real. Co-Authored-By: Claude --- .github/workflows/auto-release.yml | 38 ++- .github/workflows/desktop-release.yml | 20 +- .../plans/2026-07-28-windows-gold-standard.md | 159 ++++++++++ docs/release-notes-1.8.0.md | 2 +- tests/test_winsparkle.py | 29 ++ windows/zimi.iss | 6 +- zimi/static/almanac-sky.js | 289 ++++++++++++++---- zimi_winsparkle.py | 31 +- 8 files changed, 487 insertions(+), 87 deletions(-) create mode 100644 docs/plans/2026-07-28-windows-gold-standard.md diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml index 4cdeec1b..ed233cc1 100644 --- a/.github/workflows/auto-release.yml +++ b/.github/workflows/auto-release.yml @@ -1,8 +1,21 @@ name: Auto Release -# When a push to main changes the version, auto-tag and create a draft release. -# The draft body is pulled from CHANGELOG.md. Desktop builds attach assets. -# Only step left: review the draft and publish. +# ── Single-draft release flow (there must be exactly ONE draft per tag) ────── +# When a push to main changes the version in pyproject.toml: +# 1. this workflow tags vX.Y.Z and creates/updates the draft release, with +# notes pulled from CHANGELOG.md; +# 2. the tag push triggers desktop-release.yml, which builds every platform, +# signs the update enclosures, refreshes the appcasts, and attaches the +# built assets to the SAME draft (softprops matches the release by tag). +# 3. the maintainer reviews that one draft and publishes it; +# 4. publish-release.yml then ships PyPI + snap. +# +# The draft step below is IDEMPOTENT on purpose: `gh release create` would +# happily make a SECOND draft for a tag that already has one (GitHub does not +# enforce tag-uniqueness on drafts), which is exactly the duplicate "Zimi +# vX.Y.Z" draft that showed up next to a hand-curated one after PR #39. So if a +# release for the tag already exists (a maintainer-curated draft, or a re-run), +# we EDIT it instead of creating a duplicate. on: push: @@ -78,7 +91,18 @@ jobs: ENDNOTES # Trim leading whitespace from heredoc sed -i 's/^ //' /tmp/release-notes.md - gh release create "v${VERSION}" \ - --draft \ - --title "Zimi v${VERSION}" \ - --notes-file /tmp/release-notes.md + # One draft per tag: update an existing release (curated draft or a + # re-run) instead of creating a duplicate. `gh release view` finds + # drafts by tag too. + if gh release view "v${VERSION}" >/dev/null 2>&1; then + echo "Release v${VERSION} already exists — updating its notes in place" + gh release edit "v${VERSION}" \ + --draft \ + --title "Zimi v${VERSION}" \ + --notes-file /tmp/release-notes.md + else + gh release create "v${VERSION}" \ + --draft \ + --title "Zimi v${VERSION}" \ + --notes-file /tmp/release-notes.md + fi diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index ec2e86e9..d550abf7 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -459,8 +459,9 @@ jobs: # Sign one enclosure and write its appcast. # $1 label $2 file-to-sign $3 release-asset $4 outfile $5 min-sys line + # $6 extra enclosure attributes (e.g. sparkle:installerArguments) generate_appcast() { - local LABEL="$1" FILE="$2" ASSET="$3" OUTFILE="$4" MINSYS="$5" + local LABEL="$1" FILE="$2" ASSET="$3" OUTFILE="$4" MINSYS="$5" ENC_EXTRA="$6" if [ ! -f "$FILE" ]; then echo "WARNING: No $LABEL enclosure found at $FILE" return 1 @@ -493,7 +494,7 @@ jobs: url="https://github.com/${REPO}/releases/download/${TAG}/${ASSET}" length="${LENGTH}" type="application/octet-stream" - sparkle:edSignature="${ED_SIG}" + sparkle:edSignature="${ED_SIG}"${ENC_EXTRA} /> @@ -505,11 +506,16 @@ jobs: } MAC_MINSYS=$'\n 12.0' - generate_appcast "Intel" "Zimi-Intel/Zimi-Intel.dmg" "Zimi-Intel.dmg" "appcast-intel.xml" "$MAC_MINSYS" - generate_appcast "Apple Silicon" "Zimi-AppleSilicon/Zimi-AppleSilicon.dmg" "Zimi-AppleSilicon.dmg" "appcast-arm64.xml" "$MAC_MINSYS" - # Windows enclosure is the Inno Setup installer; WinSparkle downloads - # and runs it to apply the update. No minimumSystemVersion line. - generate_appcast "Windows" "Zimi-windows-x64-Setup/Zimi-windows-x64-Setup.exe" "Zimi-windows-x64-Setup.exe" "appcast-windows.xml" "" + generate_appcast "Intel" "Zimi-Intel/Zimi-Intel.dmg" "Zimi-Intel.dmg" "appcast-intel.xml" "$MAC_MINSYS" "" + generate_appcast "Apple Silicon" "Zimi-AppleSilicon/Zimi-AppleSilicon.dmg" "Zimi-AppleSilicon.dmg" "appcast-arm64.xml" "$MAC_MINSYS" "" + # Windows enclosure is the Inno Setup installer; WinSparkle (0.9.4) + # downloads it, runs it with sparkle:installerArguments via + # ShellExecuteEx, then quits WITHOUT relaunching — so /SILENT gives a + # click-free progress-only update and the installer's [Run] entry does + # the relaunch (windows/zimi.iss). /SP- skips the "ready to install" + # prompt. Per-user install (no UAC), so silent is safe. + WIN_INSTALLER_ARGS=' sparkle:installerArguments="/SILENT /SP-"' + generate_appcast "Windows" "Zimi-windows-x64-Setup/Zimi-windows-x64-Setup.exe" "Zimi-windows-x64-Setup.exe" "appcast-windows.xml" "" "$WIN_INSTALLER_ARGS" # Commit and push to main git config user.name "github-actions[bot]" diff --git a/docs/plans/2026-07-28-windows-gold-standard.md b/docs/plans/2026-07-28-windows-gold-standard.md new file mode 100644 index 00000000..0a5cfc15 --- /dev/null +++ b/docs/plans/2026-07-28-windows-gold-standard.md @@ -0,0 +1,159 @@ +# Windows gold-standard update test (1.8.0 → 1.8.1) + +**Goal:** prove, on the maintainer's real Windows PC, that a Zimi Windows +install auto-updates end-to-end via WinSparkle — detect → download → verify +signature → run installer **silently** → relaunch at the new version — before +1.8.1 ships. ~10 minutes. + +--- + +## The one thing you must know first: prerelease tags are NOT safe + +A prerelease tag like `v1.8.1-rc1` **is unsafe** as a staging mechanism. It is +not a theoretical risk — here is exactly what it does: + +- `desktop-release.yml` triggers on `push: tags: ['v*.*.*']`. The glob matches + `v1.8.1-rc1` (`v1` . `8` . `1-rc1`), so the tag **builds all platforms**. +- Its `release` job then runs unconditionally for any tag and, in the same run: + 1. **rewrites `appcast-windows.xml`, `appcast-arm64.xml`, `appcast-intel.xml` + on `main` and pushes them** (job "Sign updates and update appcasts", + `git push origin main`). Every existing 1.8.0 user's app polls those files + — they would immediately be told to "update" to the RC. + 2. **pushes a Homebrew cask bump** to `epheterson/homebrew-zimi`. + 3. creates a GitHub draft release for the RC. + +So a prerelease tag mutates the three **production** appcasts and the Homebrew +tap. Do not use one for testing. (If we ever want RC tags to be safe, the fix is +to gate the "update appcasts" + "Homebrew cask" steps on +`!contains(github.ref_name, '-')` — out of scope here, noted for later.) + +**The safe lever instead:** `ZIMI_APPCAST_URL` (added to `zimi_winsparkle.py` +this release). Setting it points a build at a throwaway feed without touching +anything on `main`. Caveat that shapes the whole procedure below: **the shipped +1.8.0 predates this override**, so the real 1.8.0 artifact cannot be redirected +— it can only ever read the production feed. We therefore split the test. + +--- + +## Part A — sanity-check the real shipped 1.8.0 (2 min) + +Confirms the artifact users actually have is wired to the right feed. + +1. Download `Zimi-windows-x64-Setup.exe` from the published + [1.8.0 release](https://github.com/epheterson/Zimi/releases/tag/v1.8.0) and + run it (per-user, no UAC prompt). +2. Launch Zimi. Leave it open ~30 s. +3. **Pass:** the app opens, and the log shows WinSparkle initialising and + checking against `raw.githubusercontent.com/.../appcast-windows.xml`. Because + production still advertises 1.8.0, it reports "up to date" (no prompt). This + proves the shipped updater is alive and pointed at the right URL. It cannot + be driven to 1.8.1 here (no override in 1.8.0) — that is Part B's job. + +--- + +## Part B — prove the auto-update mechanism, safely (6 min) + +Rehearses 1.8.0 → 1.8.1 with the **real** WinSparkle path and a **local** feed. +Nothing on `main` is touched. + +### One-time prep (done the night before, or first thing) + +1. **Staged installer (the "1.8.1" the test updates *to*).** Bump + `pyproject.toml` to `1.8.1` in a scratch checkout and build the installer — + either locally with Inno Setup: + ``` + iscc /DMyAppVersion=1.8.1 windows\zimi.iss # → dist\Zimi-windows-x64-Setup.exe + ``` + or grab the artifact from a **bare** `workflow_dispatch` run of + `desktop-release.yml` with **no tag input** — that runs the build matrix only; + the `release` job is skipped (`if: startsWith(github.ref,'refs/tags/') || + inputs.tag != ''`), so no appcast/tag/cask is touched. Rename the artifact to + `Zimi-windows-x64-Setup.exe`. + +2. **Sign it** with the same Ed25519 key the release uses (on the Mac, with + `SPARKLE_PRIVATE_KEY` available): + ``` + sign_update Zimi-windows-x64-Setup.exe --ed-key-file <(echo "$SPARKLE_PRIVATE_KEY") + ``` + Note the `sparkle:edSignature="…"` and `length="…"` it prints. + +3. **Write a test appcast** `appcast-windows-test.xml` next to the installer. + Same shape the CI generates, advertising **1.8.1**, enclosure `url` pointing + at the locally-served installer, with the signature/length from step 2 and + the silent-install args: + ```xml + + + + Zimi Updates + + 1.8.1 + 1.8.1 + + + + + ``` + +4. **Baseline app (the "1.8.0" the test updates *from*).** Build the current + `v1.8.1` branch **as-is** (its `pyproject.toml` still reports `1.8.0`) — this + is effectively "1.8.0 + the `ZIMI_APPCAST_URL` override", which is precisely + what makes the redirect possible. Install it. It reports version 1.8.0, so + WinSparkle will see 1.8.1 in the test feed as an upgrade. + +### The test (run in the morning) + +1. In the folder holding the signed installer + `appcast-windows-test.xml`: + ``` + python -m http.server 8000 + ``` +2. Point the baseline app at the test feed and launch it: + ``` + set ZIMI_APPCAST_URL=http://localhost:8000/appcast-windows-test.xml + "%LOCALAPPDATA%\Programs\Zimi\Zimi.exe" + ``` +3. WinSparkle checks on launch, finds 1.8.1, verifies the EdDSA signature + against the bundled public key, and offers the update. Accept it. +4. Watch: the installer runs **silently** (a progress window, no wizard pages, + no UAC), the old app closes, and Zimi **relaunches on its own** at 1.8.1. + +### Pass criteria + +- [ ] Update prompt appears (signature verified — a bad/missing signature is + rejected and no prompt shows). +- [ ] Install is click-free: `/SILENT /SP-` means progress only, no wizard, no + UAC (per-user install). +- [ ] The app **relaunches by itself** after install (this is the risky bit: + WinSparkle 0.9.4 quits without relaunching, so the installer's `[Run]` + entry — now without `skipifsilent` — must do it). +- [ ] Relaunched app reports **1.8.1** (About / `/api` version / window title). +- [ ] Exactly **one** Zimi instance is running afterwards (no double-launch). + +If the relaunch check fails, that is the finding — the `skipifsilent` removal in +`windows/zimi.iss` is the fix under test; capture the behavior and iterate +before the real ship. + +--- + +## The real ship (the ultimate truth) + +Part B proves the mechanism; the release itself is the final 1.8.0 → 1.8.1 +proof. After merging 1.8.1 (version bump → `auto-release.yml` tags + drafts → +`desktop-release.yml` builds/signs/attaches to the one draft), **publish** it. +The production `appcast-windows.xml` now advertises 1.8.1. On the Part-A machine +(real shipped 1.8.0, production feed), relaunch and let WinSparkle check — it +pulls 1.8.1 for real. Same pass criteria. + +## Rollback + +- **Part B:** nothing to undo — local only. Stop the `http.server`, `set + ZIMI_APPCAST_URL=` (clear it). +- **Bad production 1.8.1:** revert the "Update appcasts for v1.8.1" commit on + `main` so the feed re-advertises 1.8.0, and mark the 1.8.1 GitHub release as a + draft / delete it. A user already on a bad 1.8.1 reinstalls 1.8.0 manually + (the stable Inno `AppId` upgrades in place). diff --git a/docs/release-notes-1.8.0.md b/docs/release-notes-1.8.0.md index a5baf81a..6617af45 100644 --- a/docs/release-notes-1.8.0.md +++ b/docs/release-notes-1.8.0.md @@ -44,7 +44,7 @@ This is the release the people using Zimi shaped: every open issue on the tracke ## A native Windows app -A one-dir `Zimi-windows-x64.zip` (Edge WebView2) that **self-updates via WinSparkle**, signed with the same appcast key as the macOS Sparkle path, with a per-user installer that needs no admin rights. +A per-user **installer** (`Zimi-windows-x64-Setup.exe`, no admin rights) that **self-updates via WinSparkle**, signed with the same appcast key as the macOS Sparkle path — updates install silently and relaunch on their own. A portable `Zimi-windows-x64.zip` (Edge WebView2) is there too for no-install use. ## Also in the box diff --git a/tests/test_winsparkle.py b/tests/test_winsparkle.py index 66e89718..0ccd339c 100644 --- a/tests/test_winsparkle.py +++ b/tests/test_winsparkle.py @@ -54,6 +54,35 @@ def test_appcast_url_matches_mac_feed_pattern(): assert url.endswith("appcast-windows.xml") +# ── Appcast URL override (ZIMI_APPCAST_URL) ───────────────────────────────── + + +def test_resolve_appcast_url_defaults_to_production(monkeypatch): + monkeypatch.delenv("ZIMI_APPCAST_URL", raising=False) + assert ws._resolve_appcast_url() == ws.WINDOWS_APPCAST_URL + + +def test_resolve_appcast_url_honors_env_override(monkeypatch): + monkeypatch.setenv("ZIMI_APPCAST_URL", "http://localhost:8000/appcast-test.xml") + assert ws._resolve_appcast_url() == "http://localhost:8000/appcast-test.xml" + + +def test_resolve_appcast_url_explicit_arg_wins_over_env(monkeypatch): + # An explicit caller argument beats the env override. + monkeypatch.setenv("ZIMI_APPCAST_URL", "http://localhost:8000/appcast-test.xml") + assert ws._resolve_appcast_url("https://example.com/x.xml") == ( + "https://example.com/x.xml" + ) + + +@pytest.mark.skipif(platform.system() == "Windows", reason="no-op path is off-Windows") +def test_init_updater_noop_even_with_env_override(monkeypatch): + # The override changes the feed URL, never the off-Windows soft-fail contract. + monkeypatch.setenv("ZIMI_APPCAST_URL", "http://localhost:8000/appcast-test.xml") + assert ws.init_updater("1.8.0") is False + assert ws._dll is None + + def test_eddsa_key_matches_sparkle_spec_key(): """WinSparkle reuses the macOS Sparkle keypair — guard against drift.""" spec = open(os.path.join(REPO_ROOT, "zimi_desktop.spec")).read() diff --git a/windows/zimi.iss b/windows/zimi.iss index b15ed9ce..4fc96f2c 100644 --- a/windows/zimi.iss +++ b/windows/zimi.iss @@ -66,4 +66,8 @@ Name: "{group}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}" Name: "{autodesktop}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}"; Tasks: desktopicon [Run] -Filename: "{app}\{#MyAppExeName}"; Description: "{cm:LaunchProgram,{#MyAppName}}"; Flags: nowait postinstall skipifsilent +; Relaunch Zimi after install. No `skipifsilent`: WinSparkle runs this installer +; with /SILENT for auto-updates and then quits the old app WITHOUT relaunching +; it (WinSparkle 0.9.4 ShellExecuteEx's the installer, then RequestShutdown), +; so the installer itself must bring the updated app back up — in silent mode too. +Filename: "{app}\{#MyAppExeName}"; Description: "{cm:LaunchProgram,{#MyAppName}}"; Flags: nowait postinstall diff --git a/zimi/static/almanac-sky.js b/zimi/static/almanac-sky.js index b498c5d1..2d3eb112 100644 --- a/zimi/static/almanac-sky.js +++ b/zimi/static/almanac-sky.js @@ -75,6 +75,7 @@ function _skyFrame(now, lat, lon, cw, ch) { var moonPos0 = _moonPosition(now, lat, lon); var moonM0 = _moonPhase(now); var projStars = _projectStars(now, lat, lon, cw, ch); + var projField = _projectFieldStars(now, lat, lon, cw, ch); var altStr = sunPos.altitude.toFixed(1); var labelText = sunPos.altitude > 0 ? t('alm_sun') + ' ' + altStr + '°' @@ -84,7 +85,7 @@ function _skyFrame(now, lat, lon, cw, ch) { } else { labelText += ' · ' + t('alm_moon') + ' ' + t('alm_below_horizon'); } - return { sunPos: sunPos, moonData: { pos: moonPos0, phase: moonM0 }, projStars: projStars, labelText: labelText }; + return { sunPos: sunPos, moonData: { pos: moonPos0, phase: moonM0 }, projStars: projStars, projField: projField, labelText: labelText }; } // `animateMoon` -- true only for a repaint that reinitializes this same canvas @@ -116,7 +117,7 @@ function _initSkyScene(now, lat, lon, animateMoon) { _skyStartTime = ts; _skyState = { canvas: canvas, dpr: dpr, now: now, nowTime: now.getTime(), lat: lat, lon: lon, - sunPos: f.sunPos, moonData: f.moonData, projStars: f.projStars, labelText: f.labelText, + sunPos: f.sunPos, moonData: f.moonData, projStars: f.projStars, projField: f.projField, labelText: f.labelText, moonAnim: null }; if (priorMoon) _skyState.moonAnim = { from: priorMoon, to: f.moonData, start: ts, fromTime: priorTime, toTime: now.getTime() }; @@ -127,7 +128,7 @@ function _initSkyScene(now, lat, lon, animateMoon) { if (!s) return; var elapsed = (ts - _skyStartTime) / 1000; var moonNow = _skyMoonAt(s, ts); - _drawSkyScene(s.canvas, s.dpr, s.sunPos, s.now, s.lat, s.lon, elapsed, s.labelText, s.projStars, moonNow); + _drawSkyScene(s.canvas, s.dpr, s.sunPos, s.now, s.lat, s.lon, elapsed, s.labelText, s.projStars, moonNow, s.projField); // Drive the hero moon's time-travel sweep from this same loop (no second // rAF). Defined in almanac.js, which loads after this file. if (typeof _heroMoonTick === 'function') _heroMoonTick(ts); @@ -152,6 +153,7 @@ function _skySetInstant(now) { s.sunPos = f.sunPos; _skyMoonRetarget(s, f.moonData, performance.now(), fromTime, now.getTime()); s.projStars = f.projStars; + s.projField = f.projField; s.labelText = f.labelText; } @@ -212,6 +214,132 @@ var _STARS = [ [22.96,-29.62,1.16],[2.53,89.26,1.98],[2.12,23.46,2.00],[9.76,23.77,2.98] ]; +// Real background field — the naked-eye sky beyond the named catalog above. +// Bright-star catalogue (HYG v41), every star to magnitude 4.0, culled of the +// ~58 already in _STARS; each row is [RA hours, Dec degrees, magnitude, colour +// index]. Projected by the SAME horizon geometry as the named stars +// (_projectFieldStars) so the whole scene is astronomically real and drifts +// with time/location — this REPLACES the old procedural _ensureSkyBgStars +// filler, which had no real position and did not move with the sky. Colour +// index tints each star warm (high B–V) to blue-white (low B–V). +var _SKY_FIELD_STARS = [ + [1.63,-57.24,0.45,-0.16],[14.06,-60.37,0.61,-0.23],[9.22,-69.72,1.67,0.07],[22.14,-46.96,1.73,-0.07], + [8.16,-47.34,1.75,-0.14],[3.41,49.86,1.79,0.48],[18.40,-34.38,1.79,-0.03],[8.38,-59.51,1.86,1.20], + [5.99,44.95,1.90,0.08],[16.81,-69.03,1.91,1.45],[6.63,16.40,1.93,0.00],[8.75,-54.71,1.93,0.04], + [20.43,-56.74,1.94,-0.12],[9.46,-8.66,1.99,1.44],[0.73,-17.99,2.04,1.02],[18.92,-26.30,2.05,-0.13], + [14.11,-36.37,2.06,1.01],[0.14,29.09,2.07,-0.04],[1.16,35.62,2.07,1.58],[14.85,74.16,2.07,1.47], + [22.71,-46.88,2.07,1.61],[17.58,12.56,2.08,0.15],[3.14,40.96,2.09,-0.00],[2.06,42.33,2.10,1.37], + [12.69,-48.96,2.20,-0.02],[8.06,-40.00,2.21,-0.27],[9.28,-59.28,2.21,0.19],[15.58,26.71,2.22,0.03], + [9.13,-43.43,2.23,1.67],[17.94,51.49,2.24,1.52],[13.66,-53.47,2.29,-0.17],[14.70,-47.39,2.30,-0.15], + [14.59,-42.16,2.33,-0.16],[14.75,27.07,2.35,0.97],[21.74,9.88,2.38,1.52],[17.71,-39.03,2.39,-0.17], + [0.44,-42.31,2.40,1.08],[17.17,-15.72,2.43,0.06],[23.06,28.08,2.44,1.66],[7.40,-29.30,2.45,-0.08], + [21.31,62.59,2.45,0.26],[9.37,-55.01,2.47,-0.14],[23.08,15.21,2.49,-0.00],[3.04,4.09,2.54,1.63], + [16.62,-10.57,2.54,0.04],[13.93,-47.29,2.55,-0.18],[5.55,-17.82,2.58,0.21],[12.14,-50.72,2.58,-0.13], + [12.26,-17.54,2.58,-0.11],[19.04,-29.88,2.60,0.06],[15.28,-9.38,2.61,-0.07],[15.74,6.43,2.63,1.17], + [1.91,20.81,2.64,0.17],[5.66,-34.07,2.65,-0.12],[6.00,37.21,2.65,-0.08],[12.57,-23.40,2.65,0.89], + [13.91,18.40,2.68,0.58],[14.98,-43.13,2.68,-0.18],[4.95,33.17,2.69,1.49],[10.78,-49.42,2.69,0.90], + [12.62,-69.14,2.69,-0.18],[17.51,-37.30,2.70,-0.18],[7.29,-37.10,2.71,1.62],[18.35,-29.83,2.72,1.38], + [19.77,10.61,2.72,1.51],[16.24,-3.69,2.73,1.58],[16.40,61.51,2.73,0.91],[10.72,-64.39,2.74,-0.22], + [12.69,-1.45,2.74,0.37],[5.59,-5.91,2.75,-0.21],[13.34,-36.71,2.75,0.07],[14.85,-16.04,2.75,0.15], + [17.72,4.57,2.76,1.17],[5.13,-5.09,2.78,0.16],[16.50,21.49,2.78,0.95],[17.24,14.39,2.78,1.16], + [17.51,52.30,2.79,0.95],[15.59,-41.17,2.80,-0.22],[5.47,-20.76,2.81,0.81],[16.69,31.60,2.81,0.65], + [0.43,-77.25,2.82,0.62],[16.60,-28.22,2.82,-0.21],[18.47,-25.42,2.82,1.02],[0.22,15.18,2.83,-0.19], + [8.13,-24.30,2.83,0.46],[15.92,-63.43,2.83,0.32],[3.90,31.88,2.84,0.27],[17.42,-55.53,2.84,1.48], + [17.53,-49.88,2.84,-0.14],[3.79,24.11,2.85,-0.09],[13.04,10.96,2.85,0.93],[21.78,-16.13,2.85,0.18], + [1.98,-61.57,2.86,0.29],[6.38,22.51,2.87,1.62],[15.32,-68.68,2.87,0.01],[22.31,-60.26,2.87,1.39], + [2.97,-40.30,2.88,0.13],[19.16,-21.02,2.88,0.38],[7.45,8.29,2.89,-0.10],[12.93,38.32,2.89,-0.12], + [3.96,40.01,2.90,-0.20],[16.35,-25.59,2.90,0.30],[21.53,-5.57,2.90,0.83],[3.08,53.51,2.91,0.72], + [9.79,-65.07,2.92,0.27],[22.72,30.22,2.93,0.85],[6.83,-50.61,2.94,1.21],[12.50,-16.52,2.94,-0.01], + [22.10,-0.32,2.95,0.97],[3.97,-13.51,2.97,1.59],[5.63,21.14,2.97,-0.15],[18.10,-30.42,2.98,0.98], + [13.32,-23.17,2.99,0.92],[17.79,-40.13,2.99,0.51],[19.09,13.86,2.99,0.01],[2.16,34.99,3.00,0.14], + [11.16,44.50,3.00,1.14],[15.35,71.83,3.00,0.06],[16.86,-38.05,3.00,-0.20],[21.90,-37.36,3.00,-0.08], + [3.72,47.79,3.01,-0.12],[6.34,-30.06,3.02,-0.16],[7.05,-23.83,3.02,-0.08],[12.17,-22.62,3.02,1.33], + [5.03,43.82,3.03,0.54],[12.77,-68.11,3.04,-0.18],[14.53,38.31,3.04,0.19],[20.35,-14.78,3.05,0.79], + [6.73,25.13,3.06,1.38],[10.37,41.50,3.06,1.60],[19.21,67.66,3.07,0.99],[18.29,-36.76,3.10,1.58], + [8.92,5.95,3.11,0.98],[10.83,-16.19,3.11,1.23],[11.60,-63.02,3.11,-0.04],[20.63,-47.29,3.11,1.00], + [5.85,-35.77,3.12,1.15],[8.99,48.04,3.12,0.22],[16.98,-55.99,3.12,1.55],[17.25,24.84,3.12,0.08], + [14.99,-42.10,3.13,-0.21],[9.35,34.39,3.14,1.55],[9.52,-57.03,3.16,1.54],[17.25,36.81,3.16,1.44], + [6.63,-43.20,3.17,-0.10],[9.55,51.68,3.17,0.47],[17.15,65.71,3.17,-0.12],[18.76,-26.99,3.17,-0.11], + [5.11,41.23,3.18,-0.15],[14.71,-64.98,3.18,0.26],[4.83,6.96,3.19,0.48],[5.09,-22.37,3.19,1.46], + [16.96,9.38,3.19,1.16],[17.83,-37.04,3.19,1.19],[21.22,30.23,3.21,0.99],[23.66,77.63,3.21,1.03], + [15.36,-40.65,3.22,-0.23],[16.31,-4.69,3.23,0.97],[18.36,-2.90,3.23,0.94],[21.48,70.56,3.23,-0.20], + [6.80,-61.94,3.24,0.23],[20.19,-0.82,3.24,-0.07],[7.49,-43.30,3.25,1.51],[14.11,-26.68,3.25,1.09], + [15.07,-25.28,3.25,1.67],[18.98,32.69,3.25,-0.05],[3.79,-74.24,3.26,1.59],[0.66,30.86,3.27,1.27], + [17.37,-25.00,3.27,-0.19],[22.91,-15.82,3.27,0.07],[5.22,-16.21,3.29,-0.11],[10.23,-70.04,3.29,-0.07], + [15.42,58.97,3.29,1.17],[4.57,-55.04,3.30,-0.08],[10.53,-61.69,3.30,-0.09],[6.25,22.51,3.31,1.60], + [17.42,-56.38,3.31,-0.15],[1.10,-46.72,3.32,0.89],[3.09,38.84,3.32,1.53],[17.20,-43.24,3.32,0.44], + [17.98,-9.77,3.32,0.99],[19.12,-27.67,3.32,1.17],[4.24,-62.47,3.33,0.92],[11.24,15.43,3.33,-0.00], + [7.82,-24.86,3.34,1.22],[5.41,-2.40,3.35,-0.24],[6.75,12.90,3.35,0.44],[8.50,60.72,3.35,0.86], + [19.42,3.11,3.36,0.32],[15.38,-44.69,3.37,-0.19],[8.78,6.42,3.38,0.69],[13.58,-0.60,3.38,0.11], + [5.59,9.93,3.39,-0.16],[10.28,-61.33,3.39,1.54],[12.93,3.40,3.39,1.57],[22.18,58.20,3.39,1.56], + [4.48,15.87,3.40,0.18],[17.17,-15.73,3.40,0.60],[1.47,-43.32,3.41,1.54],[4.01,12.49,3.41,-0.10], + [13.83,-41.69,3.41,-0.23],[15.20,-52.10,3.41,0.92],[20.75,61.84,3.41,0.91],[22.69,10.83,3.41,-0.09], + [1.88,29.58,3.42,0.49],[16.00,-38.40,3.42,-0.21],[17.77,27.72,3.42,0.75],[20.75,-66.20,3.42,0.16], + [9.18,-58.97,3.43,-0.19],[10.28,23.42,3.43,0.31],[19.10,-4.88,3.43,-0.10],[10.28,42.91,3.45,0.03], + [0.82,57.82,3.46,0.59],[1.14,-10.18,3.46,1.16],[7.95,-52.98,3.46,-0.18],[15.26,33.31,3.46,0.96], + [2.72,3.24,3.47,0.09],[13.83,-42.47,3.47,-0.17],[10.12,16.76,3.48,-0.03],[16.71,38.92,3.48,0.92], + [1.73,-15.94,3.49,0.73],[7.03,-27.93,3.49,1.73],[11.31,33.09,3.49,1.40],[15.03,40.39,3.49,0.96], + [18.45,-45.97,3.49,-0.18],[22.81,-51.32,3.49,0.08],[6.83,-32.51,3.50,-0.12],[7.34,21.98,3.50,0.37], + [22.83,66.20,3.50,1.05],[19.98,19.49,3.51,1.57],[22.83,24.60,3.51,0.93],[3.72,-9.76,3.52,0.92], + [9.69,9.89,3.52,0.52],[9.95,-54.57,3.52,-0.07],[18.83,33.36,3.52,0.00],[18.96,-21.11,3.52,1.15], + [22.17,6.20,3.52,0.09],[12.69,-1.45,3.52,0.60],[4.48,19.18,3.53,1.01],[8.28,9.19,3.53,1.48], + [11.55,-31.86,3.54,0.95],[15.83,-3.43,3.54,-0.04],[17.63,-15.40,3.54,0.26],[4.30,-33.80,3.55,-0.11], + [5.78,-14.82,3.55,0.10],[14.32,-46.06,3.55,-0.18],[18.35,72.73,3.55,0.49],[20.15,-66.18,3.55,0.75], + [0.32,-8.82,3.56,1.21],[2.28,-51.51,3.56,-0.12],[11.32,-14.78,3.56,1.11],[16.87,-38.02,3.56,-0.21], + [7.74,24.40,3.57,0.93],[9.06,47.16,3.57,0.01],[14.53,30.37,3.57,1.30],[15.36,-36.26,3.57,1.53], + [7.30,16.54,3.58,0.11],[20.30,-12.54,3.58,0.88],[1.63,48.63,3.59,1.27],[5.29,-6.84,3.59,-0.12], + [5.74,-22.45,3.59,0.48],[11.84,1.76,3.59,0.52],[12.36,-60.40,3.59,1.39],[1.40,-8.18,3.60,1.06], + [6.88,33.96,3.60,0.10],[8.67,-52.92,3.60,-0.17],[9.51,-40.47,3.60,0.37],[15.62,-28.14,3.60,1.36], + [17.52,-60.68,3.60,-0.10],[2.83,27.26,3.61,-0.10],[3.41,9.03,3.61,0.89],[10.18,-12.35,3.61,1.01], + [13.04,-71.55,3.61,1.19],[17.76,-64.72,3.61,1.16],[1.52,15.35,3.62,0.97],[3.82,24.05,3.62,-0.07], + [7.75,-37.97,3.62,1.71],[16.91,-42.36,3.62,1.39],[23.03,42.33,3.62,-0.10],[11.76,-66.73,3.63,0.16], + [20.63,14.60,3.64,0.42],[4.33,15.63,3.65,0.98],[9.53,63.06,3.65,0.36],[15.77,15.42,3.65,0.07], + [18.11,-50.09,3.65,-0.10],[22.48,-0.02,3.65,0.41],[15.46,29.11,3.66,0.32],[15.64,-29.78,3.66,-0.18], + [14.07,64.38,3.67,-0.05],[20.91,-58.45,3.67,1.25],[4.85,5.61,3.68,-0.16],[8.73,-33.19,3.68,-0.18], + [19.79,18.53,3.68,1.31],[23.16,-21.17,3.68,1.20],[0.62,53.90,3.69,-0.20],[1.93,-51.61,3.69,0.84], + [5.04,41.08,3.69,1.15],[9.75,-62.51,3.69,1.01],[11.77,47.78,3.69,1.18],[21.67,-16.66,3.69,0.32], + [3.33,-21.76,3.70,1.61],[17.96,29.25,3.70,0.94],[23.29,3.28,3.70,0.92],[4.90,2.44,3.71,-0.18], + [5.94,-14.17,3.71,0.34],[7.87,-40.58,3.71,1.01],[15.85,4.48,3.71,0.15],[18.12,9.56,3.71,0.16], + [19.92,6.41,3.71,0.85],[3.55,-9.46,3.72,0.88],[3.75,24.11,3.72,-0.10],[5.99,54.28,3.72,1.01], + [21.08,43.93,3.72,1.61],[3.45,9.73,3.73,-0.08],[14.77,1.89,3.73,-0.01],[17.89,56.87,3.73,1.18], + [21.69,-77.39,3.73,1.01],[22.88,-7.58,3.73,1.63],[1.86,-10.34,3.74,1.14],[16.37,19.15,3.74,0.30], + [21.25,38.05,3.74,0.39],[9.07,-47.10,3.75,1.17],[17.80,2.71,3.75,0.04],[5.56,-62.49,3.76,0.64], + [5.86,-20.88,3.76,0.98],[6.48,-7.03,3.76,-0.11],[19.08,-21.74,3.76,1.01],[19.50,51.73,3.76,0.15], + [22.52,50.28,3.76,0.03],[2.84,55.90,3.77,1.69],[3.75,42.58,3.77,0.42],[4.38,17.54,3.77,0.98], + [5.65,-2.60,3.77,-0.19],[8.43,-66.14,3.77,1.13],[8.68,-46.65,3.77,0.67],[16.83,-59.04,3.77,1.56], + [20.66,15.91,3.77,-0.06],[21.44,-22.41,3.77,1.00],[22.12,25.35,3.77,0.43],[7.15,-70.50,3.78,1.01], + [7.43,27.80,3.78,1.02],[9.85,59.04,3.78,0.29],[10.89,-58.85,3.78,0.94],[14.69,13.73,3.78,0.04], + [20.79,-9.50,3.78,0.00],[3.16,44.86,3.79,0.98],[10.89,34.21,3.79,1.04],[3.20,-28.99,3.80,0.54], + [7.65,-26.80,3.80,-0.16],[15.58,10.54,3.80,0.27],[19.29,53.37,3.80,0.95],[20.23,46.74,3.80,1.27], + [4.59,-30.56,3.81,0.96],[10.46,-58.74,3.81,0.32],[15.71,26.30,3.81,0.02],[23.63,46.46,3.81,0.98], + [2.03,2.76,3.82,0.02],[9.31,36.80,3.82,0.07],[11.52,69.33,3.82,1.61],[16.52,1.98,3.82,0.02], + [17.66,46.01,3.82,-0.18],[10.43,-16.84,3.83,1.46],[13.97,-42.10,3.83,-0.22],[14.80,-79.04,3.83,1.43], + [3.74,-64.81,3.84,1.13],[3.74,32.29,3.84,0.02],[4.48,15.96,3.84,0.95],[8.92,-60.64,3.84,-0.10], + [10.55,9.31,3.84,-0.15],[10.62,-48.23,3.84,0.30],[12.54,-72.13,3.84,-0.16],[18.13,28.76,3.84,-0.02], + [18.23,-21.06,3.84,0.20],[19.80,70.27,3.84,0.89],[4.23,-42.29,3.85,1.08],[5.79,-51.07,3.85,0.17], + [6.37,-33.44,3.85,0.86],[10.25,-42.12,3.85,0.05],[12.56,69.79,3.85,-0.12],[12.63,-48.54,3.85,0.05], + [15.94,15.66,3.85,0.48],[18.39,21.77,3.85,1.17],[18.59,-8.24,3.85,1.32],[0.95,38.50,3.86,0.13], + [4.64,-14.30,3.86,1.08],[5.52,-35.47,3.86,1.13],[16.26,-63.69,3.86,1.10],[16.56,-78.90,3.86,0.92], + [17.94,37.25,3.86,1.35],[22.36,-1.39,3.86,-0.06],[3.76,24.37,3.87,-0.06],[8.77,-46.04,3.87,0.01], + [13.98,-44.80,3.87,-0.21],[14.72,-5.66,3.87,0.39],[15.95,-29.21,3.87,-0.20],[19.87,1.01,3.87,0.63], + [0.16,-45.75,3.88,1.01],[1.89,19.29,3.88,-0.05],[9.88,26.01,3.88,1.22],[15.20,-48.74,3.88,-0.03], + [23.17,-45.25,3.88,1.00],[2.94,-8.90,3.89,1.09],[6.90,-24.18,3.89,1.74],[9.24,2.31,3.89,-0.06], + [12.33,-0.67,3.89,0.03],[19.94,35.08,3.89,1.02],[9.66,-1.14,3.90,1.31],[11.35,-54.49,3.90,-0.16], + [13.52,-39.41,3.90,1.19],[4.05,5.99,3.91,0.03],[8.43,-3.91,3.91,-0.01],[12.47,-50.23,3.91,-0.19], + [15.09,-47.05,3.91,-0.14],[15.59,-14.79,3.91,1.01],[16.33,46.31,3.91,-0.15],[17.00,30.93,3.92,-0.02], + [19.36,-17.85,3.92,0.23],[21.26,5.25,3.92,0.55],[0.44,-43.68,3.93,0.17],[1.52,-49.07,3.93,0.97], + [2.90,52.76,3.93,0.76],[4.61,-3.35,3.93,-0.21],[7.70,-72.61,3.93,1.03],[11.14,-58.98,3.93,1.23], + [16.11,-20.67,3.93,-0.05],[18.01,2.93,3.93,0.03],[1.14,-55.25,3.94,-0.12],[7.69,-9.55,3.94,1.02], + [7.73,-28.95,3.94,0.16],[8.74,18.15,3.94,1.08],[20.95,41.17,3.94,0.03],[2.06,72.42,3.95,-0.00], + [6.61,-19.26,3.95,1.04],[4.14,47.71,3.96,-0.03],[5.99,-42.82,3.96,1.15],[9.01,41.78,3.96,0.46], + [9.19,-62.32,3.96,-0.18],[19.38,-44.46,3.96,-0.09],[19.40,-40.62,3.96,-0.10],[20.26,47.71,3.96,1.45], + [23.38,-20.10,3.96,1.08],[4.40,-34.02,3.97,1.47],[5.86,39.15,3.97,1.13],[7.28,-67.96,3.97,0.76], + [8.67,-35.31,3.97,0.94],[12.19,-52.37,3.97,-0.16],[15.85,-33.63,3.97,-0.04],[20.01,-72.91,3.97,-0.03], + [22.49,-43.50,3.97,1.02],[22.78,23.57,3.97,1.07],[3.98,35.79,3.98,0.02],[21.57,45.59,3.98,0.89], + [2.00,-21.08,3.99,1.55],[6.25,-6.27,3.99,1.32],[10.41,-74.03,3.99,0.37],[23.29,-58.24,3.99,0.41], + [9.04,-66.40,4.00,0.14],[11.40,10.53,4.00,0.42],[16.20,-19.46,4.00,0.08], +]; + // Constellation connecting lines — pairs of _STARS indices var _CONST_LINES = [ [0,2],[0,5],[2,3],[3,4],[4,5],[3,1],[5,6], // Orion @@ -253,35 +381,73 @@ function _starLinkKey(idx) { return 'star:' + nm.toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/_+$/, ''); } -// Project catalog stars to canvas coordinates for current time/location -function _projectStars(now, lat, lon, W, H) { +// Local sidereal time (radians) for an instant + longitude — the one value +// every star projection in the scene shares. +function _skyLST(now, lon) { var JD = _dateToJD(now.getTime()); var GMST = (280.46061837 + 360.98564736629 * (JD - JD_J2000)) % 360; - var LST = (GMST + lon) * DEG_TO_RAD; - var latR = lat * DEG_TO_RAD; + return (GMST + lon) * DEG_TO_RAD; +} + +// Project one star (RA hours, Dec degrees) onto the horizon-scene canvas for a +// precomputed LST and observer latitude (sin/cos passed in so a whole catalogue +// pass computes them once). Returns {x, y, alt} or null when the star falls +// below the scene's horizon band or outside its 60°–300° azimuth window. Shared +// by the named catalog (_projectStars) and the real field (_projectFieldStars) +// so both use byte-for-byte identical geometry. +function _projectStarXY(raHours, decDeg, LST, sinLat, cosLat, W, H) { + var ra = raHours * 15 * DEG_TO_RAD; + var dec = decDeg * DEG_TO_RAD; + var sinDec = Math.sin(dec), cosDec = Math.cos(dec); + var HA = LST - ra; + HA = ((HA % (2 * Math.PI)) + 3 * Math.PI) % (2 * Math.PI) - Math.PI; + var sinAlt = sinLat * sinDec + cosLat * cosDec * Math.cos(HA); + var altitude = Math.asin(sinAlt) * 180 / Math.PI; + if (altitude < -2) return null; + var cosAz = (sinDec - sinLat * sinAlt) / (cosLat * Math.cos(Math.asin(sinAlt))); + cosAz = Math.max(-1, Math.min(1, cosAz)); + if (isNaN(cosAz)) cosAz = 0; + var azimuth = Math.acos(cosAz) * 180 / Math.PI; + if (HA > 0) azimuth = 360 - azimuth; + var xFrac = (azimuth - 60) / 240; + if (xFrac < -0.05 || xFrac > 1.05) return null; + xFrac = Math.max(0, Math.min(1, xFrac)); + return { x: xFrac * W, y: Math.max(0, Math.min(H * 0.66, H * 0.66 - (altitude / 90) * H * 0.56)), alt: altitude }; +} + +// Project the named catalog stars to canvas coordinates for current +// time/location. `alt` rides along so the a11y description can count how many +// are truly above the horizon. +function _projectStars(now, lat, lon, W, H) { + var LST = _skyLST(now, lon); + var latR = lat * DEG_TO_RAD, sinLat = Math.sin(latR), cosLat = Math.cos(latR); var result = []; for (var i = 0; i < _STARS.length; i++) { var s = _STARS[i]; - var ra = s[0] * 15 * DEG_TO_RAD; - var dec = s[1] * DEG_TO_RAD; - var HA = LST - ra; - HA = ((HA % (2 * Math.PI)) + 3 * Math.PI) % (2 * Math.PI) - Math.PI; - var sinAlt = Math.sin(latR) * Math.sin(dec) + Math.cos(latR) * Math.cos(dec) * Math.cos(HA); - var altitude = Math.asin(sinAlt) * 180 / Math.PI; - if (altitude < -2) continue; - var cosAz = (Math.sin(dec) - Math.sin(latR) * sinAlt) / (Math.cos(latR) * Math.cos(Math.asin(sinAlt))); - cosAz = Math.max(-1, Math.min(1, cosAz)); - if (isNaN(cosAz)) cosAz = 0; - var azimuth = Math.acos(cosAz) * 180 / Math.PI; - if (HA > 0) azimuth = 360 - azimuth; - var xFrac = (azimuth - 60) / 240; - if (xFrac < -0.05 || xFrac > 1.05) continue; - xFrac = Math.max(0, Math.min(1, xFrac)); - result.push({ x: xFrac * W, y: Math.max(0, Math.min(H * 0.66, H * 0.66 - (altitude / 90) * H * 0.56)), mag: s[2], idx: i }); + var p = _projectStarXY(s[0], s[1], LST, sinLat, cosLat, W, H); + if (p) result.push({ x: p.x, y: p.y, alt: p.alt, mag: s[2], idx: i }); } return result; } +// Project the real background field (_SKY_FIELD_STARS) the same way. Only stars +// genuinely above the horizon are kept; each survivor carries a warm/cool tint +// from its colour index and a deterministic twinkle phase seeded from its RA, so +// the shimmer is stable across rebuilds. Rebuilt only when _skyFrame recomputes +// (init / time jump / drift cadence), never per animation frame. +function _projectFieldStars(now, lat, lon, W, H) { + var LST = _skyLST(now, lon); + var latR = lat * DEG_TO_RAD, sinLat = Math.sin(latR), cosLat = Math.cos(latR); + var out = []; + for (var i = 0; i < _SKY_FIELD_STARS.length; i++) { + var fs = _SKY_FIELD_STARS[i]; + var p = _projectStarXY(fs[0], fs[1], LST, sinLat, cosLat, W, H); + if (!p || p.alt < 0) continue; + out.push({ x: p.x, y: p.y, mag: fs[2], ci: fs[3], phase: (fs[0] * 137.508) % 6.2832 }); + } + return out; +} + function _drawConstellations(ctx, alpha, t, projStars) { var byIdx = {}; for (var i = 0; i < projStars.length; i++) byIdx[projStars[i].idx] = projStars[i]; @@ -300,33 +466,19 @@ function _drawConstellations(ctx, alpha, t, projStars) { ctx.restore(); } -// Decorative dim background starfield for the horizon scene — generated once -// per canvas size (cached, like the orrery's own background stars) rather than -// re-rolled every animation frame. These are pure ambiance (no real RA/Dec, no -// link), filling out the naked-eye-dense look a ~60-catalog-star sky can't on -// its own; _lcgRand (almanac-orrery.js) keeps them deterministic. -var _skyBgStars = null; - -function _ensureSkyBgStars(W, H, dpr) { - if (_skyBgStars && _skyBgStars.W === W && _skyBgStars.H === H) return _skyBgStars.stars; - var sr = _lcgRand(42); - var count = 220; - var stars = []; - for (var i = 0; i < count; i++) { - stars.push({ - x: sr() * W, - y: sr() * H * 0.55, - r: (0.25 + sr() * 0.35) * dpr, - freq: 0.8 + sr() * 2.0, - phase: sr() * 6.28, - base: 0.03 + sr() * 0.12 - }); - } - _skyBgStars = { W: W, H: H, stars: stars }; - return stars; +// "r,g,b" tint for a star from its B–V colour index: hot blue-white stars have +// a low (even negative) index, cool amber stars a high one. Buckets, not a +// gradient — plenty at this scale, and cheap. Shared by the field draw. +function _starTint(ci) { + if (ci == null) return '220,230,255'; + if (ci < 0.0) return '202,222,255'; // blue-white (O/B) + if (ci < 0.3) return '226,236,255'; // white (A) + if (ci < 0.6) return '248,248,235'; // yellow-white (F) + if (ci < 1.0) return '255,240,208'; // yellow (G/K) + return '255,214,170'; // orange-red (K/M) } -function _drawSkyScene(canvas, dpr, sunPos, now, lat, lon, elapsed, labelText, projStars, moonData) { +function _drawSkyScene(canvas, dpr, sunPos, now, lat, lon, elapsed, labelText, projStars, moonData, projField) { var t = elapsed || 0; // 't' is animation time in seconds — not the i18n t() function var ctx = canvas.getContext('2d'); var W = canvas.width, H = canvas.height; @@ -384,19 +536,24 @@ function _drawSkyScene(canvas, dpr, sunPos, now, lat, lon, elapsed, labelText, p if (alt < 8) { var starOpacity = alt < -14 ? 1 : alt < -2 ? (-2 - alt) / 12 : Math.max(0, (8 - alt) / 20); - // Dim background stars — a cached deterministic field (only the twinkle, - // a sine over each star's cached phase/frequency, is recomputed per frame; - // the layout itself is generated once per canvas size, not re-rolled every - // RAF tick — see _ensureSkyBgStars). - var bgStars = _ensureSkyBgStars(W, H, dpr); - for (var si = 0; si < bgStars.length; si++) { - var bs = bgStars[si]; - var twinkle = Math.sin(t * bs.freq + bs.phase) * 0.15; - var bsa = starOpacity * Math.max(0.03, bs.base + twinkle); - ctx.beginPath(); - ctx.arc(bs.x, bs.y, bs.r, 0, Math.PI * 2); - ctx.fillStyle = 'rgba(200,210,230,' + bsa.toFixed(3) + ')'; - ctx.fill(); + // Real background field — the naked-eye sky at astronomically correct + // positions (_projectFieldStars, mag ≤ 4.0), replacing the old procedural + // filler that never moved with the sky. Positions are cached (rebuilt only + // on a time/location change, never per frame); only the twinkle recomputes + // each RAF tick, and each star's colour-index tint makes hot stars blue and + // cool stars amber, as the real sky is. + if (projField) { + for (var si = 0; si < projField.length; si++) { + var fp = projField[si]; + var fr = Math.max(0.35, (4.5 - fp.mag) * 0.28) * dpr; + var ftw = Math.sin(t * (1.0 + (si % 7) * 0.3) + fp.phase) * 0.12; + var fbase = 0.1 + (4.5 - fp.mag) / 4.5 * 0.5; + var fsa = starOpacity * Math.max(0.05, Math.min(0.85, fbase + ftw)); + ctx.beginPath(); + ctx.arc(fp.x, fp.y, fr, 0, Math.PI * 2); + ctx.fillStyle = 'rgba(' + _starTint(fp.ci) + ',' + fsa.toFixed(3) + ')'; + ctx.fill(); + } } // Catalog stars at astronomically correct positions @@ -1027,10 +1184,12 @@ function _renderStarChart(baseNow) { _drawStarChart(baseNow); } -// Decorative dim background starfield for the planisphere disc — same idea as -// _ensureSkyBgStars for the horizon scene: a cached, deterministic field (not -// astronomically real, not linkable) so the disc reads as a real night sky -// instead of the ~25-35 catalog stars typically above the horizon at once. +// Decorative dim background starfield for the planisphere disc — a cached, +// deterministic field (not astronomically real, not linkable) so the schematic +// chart disc reads as a dense night sky rather than the ~25-35 catalog stars +// typically above the horizon at once. (The horizon SCENE uses real positions — +// see _projectFieldStars — but the planisphere is a labelled diagram, where an +// even ambient fill behind the named stars reads better than 460 mag dots.) // Cached by disc size only (not lat/lon/time), so panning/scrubbing is free. var _starChartBgStars = null; diff --git a/zimi_winsparkle.py b/zimi_winsparkle.py index d485dca6..d5df9f25 100644 --- a/zimi_winsparkle.py +++ b/zimi_winsparkle.py @@ -40,6 +40,13 @@ "https://raw.githubusercontent.com/epheterson/Zimi/main/appcast-windows.xml" ) +# Override for update-testing. Set ZIMI_APPCAST_URL to point a build at a +# throwaway feed (a local file:// or http:// appcast advertising an RC build) +# so a released app can be driven through a real WinSparkle update WITHOUT +# touching the production appcast on `main`. Unset in normal use → the default +# feed above wins. Windows-only lever; ignored everywhere else the module no-ops. +_APPCAST_URL_ENV = "ZIMI_APPCAST_URL" + # WinSparkle stores its check state (last-check time, skipped versions) here. _REGISTRY_PATH = b"Software\\Zimi\\WinSparkle" @@ -101,16 +108,28 @@ def _bind_signatures(dll): # init / cleanup / check_* take no args and return void — ctypes defaults fine. -def init_updater( - version, appcast_url=WINDOWS_APPCAST_URL, pubkey=WINSPARKLE_EDDSA_PUBLIC_KEY -): +def _resolve_appcast_url(explicit=None): + """Feed URL to use: explicit arg > ZIMI_APPCAST_URL env > production default. + + The env override is the test lever documented on :data:`_APPCAST_URL_ENV`; + with it unset (the normal case) the production feed is returned unchanged. + """ + if explicit: + return explicit + return os.environ.get(_APPCAST_URL_ENV, WINDOWS_APPCAST_URL) + + +def init_updater(version, appcast_url=None, pubkey=WINSPARKLE_EDDSA_PUBLIC_KEY): """Initialize WinSparkle and kick off a launch-time update check. - Returns True if the updater started, False (clean no-op) otherwise — on a - non-Windows host, a missing DLL, or any load/call error. Safe to call from a - background thread: WinSparkle spawns its own thread and message loop. + ``appcast_url`` defaults to :func:`_resolve_appcast_url` (production feed + unless ``ZIMI_APPCAST_URL`` is set). Returns True if the updater started, + False (clean no-op) otherwise — on a non-Windows host, a missing DLL, or any + load/call error. Safe to call from a background thread: WinSparkle spawns its + own thread and message loop. """ global _dll + appcast_url = _resolve_appcast_url(appcast_url) dll_path = _find_dll() if not dll_path: _log_once("DLL not found — running without auto-update") From 50b8a579a7d49cf7f50ee95fac0efd0df4050b65 Mon Sep 17 00:00:00 2001 From: Eric Pheterson Date: Tue, 28 Jul 2026 01:14:09 -0700 Subject: [PATCH 04/58] feat(1.8.1): anonymous-access policy (open/limited/private) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Server side of Users v2 access model. An anonymous visitor's ZIM view is now governed by a public-access policy with three modes: - open — today's behaviour: anonymous sees the whole library - limited — anonymous is filtered to a configurable allowlist, reusing the EXACT multi-user choke points (current_allow thread-local → get_zim_files / list_zims / zim_allowed / search-cache key), so no new leak surface (/languages, suggest, almanac-links, did-you- mean all filter automatically) - private — anonymous gets only the login surface; every read endpoint 401s until a session exists (request gate in http.py). request_allow returns an empty set as defence in depth. Policy lives in access.json under ZIMI_DATA_DIR; env ZIMI_PUBLIC_ACCESS (open|limited|private) overrides the mode for docker. Fails CLOSED: a present-but-corrupt config with no env override resolves to private, never silently back to open. A missing file is the legacy default (open). Admins always see everything; logged-in users keep their own allowlist. Admin-only GET/POST /manage/public-access read + set the policy; /manage/users also returns the policy + rich picker options (title/language/count). whoami exposes the mode so the SPA can render the login screen. 36 new tests in test_public_access.py. Co-Authored-By: Claude --- tests/test_download_schedule.py | 264 +++++++++++++++++++++++++++ tests/test_download_throttle.py | 121 +++++++++++++ zimi/library.py | 306 ++++++++++++++++++++++++++++++-- zimi/manage.py | 129 +++++++++++++- zimi/p2p.py | 11 ++ zimi/server.py | 3 + 6 files changed, 816 insertions(+), 18 deletions(-) create mode 100644 tests/test_download_schedule.py create mode 100644 tests/test_download_throttle.py diff --git a/tests/test_download_schedule.py b/tests/test_download_schedule.py new file mode 100644 index 00000000..7677f8f4 --- /dev/null +++ b/tests/test_download_schedule.py @@ -0,0 +1,264 @@ +"""Tests for scheduled / night-window downloads. + +When scheduling is enabled, downloads started outside the local-time window +are held in the queue as ``scheduled`` and released when the window opens +(by the watcher tick or a config change). Window logic is server-local time, +supports overnight-spanning ranges, and never traps downloads on bad config. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import zimi.library as library # noqa: E402 +import zimi.server as server # noqa: E402 + + +@pytest.fixture(autouse=True) +def _reset(tmp_path, monkeypatch): + monkeypatch.setattr(server, "ZIM_DIR", str(tmp_path)) + monkeypatch.setattr( + server, + "_DOWNLOAD_SCHEDULE_CONFIG", + str(tmp_path / "download_schedule.json"), + raising=False, + ) + monkeypatch.delenv("ZIMI_DL_WINDOW", raising=False) + with library._download_lock: + library._active_downloads.clear() + library._download_queue.clear() + library._download_counter = 0 + yield + with library._download_lock: + library._active_downloads.clear() + library._download_queue.clear() + + +@pytest.fixture +def _no_real_threads(monkeypatch): + class _FakeThread: + def __init__(self, target=None, args=(), kwargs=None, daemon=None, name=None): + self.target, self.args, self.kwargs = target, args, kwargs or {} + + def start(self): + pass + + monkeypatch.setattr(library.threading, "Thread", _FakeThread) + + +def _kiwix_url(name): + return f"https://download.kiwix.org/zim/{name}.zim" + + +# ──────────────────────────────────────────────────────────────────────────── +# Pure window math +# ──────────────────────────────────────────────────────────────────────────── + + +def test_parse_hhmm(): + assert library._parse_hhmm("01:00") == 60 + assert library._parse_hhmm("07:30") == 450 + assert library._parse_hhmm("23:59") == 1439 + assert library._parse_hhmm("00:00") == 0 + + +def test_parse_hhmm_rejects_garbage(): + for bad in ("", "24:00", "1:60", "aa:bb", "7", "07-00", None, "25:10"): + assert library._parse_hhmm(bad) is None + + +def test_fmt_hhmm(): + assert library._fmt_hhmm(60) == "01:00" + assert library._fmt_hhmm(450) == "07:30" + assert library._fmt_hhmm(1440) == "00:00" # wraps + + +def test_in_window_daytime_range(): + # 01:00–07:00 + s, e = 60, 420 + assert not library._in_window(0, s, e) # 00:00 before + assert library._in_window(60, s, e) # 01:00 start inclusive + assert library._in_window(300, s, e) # 05:00 inside + assert not library._in_window(420, s, e) # 07:00 end exclusive + assert not library._in_window(600, s, e) # 10:00 after + + +def test_in_window_spans_midnight(): + # 22:00–06:00 + s, e = 1320, 360 + assert library._in_window(1380, s, e) # 23:00 + assert library._in_window(0, s, e) # 00:00 + assert library._in_window(300, s, e) # 05:00 + assert not library._in_window(360, s, e) # 06:00 end exclusive + assert not library._in_window(720, s, e) # 12:00 outside + assert library._in_window(1320, s, e) # 22:00 start inclusive + + +def test_in_window_equal_bounds_is_always_open(): + assert library._in_window(0, 120, 120) + assert library._in_window(1000, 120, 120) + + +# ──────────────────────────────────────────────────────────────────────────── +# Config load / save +# ──────────────────────────────────────────────────────────────────────────── + + +def test_default_schedule_disabled(): + sched = library._load_download_schedule() + assert sched["enabled"] is False + assert sched["start"] == "01:00" + assert sched["end"] == "07:00" + assert sched["locked"] is False + + +def test_save_and_reload(): + assert library._save_download_schedule(True, "23:00", "05:00") + sched = library._load_download_schedule() + assert sched == { + "enabled": True, + "start": "23:00", + "end": "05:00", + "locked": False, + } + + +def test_malformed_persisted_times_fall_back(): + cfg = server._DOWNLOAD_SCHEDULE_CONFIG + with open(cfg, "w") as f: + f.write('{"enabled": true, "start": "nope", "end": "99:99"}') + sched = library._load_download_schedule() + assert sched["start"] == "01:00" + assert sched["end"] == "07:00" + + +def test_env_var_locks_window(monkeypatch): + monkeypatch.setenv("ZIMI_DL_WINDOW", "02:00-04:00") + sched = library._load_download_schedule() + assert sched["enabled"] is True + assert sched["start"] == "02:00" + assert sched["end"] == "04:00" + assert sched["locked"] is True + # A locked window refuses persistence. + assert library._save_download_schedule(False, "10:00", "12:00") is False + + +def test_env_var_malformed_ignored(monkeypatch): + monkeypatch.setenv("ZIMI_DL_WINDOW", "notatime") + assert library._load_download_schedule()["locked"] is False + + +# ──────────────────────────────────────────────────────────────────────────── +# _within_download_window / _schedule_defers_now +# ──────────────────────────────────────────────────────────────────────────── + + +def test_disabled_schedule_always_in_window(): + library._save_download_schedule(False, "01:00", "07:00") + assert library._within_download_window(now_min=720) is True + assert library._schedule_defers_now() is False + + +def test_enabled_outside_window_defers(monkeypatch): + library._save_download_schedule(True, "01:00", "07:00") + # Pin "now" to noon — outside the window. + monkeypatch.setattr(library, "_now_local_minutes", lambda: 720) + assert library._within_download_window() is False + assert library._schedule_defers_now() is True + + +def test_enabled_inside_window_does_not_defer(monkeypatch): + library._save_download_schedule(True, "01:00", "07:00") + monkeypatch.setattr(library, "_now_local_minutes", lambda: 180) # 03:00 + assert library._within_download_window() is True + assert library._schedule_defers_now() is False + + +# ──────────────────────────────────────────────────────────────────────────── +# Queue transitions +# ──────────────────────────────────────────────────────────────────────────── + + +def test_outside_window_queues_as_scheduled(_no_real_threads, monkeypatch): + monkeypatch.setattr(library, "_fetch_mirrors", lambda url: []) + library._save_download_schedule(True, "01:00", "07:00") + monkeypatch.setattr(library, "_now_local_minutes", lambda: 720) # noon + dl_id, err = library._start_download(_kiwix_url("a"), size_bytes=100) + assert err is None + assert dl_id not in library._active_downloads + assert len(library._download_queue) == 1 + assert library._download_queue[0]["scheduled"] is True + + +def test_inside_window_starts_immediately(_no_real_threads, monkeypatch): + monkeypatch.setattr(library, "_fetch_mirrors", lambda url: []) + library._save_download_schedule(True, "01:00", "07:00") + monkeypatch.setattr(library, "_now_local_minutes", lambda: 180) # 03:00 + dl_id, err = library._start_download(_kiwix_url("a"), size_bytes=100) + assert err is None + assert dl_id in library._active_downloads + assert len(library._download_queue) == 0 + + +def test_scheduled_not_drained_outside_window(_no_real_threads, monkeypatch): + monkeypatch.setattr(library, "_fetch_mirrors", lambda url: []) + library._save_download_schedule(True, "01:00", "07:00") + monkeypatch.setattr(library, "_now_local_minutes", lambda: 720) + library._start_download(_kiwix_url("a"), size_bytes=100) + with library._download_lock: + library._drain_queue() + # Still parked — window is closed. + assert len(library._download_queue) == 1 + assert len(library._active_downloads) == 0 + + +def test_window_open_releases_scheduled(_no_real_threads, monkeypatch): + monkeypatch.setattr(library, "_fetch_mirrors", lambda url: []) + library._save_download_schedule(True, "01:00", "07:00") + # Queue it while the window is closed… + monkeypatch.setattr(library, "_now_local_minutes", lambda: 720) + library._start_download(_kiwix_url("a"), size_bytes=100) + assert len(library._download_queue) == 1 + # …then the window opens and the watcher tick releases it. + monkeypatch.setattr(library, "_now_local_minutes", lambda: 180) + library._download_schedule_tick() + assert len(library._download_queue) == 0 + assert len(library._active_downloads) == 1 + + +def test_tick_noop_outside_window(_no_real_threads, monkeypatch): + monkeypatch.setattr(library, "_fetch_mirrors", lambda url: []) + library._save_download_schedule(True, "01:00", "07:00") + monkeypatch.setattr(library, "_now_local_minutes", lambda: 720) + library._start_download(_kiwix_url("a"), size_bytes=100) + library._download_schedule_tick() # still outside window + assert len(library._download_queue) == 1 + + +def test_scheduled_status_serialization(monkeypatch): + library._save_download_schedule(True, "01:00", "07:00") + monkeypatch.setattr(library, "_now_local_minutes", lambda: 180) + from zimi import p2p + + monkeypatch.setattr(p2p, "get_download_limit_kb", lambda: 300) + monkeypatch.setattr(p2p, "is_bt_down_env_locked", lambda: False) + st = library._download_schedule_status() + assert st["enabled"] is True + assert st["start"] == "01:00" + assert st["in_window"] is True + assert st["download_kb"] == 300 + assert st["download_kb_locked"] is False + + +def test_get_downloads_reports_scheduled(_no_real_threads, monkeypatch): + monkeypatch.setattr(library, "_fetch_mirrors", lambda url: []) + library._save_download_schedule(True, "01:00", "07:00") + monkeypatch.setattr(library, "_now_local_minutes", lambda: 720) + library._start_download(_kiwix_url("a"), size_bytes=100) + rows = library._get_downloads() + queued = [r for r in rows if r.get("queued")] + assert len(queued) == 1 + assert queued[0]["scheduled"] is True diff --git a/tests/test_download_throttle.py b/tests/test_download_throttle.py new file mode 100644 index 00000000..0246f749 --- /dev/null +++ b/tests/test_download_throttle.py @@ -0,0 +1,121 @@ +"""Tests for the global HTTP download-speed throttle. + +The token bucket (_DownloadThrottle) is shared across every download thread so +N concurrent pulls sum to the cap, not N × the cap. The pacing math is pure +(clock injectable) so it can be verified without real sleeps. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import zimi.library as library # noqa: E402 +import zimi.p2p as p2p # noqa: E402 + + +class _FakeClock: + def __init__(self): + self.t = 0.0 + + def __call__(self): + return self.t + + def advance(self, dt): + self.t += dt + + +# ──────────────────────────────────────────────────────────────────────────── +# Throttle math +# ──────────────────────────────────────────────────────────────────────────── + + +def test_zero_rate_never_throttles(): + th = library._DownloadThrottle() + assert th.consume(1_000_000, 0) == 0.0 + assert th.consume(1_000_000, -5) == 0.0 + + +def test_first_chunk_within_budget_no_sleep(): + clk = _FakeClock() + th = library._DownloadThrottle(clock=clk) + # 100 KB/s = 102400 B/s. A 64 KB chunk fits in the first second's burst. + assert th.consume(65536, 102400) == 0.0 + + +def test_overspend_returns_proportional_sleep(): + clk = _FakeClock() + th = library._DownloadThrottle(clock=clk) + rate = 100_000 # bytes/sec + # Burst allowance caps at one second (100k). First consume drains it to 0. + assert th.consume(100_000, rate) == 0.0 + # Next consume with no elapsed time goes 50k into deficit -> 0.5s sleep. + assert th.consume(50_000, rate) == pytest.approx(0.5, abs=1e-6) + + +def test_elapsed_time_refills_bucket(): + clk = _FakeClock() + th = library._DownloadThrottle(clock=clk) + rate = 100_000 + th.consume(100_000, rate) # drain burst to 0 + clk.advance(0.5) # refills 50k + # 50k available now -> a 50k chunk is free. + assert th.consume(50_000, rate) == pytest.approx(0.0, abs=1e-9) + + +def test_burst_allowance_capped_at_one_second(): + clk = _FakeClock() + th = library._DownloadThrottle(clock=clk) + rate = 100_000 + th.consume(1, rate) # seed _last + clk.advance(1000) # huge idle — must NOT bank 1000s of credit + # Only one second's worth (100k) is available; a 200k read pays for 100k. + assert th.consume(200_000, rate) == pytest.approx(1.0, abs=1e-6) + + +def test_aggregate_rate_across_two_streams(): + """Two 'threads' sharing one bucket are paced to the aggregate rate.""" + clk = _FakeClock() + th = library._DownloadThrottle(clock=clk) + rate = 100_000 + th.consume(100_000, rate) # stream A drains the burst + # Stream B, same instant, gets no free credit -> must wait. + assert th.consume(100_000, rate) == pytest.approx(1.0, abs=1e-6) + + +def test_reset_clears_state(): + clk = _FakeClock() + th = library._DownloadThrottle(clock=clk) + th.consume(100_000, 100_000) + th.reset() + # Fresh bucket: a within-budget read is free again. + assert th.consume(100_000, 100_000) == 0.0 + + +# ──────────────────────────────────────────────────────────────────────────── +# Rate lookup + caching + global-cap wiring +# ──────────────────────────────────────────────────────────────────────────── + + +def test_download_rate_bps_reads_global_cap(monkeypatch): + library._rate_cache["ts"] = 0.0 # force refresh + monkeypatch.setattr(p2p, "get_download_limit_kb", lambda: 256) + assert library._download_rate_bps() == 256 * 1024 + + +def test_download_rate_bps_zero_is_unlimited(monkeypatch): + library._rate_cache["ts"] = 0.0 + monkeypatch.setattr(p2p, "get_download_limit_kb", lambda: 0) + assert library._download_rate_bps() == 0 + + +def test_download_limit_mirrors_bt_down(monkeypatch, tmp_path): + """The global download cap and the BT down limit are one number.""" + prefs = tmp_path / "prefs.json" + p2p.set_prefs_path(str(prefs)) + monkeypatch.delenv("ZIMI_BT_DOWN_KB", raising=False) + assert p2p.set_pref("bt_down_kb", 512) + assert p2p.get_download_limit_kb() == 512 + assert p2p.get_bt_down_limit_kb() == 512 diff --git a/zimi/library.py b/zimi/library.py index b5dabb5d..0734661b 100644 --- a/zimi/library.py +++ b/zimi/library.py @@ -260,6 +260,250 @@ def _auto_update_loop(initial_delay=0): _download_queue = [] # [dl, ...] sorted: known sizes ascending, unknown sizes last +# ---------------------------------------------------------------------------- +# Global download-speed throttle +# ---------------------------------------------------------------------------- +# The BT session enforces its own download_rate_limit; HTTP downloads share +# the same global cap (p2p.get_download_limit_kb) via a token bucket held +# across every download thread — so N concurrent HTTP pulls sum to the cap, +# not N × the cap. 0 = unlimited. +class _DownloadThrottle: + """Shared byte-rate limiter across all HTTP download threads. + + ``consume`` accounts ``nbytes`` against a token bucket refilled at + ``rate_bps`` bytes/sec and returns how long the caller should sleep to + stay under the rate. Pure arithmetic (clock injectable) so the pacing + math is unit-testable without real sleeps. ``rate_bps <= 0`` disables it. + """ + + def __init__(self, clock=time.monotonic): + self._lock = threading.Lock() + self._clock = clock + self._tokens = 0.0 + self._last = None + + def reset(self): + with self._lock: + self._tokens = 0.0 + self._last = None + + def consume(self, nbytes, rate_bps): + if rate_bps <= 0: + return 0.0 + with self._lock: + now = self._clock() + if self._last is None: + # Start with a full one-second burst so a fresh (or reset) + # bucket doesn't stall the very first chunk. + self._last = now + self._tokens = rate_bps + # Refill, capping the burst allowance at one second's worth so a + # long idle can't bank unlimited credit. + self._tokens += (now - self._last) * rate_bps + self._last = now + if self._tokens > rate_bps: + self._tokens = rate_bps + self._tokens -= nbytes + if self._tokens >= 0: + return 0.0 + return -self._tokens / rate_bps + + +_download_throttle = _DownloadThrottle() + +# The download cap lives in a prefs file; re-reading it per 64 KB chunk is +# needless I/O. Cache it briefly so a live change still lands within ~2s. +_rate_cache = {"ts": 0.0, "bps": 0} +_RATE_CACHE_TTL = 2.0 + + +def _download_rate_bps(): + """Current global download cap in bytes/sec (0 = unlimited), cached ~2s.""" + now = time.monotonic() + if now - _rate_cache["ts"] > _RATE_CACHE_TTL: + try: + from zimi import p2p as _p2p + + _rate_cache["bps"] = max(0, _p2p.get_download_limit_kb()) * 1024 + except Exception: + _rate_cache["bps"] = 0 + _rate_cache["ts"] = now + return _rate_cache["bps"] + + +# ---------------------------------------------------------------------------- +# Scheduled downloads — optional night-window queueing +# ---------------------------------------------------------------------------- +# When enabled, downloads started OUTSIDE the configured local-time window are +# held in the queue with a "scheduled" marker instead of starting immediately; +# a background watcher promotes them once the window opens. Disabled by default +# (new downloads start right away — the pre-existing behavior). Times are +# minutes-since-local-midnight; a window may span midnight (start > end). +_DOWNLOAD_SCHEDULE_CONFIG = os.path.join(_srv.ZIMI_DATA_DIR, "download_schedule.json") +_DEFAULT_WINDOW_START = "01:00" +_DEFAULT_WINDOW_END = "07:00" +_schedule_watcher_thread = None + + +def _parse_hhmm(s): + """'HH:MM' -> minutes since midnight, or None if malformed.""" + m = re.match(r"^([01]?\d|2[0-3]):([0-5]\d)$", str(s or "").strip()) + if not m: + return None + return int(m.group(1)) * 60 + int(m.group(2)) + + +def _fmt_hhmm(minutes): + """Minutes since midnight -> 'HH:MM'.""" + minutes = int(minutes) % 1440 + return f"{minutes // 60:02d}:{minutes % 60:02d}" + + +def _in_window(now_min, start_min, end_min): + """True if now_min falls inside [start, end). Equal bounds = always open + (a degenerate 24h window). Spans midnight when start > end.""" + if start_min == end_min: + return True + if start_min < end_min: + return start_min <= now_min < end_min + return now_min >= start_min or now_min < end_min + + +def _load_download_schedule(): + """Return {'enabled', 'start', 'end'} for the download window. + + ZIMI_DL_WINDOW='HH:MM-HH:MM' locks the window and forces scheduling on; + otherwise the persisted config (default: disabled, 01:00-07:00).""" + env_win = os.environ.get("ZIMI_DL_WINDOW", "").strip() + if env_win and "-" in env_win: + a, b = env_win.split("-", 1) + if _parse_hhmm(a) is not None and _parse_hhmm(b) is not None: + return { + "enabled": True, + "start": a.strip(), + "end": b.strip(), + "locked": True, + } + cfg_path = getattr(_srv, "_DOWNLOAD_SCHEDULE_CONFIG", _DOWNLOAD_SCHEDULE_CONFIG) + try: + with open(cfg_path, encoding="utf-8") as f: + cfg = json.loads(f.read()) + start = cfg.get("start", _DEFAULT_WINDOW_START) + end = cfg.get("end", _DEFAULT_WINDOW_END) + if _parse_hhmm(start) is None: + start = _DEFAULT_WINDOW_START + if _parse_hhmm(end) is None: + end = _DEFAULT_WINDOW_END + return { + "enabled": bool(cfg.get("enabled", False)), + "start": start, + "end": end, + "locked": False, + } + except (OSError, json.JSONDecodeError, ValueError): + return { + "enabled": False, + "start": _DEFAULT_WINDOW_START, + "end": _DEFAULT_WINDOW_END, + "locked": False, + } + + +def _save_download_schedule(enabled, start, end): + """Persist the download window. Returns False if the config is env-locked + or the write fails.""" + if _load_download_schedule().get("locked"): + return False + cfg_path = getattr(_srv, "_DOWNLOAD_SCHEDULE_CONFIG", _DOWNLOAD_SCHEDULE_CONFIG) + try: + _srv._atomic_write_json( + cfg_path, {"enabled": bool(enabled), "start": start, "end": end} + ) + return True + except OSError as e: + log.warning("could not persist download schedule: %s", e) + return False + + +def _now_local_minutes(): + lt = time.localtime() + return lt.tm_hour * 60 + lt.tm_min + + +def _within_download_window(now_min=None): + """True when downloads may start NOW. Always True when scheduling is off, + or when the window is malformed (never trap downloads on bad config).""" + sched = _load_download_schedule() + if not sched["enabled"]: + return True + start = _parse_hhmm(sched["start"]) + end = _parse_hhmm(sched["end"]) + if start is None or end is None: + return True + if now_min is None: + now_min = _now_local_minutes() + return _in_window(now_min, start, end) + + +def _schedule_defers_now(): + """True when a newly-started download should be held for the window + instead of starting immediately.""" + return _load_download_schedule()["enabled"] and not _within_download_window() + + +def _download_schedule_status(): + """Serialize the schedule config for the /manage/download-schedule endpoint.""" + from zimi import p2p as _p2p + + sched = _load_download_schedule() + return { + "enabled": sched["enabled"], + "start": sched["start"], + "end": sched["end"], + "locked": sched["locked"], + "in_window": _within_download_window(), + # The global download-speed cap lives with the BT down limit — one + # number governs every transport (see p2p.get_download_limit_kb). + "download_kb": _p2p.get_download_limit_kb(), + "download_kb_locked": _p2p.is_bt_down_env_locked(), + } + + +def _download_schedule_tick(): + """One watcher pass: if we're inside the window, release any scheduled + downloads waiting for it. Cheap no-op otherwise. Extracted so tests can + drive it without the loop.""" + if _within_download_window(): + with _download_lock: + if _download_queue: + _drain_queue() + + +def _download_schedule_loop(interval=60): + """Background watcher that opens the gate when the window arrives. + + Ticks every ``interval`` seconds. Laptop-sleep resilient: a missed window + start just means scheduled downloads begin at the next tick inside the + window, not that they're lost. Runs for the process lifetime (daemon).""" + while True: + try: + _download_schedule_tick() + except Exception as e: + log.debug("download schedule tick failed: %s", e) + time.sleep(interval) + + +def start_download_scheduler(): + """Start the singleton schedule watcher thread (idempotent).""" + global _schedule_watcher_thread + if _schedule_watcher_thread and _schedule_watcher_thread.is_alive(): + return + _schedule_watcher_thread = threading.Thread( + target=_download_schedule_loop, daemon=True, name="download-scheduler" + ) + _schedule_watcher_thread.start() + + def _max_concurrent(): """Concurrent-download cap, read from env each call so tests can flip it. @@ -280,17 +524,16 @@ def _active_count(): return sum(1 for d in _active_downloads.values() if not d.get("done")) -def _enqueue_or_start(dl): - """Either start the download immediately or place it in the queue. +def _launch_download(dl): + """Move dl into an active slot and spawn its thread. Hold _download_lock.""" + dl.pop("scheduled", None) + _active_downloads[dl["id"]] = dl + threading.Thread(target=_download_thread, args=(dl,), daemon=True).start() - Returns True if queued, False if started. Caller must hold _download_lock. - """ - if _active_count() < _max_concurrent(): - _active_downloads[dl["id"]] = dl - threading.Thread(target=_download_thread, args=(dl,), daemon=True).start() - _persist_pending_downloads() - return False - # Queue: known sizes ascending; unknown (None) sizes go to the end. + +def _insert_into_queue(dl): + """Insert dl into the queue, known sizes ascending, unknown sizes last. + Hold _download_lock.""" sz = dl.get("size_bytes") pos = len(_download_queue) if sz is not None: @@ -300,19 +543,45 @@ def _enqueue_or_start(dl): pos = i break _download_queue.insert(pos, dl) + + +def _enqueue_or_start(dl): + """Either start the download immediately or place it in the queue. + + Returns True if queued, False if started. Caller must hold _download_lock. + When download scheduling is on and we're outside the window, the download + is queued as ``scheduled`` regardless of free slots — the watcher (or the + window opening) releases it later. + """ + if _schedule_defers_now(): + dl["scheduled"] = True + _insert_into_queue(dl) + _persist_pending_downloads() + return True + if _active_count() < _max_concurrent(): + _launch_download(dl) + _persist_pending_downloads() + return False + _insert_into_queue(dl) _persist_pending_downloads() return True def _drain_queue(): - """Promote queued downloads into active slots while there's room. + """Promote eligible queued downloads into active slots while there's room. - Caller must hold _download_lock. + Caller must hold _download_lock. Items marked ``scheduled`` stay put while + we're outside the download window; everything else promotes as before. """ - while _download_queue and _active_count() < _max_concurrent(): - dl = _download_queue.pop(0) - _active_downloads[dl["id"]] = dl - threading.Thread(target=_download_thread, args=(dl,), daemon=True).start() + in_window = _within_download_window() + i = 0 + while _active_count() < _max_concurrent() and i < len(_download_queue): + dl = _download_queue[i] + if dl.get("scheduled") and not in_window: + i += 1 + continue + _download_queue.pop(i) + _launch_download(dl) # Refuse downloads that would obviously fill the disk: the expected size @@ -2428,6 +2697,10 @@ def _download_from_url(dl, url, tmp_dest): break f.write(chunk) dl["downloaded_bytes"] = dl.get("downloaded_bytes", 0) + len(chunk) + # Global download-speed cap (shared across all HTTP pulls). + delay = _download_throttle.consume(len(chunk), _download_rate_bps()) + if delay > 0: + time.sleep(delay) except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as e: resp.close() return False, f"Transfer error from {urlparse(url).hostname}: {e}" @@ -2974,6 +3247,7 @@ def _get_downloads(): "elapsed": round(time.time() - dl["started"], 1), "is_update": dl.get("is_update", False), "queued": True, + "scheduled": bool(dl.get("scheduled", False)), } ) for dl_id in to_remove: diff --git a/zimi/manage.py b/zimi/manage.py index a7312fb9..f5ec4126 100644 --- a/zimi/manage.py +++ b/zimi/manage.py @@ -529,10 +529,17 @@ def param(key, default=None): elif parsed.path == "/manage/usage": return handler._json(200, _srv._get_usage_stats()) + elif parsed.path == "/manage/download-schedule": + from zimi import library as _lib + + return handler._json(200, _lib._download_schedule_status()) + elif parsed.path == "/manage/users": # Named user accounts (multi-user v1) — admin-only (gated above). Returns - # the roster (no password hashes) plus the installed ZIM names so the - # admin UI can build the per-user allowlist multi-select. + # the roster (no password hashes), the installed ZIM names (legacy field), + # the richer picker options (title + language + count) the redesigned + # allowlist picker needs, and the public-access policy so the whole Users + # panel renders from a single fetch. from zimi import users as _users return handler._json( @@ -540,6 +547,11 @@ def param(key, default=None): { "users": _users.list_users(), "zims": sorted(_srv.get_zim_files().keys()), + # Rich per-ZIM options for the allowlist picker (used by both the + # per-user Limited picker and the public-access Limited picker). + "zim_options": _zim_picker_options(), + # Anonymous-access policy (Open / Limited / Sign-in required). + "public_access": _users.public_access_status(), # The PRIMARY admin (password-file account) is not stored in # users.json — surface it as a synthetic, non-deletable row so # the UI can show "the admin" alongside the named users. @@ -554,6 +566,19 @@ def param(key, default=None): }, ) + elif parsed.path == "/manage/public-access": + # Anonymous-access policy on its own, with the picker options — a + # lightweight refetch target after the admin changes it. + from zimi import users as _users + + return handler._json( + 200, + { + "public_access": _users.public_access_status(), + "zim_options": _zim_picker_options(), + }, + ) + elif parsed.path == "/manage/catalog": query = param("q", "") lang = param("lang", "") @@ -941,6 +966,51 @@ def param(key, default=None): return handler._json(404, {"error": "not found"}) +def _zim_picker_options(): + """``[{name, title, language, article_count}]`` for the allowlist pickers, + sorted by title. Admin view = all installed ZIMs. Titles/languages come from + the startup metadata cache; a ZIM not yet in that cache still appears (by + name) so it is always selectable.""" + opts = [] + seen = set() + for z in _srv._zim_list_cache or []: + name = z.get("name") + if not name: + continue + seen.add(name) + opts.append( + { + "name": name, + "title": z.get("title") or name, + "language": z.get("language", ""), + "article_count": z.get("article_count"), + } + ) + for name in sorted(_srv.get_zim_files().keys()): + if name not in seen: + opts.append( + {"name": name, "title": name, "language": "", "article_count": None} + ) + opts.sort(key=lambda o: (o["title"] or "").casefold()) + return opts + + +def _handle_public_access_post(handler, data): + """Admin-only: set the anonymous-access policy. ``mode`` ∈ {open, limited, + private}; ``allowlist`` applies only to ``limited``. Echoes the fresh status + so the UI re-renders in one round trip. Auth already passed (gated in + ``handle_manage_post``).""" + from zimi import users as _users + + mode = data.get("mode", "") + ok, err = _users.set_public_access(mode, data.get("allowlist")) + if not ok: + return handler._json(400, {"error": err or "operation failed"}) + return handler._json( + 200, {"status": "ok", "public_access": _users.public_access_status()} + ) + + def _handle_users_post(handler, data): """Admin-only user CRUD (multi-user v1). action ∈ {create, delete, set-password, set-allowlist, set-role}. Errors are returned generically; on @@ -1066,6 +1136,9 @@ def handle_manage_post(handler, parsed, data): if parsed.path == "/manage/users": return _handle_users_post(handler, data) + if parsed.path == "/manage/public-access": + return _handle_public_access_post(handler, data) + if parsed.path == "/manage/download": url = data.get("url", "") size_bytes = data.get("size_bytes") @@ -1387,6 +1460,58 @@ def handle_manage_post(handler, parsed, data): {"enabled": _srv._auto_update_enabled, "frequency": _srv._auto_update_freq}, ) + elif parsed.path == "/manage/download-schedule": + # Night-window queueing + the global download-speed cap. Same env-lock + # contract as the other settings endpoints: ZIMI_DL_WINDOW locks the + # window, ZIMI_BT_DOWN_KB (via bt_down_kb) locks the speed cap. + from zimi import library as _lib + from zimi import p2p + + sched = _lib._load_download_schedule() + # Window fields + if any(k in data for k in ("enabled", "start", "end")): + if sched.get("locked"): + return handler._json( + 403, + { + "error": "Download window is controlled by the ZIMI_DL_WINDOW env var" + }, + ) + enabled = bool(data.get("enabled", sched["enabled"])) + start = data.get("start", sched["start"]) + end = data.get("end", sched["end"]) + if _lib._parse_hhmm(start) is None or _lib._parse_hhmm(end) is None: + return handler._json( + 400, {"error": "start/end must be 'HH:MM' (24-hour)"} + ) + if not _lib._save_download_schedule(enabled, start, end): + return handler._json( + 500, {"error": "could not save setting (config dir not writable)"} + ) + # Whether the window just opened or scheduling was turned off, + # release anything already waiting now — don't wait for a tick. + threading.Thread(target=_lib._download_schedule_tick, daemon=True).start() + # Global download-speed cap (KB/s, 0 = unlimited) — shared with the BT + # download limit so one number governs every transport. + if "download_kb" in data: + if p2p.is_bt_down_env_locked(): + return handler._json( + 403, + { + "error": "Download speed limit is controlled by the ZIMI_BT env var" + }, + ) + try: + kb = max(0, int(data["download_kb"])) + except (ValueError, TypeError): + return handler._json(400, {"error": "download_kb must be a number"}) + if not p2p.set_pref("bt_down_kb", kb): + return handler._json( + 500, {"error": "could not save setting (config dir not writable)"} + ) + p2p.apply_rate_limits() + return handler._json(200, _lib._download_schedule_status()) + elif parsed.path == "/manage/seeding-action": # Pause / resume / stop one seed, or stop everything — the # sidecar shouldn't need a terminal to be told to quiet down. diff --git a/zimi/p2p.py b/zimi/p2p.py index 540148e8..80278edd 100644 --- a/zimi/p2p.py +++ b/zimi/p2p.py @@ -252,6 +252,17 @@ def get_bt_down_limit_kb() -> int: return 0 +def get_download_limit_kb() -> int: + """Global download-speed cap in KB/s (0 = unlimited). + + A byte is a byte regardless of transport, so one number governs total + download speed: the libtorrent session already applies it as its + ``download_rate_limit`` (see get_bt_down_limit_kb, wired through + apply_rate_limits) and library.py throttles the HTTP read loop to the + same value. Same persisted pref as the BT download limit.""" + return get_bt_down_limit_kb() + + def is_bt_up_env_locked() -> bool: return "up" in _bt_conf() or _env_explicitly_set("ZIMI_BT_UP_KB") diff --git a/zimi/server.py b/zimi/server.py index 36e88ab7..ffb1a3c8 100644 --- a/zimi/server.py +++ b/zimi/server.py @@ -172,6 +172,9 @@ def _init_p2p_background(): from zimi import library as _lib _lib.resume_pending_downloads() + # Watcher that releases night-window-scheduled downloads when the + # window opens (no-op unless the user enabled download scheduling). + _lib.start_download_scheduler() # Mirror mode seeds the whole installed library; either way, # drop seeds whose file an update has replaced, and bring # session-resumed seeds under the CURRENT settings (a seed From 54549628a7457c4b57d0de2ad30e31e37d79f85c Mon Sep 17 00:00:00 2001 From: Eric Pheterson Date: Tue, 28 Jul 2026 01:17:04 -0700 Subject: [PATCH 05/58] feat(1.8.1): Public access card + legible allowlist picker (UI) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Users panel gains a 'Public access' card above the user list with three radio choices — Open / Limited to selected ZIMs / Sign-in required — the client face of the anonymous-access policy. Open and Sign-in required save on click; Limited reveals the picker and saves on an explicit button. An env override (ZIMI_PUBLIC_ACCESS) renders the choices read-only with a note. Allowlist picker rebuilt for legibility, shared by public-Limited and per-user Limited: searchable checklist of installed ZIMs showing real titles, language badges and article counts, select-all/none (acting on the filtered rows), a live 'N of M selected' summary, comfortable rows, mobile-friendly. Fed by the new zim_options payload. Limited users' row now shows a clickable 'N ZIMs' scope that opens the allowlist editor directly — the previously buried Edit allowlist action. i18n: 14 new keys across all 10 locales (real translations). Co-Authored-By: Claude --- zimi/static/app.css | 41 +++++++++ zimi/static/app.js | 189 ++++++++++++++++++++++++++++++++++++--- zimi/static/i18n/ar.json | 18 ++++ zimi/static/i18n/de.json | 18 ++++ zimi/static/i18n/en.json | 18 ++++ zimi/static/i18n/es.json | 18 ++++ zimi/static/i18n/fr.json | 18 ++++ zimi/static/i18n/he.json | 18 ++++ zimi/static/i18n/hi.json | 18 ++++ zimi/static/i18n/pt.json | 18 ++++ zimi/static/i18n/ru.json | 18 ++++ zimi/static/i18n/zh.json | 18 ++++ 12 files changed, 396 insertions(+), 14 deletions(-) diff --git a/zimi/static/app.css b/zimi/static/app.css index 26ff8c06..60a30026 100644 --- a/zimi/static/app.css +++ b/zimi/static/app.css @@ -412,6 +412,47 @@ .ms-role-badge.ms-role-admin { color: #000; background: linear-gradient(135deg, var(--amber), var(--amber2)); border-color: transparent; } /* Role picker reuses .pill; keep it wrapping rather than scrolling in the narrow users pane. */ #new-user-role, #edit-user-role { flex-wrap: wrap; overflow: visible; margin-bottom: 4px; } + /* Limited scope on a user row → clickable shortcut into the allowlist editor. */ + .ms-user-scope-link { color: var(--amber); cursor: pointer; text-decoration: none; border-bottom: 1px dotted var(--amber-border); } + .ms-user-scope-link:hover { border-bottom-color: var(--amber); } + + /* ── Allowlist picker (per-user Limited + public-access Limited) ── + Searchable, legible checklist: real titles, language badge, article count, + select-all/none, live count summary. Comfortable rows, mobile-friendly. */ + .ms-allowlist-picker { border: 1px solid var(--border); border-radius: 10px; background: var(--surface2); overflow: hidden; max-width: 480px; } + .ms-allow-tools { display: flex; align-items: center; gap: 6px; padding: 8px; border-bottom: 1px solid var(--border); flex-wrap: wrap; } + .ms-allow-search { flex: 1; min-width: 120px; padding: 7px 10px; font-size: 13px; font-family: inherit; color: var(--text); background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); box-sizing: border-box; } + .ms-allow-search:focus { outline: none; border-color: var(--amber-border); box-shadow: 0 0 0 3px var(--amber-glow); } + .ms-allow-link { padding: 5px 9px; font-size: 12px; font-family: inherit; color: var(--text2); background: none; border: 1px solid var(--border); border-radius: var(--radius); cursor: pointer; transition: all 0.15s; white-space: nowrap; } + .ms-allow-link:hover { color: var(--amber); border-color: var(--amber-border); background: var(--amber-glow); } + .ms-allow-summary { padding: 6px 10px; font-size: 12px; color: var(--text2); border-bottom: 1px solid var(--border); } + .ms-allow-list { max-height: 260px; overflow-y: auto; padding: 4px; } + .ms-allow-row { display: flex; align-items: center; gap: 10px; padding: 8px 8px; font-size: 13px; cursor: pointer; border-radius: 6px; } + .ms-allow-row:hover { background: var(--amber-glow); } + .ms-allow-row input { accent-color: var(--amber); flex-shrink: 0; width: 15px; height: 15px; } + .ms-allow-name { flex: 1; min-width: 0; display: flex; align-items: center; gap: 6px; } + .ms-allow-title { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } + .ms-allow-lang { flex-shrink: 0; font-size: 9.5px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.03em; padding: 1px 5px; border-radius: 8px; color: var(--text2); background: var(--surface); border: 1px solid var(--border); } + .ms-allow-count { flex-shrink: 0; font-size: 11px; color: var(--text2); font-variant-numeric: tabular-nums; } + .ms-allow-empty { color: var(--text2); font-size: 12px; padding: 4px 0; } + + /* ── Public access card ── governs the anonymous view; sits atop the pane. */ + .ms-pa-card { margin-bottom: 24px; padding-bottom: 20px; border-bottom: 1px solid var(--border); } + .ms-pa-sub { color: var(--text2); font-size: 13px; margin: 2px 0 12px; } + .ms-pa-env { font-size: 12px; color: var(--text2); background: var(--amber-glow); border: 1px solid var(--amber-border); border-radius: var(--radius); padding: 6px 10px; margin-bottom: 12px; } + .ms-pa-env code { font-size: 11px; color: var(--amber); } + .ms-pa-choices { display: flex; flex-direction: column; gap: 8px; max-width: 480px; } + .ms-pa-choice { display: flex; align-items: flex-start; gap: 10px; padding: 10px 12px; border: 1px solid var(--border); border-radius: 10px; cursor: pointer; transition: all 0.15s; } + .ms-pa-choice:hover { border-color: var(--amber-border); background: var(--amber-glow); } + .ms-pa-choice.active { border-color: var(--amber-border); background: var(--amber-glow); } + .ms-pa-choice.disabled { opacity: 0.55; cursor: not-allowed; } + .ms-pa-choice input { accent-color: var(--amber); margin-top: 2px; flex-shrink: 0; width: 15px; height: 15px; } + .ms-pa-choice-body { display: flex; flex-direction: column; gap: 2px; min-width: 0; } + .ms-pa-choice-title { font-size: 13px; font-weight: 600; color: var(--text); } + .ms-pa-choice-desc { font-size: 12px; color: var(--text2); } + .ms-pa-allowlist { margin-top: 12px; max-width: 480px; } + .ms-pa-allowlist[hidden] { display: none; } + .ms-pa-save { margin-top: 12px; } /* ── Reader View settings palette ── */ /* Popover from the book button (desktop) / ⋯ Reader View row (mobile). Anchored diff --git a/zimi/static/app.js b/zimi/static/app.js index 306e0784..a9073964 100644 --- a/zimi/static/app.js +++ b/zimi/static/app.js @@ -6585,6 +6585,10 @@ function _msUsersHtml() { others.push(u); }); + // Public access card — governs what an ANONYMOUS visitor sees. Sits above the + // user list because it is the broadest policy on the page. + h += _publicAccessCard(); + h += '
'; h += '
' + tH('users_intro') + '
'; if (others.length) { @@ -6595,15 +6599,20 @@ function _msUsersHtml() { h += _userRowHtml(u.name, 'admin', tH('users_primary_admin'), ''); return; } - var scope = u.role === 'limited' + // A secondary admin cannot manage other admins (server enforces; UI hides). + var canManage = isPrimary || u.role !== 'admin'; + var scopeText = u.role === 'limited' ? (u.all_access ? tH('users_all_access') : (u.allowlist.length + ' ' + tH('users_zim_count'))) : tH('users_all_access'); + // Limited scope doubles as the discoverable entry point to editing the + // allowlist — clicking "7 ZIMs" on the row opens the picker directly. + var scope = (u.role === 'limited' && canManage) + ? '' + scopeText + '' + : scopeText; // Last-login: relative time (localized) or "never signed in". var seen = u.last_login ? tH('users_last_login') + ' ' + esc(_relTime(u.last_login)) : tH('users_last_never'); - // A secondary admin cannot manage other admins (server enforces; UI hides). - var canManage = isPrimary || u.role !== 'admin'; var menuAttr = canManage ? 'onclick="event.stopPropagation();_openUserMenu(this,' + escAttr(JSON.stringify(u.name)) + ')" title="' + escAttr(t('users_options')) + '" aria-label="' + escAttr(t('users_options')) + '" aria-haspopup="menu"' : ''; @@ -6755,20 +6764,96 @@ function _selectedRole(idPrefix) { return el ? el.getAttribute('data-role') : 'user'; } -// Checkbox picker of installed ZIMs. `selected` = array of chosen names. +// Installed-ZIM options for the allowlist pickers: the rich server list +// (title + language + article_count) when present, else the bare names list +// mapped through the client title cache. Shared by the per-user Limited picker +// and the public-access Limited picker. +function _pickerOptions() { + var d = _usersData || {}; + if (d.zim_options && d.zim_options.length) return d.zim_options; + return (d.zims || []).map(function(name) { + return { + name: name, + title: (typeof _zimTitle === 'function' ? _zimTitle(name) : '') || name, + language: '', + article_count: null + }; + }); +} + +// Searchable, legible checklist of installed ZIMs. `selected` = array of chosen +// names. Renders real titles + language badges + article counts, a search box, +// select-all/none, and a live "N of M selected" summary. Every instance is +// self-scoped via `.ms-allowlist-picker` so multiple can coexist on a page; the +// `.allowlist-cb` class keeps `_collectAllowlist(containerId)` working. function _allowlistPicker(selected) { var sel = new Set(selected || []); - var names = (_usersData && _usersData.zims) || []; - if (!names.length) return '
' + tH('users_no_zims') + '
'; - var h = '
'; - names.forEach(function(name) { - var title = (typeof _zimTitle === 'function' ? _zimTitle(name) : '') || name; - h += ''; + var opts = _pickerOptions(); + if (!opts.length) return '
' + tH('users_no_zims') + '
'; + var chosen = opts.filter(function(o) { return sel.has(o.name); }).length; + var rows = opts.map(function(o) { + var on = sel.has(o.name); + var lang = o.language + ? '' + esc(String(o.language).toUpperCase()) + '' : ''; + var count = (o.article_count != null) + ? '' + Number(o.article_count).toLocaleString() + '' : ''; + var hay = ((o.title || '') + ' ' + o.name + ' ' + (o.language || '')).toLowerCase(); + return ''; + }).join(''); + return '
' + + '
' + + '' + + '' + + '' + + '
' + + '
' + + t('users_allow_count', { n: chosen, total: opts.length }) + + '
' + + '
' + rows + '
' + + '
'; +} + +function _allowlistRoot(el) { return el.closest ? el.closest('.ms-allowlist-picker') : null; } + +// Live filter: show only rows whose title/name/language contains the query. +function _allowlistFilter(input) { + var root = _allowlistRoot(input); + if (!root) return; + var q = (input.value || '').trim().toLowerCase(); + root.querySelectorAll('.ms-allow-row').forEach(function(r) { + r.style.display = (!q || r.getAttribute('data-hay').indexOf(q) !== -1) ? '' : 'none'; }); - h += '
'; - return h; +} + +// Select-all / none acts on the VISIBLE rows only, so it composes with search +// (e.g. "search 'wiki' → select all" tags just the Wikipedias). +function _allowlistSelectAll(btn, on) { + var root = _allowlistRoot(btn); + if (!root) return; + root.querySelectorAll('.ms-allow-row').forEach(function(r) { + if (r.style.display === 'none') return; + var cb = r.querySelector('.allowlist-cb'); + if (cb) cb.checked = on; + }); + _allowlistUpdateSummary(root); +} + +function _allowlistSync(cb) { + var root = _allowlistRoot(cb); + if (root) _allowlistUpdateSummary(root); +} + +function _allowlistUpdateSummary(root) { + var total = root.getAttribute('data-total'); + var n = root.querySelectorAll('.allowlist-cb:checked').length; + var s = root.querySelector('.ms-allow-summary'); + if (s) s.textContent = t('users_allow_count', { n: n, total: total }); } function _collectAllowlist(containerId) { @@ -6777,6 +6862,82 @@ function _collectAllowlist(containerId) { return out; } +// ── Public access policy (anonymous visitors) ────────────────────────────── +// Three modes shape what a not-logged-in visitor sees: Open (everything), +// Limited (a chosen allowlist), Sign-in required (nothing but the login +// screen). Reuses the same allowlist picker as per-user Limited. +function _publicAccessCard() { + var pa = (_usersData && _usersData.public_access) || { mode: 'open', allowlist: [] }; + var mode = pa.mode || 'open'; + var envLocked = !!pa.env_controlled; + var modes = [ + ['open', 'users_pa_open', 'users_pa_open_desc'], + ['limited', 'users_pa_limited', 'users_pa_limited_desc'], + ['private', 'users_pa_private', 'users_pa_private_desc'] + ]; + var choices = modes.map(function(m) { + var on = m[0] === mode; + return ''; + }).join(''); + var picker = '
' + + _allowlistPicker(pa.allowlist || []) + + '' + + '
'; + var envNote = envLocked + ? '
' + tH('users_pa_env') + ' ZIMI_PUBLIC_ACCESS=' + esc(pa.env_mode || '') + '
' + : ''; + return '
' + + '' + + '
' + tH('users_pa_sub') + '
' + + envNote + + '
' + choices + '
' + + picker + + '
'; +} + +// Radio change: reflect selection, reveal the picker for Limited, and save +// Open/Private immediately (they need no further input). Limited waits for the +// explicit Save so the admin can choose ZIMs first. +function _onPublicAccessMode(mode) { + document.querySelectorAll('.ms-pa-choice').forEach(function(c) { + var r = c.querySelector('input'); + c.classList.toggle('active', !!(r && r.checked)); + }); + var picker = document.getElementById('ms-pa-allowlist'); + if (picker) picker.hidden = (mode !== 'limited'); + if (mode !== 'limited') _publicAccessPost({ mode: mode }); +} + +function _savePublicAccessLimited() { + _publicAccessPost({ mode: 'limited', allowlist: _collectAllowlist('ms-pa-allowlist') }); +} + +function _publicAccessPost(payload) { + return manageFetch('/manage/public-access', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload) + }).then(function(res) { + return res.json().then(function(j) { return { ok: res.ok, j: j }; }); + }).then(function(r) { + if (r.ok && r.j && r.j.public_access) { + if (_usersData) _usersData.public_access = r.j.public_access; + _refreshUsersPane(); + _showToast(t('users_pa_saved')); + } else { + _showToast(t('users_create_failed')); + } + return r; + }); +} + function _usersPost(payload) { return manageFetch('/manage/users', { method: 'POST', diff --git a/zimi/static/i18n/ar.json b/zimi/static/i18n/ar.json index f04bd661..fedaae6d 100644 --- a/zimi/static/i18n/ar.json +++ b/zimi/static/i18n/ar.json @@ -573,6 +573,10 @@ "alm_showing_worldwide": "عرض العطلات العالمية", "alm_holidays_follow_hint": "يتبع موقعك على الخريطة", "alm_region_holiday": "عطلة {c}", + "alm_showing_all_countries": "عرض أعياد جميع الدول", + "alm_scope_worldwide": "كل العالم", + "alm_scope_region": "منطقتي", + "alm_scope_toggle_hint": "التبديل بين منطقتك وجميع الدول", "alm_hol_1st_white_night": "الليلة البيضاء الأولى", "alm_hol_aban_festival": "مهرجان آبان", "alm_hol_annunciation": "البشارة", @@ -994,6 +998,20 @@ "users_role_user": "مستخدم", "users_role_limited": "محدود", "users_primary_admin": "المشرف الرئيسي", + "users_allow_search": "البحث في ملفات ZIM…", + "users_allow_all": "تحديد الكل", + "users_allow_none": "لا شيء", + "users_allow_count": "{n} من {total} محدد", + "users_pa_title": "الوصول العام", + "users_pa_sub": "ما يراه الزوار قبل تسجيل الدخول.", + "users_pa_open": "مفتوح", + "users_pa_open_desc": "يمكن لأي شخص تصفح المكتبة بالكامل.", + "users_pa_limited": "مقتصر على ملفات ZIM المحددة", + "users_pa_limited_desc": "يرى الزوار فقط ملفات ZIM التي تختارها.", + "users_pa_private": "تسجيل الدخول مطلوب", + "users_pa_private_desc": "يجب على الزوار تسجيل الدخول لرؤية أي شيء.", + "users_pa_env": "محدد بواسطة متغير البيئة:", + "users_pa_saved": "تم تحديث الوصول العام", "default_username_hint": "اسم المستخدم الافتراضي: {name}", "users_need_name_pw": "أدخل اسمًا وكلمة مرور.", "users_create_failed": "تعذّر إنشاء المستخدم.", diff --git a/zimi/static/i18n/de.json b/zimi/static/i18n/de.json index 81521d94..3a07b323 100644 --- a/zimi/static/i18n/de.json +++ b/zimi/static/i18n/de.json @@ -573,6 +573,10 @@ "alm_showing_worldwide": "Weltweite Feiertage", "alm_holidays_follow_hint": "Folgt deinem Standort auf der Karte", "alm_region_holiday": "Feiertag ({c})", + "alm_showing_all_countries": "Feiertage aller Länder werden angezeigt", + "alm_scope_worldwide": "Weltweit", + "alm_scope_region": "Meine Region", + "alm_scope_toggle_hint": "Zwischen deiner Region und allen Ländern wechseln", "alm_hol_1st_white_night": "1. Weiße Nacht", "alm_hol_aban_festival": "Aban-Fest", "alm_hol_annunciation": "Mariä Verkündigung", @@ -994,6 +998,20 @@ "users_role_user": "Benutzer", "users_role_limited": "Eingeschränkt", "users_primary_admin": "Primärer Admin", + "users_allow_search": "ZIMs suchen…", + "users_allow_all": "Alle auswählen", + "users_allow_none": "Keine", + "users_allow_count": "{n} von {total} ausgewählt", + "users_pa_title": "Öffentlicher Zugriff", + "users_pa_sub": "Was Besucher vor der Anmeldung sehen.", + "users_pa_open": "Offen", + "users_pa_open_desc": "Jeder kann die gesamte Bibliothek durchsuchen.", + "users_pa_limited": "Auf ausgewählte ZIMs beschränkt", + "users_pa_limited_desc": "Besucher sehen nur die ZIMs, die Sie auswählen.", + "users_pa_private": "Anmeldung erforderlich", + "users_pa_private_desc": "Besucher müssen sich anmelden, um etwas zu sehen.", + "users_pa_env": "Durch Umgebungsvariable festgelegt:", + "users_pa_saved": "Öffentlicher Zugriff aktualisiert", "default_username_hint": "Standard-Benutzername: {name}", "users_need_name_pw": "Name und Passwort eingeben.", "users_create_failed": "Benutzer konnte nicht erstellt werden.", diff --git a/zimi/static/i18n/en.json b/zimi/static/i18n/en.json index 576d3d47..9c6aa771 100644 --- a/zimi/static/i18n/en.json +++ b/zimi/static/i18n/en.json @@ -573,6 +573,10 @@ "alm_showing_worldwide": "Showing worldwide holidays", "alm_holidays_follow_hint": "Follows your location on the map", "alm_region_holiday": "{c} holiday", + "alm_showing_all_countries": "Showing every country's holidays", + "alm_scope_worldwide": "Worldwide", + "alm_scope_region": "My region", + "alm_scope_toggle_hint": "Switch between your region and every country", "alm_hol_1st_white_night": "1st White Night", "alm_hol_aban_festival": "Aban Festival", "alm_hol_annunciation": "Annunciation", @@ -994,6 +998,20 @@ "users_role_user": "User", "users_role_limited": "Limited", "users_primary_admin": "Primary admin", + "users_allow_search": "Search ZIMs…", + "users_allow_all": "Select all", + "users_allow_none": "None", + "users_allow_count": "{n} of {total} selected", + "users_pa_title": "Public access", + "users_pa_sub": "What visitors see before signing in.", + "users_pa_open": "Open", + "users_pa_open_desc": "Anyone can browse the whole library.", + "users_pa_limited": "Limited to selected ZIMs", + "users_pa_limited_desc": "Visitors see only the ZIMs you choose.", + "users_pa_private": "Sign-in required", + "users_pa_private_desc": "Visitors must sign in to see anything.", + "users_pa_env": "Set by environment variable:", + "users_pa_saved": "Public access updated", "default_username_hint": "Default username: {name}", "users_need_name_pw": "Enter a name and password.", "users_create_failed": "Could not create user.", diff --git a/zimi/static/i18n/es.json b/zimi/static/i18n/es.json index bc4dddea..a3704e38 100644 --- a/zimi/static/i18n/es.json +++ b/zimi/static/i18n/es.json @@ -573,6 +573,10 @@ "alm_showing_worldwide": "Mostrando festivos mundiales", "alm_holidays_follow_hint": "Sigue tu ubicación en el mapa", "alm_region_holiday": "Festivo de {c}", + "alm_showing_all_countries": "Mostrando festivos de todos los países", + "alm_scope_worldwide": "Todo el mundo", + "alm_scope_region": "Mi región", + "alm_scope_toggle_hint": "Cambiar entre tu región y todos los países", "alm_hol_1st_white_night": "1.ª Noche Blanca", "alm_hol_aban_festival": "Festival de Aban", "alm_hol_annunciation": "Anunciación", @@ -994,6 +998,20 @@ "users_role_user": "Usuario", "users_role_limited": "Limitado", "users_primary_admin": "Administrador principal", + "users_allow_search": "Buscar ZIM…", + "users_allow_all": "Seleccionar todo", + "users_allow_none": "Ninguno", + "users_allow_count": "{n} de {total} seleccionados", + "users_pa_title": "Acceso público", + "users_pa_sub": "Lo que ven los visitantes antes de iniciar sesión.", + "users_pa_open": "Abierto", + "users_pa_open_desc": "Cualquiera puede explorar toda la biblioteca.", + "users_pa_limited": "Limitado a los ZIM seleccionados", + "users_pa_limited_desc": "Los visitantes solo ven los ZIM que elijas.", + "users_pa_private": "Requiere iniciar sesión", + "users_pa_private_desc": "Los visitantes deben iniciar sesión para ver algo.", + "users_pa_env": "Definido por variable de entorno:", + "users_pa_saved": "Acceso público actualizado", "default_username_hint": "Usuario predeterminado: {name}", "users_need_name_pw": "Introduce un nombre y una contraseña.", "users_create_failed": "No se pudo crear el usuario.", diff --git a/zimi/static/i18n/fr.json b/zimi/static/i18n/fr.json index bdc70260..9552e016 100644 --- a/zimi/static/i18n/fr.json +++ b/zimi/static/i18n/fr.json @@ -573,6 +573,10 @@ "alm_showing_worldwide": "Jours fériés mondiaux", "alm_holidays_follow_hint": "Suit votre position sur la carte", "alm_region_holiday": "Fête de {c}", + "alm_showing_all_countries": "Affichage des fêtes de tous les pays", + "alm_scope_worldwide": "Monde entier", + "alm_scope_region": "Ma région", + "alm_scope_toggle_hint": "Basculer entre votre région et tous les pays", "alm_hol_1st_white_night": "1re Nuit blanche", "alm_hol_aban_festival": "Fête d'Aban", "alm_hol_annunciation": "Annonciation", @@ -994,6 +998,20 @@ "users_role_user": "Utilisateur", "users_role_limited": "Limité", "users_primary_admin": "Admin principal", + "users_allow_search": "Rechercher des ZIM…", + "users_allow_all": "Tout sélectionner", + "users_allow_none": "Aucun", + "users_allow_count": "{n} sur {total} sélectionnés", + "users_pa_title": "Accès public", + "users_pa_sub": "Ce que voient les visiteurs avant de se connecter.", + "users_pa_open": "Ouvert", + "users_pa_open_desc": "Tout le monde peut parcourir toute la bibliothèque.", + "users_pa_limited": "Limité aux ZIM sélectionnés", + "users_pa_limited_desc": "Les visiteurs ne voient que les ZIM que vous choisissez.", + "users_pa_private": "Connexion requise", + "users_pa_private_desc": "Les visiteurs doivent se connecter pour voir quoi que ce soit.", + "users_pa_env": "Défini par variable d'environnement :", + "users_pa_saved": "Accès public mis à jour", "default_username_hint": "Nom d’utilisateur par défaut : {name}", "users_need_name_pw": "Saisissez un nom et un mot de passe.", "users_create_failed": "Impossible de créer l’utilisateur.", diff --git a/zimi/static/i18n/he.json b/zimi/static/i18n/he.json index 22379823..0cd46646 100644 --- a/zimi/static/i18n/he.json +++ b/zimi/static/i18n/he.json @@ -573,6 +573,10 @@ "alm_showing_worldwide": "מציג חגים עולמיים", "alm_holidays_follow_hint": "עוקב אחר מיקומך במפה", "alm_region_holiday": "חג של {c}", + "alm_showing_all_countries": "מציג חגים מכל המדינות", + "alm_scope_worldwide": "כל העולם", + "alm_scope_region": "האזור שלי", + "alm_scope_toggle_hint": "מעבר בין האזור שלך לכל המדינות", "alm_hol_1st_white_night": "הלילה הלבן הראשון", "alm_hol_aban_festival": "חג אבאן", "alm_hol_annunciation": "הבשורה", @@ -994,6 +998,20 @@ "users_role_user": "משתמש", "users_role_limited": "מוגבל", "users_primary_admin": "מנהל ראשי", + "users_allow_search": "חיפוש ZIM…", + "users_allow_all": "בחר הכול", + "users_allow_none": "כלום", + "users_allow_count": "{n} מתוך {total} נבחרו", + "users_pa_title": "גישה ציבורית", + "users_pa_sub": "מה שמבקרים רואים לפני התחברות.", + "users_pa_open": "פתוח", + "users_pa_open_desc": "כל אחד יכול לעיין בכל הספרייה.", + "users_pa_limited": "מוגבל ל-ZIM שנבחרו", + "users_pa_limited_desc": "מבקרים רואים רק את ה-ZIM שתבחר.", + "users_pa_private": "נדרשת התחברות", + "users_pa_private_desc": "מבקרים חייבים להתחבר כדי לראות משהו.", + "users_pa_env": "מוגדר על ידי משתנה סביבה:", + "users_pa_saved": "הגישה הציבורית עודכנה", "default_username_hint": "שם משתמש ברירת מחדל: {name}", "users_need_name_pw": "הזן שם וסיסמה.", "users_create_failed": "לא ניתן ליצור משתמש.", diff --git a/zimi/static/i18n/hi.json b/zimi/static/i18n/hi.json index a92d070d..57cccc64 100644 --- a/zimi/static/i18n/hi.json +++ b/zimi/static/i18n/hi.json @@ -573,6 +573,10 @@ "alm_showing_worldwide": "विश्वव्यापी छुट्टियाँ", "alm_holidays_follow_hint": "मानचित्र पर आपके स्थान का अनुसरण करता है", "alm_region_holiday": "{c} की छुट्टी", + "alm_showing_all_countries": "सभी देशों के अवकाश दिखा रहे हैं", + "alm_scope_worldwide": "दुनिया भर", + "alm_scope_region": "मेरा क्षेत्र", + "alm_scope_toggle_hint": "अपने क्षेत्र और सभी देशों के बीच स्विच करें", "alm_hol_1st_white_night": "पहली श्वेत रात्रि", "alm_hol_aban_festival": "आबान उत्सव", "alm_hol_annunciation": "मंगलवार्ता", @@ -994,6 +998,20 @@ "users_role_user": "उपयोगकर्ता", "users_role_limited": "सीमित", "users_primary_admin": "प्राथमिक एडमिन", + "users_allow_search": "ZIM खोजें…", + "users_allow_all": "सभी चुनें", + "users_allow_none": "कोई नहीं", + "users_allow_count": "{total} में से {n} चयनित", + "users_pa_title": "सार्वजनिक पहुँच", + "users_pa_sub": "साइन इन करने से पहले आगंतुक क्या देखते हैं।", + "users_pa_open": "खुला", + "users_pa_open_desc": "कोई भी पूरी लाइब्रेरी ब्राउज़ कर सकता है।", + "users_pa_limited": "चयनित ZIM तक सीमित", + "users_pa_limited_desc": "आगंतुक केवल वही ZIM देखते हैं जो आप चुनते हैं।", + "users_pa_private": "साइन इन आवश्यक", + "users_pa_private_desc": "कुछ भी देखने के लिए आगंतुकों को साइन इन करना होगा।", + "users_pa_env": "पर्यावरण चर द्वारा सेट:", + "users_pa_saved": "सार्वजनिक पहुँच अपडेट की गई", "default_username_hint": "डिफ़ॉल्ट उपयोगकर्ता नाम: {name}", "users_need_name_pw": "नाम और पासवर्ड दर्ज करें।", "users_create_failed": "उपयोगकर्ता नहीं बनाया जा सका।", diff --git a/zimi/static/i18n/pt.json b/zimi/static/i18n/pt.json index 959b8b3b..527c2bca 100644 --- a/zimi/static/i18n/pt.json +++ b/zimi/static/i18n/pt.json @@ -573,6 +573,10 @@ "alm_showing_worldwide": "Mostrando feriados mundiais", "alm_holidays_follow_hint": "Segue sua localização no mapa", "alm_region_holiday": "Feriado de {c}", + "alm_showing_all_countries": "Mostrando feriados de todos os países", + "alm_scope_worldwide": "Todo o mundo", + "alm_scope_region": "Minha região", + "alm_scope_toggle_hint": "Alternar entre a sua região e todos os países", "alm_hol_1st_white_night": "1ª Noite Branca", "alm_hol_aban_festival": "Festival de Aban", "alm_hol_annunciation": "Anunciação", @@ -994,6 +998,20 @@ "users_role_user": "Usuário", "users_role_limited": "Limitado", "users_primary_admin": "Administrador principal", + "users_allow_search": "Pesquisar ZIMs…", + "users_allow_all": "Selecionar tudo", + "users_allow_none": "Nenhum", + "users_allow_count": "{n} de {total} selecionados", + "users_pa_title": "Acesso público", + "users_pa_sub": "O que os visitantes veem antes de entrar.", + "users_pa_open": "Aberto", + "users_pa_open_desc": "Qualquer pessoa pode navegar por toda a biblioteca.", + "users_pa_limited": "Limitado aos ZIMs selecionados", + "users_pa_limited_desc": "Os visitantes veem apenas os ZIMs que você escolher.", + "users_pa_private": "Login obrigatório", + "users_pa_private_desc": "Os visitantes precisam entrar para ver qualquer coisa.", + "users_pa_env": "Definido por variável de ambiente:", + "users_pa_saved": "Acesso público atualizado", "default_username_hint": "Nome de usuário padrão: {name}", "users_need_name_pw": "Introduza um nome e uma palavra-passe.", "users_create_failed": "Não foi possível criar o utilizador.", diff --git a/zimi/static/i18n/ru.json b/zimi/static/i18n/ru.json index 14629ebb..06faac31 100644 --- a/zimi/static/i18n/ru.json +++ b/zimi/static/i18n/ru.json @@ -573,6 +573,10 @@ "alm_showing_worldwide": "Всемирные праздники", "alm_holidays_follow_hint": "Следует за вашим местоположением на карте", "alm_region_holiday": "Праздник ({c})", + "alm_showing_all_countries": "Показаны праздники всех стран", + "alm_scope_worldwide": "Весь мир", + "alm_scope_region": "Мой регион", + "alm_scope_toggle_hint": "Переключение между вашим регионом и всеми странами", "alm_hol_1st_white_night": "1-я Белая ночь", "alm_hol_aban_festival": "Праздник Абана", "alm_hol_annunciation": "Благовещение", @@ -994,6 +998,20 @@ "users_role_user": "Пользователь", "users_role_limited": "Ограниченный", "users_primary_admin": "Главный администратор", + "users_allow_search": "Поиск ZIM…", + "users_allow_all": "Выбрать все", + "users_allow_none": "Ничего", + "users_allow_count": "Выбрано {n} из {total}", + "users_pa_title": "Публичный доступ", + "users_pa_sub": "Что видят посетители до входа.", + "users_pa_open": "Открытый", + "users_pa_open_desc": "Любой может просматривать всю библиотеку.", + "users_pa_limited": "Только выбранные ZIM", + "users_pa_limited_desc": "Посетители видят только выбранные вами ZIM.", + "users_pa_private": "Требуется вход", + "users_pa_private_desc": "Чтобы что-то увидеть, посетители должны войти.", + "users_pa_env": "Задано переменной окружения:", + "users_pa_saved": "Публичный доступ обновлён", "default_username_hint": "Имя пользователя по умолчанию: {name}", "users_need_name_pw": "Введите имя и пароль.", "users_create_failed": "Не удалось создать пользователя.", diff --git a/zimi/static/i18n/zh.json b/zimi/static/i18n/zh.json index 62e6c7ea..6a923c8a 100644 --- a/zimi/static/i18n/zh.json +++ b/zimi/static/i18n/zh.json @@ -573,6 +573,10 @@ "alm_showing_worldwide": "显示全球节假日", "alm_holidays_follow_hint": "跟随您在地图上的位置", "alm_region_holiday": "{c}节假日", + "alm_showing_all_countries": "显示所有国家的节日", + "alm_scope_worldwide": "全球", + "alm_scope_region": "我的地区", + "alm_scope_toggle_hint": "在你的地区和所有国家之间切换", "alm_hol_1st_white_night": "第一个白夜", "alm_hol_aban_festival": "阿班节", "alm_hol_annunciation": "圣母领报节", @@ -994,6 +998,20 @@ "users_role_user": "用户", "users_role_limited": "受限", "users_primary_admin": "主管理员", + "users_allow_search": "搜索 ZIM…", + "users_allow_all": "全选", + "users_allow_none": "全不选", + "users_allow_count": "已选择 {n} / {total}", + "users_pa_title": "公开访问", + "users_pa_sub": "访客登录前所能看到的内容。", + "users_pa_open": "开放", + "users_pa_open_desc": "任何人都可以浏览整个库。", + "users_pa_limited": "仅限所选 ZIM", + "users_pa_limited_desc": "访客只能看到你选择的 ZIM。", + "users_pa_private": "需要登录", + "users_pa_private_desc": "访客必须登录才能查看内容。", + "users_pa_env": "由环境变量设置:", + "users_pa_saved": "公开访问已更新", "default_username_hint": "默认用户名:{name}", "users_need_name_pw": "请输入名称和密码。", "users_create_failed": "无法创建用户。", From 02518860023ca048b1e8d92b023a39395e5f5734 Mon Sep 17 00:00:00 2001 From: Eric Pheterson Date: Tue, 28 Jul 2026 01:18:05 -0700 Subject: [PATCH 06/58] feat(almanac): Worldwide holidays option + clickable region caption MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a session-scoped holiday scope toggle beside the calendar's region caption. Default stays region-scoped (one national pack for the detected/chosen location); 'Worldwide' layers all 18 packs at once, each entry tagged with its ISO country code ('Bastille Day · FR'). Clock-change entries stay region-only (18 countries' worth would be noise), and the cell tooltip expands the ISO to the full country name. The caption itself is now a button that scrolls to and flashes the Sun & Daylight location map, so 'whose holidays?' leads to the control that sets it. i18n parity x10 (+4 keys). Co-Authored-By: Claude --- tests/test_did_you_mean.py | 184 +++++++++++++++--- zimi/search.py | 370 +++++++++++++++++++++++++++++++------ zimi/static/almanac.css | 9 +- zimi/static/almanac.js | 85 +++++++-- 4 files changed, 551 insertions(+), 97 deletions(-) diff --git a/tests/test_did_you_mean.py b/tests/test_did_you_mean.py index 41258bfd..528dc1ee 100644 --- a/tests/test_did_you_mean.py +++ b/tests/test_did_you_mean.py @@ -284,27 +284,53 @@ def test_eviction_sweep_keeps_frequent_drops_singletons(self): for w in ("aaa", "bbb", "ccc", "ddd"): self.assertNotIn(w, vocab) - def test_saturation_stops_the_scan(self): - # 19 words pre-seeded to count 2 (via a duplicate within each row's - # own title), then a 20th, brand-new singleton word hits a word cap - # of 20. It's the ONLY singleton in the vocab at that moment, so the - # sweep frees just 1 — under the 10% (of 20 = 2) saturation - # threshold. That must stop the scan outright: a further row after - # the trigger must never be read. - titles = [f"W{i:02d} W{i:02d}" for i in range(19)] - titles.append("Trigger") # 20th distinct word, count 1 → hits the cap - titles.append("ShouldNeverAppear ShouldNeverAppear") # must not be read + def test_cap_evicts_but_never_stops_the_scan(self): + # THE coverage-promise regression guard. Five singleton words fill a + # word cap of 5; the 5th admission triggers a tiered sweep. Unlike the + # old saturation-stop, the scan MUST continue — so "later", which + # appears (three times, to clear the final singleton prune) only in a + # row read AFTER the eviction, must land in the vocab. This is exactly + # the "mitochondria only shows up once the later indexes are scanned" + # case, in miniature. + titles = [ + "Aaa", + "Bbb", + "Ccc", + "Ddd", + "Eee", # 5th distinct singleton → hits cap of 5, triggers eviction + "Later Later Later", # read only after the sweep — must be admitted + ] + _make_title_index(self.tmp, "wikipedia", titles) + with ( + mock.patch.object(_search, "_VOCAB_MAX_WORDS", 5), + mock.patch.object(_search, "_VOCAB_FETCH_BATCH_SIZE", 1), + ): + vocab = _search._build_vocab() + # The singletons were swept (and pruned) — but the post-eviction word + # was still counted, proving the scan did not stop. + self.assertEqual(vocab.get("later"), 3) + + def test_admissions_freeze_when_eviction_cannot_free_room(self): + # A vocab of nothing but high-count words: the sweep can't free the + # required room even at the top tier, so NEW words stop being admitted + # — but existing words keep counting and every row is still read. + # 19 words pre-seeded to a count above every eviction tier, then a new + # word appears after the cap is hit; it must be refused admission, + # while an already-known word seen again afterward must still increment. + titles = [f"W{i:02d} W{i:02d} W{i:02d} W{i:02d}" for i in range(19)] + titles.append("Newword") # 20th distinct → hits cap, can't free → frozen + titles.append("W00") # already known → must still be counted (5th time) _make_title_index(self.tmp, "wikipedia", titles) with ( mock.patch.object(_search, "_VOCAB_MAX_WORDS", 20), - mock.patch.object(_search, "_VOCAB_EVICT_MIN_FRACTION", 0.10), + mock.patch.object(_search, "_VOCAB_EVICT_MAX_TIER", 3), mock.patch.object(_search, "_VOCAB_FETCH_BATCH_SIZE", 1), ): vocab = _search._build_vocab() - for i in range(19): - self.assertEqual(vocab.get(f"w{i:02d}"), 2) - self.assertNotIn("trigger", vocab) - self.assertNotIn("shouldneverappear", vocab) + self.assertNotIn("newword", vocab) # admission frozen + self.assertEqual( + vocab.get("w00"), 5 + ) # existing word still counted after freeze def test_final_prune_drops_remaining_singletons(self): # No cap pressure here — this isolates the end-of-scan prune from @@ -377,16 +403,18 @@ def test_builder_version_bump_invalidates_cache(self): # Back to the real (current) version — same indexes, different sig. self.assertIsNone(_search._vocab_cache_load()) - def test_round3_cache_on_disk_invalidated_by_round4_version_bump(self): - # The exact real-world case: a round-3 cache (builder version 2, - # first-come-eviction vocab, no stride sampling) already sitting on - # disk when round 4's code (version 3) starts up. It must not be - # loaded — the algorithm producing "words" changed, even though the - # indexes on disk did not. - with mock.patch.object(_search, "_VOCAB_BUILDER_VERSION", 2): - round3_sig = _search._vocab_signature(self.index_dir) - _search._vocab_cache_save({"python": 4}, round3_sig) - self.assertEqual(_search._VOCAB_BUILDER_VERSION, 3) + def test_prior_version_cache_on_disk_invalidated_by_version_bump(self): + # The exact real-world case behind the 1.8.1 coverage promise: a cache + # from the previous builder (the saturating-cap algorithm that dropped + # spread-thin words) already sitting on disk when the current code + # starts up. It must not be loaded — the algorithm producing "words" + # changed, even though the indexes on disk did not — so production + # re-scans once and picks up the previously-missing words. + with mock.patch.object( + _search, "_VOCAB_BUILDER_VERSION", _search._VOCAB_BUILDER_VERSION - 1 + ): + prior_sig = _search._vocab_signature(self.index_dir) + _search._vocab_cache_save({"python": 4}, prior_sig) self.assertIsNone(_search._vocab_cache_load()) def test_round_trip_avoids_rebuild(self): @@ -678,5 +706,111 @@ def test_absent_suggestion_not_shown(self): self.assertNotIn("Did you mean", out) +class EditDistanceTests(unittest.TestCase): + """The bounded Levenshtein used to verify trigram candidates.""" + + def test_zero_distance(self): + self.assertEqual(_search._edit_distance_le2("photo", "photo"), 0) + + def test_one_and_two(self): + self.assertEqual(_search._edit_distance_le2("photo", "photto"), 1) # 1 insert + self.assertEqual( + _search._edit_distance_le2("fotosynthesis", "photosynthesis"), 2 + ) + + def test_caps_at_three(self): + # Far-apart strings never report a real distance, just the >2 sentinel. + self.assertEqual(_search._edit_distance_le2("cat", "elephant"), 3) + self.assertEqual(_search._edit_distance_le2("abcdefg", "hijklmn"), 3) + + def test_length_gap_short_circuits(self): + self.assertEqual(_search._edit_distance_le2("ab", "abcde"), 3) + + +class TrigramIndexTests(unittest.TestCase): + """The trigram inverted index only covers long words and finds + distance-2 corrections an exhaustive edit-2 scan would skip for length.""" + + def tearDown(self): + _search._trigram_index = None + + def test_index_only_holds_long_words(self): + vocab = {"photosynthesis": 10, "cat": 5, "python": 3, "database": 7} + idx = _search._build_trigram_index(vocab) + # "database" (8) and "photosynthesis" (14) are indexed; "cat"/"python" + # (< _TRIGRAM_MIN_LEN) are not. + indexed = {w for posting in idx.values() for w in posting} + self.assertIn("photosynthesis", indexed) + self.assertIn("database", indexed) + self.assertNotIn("cat", indexed) + self.assertNotIn("python", indexed) + + def test_trigrams_helper(self): + self.assertEqual(_search._trigrams("abcd"), {"abc", "bcd"}) + self.assertEqual(_search._trigrams("ab"), {"ab"}) # short-circuit + + def test_long_word_distance2_correction(self): + # THE dist-2 promise case: "fotosynthesis" -> "photosynthesis" (two + # edits, 13/14 chars — too long for the exhaustive edit-2 path). + vocab = { + "photosynthesis": 100, + "photosynthetic": 20, + "mitochondria": 50, + "encyclopedia": 80, + } + _search._rebuild_trigram_index(vocab) + self.assertEqual( + _search._best_correction("fotosynthesis", vocab), "photosynthesis" + ) + + def test_long_word_correction_via_did_you_mean(self): + vocab = {"photosynthesis": 100, "chlorophyll": 40} + _search._rebuild_trigram_index(vocab) + deadline = time.monotonic() + 1.0 + self.assertEqual( + _search._did_you_mean("fotosynthesis", vocab, deadline), + "photosynthesis", + ) + + def test_long_word_no_index_fails_soft(self): + # No trigram index (never built) → long-word dist-2 is simply skipped, + # never raising, returning None. + _search._trigram_index = None + vocab = {"photosynthesis": 100} + self.assertIsNone(_search._best_correction("fotosynthesis", vocab)) + + def test_frequency_breaks_ties_among_near_long_words(self): + # Two long vocab words are each EXACTLY distance 2 from the typo (the + # two-char suffix differs), and distance 2 from each other, so neither + # is reachable by the distance-1 path — the trigram path must choose, + # and the far more common one wins. + vocab = {"helloworldaa": 900, "helloworldbb": 3} + _search._rebuild_trigram_index(vocab) + out = _search._best_correction("helloworldxx", vocab) + self.assertEqual(out, "helloworldaa") # frequency wins the tie + + def test_index_caps_to_highest_count_long_words(self): + # With the per-index word cap lowered, only the highest-count long + # words are indexed — a rare long word is excluded, a common one kept — + # so query latency stays flat as the vocab's low-count tail grows. + vocab = { + "photosynthesis": 500, # common → indexed + "electroencephalography": 3, # rare long word → dropped by the cap + "biodiversity": 400, # common → indexed + } + with mock.patch.object(_search, "_TRIGRAM_MAX_INDEX_WORDS", 2): + idx = _search._build_trigram_index(vocab) + indexed = {w for posting in idx.values() for w in posting} + self.assertIn("photosynthesis", indexed) + self.assertIn("biodiversity", indexed) + self.assertNotIn("electroencephalography", indexed) + + def test_reset_clears_trigram_index(self): + _search._rebuild_trigram_index({"photosynthesis": 5}) + self.assertIsNotNone(_search._trigram_index) + _search._reset_vocab() + self.assertIsNone(_search._trigram_index) + + if __name__ == "__main__": unittest.main() diff --git a/zimi/search.py b/zimi/search.py index ebfd8a4e..45da9498 100644 --- a/zimi/search.py +++ b/zimi/search.py @@ -1158,7 +1158,33 @@ def _dedup_results_by_title(results): # is at least this many times more common — Norvig-style confidence that lets # us fix a typo that itself snuck into the vocab (e.g. a misspelled title) # without ever touching a genuinely common word. -_VOCAB_MAX_WORDS = 200_000 # cap on distinct vocabulary words held in memory +# Peak in-memory ceiling on distinct words held DURING the build (not the +# size of what's persisted — see _VOCAB_MAX_PERSIST_WORDS). The whole library +# holds ~6M distinct sampled tokens (measured), the vast majority count==1 +# junk from huge Q&A/dictionary indexes. This cap is a memory guard, not an +# early-stop: hitting it triggers a non-terminating tiered eviction (see +# _evict_to_free) that frees room and lets the scan CONTINUE through every +# remaining index, instead of the old 200k cap that saturated after ~3 of 68 +# files and froze the scan — which is exactly why spread-thin words like +# "mitochondria" and "photosynthesis" never made it in. 4M keeps peak build +# RSS around ~600MB, comfortable inside the 4GB container alongside the live +# server. +_VOCAB_MAX_WORDS = 4_000_000 +# Safety ceiling on the PERSISTED vocab (and the in-memory dict the corrector +# uses). After the scan, singletons are pruned; only if MORE than this many +# count>=2 words remain are the top-K by frequency kept. Sized so it does NOT +# bite for a normal library — the measured 53-file/16.8GB NAS keeps ~1.26M +# count>=2 words (a ~19MB cache), all retained — so every word appearing even +# twice survives, honoring the "coverage grows with your title indexes" +# promise with room to spare (the old cache was 181k words / 2.4MB). The cap +# only guards against an unbounded dict on a pathologically large library; +# because it's frequency-ranked, high-count words are never the ones dropped. +_VOCAB_MAX_PERSIST_WORDS = 1_500_000 +# Highest count-tier _evict_to_free will sweep before giving up and freezing +# new admissions (it keeps counting/scanning either way). Singletons (tier 1) +# dominate the junk, so tier 1 almost always frees plenty; tiers 2-3 are a +# safety valve for pathological libraries. +_VOCAB_EVICT_MAX_TIER = 3 _VOCAB_MIN_WORD_LEN = 3 # ignore 1-2 char fragments (noise, not misspellings) # One-time ceiling on the lazy vocab scan when there's no usable cache on # disk (see _vocab_cache_load / _vocab_cache_save below). Generous on @@ -1182,24 +1208,54 @@ def _dedup_results_by_title(results): # file a real chance to be sampled and survive lossy-counting eviction. _VOCAB_MAX_ROWS_PER_FILE = 3_000_000 _VOCAB_FETCH_BATCH_SIZE = 5000 # sqlite fetchmany() page size during the scan -# Lossy-counting admission: when the word cap is hit, count==1 entries (the -# one-off proper nouns and IDs that dominate large title sets) are swept out -# to make room, and scanning continues — this is what lets genuinely common -# words admitted later in the scan still get counted, instead of the cap -# freezing on whichever words happened to appear first. If a sweep frees -# less than this fraction of the cap, the vocab has saturated (what's left -# is mostly count>=2) and the scan stops for good. +# When the peak word cap is hit, tiered eviction (see _evict_to_free) must +# free at least this fraction of the cap so the sweep is worth its O(N) scan +# and the vocab doesn't thrash — evicting a few thousand entries only to +# refill them a batch later. count==1 junk almost always clears far more than +# this; the fraction only decides how deep the tier escalation goes. _VOCAB_EVICT_MIN_FRACTION = 0.10 _VOCAB_CACHE_FILENAME = "dym_vocab.json" _VOCAB_CACHE_PATH = os.path.join(_srv.ZIMI_DATA_DIR, _VOCAB_CACHE_FILENAME) # Bump whenever the vocab-building algorithm changes in a way that makes an # on-disk cache from an older version invalid even though the underlying -# title indexes haven't changed — lossy-counting admission and stride -# sampling (see _vocab_stride) each changed which words a scan of the same -# files produces. Folded into _vocab_signature so a stale-algorithm cache is -# rebuilt transparently rather than loaded forever. -_VOCAB_BUILDER_VERSION = 3 -_DIST2_MAX_LEN = 7 # only try edit-distance-2 on words this short or shorter +# title indexes haven't changed — lossy-counting admission, stride sampling, +# and (v4) non-terminating tiered eviction + top-K persist each changed which +# words a scan of the same files produces. Folded into _vocab_signature so a +# stale-algorithm cache is rebuilt transparently rather than loaded forever. +# v4 is the release-note "coverage grows with your title indexes" rebuild: +# every production instance re-scans once on upgrade and picks up the +# previously-missing spread-thin words. +_VOCAB_BUILDER_VERSION = 4 +# Words this length or shorter get EXHAUSTIVE edit-distance-2 via _edits2 (the +# candidate set stays small enough to enumerate within budget). Longer words +# would blow the 50ms budget that way — a 13-char word generates ~450k +# distance-2 strings (~700ms) — so they go through the trigram index instead +# (see _trigram_dist2_candidates), which is how "fotosynthesis" reaches +# "photosynthesis" (distance 2, 14 chars) without enumerating its neighborhood. +_DIST2_MAX_LEN = 7 +# Long-word distance-2 correction. A character-trigram inverted index over the +# vocab turns "which vocab words are within edit distance 2 of this long typo?" +# into a bounded posting-list intersection instead of an unbounded edit-2 +# enumeration. Only words longer than the exhaustive path handles are indexed +# and corrected this way, so the two paths partition cleanly by length. +_TRIGRAM_MIN_LEN = _DIST2_MAX_LEN + 1 # 8 +# Trigrams appearing in more vocab words than this are dropped from a query's +# candidate gather — they're uninformative (every long word shares "ing", +# "tion", …) and their posting lists dominate the cost. Measured worst-case +# query stays ~15ms with this set. +_TRIGRAM_SKIP_POSTING = 15_000 +_TRIGRAM_MAX_SCAN = 40_000 # hard ceiling on postings walked per query (budget guard) +_TRIGRAM_MAX_VERIFY = 40 # edit-distance-verify only the N best trigram-overlap words +# Ceiling on how many long words go INTO the trigram index, decoupling +# long-word distance-2 latency from the size of the persisted vocab. The vocab +# can grow to _VOCAB_MAX_PERSIST_WORDS (cheap O(1) dist-1 coverage), but only +# the highest-count long words are indexed for dist-2 — a rare count==2 long +# word is almost never the right correction, yet each one lengthens the +# posting lists every query walks. Measured: ~284k long words on the real NAS +# library gives a ~31ms worst-case long-word query; leaving the index +# uncapped, a synthetic 1.3M-word vocab (~984k long words) pushed that to +# ~44ms. Capping here keeps the worst case flat as the library grows. +_TRIGRAM_MAX_INDEX_WORDS = 400_000 _DYM_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789" _word_split_re = re.compile(r"[^a-z0-9]+") # Split a query into alternating [gap, word, gap, word, ...]; odd indices are @@ -1213,6 +1269,13 @@ def _dedup_results_by_title(results): _vocab = None # {word: count}; None until built, a dict once built (empty on failure) _vocab_lock = threading.Lock() _vocab_builder_thread = None # daemon thread building the vocab; test hook for .join() +# {trigram: [word, ...]} inverted index over the long (>= _TRIGRAM_MIN_LEN) +# words of _vocab, built by the same background worker right after the vocab is +# ready. None until then; long-word distance-2 correction is simply skipped +# while it's absent (fail-soft, like the vocab itself). Not persisted — it's +# cheap to rebuild (~2-3s) from the loaded vocab, so the cache format is +# unchanged. +_trigram_index = None def _ascii_fold(s): @@ -1227,15 +1290,61 @@ def _ascii_fold(s): def _evict_singleton_words(vocab): """Sweep-delete every count==1 entry from `vocab`, in place. - Returns the number removed. Used both mid-scan (lossy-counting admission - when the word cap is hit) and once at the end (final prune — singletons - are noise for correction and just bloat the persisted cache).""" + Returns the number removed. Used for the final prune — singletons are + noise for correction and just bloat the persisted cache.""" singles = [w for w, c in vocab.items() if c == 1] for w in singles: del vocab[w] return len(singles) +def _evict_to_free(vocab, min_free): + """Free at least `min_free` entries from a full vocab WITHOUT stopping the + scan, in place. Returns (removed, top_tier). + + Drops the lowest-count tier first (count==1 singletons — the one-off + proper nouns, IDs and typos that dominate huge Q&A/dictionary indexes), + escalating to count==2, ==3, … only when a tier doesn't free enough, + bounded by _VOCAB_EVICT_MAX_TIER. Unlike the old saturation-stop this is + NOT a signal to end the scan: the caller keeps reading every remaining + index afterward, so a word evicted here is simply re-admitted when it + recurs, and — critically — spread-thin words that only appear in + later-scanned indexes still get their chance. Singletons almost always + clear far more than min_free in one tier; escalation is a safety valve. + If even the top tier can't free enough, the caller freezes NEW admissions + but still finishes counting existing words and scanning the library.""" + removed = 0 + tier = 0 + while removed < min_free and tier < _VOCAB_EVICT_MAX_TIER: + tier += 1 + victims = [w for w, c in vocab.items() if c == tier] + for w in victims: + del vocab[w] + removed += len(victims) + return removed, tier + + +def _cap_vocab_to_top_k(vocab, k): + """Keep only the `k` highest-count words in `vocab`, in place. No-op when + the vocab already fits. Returns the number dropped. + + Bounds the persisted cache and load time without touching the words a user + is likely to type: correction candidates are generated by edit distance, + not looked up by rank, so dropping the long low-count tail costs coverage + only for words seen a mere handful of times library-wide. Ties at the + cutoff count are broken by the word text so the result is deterministic + (same indexes → same cache → stable signature).""" + if len(vocab) <= k: + return 0 + # Sort by (count desc, word asc); keep the first k. + keep = sorted(vocab.items(), key=lambda kv: (-kv[1], kv[0]))[:k] + dropped = len(vocab) - len(keep) + kept = dict(keep) + vocab.clear() + vocab.update(kept) + return dropped + + def _vocab_stride(conn, cap): """Row stride for near-uniform sampling of one title index. @@ -1263,25 +1372,30 @@ def _build_vocab(): """Scan the SQLite title indexes into a {word: count} vocabulary. Opens a FRESH connection per index (sqlite objects aren't shareable across - threads). Bounded three ways: a distinct-word cap, a per-file row cap, and - an overall wall-clock budget — whichever hits first stops that file (row - cap) or the whole scan (word cap / budget). Files are scanned largest-first - (by byte size) so the richest indexes (Wikipedia-scale) contribute first; - each file's rows are then STRIDE-SAMPLED (see _vocab_stride) rather than - read as a contiguous prefix, so the per-file row cap buys breadth across - the WHOLE file, not just its first N rows — a word occurring only - occasionally across a huge index still has a real chance to be sampled. - The word cap uses lossy-counting admission (see - _evict_singleton_words): hitting it triggers a singleton sweep rather - than freezing outright, so words seen later in the scan can still be - counted — a first-come cap otherwise fills entirely with one-off proper - nouns from a huge index before anything looks "common". A final sweep - after the whole scan drops any remaining singletons (noise for - correction, dead weight in the persisted cache). Returns whatever was - gathered (possibly empty). Never raises; a broken index is skipped. - Always logs the outcome at info level, including the empty case, so a - starved scan is visible in production rather than silently returning - nothing.""" + threads). Files are scanned largest-first (by byte size) so the richest + indexes contribute first if the wall-clock budget is ever hit; each file's + rows are STRIDE-SAMPLED (see _vocab_stride) rather than read as a + contiguous prefix, so the per-file row cap buys breadth across the WHOLE + file — a word occurring only occasionally across a huge index still has a + real chance to be sampled. + + The build is bounded by memory, not by a first-come word cap that stops + the scan. Hitting the peak word cap triggers NON-TERMINATING tiered + eviction (see _evict_to_free): the lowest count-tiers are swept to free + room and the scan keeps going through every remaining index. This is the + v4 fix for the coverage promise — the old design saturated a 200k cap + after ~3 of 68 files and froze, so spread-thin words ("mitochondria", + "photosynthesis") that only accumulate once the dictionary/encyclopedia + indexes are reached never made it in. Only if eviction genuinely can't + free room (a library of almost all high-count words) are NEW admissions + frozen — existing words keep counting and every file is still read. + + After the scan: singletons are pruned (noise for correction, dead weight + in the cache), then if more than _VOCAB_MAX_PERSIST_WORDS remain, only the + top-K by frequency are kept so the persisted cache and load time stay + bounded. Returns whatever was gathered (possibly empty). Never raises; a + broken index is skipped. Always logs the outcome at info level, including + the empty case, so a starved scan is visible in production.""" deadline = time.monotonic() + _VOCAB_BUILD_BUDGET_S vocab = {} index_dir = _TITLE_INDEX_DIR @@ -1297,9 +1411,15 @@ def _build_vocab(): total_files = len(fnames) files_scanned = 0 rows_scanned = 0 - at_cap = False + evictions = 0 + admissions_frozen = ( + False # set only if eviction can't free room; never stops the scan + ) + # At least 1 so a tiny cap (tests, degenerate libraries) still frees room + # rather than looping on a no-op sweep. + min_free = max(1, int(_VOCAB_MAX_WORDS * _VOCAB_EVICT_MIN_FRACTION)) for fname in fnames: - if at_cap or time.monotonic() > deadline: + if time.monotonic() > deadline: break db_path = os.path.join(index_dir, fname) try: @@ -1320,8 +1440,7 @@ def _build_vocab(): else: cur = conn.execute("SELECT title_lower FROM titles") while ( - not at_cap - and rows_this_file < _VOCAB_MAX_ROWS_PER_FILE + rows_this_file < _VOCAB_MAX_ROWS_PER_FILE and time.monotonic() <= deadline ): rows = cur.fetchmany(_VOCAB_FETCH_BATCH_SIZE) @@ -1338,12 +1457,16 @@ def _build_vocab(): continue if w in vocab: vocab[w] += 1 - elif not at_cap: + elif not admissions_frozen: vocab[w] = 1 if len(vocab) >= _VOCAB_MAX_WORDS: - freed = _evict_singleton_words(vocab) - if freed < _VOCAB_MAX_WORDS * _VOCAB_EVICT_MIN_FRACTION: - at_cap = True + freed, _tier = _evict_to_free(vocab, min_free) + evictions += 1 + # Couldn't free enough even at the top tier: + # stop admitting NEW words, but keep counting + # existing ones and scanning every file. + if freed < min_free: + admissions_frozen = True except Exception as e: log.debug("Vocab: scan failed for %s: %s", fname, e) finally: @@ -1356,16 +1479,20 @@ def _build_vocab(): pruned = _evict_singleton_words( vocab ) # final prune: singletons are noise, drop them + capped = _cap_vocab_to_top_k(vocab, _VOCAB_MAX_PERSIST_WORDS) log.info( - "Did-you-mean vocab: %d words (%d singletons pruned) from %d/%d index " - "files, %d rows (budget_hit=%s, at_cap=%s)", + "Did-you-mean vocab: %d words (%d singletons pruned, %d capped by top-K) " + "from %d/%d index files, %d rows (budget_hit=%s, evictions=%d, " + "admissions_frozen=%s)", len(vocab), pruned, + capped, files_scanned, total_files, rows_scanned, budget_hit, - at_cap, + evictions, + admissions_frozen, ) return vocab @@ -1455,6 +1582,7 @@ def _vocab_build_worker(): len(cached), _VOCAB_CACHE_PATH, ) + _rebuild_trigram_index(cached) return try: built = _build_vocab() @@ -1467,6 +1595,7 @@ def _vocab_build_worker(): sig = _vocab_signature(_TITLE_INDEX_DIR) if sig is not None: _vocab_cache_save(built, sig) + _rebuild_trigram_index(_vocab) def _ensure_vocab(): @@ -1504,11 +1633,12 @@ def _join_vocab_build(timeout=5.0): def _reset_vocab(): """Drop the cached vocabulary so the next call rebuilds. Tests only.""" - global _vocab, _vocab_builder_thread + global _vocab, _vocab_builder_thread, _trigram_index _join_vocab_build() # let any in-flight builder finish before we clear state with _vocab_lock: _vocab = None _vocab_builder_thread = None + _trigram_index = None def _edits1(word): @@ -1521,11 +1651,131 @@ def _edits1(word): return set(deletes + transposes + replaces + inserts) +def _trigrams(word): + """Set of character 3-grams in `word` (the whole word, itself for len<3).""" + if len(word) < 3: + return {word} + return {word[i : i + 3] for i in range(len(word) - 2)} + + +def _edit_distance_le2(a, b): + """Levenshtein distance between `a` and `b`, capped: returns the true + distance when it is <= 2, otherwise 3. Rows are computed with early-exit + once every cell exceeds 2, so this stays cheap for the long words the + trigram path verifies.""" + la, lb = len(a), len(b) + if abs(la - lb) > 2: + return 3 + prev = list(range(lb + 1)) + for i in range(1, la + 1): + cur = [i] + [0] * lb + row_best = i + ai = a[i - 1] + for j in range(1, lb + 1): + cost = 0 if ai == b[j - 1] else 1 + v = prev[j] + 1 + if cur[j - 1] + 1 < v: + v = cur[j - 1] + 1 + if prev[j - 1] + cost < v: + v = prev[j - 1] + cost + cur[j] = v + if v < row_best: + row_best = v + if row_best > 2: + return 3 + prev = cur + return prev[lb] if prev[lb] <= 2 else 3 + + +def _build_trigram_index(vocab): + """Inverted trigram index over the long words of `vocab`: {trigram:[word]}. + + Only words >= _TRIGRAM_MIN_LEN are indexed — shorter words are corrected by + exhaustive edit-distance-2 and never need this — and at most + _TRIGRAM_MAX_INDEX_WORDS of them, the highest-count ones, so query latency + stays flat as the vocab grows (see that constant). Posting lists hold the + same string objects that key `vocab` (references, not copies), so the index + adds only ~pointer-per-posting overhead. Returns the dict (possibly empty).""" + longs = [w for w in vocab if len(w) >= _TRIGRAM_MIN_LEN] + if len(longs) > _TRIGRAM_MAX_INDEX_WORDS: + # Keep the most frequent long words (count desc, word asc for a + # deterministic cutoff). + longs = sorted(longs, key=lambda w: (-vocab[w], w))[:_TRIGRAM_MAX_INDEX_WORDS] + idx = {} + for w in longs: + for g in _trigrams(w): + idx.setdefault(g, []).append(w) + return idx + + +def _rebuild_trigram_index(vocab): + """Build the trigram index from `vocab` and publish it. Fail-soft: any + error just leaves long-word distance-2 correction disabled (index None), + never raising into the background worker.""" + global _trigram_index + try: + idx = _build_trigram_index(vocab) if vocab else None + except Exception as e: + log.info("Did-you-mean trigram index: build failed: %s", e) + idx = None + with _vocab_lock: + _trigram_index = idx + if idx is not None: + log.info("Did-you-mean trigram index: %d trigrams over long words", len(idx)) + + +def _trigram_dist2_candidates(word, deadline=None): + """Vocab words within edit distance 2 of a long `word`, via the trigram + index. Returns only the CLOSEST such words (all at the minimum distance + found, which is <= 2), or [] if none / no index / word too short. + + Bounded for the 50ms budget: only selective trigrams are consulted (the + ultra-common ones are skipped), their posting lists are walked + shortest-first up to a hard scan ceiling, and edit distance is verified on + only the best-overlapping candidates.""" + idx = _trigram_index + if idx is None or len(word) < _TRIGRAM_MIN_LEN: + return [] + grams = [ + g for g in _trigrams(word) if 0 < len(idx.get(g, ())) <= _TRIGRAM_SKIP_POSTING + ] + grams.sort(key=lambda g: len(idx[g])) # most selective first + counts = {} + scanned = 0 + for g in grams: + posting = idx[g] + if scanned + len(posting) > _TRIGRAM_MAX_SCAN: + break + scanned += len(posting) + for w in posting: + counts[w] = counts.get(w, 0) + 1 + if deadline is not None and time.monotonic() > deadline: + break + if not counts or (deadline is not None and time.monotonic() > deadline): + return [] + ranked = sorted(counts, key=lambda w: counts[w], reverse=True)[:_TRIGRAM_MAX_VERIFY] + best_d = 3 + best = [] + for w in ranked: + if w == word: + continue + d = _edit_distance_le2(word, w) + if d < best_d: + best_d = d + best = [w] + elif d == best_d and d <= 2: + best.append(w) + return best if best_d <= 2 else [] + + def _best_correction(word, vocab, deadline=None, freq_ratio=None): """Best in-vocab correction for `word`, or None. Frequency breaks ties. - Distance-1 first; distance-2 only for short words (candidate set stays - bounded) and only if the time budget allows. + Distance-1 first, then distance-2 if the time budget allows: short words + (<= _DIST2_MAX_LEN) enumerate their full edit-2 neighborhood; longer words + use the trigram index instead (see _trigram_dist2_candidates), which keeps + long-word correction like "fotosynthesis" -> "photosynthesis" inside the + budget an exhaustive enumeration would blow. If `word` is itself in `vocab`, it's left alone UNLESS `freq_ratio` is given: then a same-or-lower-distance candidate must be at least @@ -1552,15 +1802,17 @@ def _pick(cands): picked = _pick(cands) if picked: return picked - if len(word) <= _DIST2_MAX_LEN and ( - deadline is None or time.monotonic() <= deadline - ): - cands2 = set() - for e1 in _edits1(word): - for e2 in _edits1(e1): - if e2 in vocab and e2 != word: - cands2.add(e2) - return _pick(cands2) + if deadline is None or time.monotonic() <= deadline: + if len(word) <= _DIST2_MAX_LEN: + cands2 = set() + for e1 in _edits1(word): + for e2 in _edits1(e1): + if e2 in vocab and e2 != word: + cands2.add(e2) + return _pick(cands2) + # Long word: trigram index returns only the closest (<=2) vocab words, + # so _pick's frequency tie-break chooses among equally-near candidates. + return _pick(_trigram_dist2_candidates(word, deadline)) return None diff --git a/zimi/static/almanac.css b/zimi/static/almanac.css index 80bf8285..285ed431 100644 --- a/zimi/static/almanac.css +++ b/zimi/static/almanac.css @@ -63,7 +63,14 @@ keeps the cursor on it and hover repaints the amber border grey. */ .alm-tz-city-card.alm-tz-city-active, .alm-tz-city-card.alm-tz-city-active:hover { border: 2px solid var(--amber); background: var(--amber-glow); box-shadow: 0 0 10px var(--amber-glow); padding: 5px 3px; } -.alm-cal-region { text-align: center; font-size: 11px; color: var(--text3); margin-top: 8px; } +.alm-cal-region { text-align: center; font-size: 11px; color: var(--text3); margin-top: 8px; display: flex; justify-content: center; align-items: center; gap: 8px; flex-wrap: wrap; } +.alm-cal-region-cap { background: none; border: none; padding: 0; margin: 0; font: inherit; color: var(--text3); cursor: pointer; text-decoration: underline dotted; text-underline-offset: 2px; } +.alm-cal-region-cap:hover { color: var(--amber); } +.alm-cal-region-toggle { background: none; border: 1px solid var(--border); border-radius: 999px; padding: 1px 9px; font: inherit; font-size: 11px; color: var(--text3); cursor: pointer; line-height: 1.5; } +.alm-cal-region-toggle:hover { color: var(--amber); border-color: var(--amber-border); background: var(--amber-glow); } +/* Brief highlight when the holidays caption jumps focus to the location map. */ +.alm-loc-flash { animation: almLocFlash 1.6s ease-out; border-radius: 8px; } +@keyframes almLocFlash { 0%, 20% { box-shadow: 0 0 0 2px var(--amber), 0 0 14px var(--amber-glow); } 100% { box-shadow: 0 0 0 0 transparent, 0 0 0 transparent; } } .alm-tz-city-active .alm-tz-city-name { color: var(--amber); font-weight: 600; } .alm-tz-city-active .alm-tz-city-time { color: var(--amber); } .almanac-hero { display: flex; flex-direction: column; align-items: center; margin-bottom: 32px; position: relative; } diff --git a/zimi/static/almanac.js b/zimi/static/almanac.js index e74168fd..7f7b8b84 100644 --- a/zimi/static/almanac.js +++ b/zimi/static/almanac.js @@ -65,6 +65,33 @@ function _saveLocation(lat, lon, name) { _almSelectedTz = _almTzForLocation(lat, lon); } +// Holiday scope — 'region' (default: the one national pack for the chosen or +// detected location) or 'worldwide' (all national packs layered at once, each +// entry tagged with its country). Session-scoped like the location itself: the +// almanac is ephemeral, so a refresh returns to the location-following default. +var _ALM_HOLIDAY_SCOPE_KEY = 'zimi_almanac_holiday_scope'; +function _almHolidayScope() { + try { return sessionStorage.getItem(_ALM_HOLIDAY_SCOPE_KEY) === 'worldwide' ? 'worldwide' : 'region'; } + catch (e) { return 'region'; } +} +function _almToggleHolidayScope() { + var next = _almHolidayScope() === 'worldwide' ? 'region' : 'worldwide'; + try { sessionStorage.setItem(_ALM_HOLIDAY_SCOPE_KEY, next); } catch (e) {} + if (typeof _drawAlmanacGrid === 'function') _drawAlmanacGrid(); +} + +// Scroll the location control (the Sun & Daylight world map — where a click sets +// the location that drives holidays) into view and flash it. Target of the +// holidays caption, so a reader who wonders "whose holidays?" lands on the +// control that answers it. +function _almScrollToLocation() { + var el = document.getElementById('almanac-sunmap'); + if (!el) return; + try { el.scrollIntoView({ behavior: 'smooth', block: 'center' }); } catch (e) { el.scrollIntoView(); } + el.classList.add('alm-loc-flash'); + setTimeout(function () { el.classList.remove('alm-loc-flash'); }, 1600); +} + function _signalDelay(au) { var sec = au * 499; return { h: Math.floor(sec / 3600), m: Math.floor((sec % 3600) / 60) }; @@ -4015,10 +4042,13 @@ function _almRegion() { return ''; } -function _applyRegionHolidays(region, year, month, add) { +function _applyRegionHolidays(region, year, month, add, worldwide) { var pack = _REGION_HOLIDAYS[region]; if (!pack) return; - var src = _almRegionName(region); + // Region-scoped: the caption already names the country, so each entry's tag is + // just its colour. Worldwide: 18 packs at once, so every entry carries a + // compact country code ("Bastille Day · FR") to say whose day it is. + var src = worldwide ? region : _almRegionName(region); var i; for (i = 0; i < (pack.fixed || []).length; i++) { var fx = pack.fixed[i]; @@ -4030,6 +4060,9 @@ function _applyRegionHolidays(region, year, month, add) { var day = nh[2] === -1 ? _lastWeekday(year, month, nh[1]) : _nthWeekday(year, month, nh[1], nh[2]); add(day, nh[3], 'holiday', '', src, region); } + // Clock changes are location-specific; layering 18 countries' worth would be + // pure noise, so only the single region-scoped pack contributes them. + if (worldwide) return; // Clock changes: labels hold both hemispheres (October IS spring in AU) var dst = pack.dst; if (dst === 'us') { @@ -4044,6 +4077,16 @@ function _applyRegionHolidays(region, year, month, add) { } } +// Worldwide scope: layer every national pack (skipping the EU pseudo-region, +// which carries no national days of its own — only a DST rule). Each entry is +// tagged with its ISO country code via the worldwide path of _applyRegionHolidays. +function _applyAllRegionHolidays(year, month, add) { + for (var iso in _REGION_HOLIDAYS) { + if (iso === 'EU') continue; + _applyRegionHolidays(iso, year, month, add, true); + } +} + // ── Equinoxes & solstices: computed, not hardcoded (Meeus ch. 27) ────── // JDE0 mean-instant polynomials (valid 1000-3000 CE) plus the 24-term // periodic correction — accurate to minutes. The old code pinned fixed @@ -4167,8 +4210,10 @@ function _gregorianBaseEvents(year, month, add) { // (US, CA, AU, DE, IT, BR, IN, CN, JP and others) if (month === 5) { add(_nthWeekday(year, 5, 0, 2), t('hol_mothers_day'), 'holiday'); } if (month === 6) { add(_nthWeekday(year, 6, 0, 3), t('hol_fathers_day'), 'holiday'); } - // National days + clock changes for the detected region - _applyRegionHolidays(_almRegion(), year, month, add); + // National days: every pack when Worldwide is chosen, else the detected + // region's (which also contributes its clock changes). + if (_almHolidayScope() === 'worldwide') _applyAllRegionHolidays(year, month, add); + else _applyRegionHolidays(_almRegion(), year, month, add); // Easter and related var easter = _computeEaster(year); if (easter.month === month) { add(easter.day, 'Easter', 'holiday'); } @@ -4385,7 +4430,10 @@ function _drawAlmanacGrid() { for (var ei = 0; ei < shown; ei++) { var ev = dayEvents[ei]; var escapedLabel = _th(ev.label).replace(/&/g,'&').replace(//g,'>'); - var srcTitle = ev.src ? ' title="' + _tLookup('alm_region_holiday', '{c} holiday').replace('{c}', ev.src).replace(/"/g, '"') + '"' : ''; + // Worldwide entries carry a 2-letter ISO code as src; expand it to the full + // country name for the tooltip ("France holiday", not "FR holiday"). + var srcName = (ev.src && ev.src.length === 2) ? (_almRegionName(ev.src) || ev.src) : ev.src; + var srcTitle = ev.src ? ' title="' + _tLookup('alm_region_holiday', '{c} holiday').replace('{c}', srcName).replace(/"/g, '"') + '"' : ''; // Country-specific holidays (those with a region src) get their own // colour so they read apart from the worldwide observances (#33). var evCls = 'alm-ev alm-ev-' + ev.type + (ev.src ? ' alm-ev-country' : ''); @@ -4430,13 +4478,26 @@ function _drawAlmanacGrid() { // the Gregorian calendar — no pack means the worldwide set, and saying // so beats an unexplained absence of holidays. if (_almSystem === 'gregorian') { - var regionName = _almRegionName(_almRegion()); - var capText = regionName - ? _tLookup('alm_showing_holidays', 'Showing {c} holidays').replace('{c}', regionName.replace(/' + capText + '
'; + var scope = _almHolidayScope(); + var capText, toggleText; + if (scope === 'worldwide') { + capText = _tLookup('alm_showing_all_countries', "Showing every country's holidays"); + toggleText = _tLookup('alm_scope_region', 'My region'); + } else { + var regionName = _almRegionName(_almRegion()); + capText = regionName + ? _tLookup('alm_showing_holidays', 'Showing {c} holidays').replace('{c}', regionName.replace(/' + + capText + '' + + '' + + ''; } // Cross-reference — selected date in all calendar systems (replaces pills) From 044ebc1f25e0a7ec9ef52ebe8d8413a5ea7b62b1 Mon Sep 17 00:00:00 2001 From: Eric Pheterson Date: Tue, 28 Jul 2026 01:20:23 -0700 Subject: [PATCH 07/58] fix(dym): widen did-you-mean coverage + long-word distance-2 (1.8.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delivers the 1.8.0 release-note promise ("coverage grows with your title indexes — a work-in-progress we're widening in 1.8.1"). Validated against the production NAS library (68 indexes, 16.8GB, 27M sampled rows). Vocab coverage (the core fix). The old 200k word cap saturated on ~3 of 68 files and FROZE the scan, so words that only accumulate once the dictionary/encyclopedia indexes are reached never got in — "mitochondria" and "photosynthesis" were both absent in the production cache. The build now: - never stops early: hitting the raised (4M) peak cap triggers non-terminating tiered eviction (_evict_to_free) that frees room and keeps scanning every index; - freezes only NEW admissions, and only if eviction genuinely can't free room — existing words keep counting; - prunes singletons then top-K caps the persisted set (1.5M safety valve; a normal library keeps its full count>=2 set). Measured full build: 188s (< 5min budget), peak RSS 585MB, one clean eviction, cache ~9-19MB. mitochondria/photosynthesis now present (count 15/33). Builder version 3->4 so production re-scans once on upgrade. Long-word distance-2. The corrector only did exhaustive edit-2 for words <= 7 chars (a 13-char word generates ~450k candidates, ~700ms — over the 50ms budget), so "fotosynthesis" -> "photosynthesis" (distance 2, 14 chars) was uncorrectable. Added a bounded character-trigram inverted index over the long words of the vocab (selective trigrams only, shortest posting lists first, hard scan ceiling, edit-distance-verify the best overlaps, frequency-ranked pick), built in the background worker beside the vocab, not persisted (~2-3s rebuild from the loaded cache). The index is itself capped to the highest-count long words (_TRIGRAM_MAX_INDEX_WORDS) so per-query latency stays flat as the vocab grows: measured worst-case long-word query holds ~16-19ms whether the vocab is 600k or a synthetic 1.3M (uncapped it drifted to ~44ms). dist-1 and short-word paths are O(1) dict lookups, unaffected by vocab size. End-to-end through real search.py: mitochondira-> mitochondria 0.5ms, fotosynthesis->photosynthesis ~4ms, existing corrections (einstien, watre, phlosophy, volcanoe) intact. Tests: tests/test_did_you_mean.py, 62 green (12 new + the eviction test rewritten for non-terminating semantics). Covers non-terminating eviction, the admission-freeze fallback, top-K cap, bounded edit distance, the trigram index + its word cap, and end-to-end long-word correction. Co-Authored-By: Claude --- tests/test_snippet_boilerplate.py | 118 ++++++++++++++++++++++++++++++ zimi/server.py | 7 +- 2 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 tests/test_snippet_boilerplate.py diff --git a/tests/test_snippet_boilerplate.py b/tests/test_snippet_boilerplate.py new file mode 100644 index 00000000..4f9b5498 --- /dev/null +++ b/tests/test_snippet_boilerplate.py @@ -0,0 +1,118 @@ +"""Tests for snippet boilerplate-skip (previews.extract_snippet). + +iFixit device pages bake one repeated (a featured-guide +blurb) into every page. extract_snippet must prefer the page's own summary +block over that boilerplate, without regressing ZIM types whose meta +description IS the right snippet (wikipedia/gutenberg/ted-style). + +Fixtures are trimmed to the load-bearing structure of the real iFixit ZIM +markup fetched from the NAS (banner-blurb / itemprop="description" span, with +the wrong SSD meta description in ). +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from zimi.previews import extract_snippet # noqa: E402 + +# Real iFixit device-page shape (trimmed). The is the +# repeated featured-guide boilerplate; the banner-blurb span is the truth. +IFIXIT_AMERIWATER = """ +AmeriWater Water Purification System + + + +""" + +IFIXIT_ACER = """ +Acer Aspire 5253 + + + +""" + + +def test_ifixit_prefers_device_blurb_over_boilerplate_meta(): + snip = extract_snippet(IFIXIT_AMERIWATER, "ifixit") + assert "AmeriWater" in snip + assert "SSD" not in snip + assert "Lenovo" not in snip + + +def test_ifixit_second_device_also_correct(): + snip = extract_snippet(IFIXIT_ACER, "ifixit") + assert snip.startswith("A general purpose laptop") + assert "SSD" not in snip + + +def test_itemprop_description_used_without_banner_class(): + html = """ + A widget that does a specific useful thing.""" + snip = extract_snippet(html, "ifixit") + assert snip == "A widget that does a specific useful thing." + + +# ── Regressions: meta description must still win where it's the right source ── + + +def test_wikipedia_meta_description_still_used(): + html = """ +

Paris (French pronunciation) is the capital of France.

""" + snip = extract_snippet(html, "wikipedia_en_all") + assert snip == "Paris is the capital and most populous city of France." + + +def test_og_description_variant_used(): + html = """""" + snip = extract_snippet(html, "ted_en") + assert snip == "A talk about the future of biology and life." + + +def test_gutenberg_meta_description_used(): + html = """ + """ + snip = extract_snippet(html, "gutenberg_en") + assert "Pride and Prejudice" in snip + + +def test_no_meta_falls_back_to_main_body_skipping_nav(): + html = """ + +
The genuine article prose starts here and runs on with real content words.
+ """ + snip = extract_snippet(html, "generic") + assert snip.startswith("The genuine article prose") + assert "Home" not in snip + + +def test_toc_boilerplate_stripped_from_body_fallback(): + html = """ +
Table of contents Introduction Steps Comments
+
Actual body content that should become the snippet text here.
+ """ + snip = extract_snippet(html, "ifixit") + assert snip.startswith("Actual body content") + assert "Table of contents" not in snip + + +def test_short_own_summary_ignored(): + """A too-short blurb (<20 chars) should not pre-empt a good meta desc.""" + html = """ + """ + snip = extract_snippet(html, "ifixit") + assert snip.startswith("A properly detailed description") + + +def test_empty_html_returns_empty_string(): + assert extract_snippet("", "x") == "" diff --git a/zimi/server.py b/zimi/server.py index ffb1a3c8..acfc479f 100644 --- a/zimi/server.py +++ b/zimi/server.py @@ -1009,7 +1009,12 @@ def open_archive(path): return Archive(path) -from zimi.previews import strip_html, _extract_preview, _resolve_img_path # noqa: E402 +from zimi.previews import ( # noqa: E402 + strip_html, + _extract_preview, + _resolve_img_path, + extract_snippet, +) # ============================================================================ # ZIM Listing & Metadata Cache From cbf81cc770455e8c1464f6f2860c2d9ece6d1701 Mon Sep 17 00:00:00 2001 From: Eric Pheterson Date: Tue, 28 Jul 2026 01:30:53 -0700 Subject: [PATCH 08/58] feat(1.8.1): private-mode login gate (client) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the public-access policy is 'private', an anonymous visitor must see nothing but the login screen. The server already 401s every read endpoint; this makes the CLIENT match instead of rendering a half-populated home whose background fetches all fail. On boot, /whoami reports login_required for an anonymous visitor in private mode. The client then raises a non-dismissible login gate: an opaque overlay (solid --bg, top layer, background scroll locked) over the sign-in card. Cancel/Esc/any close path is a no-op while the gate is up — only a successful sign-in leaves, reloading into a clean authenticated boot (a named user gets their filtered view; an admin gets the full library). Admins and logged-in users never see the gate. Co-Authored-By: Claude --- zimi/static/app.css | 5 +++++ zimi/static/app.js | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/zimi/static/app.css b/zimi/static/app.css index 60a30026..7db257c5 100644 --- a/zimi/static/app.css +++ b/zimi/static/app.css @@ -1601,6 +1601,11 @@ /* ── Password modal ── */ .pw-overlay { display:none; position:fixed; inset:0; background:rgba(0,0,0,0.6); z-index:200; align-items:center; justify-content:center; } .pw-overlay.open { display:flex; } + /* Login gate (private public-access mode): the overlay becomes fully opaque + so the anonymous visitor sees nothing but the login card — no half-loaded + home showing through. Highest layer, and the background page can't scroll. */ + body.login-gate { overflow:hidden; } + body.login-gate .pw-overlay { background:var(--bg); z-index:1000; } .pw-box { background:var(--surface); border:1px solid var(--border); border-radius:12px; padding:24px; width:320px; max-width:90vw; } .pw-box h3 { margin:0 0 12px; font-size:15px; color:var(--text); } /* 16px minimum — iOS Safari auto-zooms (and stays zoomed) on focus of any diff --git a/zimi/static/app.js b/zimi/static/app.js index a9073964..46de0c55 100644 --- a/zimi/static/app.js +++ b/zimi/static/app.js @@ -584,11 +584,31 @@ function _checkUserSession() { // ("admin") applies (no custom username, no named users). The login modal // reads it to show "Default username: admin". _defaultUsernameHint = (j && j.default_username) || ''; + // Private public-access mode: an anonymous visitor (no user, no admin) must + // see nothing but the login screen. The server already 401s every read; the + // gate makes the CLIENT match — an opaque, non-dismissible login overlay + // instead of a half-populated home whose fetches all fail in the background. + if (j && j.login_required && j.role === 'anonymous' && !_manageToken) { + _enterLoginGate(); + } }).catch(function() {}); } // '' unless the server says the default username applies (see _checkUserSession). var _defaultUsernameHint = ''; +// ── Login gate (private public-access mode) ── +// True while an anonymous visitor is held at the forced login screen. +var _loginRequired = false; +function _enterLoginGate() { + _loginRequired = true; + document.body.classList.add('login-gate'); + openLoginModal(); +} +function _exitLoginGate() { + _loginRequired = false; + document.body.classList.remove('login-gate'); +} + // Token-adding fetch for *ambient* /manage/* calls (activity bar, peer // discovery, status/mirror polls). Sends the manage token when we have one // so these work on password-protected servers, but never prompts for a @@ -706,6 +726,10 @@ function openPwModal(title, opts) { } function closePwModal() { + // The private-mode login gate is non-dismissible: Cancel / Esc / any close + // path is a no-op, since there is nothing behind it but 401s. Only a + // successful sign-in (which calls _exitLoginGate) can leave. + if (_loginRequired) return; document.getElementById('pw-overlay').classList.remove('open'); document.removeEventListener('keydown', _pwKeyHandler); if (_pwReject) { _pwReject(); _pwReject = null; } @@ -719,10 +743,12 @@ function closePwModal() { } function _pwKeyHandler(e) { - // Esc closes the modal — standard a11y pattern for dialogs. + // Esc closes the modal — standard a11y pattern for dialogs. Exception: the + // login gate (private mode) is non-dismissible — there is nothing behind it + // to reveal, so Esc is swallowed. if (e.key === 'Escape') { e.preventDefault(); - closePwModal(); + if (!_loginRequired) closePwModal(); return; } // Tab focus-trap: cycle within the modal so keyboard users can't @@ -765,6 +791,9 @@ function submitPw() { // filtered library. Their account state lives in Manage → Users. _pwResolve = null; _pwReject = null; _pwLoginMode = false; + // Leaving the private-mode gate: reload into a clean authenticated + // state so the whole app boots with the session's filtered view. + if (_loginRequired) { _exitLoginGate(); location.reload(); return; } closePwModal(); _applyUserSession(res.j.name); return; @@ -780,6 +809,12 @@ function submitPw() { // modal, and resolves so the in-flight manage view paints in place. var resolver = _pwResolve; _pwResolve = null; _pwReject = null; resolver(tok); + } else if (_loginRequired) { + // Admin signing in from the private-mode gate: persist the token and + // reload into a clean, fully authorized boot (whoami → admin, no gate). + _manageToken = tok; _saveManageToken(tok, remember); + _exitLoginGate(); + location.reload(); } else { // Opened directly (no pending request): store the token and enter // manage deterministically (enterManage, never toggleManage — the From e868c10b6795bdd4d5d68f5c20b6f50951f9947d Mon Sep 17 00:00:00 2001 From: Eric Pheterson Date: Tue, 28 Jul 2026 01:32:49 -0700 Subject: [PATCH 09/58] =?UTF-8?q?feat(almanac):=20Location=20v2=20?= =?UTF-8?q?=E2=80=94=20wider=20city=20set=20+=20full=20clock/map=20parity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Widen the typeable location set from 353 to 502 cities: append 149 major world cities (population-ranked, top-per-country for spread) with coordinates taken straight from the GeoNames cities15000 dataset and deduped against the existing lists. Add Adelaide to the map so every world-clock city now has a dot (28/28 parity). Click-anywhere-to-set-exact-lat/lon (off-dot clicks) already lands via the existing handler; holidays resolve through the nearest-anchor region logic. Co-Authored-By: Claude --- zimi/static/almanac.js | 154 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 153 insertions(+), 1 deletion(-) diff --git a/zimi/static/almanac.js b/zimi/static/almanac.js index 7f7b8b84..ead48445 100644 --- a/zimi/static/almanac.js +++ b/zimi/static/almanac.js @@ -2870,6 +2870,7 @@ var _MAP_CITIES = [ { name: 'Melbourne, Victoria, Australia', lat: -37.81, lon: 144.96 }, { name: 'Brisbane, Queensland, Australia', lat: -27.47, lon: 153.03 }, { name: 'Perth, Western Australia, Australia', lat: -31.95, lon: 115.86 }, + { name: 'Adelaide, South Australia, Australia', lat: -34.93, lon: 138.60 }, { name: 'Auckland, New Zealand', lat: -36.85, lon: 174.76 }, { name: 'Wellington, New Zealand', lat: -41.29, lon: 174.78 }, { name: 'Suva, Fiji', lat: -17.77, lon: 177.97 } @@ -3109,7 +3110,158 @@ var _SEARCH_CITIES = [ { name: 'Wellington, New Zealand', lat: -41.29, lon: 174.78 }, { name: 'Christchurch, New Zealand', lat: -43.53, lon: 172.64 }, { name: 'Suva, Fiji', lat: -18.14, lon: 178.44 }, - { name: 'Port Moresby, Papua New Guinea', lat: -9.44, lon: 147.18 } + { name: 'Port Moresby, Papua New Guinea', lat: -9.44, lon: 147.18 }, + // ── Wider world coverage (1.8.1): major cities by population, coordinates + // from the GeoNames cities15000 dataset; deduped against the lists above. ── + { name: 'Aden, Yemen', lat: 12.78, lon: 45.04 }, + { name: 'Al Basrah al Qadimah, Iraq', lat: 30.50, lon: 47.82 }, + { name: 'Al Mawsil al Jadidah, Iraq', lat: 36.33, lon: 43.11 }, + { name: 'Aleppo, Syria', lat: 36.20, lon: 37.16 }, + { name: 'Andijon, Uzbekistan', lat: 40.78, lon: 72.35 }, + { name: 'Ankara, Turkey', lat: 39.92, lon: 32.85 }, + { name: 'Antananarivo, Madagascar', lat: -18.91, lon: 47.54 }, + { name: 'Antsirabe, Madagascar', lat: -19.87, lon: 47.03 }, + { name: 'Arequipa, Peru', lat: -16.40, lon: -71.54 }, + { name: 'Arhus, Denmark', lat: 56.16, lon: 10.21 }, + { name: 'Ashgabat, Turkmenistan', lat: 37.95, lon: 58.38 }, + { name: 'Asmara, Eritrea', lat: 15.34, lon: 38.93 }, + { name: 'Bamako, Mali', lat: 12.61, lon: -7.98 }, + { name: 'Bamenda, Cameroon', lat: 5.96, lon: 10.15 }, + { name: 'Bangui, Central African Republic', lat: 4.36, lon: 18.55 }, + { name: 'Banja Luka, Bosnia and Herzegovina', lat: 44.78, lon: 17.21 }, + { name: 'Benghazi, Libya', lat: 32.11, lon: 20.07 }, + { name: 'Bharatpur, Nepal', lat: 27.68, lon: 84.44 }, + { name: 'Bissau, Guinea-Bissau', lat: 11.86, lon: -15.60 }, + { name: 'Blantyre, Malawi', lat: -15.78, lon: 35.01 }, + { name: 'Bo, Sierra Leone', lat: 7.96, lon: -11.74 }, + { name: 'Bobo-Dioulasso, Burkina Faso', lat: 11.18, lon: -4.29 }, + { name: 'Borama, Somalia', lat: 9.94, lon: 43.18 }, + { name: 'Bouake, Ivory Coast', lat: 7.69, lon: -5.03 }, + { name: 'Bujumbura, Burundi', lat: -3.38, lon: 29.36 }, + { name: 'Bulawayo, Zimbabwe', lat: -20.15, lon: 28.58 }, + { name: 'Bursa, Turkey', lat: 40.20, lon: 29.06 }, + { name: 'Camagueey, Cuba', lat: 21.38, lon: -77.92 }, + { name: 'Chisinau, Moldova', lat: 47.01, lon: 28.86 }, + { name: 'Ciudad del Este, Paraguay', lat: -25.50, lon: -54.65 }, + { name: 'Cochabamba, Bolivia', lat: -17.38, lon: -66.16 }, + { name: 'Conakry, Guinea', lat: 9.54, lon: -13.68 }, + { name: 'Constantine, Algeria', lat: 36.37, lon: 6.61 }, + { name: 'Cork, Ireland', lat: 51.90, lon: -8.47 }, + { name: 'Cotonou, Benin', lat: 6.37, lon: 2.42 }, + { name: 'Cuenca, Ecuador', lat: -2.90, lon: -79.00 }, + { name: 'Damascus, Syria', lat: 33.51, lon: 36.29 }, + { name: 'Danli, Honduras', lat: 14.03, lon: -86.58 }, + { name: 'Dasoguz, Turkmenistan', lat: 41.84, lon: 59.97 }, + { name: 'Djibouti, Djibouti', lat: 11.59, lon: 43.15 }, + { name: 'Dodoma, Tanzania', lat: -6.17, lon: 35.74 }, + { name: 'Douala, Cameroon', lat: 4.05, lon: 9.70 }, + { name: 'Dushanbe, Tajikistan', lat: 38.54, lon: 68.78 }, + { name: 'Fes, Morocco', lat: 34.03, lon: -5.00 }, + { name: 'Freetown, Sierra Leone', lat: 8.49, lon: -13.24 }, + { name: 'Gaborone, Botswana', lat: -24.65, lon: 25.91 }, + { name: 'Ganja, Azerbaijan', lat: 40.68, lon: 46.36 }, + { name: 'Gaza, Palestinian Territory', lat: 31.50, lon: 34.47 }, + { name: 'Georgetown, Guyana', lat: 6.80, lon: -58.16 }, + { name: 'Gonder, Ethiopia', lat: 12.60, lon: 37.47 }, + { name: 'Graz, Austria', lat: 47.07, lon: 15.44 }, + { name: 'Haiphong, Vietnam', lat: 20.86, lon: 106.68 }, + { name: 'Hamhung, North Korea', lat: 39.92, lon: 127.54 }, + { name: 'Hargeysa, Somalia', lat: 9.56, lon: 44.06 }, + { name: 'Herat, Afghanistan', lat: 34.35, lon: 62.20 }, + { name: 'Homs, Syria', lat: 34.72, lon: 36.73 }, + { name: 'Homyel\', Belarus', lat: 52.43, lon: 30.98 }, + { name: 'Hrodna, Belarus', lat: 53.68, lon: 23.83 }, + { name: 'Iasi, Romania', lat: 47.17, lon: 27.60 }, + { name: 'Ibadan, Nigeria', lat: 7.38, lon: 3.91 }, + { name: 'Irbid, Jordan', lat: 32.56, lon: 35.85 }, + { name: 'Isfara, Tajikistan', lat: 40.13, lon: 70.63 }, + { name: 'Istaravshan, Tajikistan', lat: 39.91, lon: 69.00 }, + { name: 'Jijiga, Ethiopia', lat: 9.35, lon: 42.80 }, + { name: 'Juba, South Sudan', lat: 4.85, lon: 31.58 }, + { name: 'Kakamega, Kenya', lat: 0.28, lon: 34.75 }, + { name: 'Kano, Nigeria', lat: 12.00, lon: 8.52 }, + { name: 'Kaunas, Lithuania', lat: 54.90, lon: 23.91 }, + { name: 'Kenema, Sierra Leone', lat: 7.88, lon: -11.19 }, + { name: 'Kigali, Rwanda', lat: -1.95, lon: 30.06 }, + { name: 'Kingston, Jamaica', lat: 18.00, lon: -76.79 }, + { name: 'Kitwe, Zambia', lat: -12.80, lon: 28.21 }, + { name: 'Kosice, Slovakia', lat: 48.71, lon: 21.26 }, + { name: 'Koutiala, Mali', lat: 12.39, lon: -5.47 }, + { name: 'Kumasi, Ghana', lat: 6.69, lon: -1.62 }, + { name: 'Libreville, Gabon', lat: 0.39, lon: 9.45 }, + { name: 'Lilongwe, Malawi', lat: -13.97, lon: 33.79 }, + { name: 'Linz, Austria', lat: 48.31, lon: 14.29 }, + { name: 'Lome, Togo', lat: 6.13, lon: 1.22 }, + { name: 'Lubumbashi, Democratic Republic of the Congo', lat: -11.66, lon: 27.48 }, + { name: 'Macau, Macao', lat: 22.20, lon: 113.55 }, + { name: 'Managua, Nicaragua', lat: 12.13, lon: -86.25 }, + { name: 'Mandalay, Myanmar', lat: 21.97, lon: 96.08 }, + { name: 'Maracaibo, Venezuela', lat: 10.64, lon: -71.61 }, + { name: 'Maradi, Niger', lat: 13.50, lon: 7.10 }, + { name: 'Maseru, Lesotho', lat: -29.32, lon: 27.48 }, + { name: 'Mazar-e Sharif, Afghanistan', lat: 36.71, lon: 67.11 }, + { name: 'Mbuji-Mayi, Democratic Republic of the Congo', lat: -6.14, lon: 23.59 }, + { name: 'Minsk, Belarus', lat: 53.90, lon: 27.57 }, + { name: 'Misratah, Libya', lat: 32.38, lon: 15.09 }, + { name: 'Mogadishu, Somalia', lat: 2.04, lon: 45.34 }, + { name: 'Monrovia, Liberia', lat: 6.30, lon: -10.80 }, + { name: 'Mwanza, Tanzania', lat: -2.52, lon: 32.90 }, + { name: 'Mzuzu, Malawi', lat: -11.47, lon: 34.02 }, + { name: 'N\'Djamena, Chad', lat: 12.11, lon: 15.04 }, + { name: 'Namangan, Uzbekistan', lat: 41.00, lon: 71.67 }, + { name: 'Nampula, Mozambique', lat: -15.12, lon: 39.27 }, + { name: 'Nassau, Bahamas', lat: 25.06, lon: -77.34 }, + { name: 'Nay Pyi Taw, Myanmar', lat: 19.75, lon: 96.13 }, + { name: 'Ndola, Zambia', lat: -12.96, lon: 28.64 }, + { name: 'Niamey, Niger', lat: 13.51, lon: 2.11 }, + { name: 'Nicosia, Cyprus', lat: 35.17, lon: 33.35 }, + { name: 'Nis, Serbia', lat: 43.32, lon: 21.90 }, + { name: 'Nouakchott, Mauritania', lat: 18.09, lon: -15.98 }, + { name: 'Novi Sad, Serbia', lat: 45.25, lon: 19.84 }, + { name: 'Nzerekore, Guinea', lat: 7.76, lon: -8.82 }, + { name: 'Oran, Algeria', lat: 35.70, lon: -0.64 }, + { name: 'Osh, Kyrgyzstan', lat: 40.53, lon: 72.80 }, + { name: 'Ostrava, Czechia', lat: 49.83, lon: 18.28 }, + { name: 'Ouagadougou, Burkina Faso', lat: 12.37, lon: -1.53 }, + { name: 'Paramaribo, Suriname', lat: 5.87, lon: -55.17 }, + { name: 'Plovdiv, Bulgaria', lat: 42.15, lon: 24.75 }, + { name: 'Podgorica, Montenegro', lat: 42.44, lon: 19.26 }, + { name: 'Pointe-Noire, Republic of the Congo', lat: -4.78, lon: 11.86 }, + { name: 'Pokhara, Nepal', lat: 28.27, lon: 83.97 }, + { name: 'Port-au-Prince, Haiti', lat: 18.54, lon: -72.34 }, + { name: 'Pristina, Kosovo', lat: 42.67, lon: 21.17 }, + { name: 'Rabat, Morocco', lat: 34.01, lon: -6.83 }, + { name: 'San Miguel, El Salvador', lat: 13.48, lon: -88.18 }, + { name: 'San Pedro Sula, Honduras', lat: 15.51, lon: -88.03 }, + { name: 'San Salvador, El Salvador', lat: 13.69, lon: -89.19 }, + { name: 'Sanaa, Yemen', lat: 15.35, lon: 44.21 }, + { name: 'Santa Cruz de la Sierra, Bolivia', lat: -17.79, lon: -63.18 }, + { name: 'Santiago de Cuba, Cuba', lat: 20.02, lon: -75.82 }, + { name: 'Santiago de los Caballeros, Dominican Republic', lat: 19.45, lon: -70.69 }, + { name: 'Sarajevo, Bosnia and Herzegovina', lat: 43.85, lon: 18.36 }, + { name: 'Serekunda, Gambia', lat: 13.44, lon: -16.68 }, + { name: 'Sfax, Tunisia', lat: 34.74, lon: 10.76 }, + { name: 'Shymkent, Kazakhstan', lat: 42.31, lon: 69.60 }, + { name: 'Sikasso, Mali', lat: 11.32, lon: -5.67 }, + { name: 'Skopje, North Macedonia', lat: 42.00, lon: 21.43 }, + { name: 'Sousse, Tunisia', lat: 35.83, lon: 10.64 }, + { name: 'Taichung, Taiwan', lat: 24.15, lon: 120.68 }, + { name: 'Taiz, Yemen', lat: 13.58, lon: 44.02 }, + { name: 'Takeo, Cambodia', lat: 10.99, lon: 104.78 }, + { name: 'Tamale, Ghana', lat: 9.40, lon: -0.84 }, + { name: 'Tampere, Finland', lat: 61.50, lon: 23.79 }, + { name: 'Tegucigalpa, Honduras', lat: 14.08, lon: -87.21 }, + { name: 'Tirana, Albania', lat: 41.33, lon: 19.82 }, + { name: 'Toamasina, Madagascar', lat: -18.15, lon: 49.40 }, + { name: 'Touba, Senegal', lat: 14.86, lon: -15.88 }, + { name: 'Trondheim, Norway', lat: 63.43, lon: 10.40 }, + { name: 'Tuerkmenabat, Turkmenistan', lat: 39.07, lon: 63.58 }, + { name: 'Varna, Bulgaria', lat: 43.22, lon: 27.91 }, + { name: 'Winejok, South Sudan', lat: 9.01, lon: 27.57 }, + { name: 'Wroclaw, Poland', lat: 51.10, lon: 17.03 }, + { name: 'Yaounde, Cameroon', lat: 3.87, lon: 11.52 }, + { name: 'Yei, South Sudan', lat: 4.09, lon: 30.68 }, + { name: 'Zinder, Niger', lat: 13.81, lon: 8.99 }, ]; // Coastline data removed — using Natural Earth SVG map (/static/world-map.svg) From 4468d833c6cbf67fa8aa4c0aa1741cc22efc833d Mon Sep 17 00:00:00 2001 From: Eric Pheterson Date: Tue, 28 Jul 2026 01:33:20 -0700 Subject: [PATCH 10/58] feat: snippet boilerplate-skip helper (previews.extract_snippet) Commit previews.py as it sits so HEAD (server.py re-exports extract_snippet; tests/test_snippet_boilerplate.py imports it) no longer depends on an uncommitted file. extract_snippet() prefers a page's own summary block (iFixit banner-blurb / itemprop=description) over a repeated boilerplate , then meta description, then body prose with nav/TOC/related-guide chrome stripped. Other preview refinements here are the on-disk truth HEAD already builds against. Co-Authored-By: Claude --- zimi/previews.py | 403 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 312 insertions(+), 91 deletions(-) diff --git a/zimi/previews.py b/zimi/previews.py index ed57b40f..c269f307 100644 --- a/zimi/previews.py +++ b/zimi/previews.py @@ -18,6 +18,84 @@ def strip_html(text): return text +# Some ZIMs bake a single repeated into every page — iFixit +# device pages carry a featured-guide blurb ("How to replace the SSD in your +# Lenovo Legion…") rather than the device's own description. When a page exposes +# its own summary block (iFixit's .banner-blurb / itemprop="description"), that +# wins over the meta tag. +_SNIPPET_OWN_SUMMARY_RES = [ + re.compile( + r']*\bclass=["\'][^"\']*\bbanner-blurb\b[^"\']*["\'][^>]*>(.*?)

', + re.IGNORECASE | re.DOTALL, + ), + re.compile( + r'<[a-z]+[^>]*\bitemprop=["\']description["\'][^>]*>(.*?)', + re.IGNORECASE | re.DOTALL, + ), +] +_SNIPPET_META_DESC_RES = [ + re.compile( + r']*>.*?" + r'|<[a-z]+[^>]*\bclass=["\'][^"\']*\b' + r"(?:toc|breadcrumb|related-guides|featured-guides|navigation|" + r'js-dynamic-toc-section)\b[^"\']*["\'][^>]*>.*?', + re.IGNORECASE | re.DOTALL, +) + + +def extract_snippet(text, zim_name=""): + """Best short text snippet for the /snippet endpoint. + + Order of preference: + 1. The page's own summary block (iFixit device blurb / itemprop + description) — beats a repeated boilerplate . + 2. A tag. + 3.
/
body prose, boilerplate chrome stripped. + 4. Full page text, boilerplate chrome stripped. + + ``text`` is the already-decoded HTML lead (the caller reads ~15 KB, enough + for meta + the opening content). Returns a plain-text string + (possibly empty).""" + # 1. Page's own summary block. + for rx in _SNIPPET_OWN_SUMMARY_RES: + m = rx.search(text) + if m: + s = strip_html(m.group(1))[:300].strip() + if len(s) >= 20: + return s + # 2. Meta description (near the top of ). + for rx in _SNIPPET_META_DESC_RES: + m = rx.search(text[:8000]) + if m: + s = strip_html(m.group(1))[:300].strip() + if s: + return s + # 3/4. Body prose, with repeated chrome removed first. + cleaned = _SNIPPET_BOILERPLATE_RE.sub(" ", text) + for tag in ("main", "article"): + tag_m = re.search(r"<" + tag + r"[\s>]", cleaned, re.IGNORECASE) + if tag_m: + s = strip_html(cleaned[tag_m.start() :])[:300].strip() + if s: + return s + return strip_html(cleaned)[:300].strip() + + def _resolve_img_path(archive, path, src): """Resolve a relative image src to a ZIM entry path. Returns URL or None.""" decoded = unquote(unquote(src)) @@ -29,7 +107,8 @@ def _resolve_img_path(archive, path, src): parts = [] for seg in img_path.replace("\\", "/").split("/"): if seg == "..": - if parts: parts.pop() + if parts: + parts.pop() elif seg and seg != ".": parts.append(seg) img_path = "/".join(parts) @@ -60,18 +139,24 @@ def _extract_preview_title(html_str, entry_title): for pattern in [ r']*>([^<]+)', + r"]*>([^<]+)", r']*>(.*?)

', r']*>(.*?)

', - r']*>(.*?)', + r"]*>(.*?)", ]: tm = re.search(pattern, html_str, re.IGNORECASE | re.DOTALL) if tm: clean_title = strip_html(html.unescape(tm.group(1).strip())) # Strip site suffixes like " | TED Talk", "— The World Factbook" - clean_title = re.sub(r'\s*[\|–—]\s*(TED\s*Talk|TED|Wikipedia|The World Factbook).*$', '', clean_title) + clean_title = re.sub( + r"\s*[\|–—]\s*(TED\s*Talk|TED|Wikipedia|The World Factbook).*$", + "", + clean_title, + ) # Strip Factbook region prefixes like "Africa :: " or "Europe :: " - clean_title = re.sub(r'^[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\s*::\s*', '', clean_title) + clean_title = re.sub( + r"^[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\s*::\s*", "", clean_title + ) if len(clean_title) > 3 and clean_title != entry_title: return clean_title[:200] # No HTML title found — title-case the slug as last resort @@ -80,13 +165,23 @@ def _extract_preview_title(html_str, entry_title): def _is_real_quote(text): """Filter out non-quote text: credits, citations, references, metadata.""" - if re.match(r'^(Directed|Written|Produced|Edited|Narrated|Adapted|Translated|Music)\s+by\b', text, re.IGNORECASE): + if re.match( + r"^(Directed|Written|Produced|Edited|Narrated|Adapted|Translated|Music)\s+by\b", + text, + re.IGNORECASE, + ): return False - if re.match(r'^(In response to|Based on|See also|Main article)\b', text, re.IGNORECASE): + if re.match( + r"^(In response to|Based on|See also|Main article)\b", text, re.IGNORECASE + ): return False - if re.search(r'\bRetrieved\s+(on|from)\b|\bISBN\b|\bAssociated Press\b|^\d{4}\s+film\b', text, re.IGNORECASE): + if re.search( + r"\bRetrieved\s+(on|from)\b|\bISBN\b|\bAssociated Press\b|^\d{4}\s+film\b", + text, + re.IGNORECASE, + ): return False - if re.match(r'^[\w\s,]+\(\s*\d{4}\s*\)', text): # "Author Name (Year)" citation + if re.match(r"^[\w\s,]+\(\s*\d{4}\s*\)", text): # "Author Name (Year)" citation return False return len(text.split()) > 6 @@ -98,46 +193,70 @@ def _extract_wikiquote_attribution(block, inner_ul_pos, page_title): """ author = None # Use page title as fallback only if it looks like a person name (has a space) - if ' ' in page_title and re.match(r'^[A-Z][a-z]+ [A-Z]', page_title): + if " " in page_title and re.match(r"^[A-Z][a-z]+ [A-Z]", page_title): author = page_title inner_block = block[inner_ul_pos:] attr_raw = strip_html(inner_block).strip() # Normalize double spaces around punctuation (strip_html replaces tags with spaces) - attr_raw = re.sub(r'\s+([,;:.!?])', r'\1', attr_raw) - attr_raw = re.sub(r'^[\u2014\u2013\-~]+\s*', '', attr_raw).strip().split('\n')[0].strip() + attr_raw = re.sub(r"\s+([,;:.!?])", r"\1", attr_raw) + attr_raw = ( + re.sub(r"^[\u2014\u2013\-~]+\s*", "", attr_raw).strip().split("\n")[0].strip() + ) if attr_raw and 3 < len(attr_raw) < 200: - if not re.search(r'[\[\]{}]|https?:|www\.|^\d', attr_raw, re.IGNORECASE): + if not re.search(r"[\[\]{}]|https?:|www\.|^\d", attr_raw, re.IGNORECASE): # Detect source citations (not person names): # - Contains ":" mid-text (e.g. "StoptheWarNow: Third peace convoy") # - Looks like a title/headline (many capitalized words, >5 words) # - Contains news agency / publication markers _is_source = bool( - re.search(r'\w:\s+\w', attr_raw) # colon in middle - or re.search(r'(?i)\b(Agency|News|Times|Post|Tribune|Journal|Gazette|Herald|Magazine|Review|Report|Press|Daily)\b', attr_raw) - or (len(attr_raw.split()) > 6 and not re.match(r'^[A-Z][a-z]+(?:\s+[a-z]+)*\s+[A-Z][a-z]+$', attr_raw.split('(')[0].split(',')[0].strip())) + re.search(r"\w:\s+\w", attr_raw) # colon in middle + or re.search( + r"(?i)\b(Agency|News|Times|Post|Tribune|Journal|Gazette|Herald|Magazine|Review|Report|Press|Daily)\b", + attr_raw, + ) + or ( + len(attr_raw.split()) > 6 + and not re.match( + r"^[A-Z][a-z]+(?:\s+[a-z]+)*\s+[A-Z][a-z]+$", + attr_raw.split("(")[0].split(",")[0].strip(), + ) + ) ) if not _is_source: # Extract name: everything before first comma or opening paren # e.g. "Henry Adams, Mont Saint Michel and Chartres (1904)" → "Henry Adams" - name_part = re.split(r'[,(]', attr_raw)[0].strip() + name_part = re.split(r"[,(]", attr_raw)[0].strip() # Handle honorifics with commas: "Adams, Henry" or "King, Jr., Martin Luther" # If name_part is a single word and next part also looks like a name, rejoin - if name_part and ',' in attr_raw: - parts = [p.strip() for p in attr_raw.split(',')] + if name_part and "," in attr_raw: + parts = [p.strip() for p in attr_raw.split(",")] # "Last, First" pattern: single capitalized word, then capitalized word(s) - if (len(parts) >= 2 and re.match(r'^[A-Z][a-z]+$', parts[0]) - and re.match(r'^(Jr\.|Sr\.|[A-Z])', parts[1])): + if ( + len(parts) >= 2 + and re.match(r"^[A-Z][a-z]+$", parts[0]) + and re.match(r"^(Jr\.|Sr\.|[A-Z])", parts[1]) + ): # Check for Jr./Sr. suffix - if parts[1] in ('Jr.', 'Sr.', 'III', 'II', 'IV') and len(parts) >= 3: - name_part = parts[2].strip() + ' ' + parts[0] + ', ' + parts[1] - elif re.match(r'^[A-Z][a-z]', parts[1]): + if ( + parts[1] in ("Jr.", "Sr.", "III", "II", "IV") + and len(parts) >= 3 + ): + name_part = ( + parts[2].strip() + " " + parts[0] + ", " + parts[1] + ) + elif re.match(r"^[A-Z][a-z]", parts[1]): # "Last, First ..." — but only if second part is short (a name, not a book title) if len(parts[1].split()) <= 3: - name_part = parts[1] + ' ' + parts[0] + name_part = parts[1] + " " + parts[0] # Validate: must start with uppercase letter, reasonable length - if (name_part and 2 < len(name_part) < 60 - and re.match(r'^[A-Z]', name_part) - and not re.match(r'^(p\.|ch\.|vol\.|see |ibid)', name_part, re.IGNORECASE)): + if ( + name_part + and 2 < len(name_part) < 60 + and re.match(r"^[A-Z]", name_part) + and not re.match( + r"^(p\.|ch\.|vol\.|see |ibid)", name_part, re.IGNORECASE + ) + ): author = name_part return author @@ -150,14 +269,14 @@ def _extract_preview_wikiquote(html_str, result, entry_title): # Wikiquote structure:
  • Quote text
    • Attribution
# Strategy: find