From ddde19c2abf1084606aaff5bc78b9b22940bb7d6 Mon Sep 17 00:00:00 2001 From: Robbie Court Date: Sun, 28 Jun 2026 11:23:18 +0100 Subject: [PATCH] Enrich and parallelise static term-page generation Generate full term pages from get_term_info (cross-references, class example downloads, graphs-for), emit Goldmark-safe HTML (full for query previews, anchors inside the hero block, blank-line section separation), and fetch concurrently via a bounded worker pool. Bumps page version 8->9 to force a full regeneration. Syncs the change from VFB2-draft so update_vfb_terms_to_static_site picks it up. --- vfbterms.py | 322 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 255 insertions(+), 67 deletions(-) diff --git a/vfbterms.py b/vfbterms.py index 193eda7..66951f1 100644 --- a/vfbterms.py +++ b/vfbterms.py @@ -9,6 +9,8 @@ import json import traceback import time +import threading +from concurrent.futures import ThreadPoolExecutor, as_completed, wait, FIRST_COMPLETED from urllib.parse import quote # Suppress the urllib3 warning about OpenSSL @@ -25,17 +27,24 @@ from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry -version = 8 +version = 9 API_BASE = "https://v3-cached.virtualflybrain.org/get_term_info" STATUS_URL = "https://vfbquery.virtualflybrain.org/status" VFB_BROWSER_BASE = "https://v2.virtualflybrain.org/org.geppetto.frontend/geppetto" # Throttle settings — stay under 20 concurrent to keep API reliable -MAX_ACTIVE_BEFORE_BACKOFF = 20 # Back off when this many queries are active +MAX_ACTIVE_BEFORE_BACKOFF = 20 # Back off when the server reports this many active queries STATUS_CHECK_INTERVAL = 10 # Seconds between status checks while waiting MAX_CAPACITY_WAIT_SECONDS = 600 # Give up waiting for capacity after this long -API_TIMEOUT_SECONDS = int(os.environ.get("VFB_API_TIMEOUT_SECONDS", "9000")) +# Concurrent fetch/generate workers. Kept below MAX_ACTIVE_BEFORE_BACKOFF so our +# own load leaves headroom for other clients; the serial loop only ever used one +# slot, leaving the server (≈20 slots) mostly idle — this is the main speed-up. +MAX_WORKERS = int(os.environ.get("VFB_MAX_WORKERS", "12")) +# Per-request timeout. The old 9000s default was a workaround for serial stalls; +# with concurrency a hung term must not pin a worker for hours. Override with +# VFB_API_TIMEOUT_SECONDS if a genuinely heavy term needs longer. +API_TIMEOUT_SECONDS = int(os.environ.get("VFB_API_TIMEOUT_SECONDS", "600")) # Known ID prefixes for internal link conversion KNOWN_PREFIXES = ( @@ -52,11 +61,12 @@ def create_session(): """Create a requests session with retry logic and connection pooling.""" session = requests.Session() retry = Retry( - total=2, - backoff_factor=120, - status_forcelist=[500, 502, 503, 504], + total=3, + backoff_factor=5, + status_forcelist=[429, 500, 502, 503, 504], + respect_retry_after_header=True, ) - adapter = HTTPAdapter(max_retries=retry, pool_connections=10, pool_maxsize=10) + adapter = HTTPAdapter(max_retries=retry, pool_connections=MAX_WORKERS, pool_maxsize=MAX_WORKERS) session.mount("https://", adapter) session.mount("http://", adapter) return session @@ -78,6 +88,37 @@ def check_server_status(): print(f"WARNING: Could not check server status: {e}") return None +# Cached, thread-safe view of server capacity so a pool of workers doesn't each +# poll /status before every request. One worker refreshes the cache at most once +# per STATUS_CHECK_INTERVAL; the rest read the cached verdict. +_status_lock = threading.Lock() +_status_cache = {"ts": 0.0, "ok": True} + +def throttle(term_id=""): + """Pause the calling worker while the server is at/over capacity. + + Reads a shared cached /status verdict (refreshed at most once per + STATUS_CHECK_INTERVAL). Returns once the server has headroom, or after + MAX_CAPACITY_WAIT_SECONDS so a single term can't block forever. With + MAX_WORKERS below MAX_ACTIVE_BEFORE_BACKOFF our own load never trips this; + it only engages when other clients are loading the server. + """ + start = time.time() + while True: + now = time.time() + with _status_lock: + if (now - _status_cache["ts"]) >= STATUS_CHECK_INTERVAL: + st = check_server_status() + _status_cache["ts"] = now + _status_cache["ok"] = True if st is None else st[0] < MAX_ACTIVE_BEFORE_BACKOFF + ok = _status_cache["ok"] + if ok: + return + if time.time() - start >= MAX_CAPACITY_WAIT_SECONDS: + print(f"WARNING: proceeding with {term_id or 'request'} after waiting for capacity") + return + time.sleep(STATUS_CHECK_INTERVAL) + def wait_for_server_capacity(term_id=""): """Block until the server has capacity below our threshold. @@ -131,8 +172,8 @@ def fetch_term_info(term_id): Checks server capacity before making the request to avoid flooding. """ - # Wait until the server isn't overloaded - wait_for_server_capacity(term_id) + # Wait until the server isn't overloaded (shared cached capacity check) + throttle(term_id) try: resp = session.get(API_BASE, params={"id": term_id}, timeout=API_TIMEOUT_SECONDS) @@ -192,6 +233,32 @@ def replace_link(match): # Match: [ ... ]( ID ) where the ID part has no spaces or parens return re.sub(r'\[([^\]]*(?:\[[^\]]*\][^\]]*)*)\]\(([^)\s]+)\)', replace_link, text) +# Markdown links nested inside a raw HTML block (e.g. the hero card

) are NOT +# processed by Hugo's Goldmark renderer — it passes HTML blocks through verbatim. +# So any `[label](id)` placed inside

…

would render as literal brackets. +# linkify_html emits real anchors instead, mapping known ids to /reports/ +# exactly like convert_internal_links does for plain markdown. +_MD_LINK_RE = re.compile(r'\[([^\]]*(?:\[[^\]]*\][^\]]*)*)\]\(([^)\s]+)\)') + +def linkify_html(text): + """Convert `[label](target)` markdown links to HTML anchors. + + For embedding in raw HTML blocks (hero card description/comment), where + Goldmark won't process markdown. Known VFB ids are rewritten to /reports/; + other targets (already-resolved URLs) are used as-is. Newlines are collapsed + to spaces so the surrounding HTML block isn't terminated by a blank line. + """ + if not text: + return "" + + def repl(match): + label = match.group(1) + identifier = match.group(2) + href = get_report_url(identifier) if is_known_id(identifier) else identifier + return f'{label}' + + return _MD_LINK_RE.sub(repl, text).replace("\r", " ").replace("\n", " ") + def format_relationships_section(relationships_text): """Format Meta.Relationships into markdown bullets. @@ -305,7 +372,16 @@ def format_synonyms_table(synonyms): return "\n".join(lines) def format_query_preview(query, term_id): - """Format a Query's preview_results as a markdown table with thumbnails.""" + """Format a Query's preview_results as an HTML table with thumbnails. + + Emitted as a complete raw HTML
rather than a markdown table wrapped + in
. Hugo's Goldmark does not process markdown + nested inside a raw HTML block, so the wrapped-markdown form leaked every + `[label](id)` cell as literal text and never rendered as a table. A full + HTML table renders verbatim under unsafe=true. The `###` heading and the + "View all" button stay outside the block, separated by blank lines so + Goldmark closes each HTML block cleanly. + """ preview = query.get("preview_results", {}) rows = preview.get("rows", []) if not rows: @@ -319,36 +395,39 @@ def format_query_preview(query, term_id): lines.append(f'### {label} ({count} total)') lines.append("") lines.append('
') - lines.append("") - lines.append("| Thumbnail | Name | Tags |") - lines.append("|-----------|------|------|") + lines.append('
') + lines.append('') + lines.append('') for row in rows: - # Extract name — may be markdown link like [name](id) + # Name — may be `[label](id)` markdown from the API; emit as an anchor + # because it lives inside a raw HTML block (Goldmark won't linkify it). name_raw = row.get("label", row.get("name", "")) - name_converted = convert_internal_links(name_raw) + name_md = convert_internal_links(name_raw) row_id = row.get("id", "") - if row_id and is_known_id(row_id) and name_converted == name_raw and name_raw: - name_converted = f'[{name_raw}]({get_report_url(row_id)})' + if row_id and is_known_id(row_id) and name_md == name_raw and name_raw: + name_md = f'[{name_raw}]({get_report_url(row_id)})' + name_html = linkify_html(name_md) if "](" in name_md else name_md - # Extract tags + # Tags tags_raw = row.get("tags", "") tags_display = tags_raw.replace("|", ", ") if tags_raw else "" - # Extract thumbnail — may be markdown image like [![alt](url)](link) + # Thumbnail — pull the URL out of `[![alt](url 'title')](link)` markdown. thumb_raw = row.get("thumbnail", "") thumb_html = "" if thumb_raw: - # Extract URL from markdown image: [![alt](url "title")](link) img_match = re.search(r'!\[[^\]]*\]\(([^\s)]+)', thumb_raw) if img_match: thumb_url = img_match.group(1) - thumb_html = f'' + thumb_html = (f'' + f'') - lines.append(f'| {thumb_html} | {name_converted} | {tags_display} |') + lines.append(f'') - lines.append("") - lines.append("") + lines.append('') + lines.append('
ThumbnailNameTags
{thumb_html}{name_html}{tags_display}
') + lines.append('') lines.append("") lines.append(f'View all {count} results in VFB →') lines.append("") @@ -427,6 +506,48 @@ def format_publications(publications): lines.append("".join(parts)) return "\n".join(lines) +def format_xrefs(xrefs, term_name): + """Format Xrefs (external cross-references) into markdown links. + + Each xref is {label, accession, link, icon}. The cached get_term_info + endpoint resolves the full external URL in `link`, so we link to it + directly rather than rebuilding from a link_base/accession template + (cf. VFBProcessTermInfoJson.xref.getLink). Link text mirrors the VFB + browser's "Cross References" section: " on ". + """ + if not xrefs: + return "" + + lines = [] + for xref in xrefs: + link = xref.get("link", "") + if not link: + continue + label = xref.get("label", "") + text = f'{term_name} on {label}' if (term_name and label) else (label or link) + accession = xref.get("accession", "") + suffix = f' ({accession})' if accession and accession not in ("None", "") else "" + lines.append(f'- [{text}]({link}){suffix}') + return "\n".join(lines) + +def format_related_tools(tools, term_id): + """Format RelatedTools into the VFB browser's "Graphs For" links. + + Each tool is {tool, label, default_args}. These launch graph/hierarchy + tools on the term inside the 3D browser; link to the term there so the + user can run them (the browser exposes the tool set once the term loads). + """ + if not tools: + return "" + + lines = [] + for t in tools: + label = t.get("label", t.get("tool", "")) + if not label: + continue + lines.append(f'- [{label}]({VFB_BROWSER_BASE}?id={term_id})') + return "\n".join(lines) + def build_hero_card(name, term_id, tags_badges, description_html, comment_html, thumbnails): """Build the hero card without blank lines that break Markdown HTML blocks.""" lines = [ @@ -480,13 +601,19 @@ def generate_page(term_data): publications = term_data.get("Publications", []) technique = term_data.get("Technique", []) images = term_data.get("Images", {}) + examples = term_data.get("Examples", {}) + xrefs = term_data.get("Xrefs", []) + related_tools = term_data.get("RelatedTools", []) # Description for front matter description = meta.get("Description", "") if not description: - # Fallback to Types text (stripped of markdown links) - types_text = meta.get("Types", "") - description = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', types_text) + # Fallback to Types text + description = meta.get("Types", "") + # Strip markdown link syntax to plain labels: the front-matter description + # becomes the tag and page summary, where raw + # `[label](id)` syntax would show literally. + description = re.sub(r'\[([^\]]+)\]\([^)]+\)', r'\1', description) # Clean for YAML description = description.replace('"', '\\"').replace("\n", " ").replace("\r", " ").strip() @@ -532,11 +659,11 @@ def generate_page(term_data): desc_html = "" if meta.get("Description"): - desc_html = f'

{convert_internal_links(meta["Description"])}

' + desc_html = f'

{linkify_html(meta["Description"])}

' comment_html = "" if comment: - comment_html = f'

{convert_internal_links(comment)}

' + comment_html = f'

{linkify_html(comment)}

' sections.append(build_hero_card(name, term_id, tags_badges, desc_html, comment_html, thumbnails)) @@ -557,6 +684,12 @@ def generate_page(term_data): syn_md = format_synonyms_table(synonyms) sections.append(f'## Alternative Names\n\n{syn_md}\n') + # ── Cross references (external databases) ── + if xrefs: + xref_md = format_xrefs(xrefs, name) + if xref_md.strip(): + sections.append(f'## Cross References\n\n{xref_md}\n') + # ── Technique ── if technique: tech_lines = "\n".join(f'- {t}' for t in technique) @@ -567,9 +700,17 @@ def generate_page(term_data): lic_md = format_licenses(licenses) sections.append(f'## License\n\n{lic_md}\n') - # ── Downloads ── - if images: - dl_md = format_downloads(images) + # ── Downloads (individual Images and class Examples) ── + # Class terms carry their downloadable volumes under Examples (one list + # per template), individuals under Images. Merge both so class pages get + # download links too, matching the browser's Available Images downloads. + download_sources = {} + for tpl, img_list in images.items(): + download_sources.setdefault(tpl, []).extend(img_list) + for tpl, ex_list in examples.items(): + download_sources.setdefault(tpl, []).extend(ex_list) + if download_sources: + dl_md = format_downloads(download_sources) if dl_md.strip(): sections.append(f'## Downloads\n\n{dl_md}\n') @@ -581,12 +722,22 @@ def generate_page(term_data): q_md = format_query_preview(q, term_id) sections.append(q_md) + # ── Graphs For (related graph/hierarchy tools) ── + if related_tools: + tools_md = format_related_tools(related_tools, term_id) + if tools_md.strip(): + sections.append(f'## Graphs For\n\n{tools_md}\n') + # ── Publications ── if publications: pub_md = format_publications(publications) sections.append(f'## References\n\n{pub_md}\n') - return "\n".join(sections) + # Join with a blank line between every section. Goldmark keeps a raw HTML + # block (e.g. the hero card) open until a blank line, so without this the + # first heading after the hero ("## Classification") gets swallowed into + # the HTML block and printed literally instead of rendered as a heading. + return "\n\n".join(sections) # ─── Term Saving ───────────────────────────────────────────────────────────── @@ -613,46 +764,79 @@ def process_group(base_path, relative_dir, label, query): chdir(target_dir) save_terms(fetch_ids(label, query)) -def save_terms(ids): - """Fetch and save term pages for a list of IDs.""" - total = len(ids) - success_count = 0 - skip_count = 0 - fail_count = 0 - for i, term_id in enumerate(ids): - try: - filename = term_id + "_v" + str(version) + ".md" - if os.path.isfile(filename): - skip_count += 1 - continue +def process_term(term_id): + """Fetch, render and write one term page. Returns a status string. - print(f"Processing {term_id} ({i+1}/{total})...") - term_data = fetch_term_info(term_id) - if term_data is None: - fail_count += 1 - continue + Runs inside a worker thread. Writes to a temp file then atomically renames + so a killed build never leaves a half-written page. Throttling against the + server's capacity happens inside fetch_term_info via throttle(). + """ + filename = term_id + "_v" + str(version) + ".md" + if os.path.isfile(filename): + return "skip" - page_content = generate_page(term_data) + term_data = fetch_term_info(term_id) + if term_data is None: + return "fail" - with open(filename, "w", encoding="utf-8") as f: - f.write(page_content) + page_content = generate_page(term_data) - success_count += 1 + tmp = f"{filename}.{os.getpid()}.{threading.get_ident()}.tmp" + with open(tmp, "w", encoding="utf-8") as f: + f.write(page_content) + os.replace(tmp, filename) - # Clean up previous version - old_filename = term_id + "_v" + str(version - 1) + ".md" - if os.path.isfile(old_filename): - os.remove(old_filename) - print(f'Removed: {old_filename}') + # Clean up previous version + old_filename = term_id + "_v" + str(version - 1) + ".md" + if os.path.isfile(old_filename): + try: + os.remove(old_filename) + except OSError: + pass + return "ok" - # Throttling is handled by wait_for_server_capacity() in fetch_term_info +def save_terms(ids): + """Fetch and save term pages for a list of IDs, concurrently. + + A bounded ThreadPoolExecutor keeps up to MAX_WORKERS requests in flight so + the server's ~20 query slots stay busy, instead of the old one-at-a-time + loop that left them idle. The in-flight window is capped so building the + future set for a 600k-id group doesn't balloon memory. + """ + total = len(ids) + counts = {"ok": 0, "skip": 0, "fail": 0} + done = 0 + window = MAX_WORKERS * 4 + progress_every = max(100, window) + def record(fut, tid): + nonlocal done + try: + status = fut.result() except Exception as e: - fail_count += 1 - print(f"ERROR processing {term_id}: {str(e)}") + status = "fail" + print(f"ERROR processing {tid}: {e}") print(traceback.format_exc()) - - print(f"\nBatch complete: {success_count} created, {skip_count} skipped (existing), {fail_count} failed out of {total} total") + counts[status] = counts.get(status, 0) + 1 + done += 1 + if status == "fail" or done % progress_every == 0: + print(f" {done}/{total} processed " + f"(created={counts['ok']} skipped={counts['skip']} failed={counts['fail']}) " + f"last={tid}:{status}") + + with ThreadPoolExecutor(max_workers=MAX_WORKERS) as ex: + inflight = {} + for term_id in ids: + inflight[ex.submit(process_term, term_id)] = term_id + if len(inflight) >= window: + finished, _ = wait(set(inflight), return_when=FIRST_COMPLETED) + for fut in finished: + record(fut, inflight.pop(fut)) + for fut in as_completed(set(inflight)): + record(fut, inflight[fut]) + + print(f"\nBatch complete: {counts['ok']} created, {counts['skip']} skipped (existing), " + f"{counts['fail']} failed out of {total} total") # ─── Testing ───────────────────────────────────────────────────────────────── @@ -698,12 +882,16 @@ def test_term_page(term_id, term_type="class"): checks["Relationships"] = "## Relationships" in page_content if term_data.get("Synonyms"): checks["Synonyms"] = "## Alternative Names" in page_content + if term_data.get("Xrefs"): + checks["Cross References"] = "## Cross References" in page_content if term_data.get("Licenses"): checks["Licenses"] = "## License" in page_content - if term_data.get("Images"): + if term_data.get("Images") or term_data.get("Examples"): checks["Downloads"] = "## Downloads" in page_content if any(q.get("preview_results", {}).get("rows") for q in term_data.get("Queries", [])): checks["Query previews"] = "table-responsive" in page_content + if term_data.get("RelatedTools"): + checks["Graphs For"] = "## Graphs For" in page_content all_pass = True for check_name, result in checks.items(): @@ -791,8 +979,8 @@ def test_report_link_regression(): checks = { "Markdown links use report path": converted == "[DNb08](/reports/FBbt_20011340)", - "Query preview preserves report link": "[DNb08](/reports/FBbt_20011340)" in query_preview, - "Plain row labels get report link": "[DNp45](/reports/FBbt_20011346)" in query_preview, + "Query preview preserves report link": 'DNb08' in query_preview, + "Plain row labels get report link": 'DNp45' in query_preview, } all_pass = True