From fce489e25848a67ec7722a0a3d46029c4ef38664 Mon Sep 17 00:00:00 2001 From: Maksim Kostromin Date: Mon, 26 Oct 2026 01:43:11 -0400 Subject: [PATCH] fix(updater): a missing historical, performance or distributions worksheet keeps the published fund ISH-1: only the holdings sheet was guarded. A missing or renamed Historical sheet wrote every history page empty and deleted the old ones, a missing Performance sheet published null returns still labelled official, and a missing Distributions sheet emptied the distribution rows. worksheetsProblem() rejects a download whose worksheet is empty where the published meta.json has history rows, official returns or distribution rows: the fund keeps its previous complete state and counts as failed. A fund that never published them passes. --- README.md | 2 +- scripts/update-data.test.ts | 38 +++++++++++++++++++++++++++++-------- scripts/update-data.ts | 25 ++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 31d18c4b0..3f8c2c3f9 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ Each fund carries a `metrics` object that powers the catalog columns shared with | `official-distribution-rate`, `official-other`, `computed-trailing-12m`, `indicated` | never emitted: iShares publishes no other yield and the updater computes none | | `null` | no yield published | -Percentages are plain numbers (`10.19` means 10.19%). An unavailable value is `null` (text fields `—`), never `0`. The returns date is the quarter end shown in the *Return As Of* column, so it can trail the NAV date. A configured filter also skips funds that do not publish the filtered metric. A fund is either fully updated or fully kept: when its download fails, or its Holdings worksheet is empty although holdings were published before, the previous files stay as they were. Physical metal trusts (IAU, IAUM, SLV) publish no securities list, so an empty Holdings worksheet is valid for them and the fund still updates NAV, net assets, returns and history (`meta.holdings.status` is `empty`). A transient failure of the fund-header request keeps the previously published 30-day SEC yield. Funds not selected for an update keep their prior metadata and data files. A fund that vanishes from the live catalog is removed only when at most three (or 2%) vanish at once; a partly parsed catalog page never deletes funds. New catalog tickers are printed as `NEW FUNDS: A, B` and appended to the job summary. Every request has a 120 s timeout, JSON files are written through a temporary file and a rename, stale pages are removed after the new `meta.json`, and a rerun with unchanged upstream data writes nothing, not even `generatedAt`. The run stops taking new funds after 25 minutes and still writes the index; it exits non-zero when every attempted fund failed. With `STORE_RAW_DOWNLOADS` the source workbook is kept as `api/ishares/raw/{TICKER}.xls`. +Percentages are plain numbers (`10.19` means 10.19%). An unavailable value is `null` (text fields `—`), never `0`. The returns date is the quarter end shown in the *Return As Of* column, so it can trail the NAV date. A configured filter also skips funds that do not publish the filtered metric. A fund is either fully updated or fully kept: when its download fails, or its Holdings worksheet is empty although holdings were published before, or its Historical, Performance or Distributions worksheet is missing or empty although that fund published history rows, returns or distribution rows before, the previous files stay as they were and the fund counts as failed. Physical metal trusts (IAU, IAUM, SLV) publish no securities list, so an empty Holdings worksheet is valid for them and the fund still updates NAV, net assets, returns and history (`meta.holdings.status` is `empty`). A transient failure of the fund-header request keeps the previously published 30-day SEC yield. Funds not selected for an update keep their prior metadata and data files. A fund that vanishes from the live catalog is removed only when at most three (or 2%) vanish at once; a partly parsed catalog page never deletes funds. New catalog tickers are printed as `NEW FUNDS: A, B` and appended to the job summary. Every request has a 120 s timeout, JSON files are written through a temporary file and a rename, stale pages are removed after the new `meta.json`, and a rerun with unchanged upstream data writes nothing, not even `generatedAt`. The run stops taking new funds after 25 minutes and still writes the index; it exits non-zero when every attempted fund failed. With `STORE_RAW_DOWNLOADS` the source workbook is kept as `api/ishares/raw/{TICKER}.xls`. ### Update controls diff --git a/scripts/update-data.test.ts b/scripts/update-data.test.ts index 779ff70a2..b0967e1c8 100644 --- a/scripts/update-data.test.ts +++ b/scripts/update-data.test.ts @@ -7,6 +7,7 @@ import { pathToFileURL } from "node:url"; import { type IndexRow, CLOSED_FUNDS, + worksheetsProblem, CONTROL_NAMES, RETURNS_BASIS, buildCatalogRow, @@ -427,6 +428,8 @@ type World = { failDownload: Set; // answers HTTP 404 for the workbook failHeader: boolean; // answers HTTP 404 for the fund header JSON weightHeader: string; // the weight column of the holdings sheet (a renamed one breaks every fund page) + distributions: boolean; // the workbook carries a Distributions sheet + dropSheets: Set; // worksheets left out of the workbook (renamed or broken by the provider) secYield: string; delayMs: number; }; @@ -435,15 +438,17 @@ const cell = (value: string) => `${value} `${rows.map((row) => `${row.map(cell).join("")}`).join("")}`; -function workbook(ticker: string, metal: boolean, weightHeader = "Weight (%)"): string { +function workbook(ticker: string, metal: boolean, weightHeader = "Weight (%)", distributions = false, dropSheets: Set = new Set()): string { const performance = [["Date", "Return"], ...monthly(2023, 2024).rows.map((r) => [r.Date, r.Return])]; const head = ["Ticker", "Name", "Asset Class", "Market Value", weightHeader, "As Of Date"]; const holdings = metal ? [head] : [head, [ticker, `${ticker} Corp`, "Equity", "100", "100", "Sep 25, 2026"]]; - return `${[ - sheetXml("Holdings", holdings), - sheetXml("Historical", [["As Of", "NAV per Share"], ["Sep 25, 2026", "50.25"], ["Sep 24, 2026", "50.00"]]), - sheetXml("Performance", performance), - ].join("")}`; + const sheets: Array<[string, string[][]]> = [ + ["Holdings", holdings], + ["Historical", [["As Of", "NAV per Share"], ["Sep 25, 2026", "50.25"], ["Sep 24, 2026", "50.00"]]], + ["Performance", performance], + ]; + if (distributions) sheets.push(["Distributions", [["Record Date", "Ex-Date", "Payable Date", "Total Distribution"], ["Sep 15, 2026", "Sep 15, 2026", "Sep 18, 2026", "0.25"]]]); + return `${sheets.filter(([name]) => !dropSheets.has(name)).map(([name, rows]) => sheetXml(name, rows)).join("")}`; } function installIsharesMock(world: World) { @@ -475,7 +480,7 @@ function installIsharesMock(world: World) { } if (url.includes("component=fundDownload")) { if (world.failDownload.has(ticker)) return new Response("nope", { status: 404 }); - return new Response(workbook(ticker, world.metal.has(ticker), world.weightHeader)); + return new Response(workbook(ticker, world.metal.has(ticker), world.weightHeader, world.distributions, world.dropSheets)); } return new Response("unexpected", { status: 500 }); } finally { @@ -486,7 +491,7 @@ function installIsharesMock(world: World) { } const newWorld = (all: string[], extra: Partial = {}): World => ({ - all, catalog: all, metal: new Set(), failDownload: new Set(), failHeader: false, weightHeader: "Weight (%)", secYield: "4.20%", delayMs: 0, ...extra, + all, catalog: all, metal: new Set(), failDownload: new Set(), failHeader: false, weightHeader: "Weight (%)", distributions: false, dropSheets: new Set(), secYield: "4.20%", delayMs: 0, ...extra, }); // Fresh feed directory per call; main gets an explicit env, so exported workflow variables never leak in @@ -709,6 +714,23 @@ describe("pipeline 2", () => { }); describe("pipeline 3", () => { + test("a missing Historical, Performance or Distributions worksheet keeps the published fund and reports it failed", async () => { + await withFeed(async ({ run, files, meta }) => { + const world = newWorld(["AAA", "BBB"], { distributions: true }); + await run(world); + const before = files(); + expect([meta("AAA").history.totalRows, meta("AAA").distributions.rows.length, meta("AAA").officialReturns.asOfDate]).toEqual([2, 1, "2024-12-28"]); + for (const sheet of ["Historical", "Performance", "Distributions"]) { + world.dropSheets = new Set([sheet]); + expect(await run(world, { TICKERS: "AAA" })).toMatchObject({ updated: 0, unchanged: 0, failed: 1 }); + expect(files()).toEqual(before); + } + // a fund that never published them passes + expect(worksheetsProblem({ history: { totalRows: 0 }, officialReturns: { asOfDate: null }, distributions: { rows: [] } }, { historyRows: 0, returnsAsOf: "", distributionRows: 0 })).toBeNull(); + expect(worksheetsProblem(undefined, { historyRows: 0, returnsAsOf: "", distributionRows: 0 })).toBeNull(); + }); + }); + test("new funds are reported, an unknown TICKERS entry is an error", async () => { await withFeed(async ({ run }) => { const world = newWorld(["AAA", "BBB", "CCC"], { catalog: ["AAA", "BBB"] }); diff --git a/scripts/update-data.ts b/scripts/update-data.ts index dffce722e..3a8595586 100755 --- a/scripts/update-data.ts +++ b/scripts/update-data.ts @@ -1643,6 +1643,12 @@ async function updateFund( if (holdingsProblem) { throw Error(`holdings guard: ${holdingsProblem}; ${prior?.ticker ? "keeping the previous state" : "no data published for this fund yet"}`); } + const worksheetProblem = worksheetsProblem(prior, { + historyRows: history?.rows.length ?? 0, + returnsAsOf: returns.asOfDate, + distributionRows: distributionsName ? worksheets[distributionsName].rows.length : 0, + }); + if (worksheetProblem) throw Error(`${worksheetProblem}; keeping the previous state`); const holdingsPages = await writePagedRows( fund.ticker, @@ -2191,6 +2197,25 @@ if (import.meta.main) { }); } +/** + * The holdings sheet has its own guard; the other worksheets of the same download must not wipe published data either. + * A Historical sheet without rows, a Performance sheet without one monthly return or a Distributions sheet without rows, + * where the published meta.json has them, is a renamed or broken worksheet: the fund keeps its previous complete state + * (history pages, official returns, distribution rows) and is reported failed. A fund that never published them passes. + */ +export function worksheetsProblem( + prior: { history?: { totalRows?: unknown }; officialReturns?: { asOfDate?: unknown }; distributions?: { rows?: unknown } } | null | undefined, + fresh: { historyRows: number; returnsAsOf: string; distributionRows: number }, +): string | null { + const publishedHistory = Number(prior?.history?.totalRows) || 0; + if (publishedHistory > 0 && fresh.historyRows === 0) return `Historical worksheet is missing or empty but ${publishedHistory} rows are published`; + const publishedReturns = prior?.officialReturns?.asOfDate; + if (typeof publishedReturns === "string" && publishedReturns && !fresh.returnsAsOf) return `Performance worksheet has no monthly returns but returns as of ${publishedReturns} are published`; + const publishedDistributions = Array.isArray(prior?.distributions?.rows) ? (prior?.distributions?.rows as unknown[]).length : 0; + if (publishedDistributions > 0 && fresh.distributionRows === 0) return `Distributions worksheet is missing or empty but ${publishedDistributions} rows are published`; + return null; +} + // shared block of .claude/tools/etfs/etf-std/holdings-guard.block.ts, applied by apply-holdings-guard.ts: never edit it in one repo /** * Sanity check of a fund's new holdings before they replace the published ones. Returns why they must be rejected