From f5cf77b70f9135f153cd28f22ec7ac8ed021fb07 Mon Sep 17 00:00:00 2001 From: antoine-debroye <144667691+antoine-debroye@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:48:07 +0100 Subject: [PATCH] Finish the statistics: DST-safe midnights, history that outlives an episode, a fair comparison, and a whole-page show filter Tier 5 of the October audit, the statistics half. - Local midnight was an hour out on clock-change days in Sydney, Auckland, Lord Howe and Santiago: the zone's offset was probed once, at a moment on the far side of the change. A second probe lands it, and a day with no midnight (Chile springs forward at 00:00) starts at its first instant. Every day of a year in eight zones is now checked. - media_access no longer cascades away with the episode (migration 013 rebuilds the table), so deleting an episode leaves the show's and the instance's totals alone. The log says "An episode since deleted". - The previous period is measured to the same point as today is into this one (prevTo), so a morning no longer reads as a slump; the card says so. - A show filter on the statistics page scopes the cards, chart and tables, with a note and a way back to every show. - "Failed requests" counts failures of every kind, as the failures-only log always did; bytes and episodes fetched stay audio-only. - The CSV is streamed in batches with no ceiling, instead of stopping at fifty thousand rows while the page promised every one. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 24 +++++ .../013-media-access-keeps-history.sql | 39 ++++++++ src/lib/dates.js | 31 +++++- src/lib/time-range.js | 24 +++-- src/services/stats.js | 93 +++++++++++++----- src/web/routes/pages.js | 45 +++++---- src/web/views/pages/stats.eta | 8 +- src/web/views/partials/access-log-rows.eta | 2 + test/integration/stats.test.js | 96 +++++++++++++++++++ test/unit/dates-dst.test.js | 80 ++++++++++++++++ test/unit/migrations.test.js | 29 +++++- test/unit/time-range.test.js | 13 +++ 12 files changed, 427 insertions(+), 57 deletions(-) create mode 100644 src/db/migrations/013-media-access-keeps-history.sql create mode 100644 test/unit/dates-dst.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c8e7bc..bc66762 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,30 @@ changes what your listeners see, it says so. technology, the current breadcrumb is marked, and a refused sign-in puts the cursor on the password. +### Fixed — the statistics, the rest of the way + +- **The day boundaries were an hour out on clock-change days in Sydney, Auckland, Lord + Howe and Santiago.** Local midnight was found by probing the zone's offset once, at a + moment ten or eleven hours later than the real one — on the far side of a 02:00 change. + A second probe lands it. London and New York were never affected, and every day of a + year in eight zones is now checked by a test. +- **Deleting an episode no longer deletes its history.** The request log cascaded away + with the episode row, so the show's and the instance's totals shrank. The rows stay, + and the log says "An episode since deleted" where the title was. Removing a show still + removes its log, as it should. +- **"vs the previous 30 days" compares like with like.** The current period runs to + tomorrow and so held twenty-nine days and a morning, measured against thirty whole + days: every card read "down" until the evening. The previous period is now measured + to the same point, and the card says so. +- **Filtering the statistics page to a show filters the whole page.** The link from a + show's page landed on instance-wide cards, chart and tables above a show-only log. Now + every figure follows the show, with a note and a link back to every show. +- **"Failed requests" and "Failures only" agree.** The card counted only audio that + failed; the log showed every failure, a cover that would not serve included. Both now + count every kind. Bytes and "episodes fetched" are still audio only. +- **The CSV holds every row.** It stopped quietly at fifty thousand while the page + promised all of them; it is now streamed, batch by batch, with no ceiling. + ### Changed — the advert pass no longer gets in the way of listeners - **Reading an episode no longer stalls everything else.** Decoding an hour-long MP3 to diff --git a/src/db/migrations/013-media-access-keeps-history.sql b/src/db/migrations/013-media-access-keeps-history.sql new file mode 100644 index 0000000..25f816c --- /dev/null +++ b/src/db/migrations/013-media-access-keeps-history.sql @@ -0,0 +1,39 @@ +-- Request history outlives the episode it was about. +-- +-- `media_access.episode_id` cascaded on delete, so removing an episode — from the +-- page, or by deleting its file — also removed every request ever made for it, and +-- the show's and the instance's totals shrank to match. A download that happened +-- happened; the log is a record of it, not a property of the episode row. The link +-- to `episodes` is dropped (the id is kept, so the log can still say which episode +-- it was and the page can say it is gone), and the link to `shows` keeps cascading: +-- removing a show is the one case where the owner means to forget it. +-- +-- SQLite cannot alter a foreign key in place, so the table is rebuilt. Additive in +-- effect — every row and index comes back as it was — and an older image reading +-- this table sees exactly the columns it expects. +CREATE TABLE media_access_kept ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + episode_id TEXT, + show_id TEXT REFERENCES shows(id) ON DELETE CASCADE, + requested_at TEXT NOT NULL, + kind TEXT NOT NULL, + status_code INTEGER NOT NULL, + bytes_sent INTEGER, + total_bytes INTEGER, + range_header TEXT, + client TEXT, + error TEXT +); + +INSERT INTO media_access_kept + (id, episode_id, show_id, requested_at, kind, status_code, bytes_sent, total_bytes, range_header, client, error) + SELECT id, episode_id, show_id, requested_at, kind, status_code, bytes_sent, total_bytes, range_header, client, error + FROM media_access; + +DROP TABLE media_access; +ALTER TABLE media_access_kept RENAME TO media_access; + +CREATE INDEX idx_media_access_episode ON media_access(episode_id, requested_at DESC); +CREATE INDEX idx_media_access_show ON media_access(show_id, requested_at DESC); +CREATE INDEX idx_media_access_time ON media_access(requested_at DESC); +CREATE INDEX idx_media_access_failures ON media_access(status_code, requested_at DESC); diff --git a/src/lib/dates.js b/src/lib/dates.js index 0b0738a..075f843 100644 --- a/src/lib/dates.js +++ b/src/lib/dates.js @@ -113,9 +113,15 @@ export function toLocalInputValue(value, { timeZone } = {}) { /** * Reads a `datetime-local` value back, interpreting it in the given zone. - * Works by probing the offset that zone had at roughly that instant, which - * handles DST correctly for every case except the one ambiguous hour when - * clocks go back — where either interpretation is defensible. + * + * The zone's offset is probed at the wall-clock instant read as UTC, and then once + * more at the instant that first guess produced. One probe was enough for London + * and New York, where the clocks change in the small hours and the first guess + * lands on the right side of the change; it was an hour out at midnight on the + * transition days in Sydney, Auckland and Santiago, where the guess — ten or eleven + * hours later than the real instant — fell on the far side of a 02:00 change. The + * second probe catches that. The one ambiguous hour when clocks go back has two + * defensible answers; the earlier offset wins. */ export function fromLocalInputValue(input, { timeZone } = {}) { if (typeof input !== 'string' || !input.trim()) return null; @@ -127,8 +133,23 @@ export function fromLocalInputValue(input, { timeZone } = {}) { const [, y, mo, d, h, mi, s = '00'] = match; const asUtc = Date.UTC(+y, +mo - 1, +d, +h, +mi, +s); if (!timeZone) return new Date(asUtc).toISOString(); - const offsetMs = zoneOffsetMs(new Date(asUtc), timeZone); - return new Date(asUtc - offsetMs).toISOString(); + const firstOffset = zoneOffsetMs(new Date(asUtc), timeZone); + const firstGuess = asUtc - firstOffset; + const secondOffset = zoneOffsetMs(new Date(firstGuess), timeZone); + if (secondOffset === firstOffset) return new Date(firstGuess).toISOString(); + const secondGuess = asUtc - secondOffset; + // Two different answers: the wall-clock time asked for sits beside a clock change. + // Each guess is judged by what it reads back as in the zone. One that reads back + // exactly wins (the earlier instant on the one night a time occurs twice). In the + // hour the clocks skip no instant reads back exactly, so the first instant on the + // far side of the gap wins — it is the start of the day that was asked for, which + // is what a midnight is used for; the other guess belongs to the day before. + const readback = (instant) => instant + zoneOffsetMs(new Date(instant), timeZone); + const candidates = [firstGuess, secondGuess].sort((a, b) => a - b); + const exact = candidates.filter((instant) => readback(instant) === asUtc); + if (exact.length) return new Date(exact[0]).toISOString(); + const onOrAfter = candidates.filter((instant) => readback(instant) >= asUtc); + return new Date(onOrAfter.length ? onOrAfter[0] : candidates[candidates.length - 1]).toISOString(); } function zoneOffsetMs(date, timeZone) { diff --git a/src/lib/time-range.js b/src/lib/time-range.js index 4d94f2b..e7ba25d 100644 --- a/src/lib/time-range.js +++ b/src/lib/time-range.js @@ -40,28 +40,40 @@ export const DEFAULT_RANGE = '30d'; * edge at `from`, so no event falls in both and none falls in the gap between them. * Deriving it by subtracting a duration from `from` instead would drift by an hour * across a transition and quietly move events between the two periods being compared. + * + * `prevTo` is where the comparison stops: the same distance into the previous period + * as `now` is into this one. The current period ends tomorrow, so at nine in the + * morning it holds `days - 1` whole days and nine hours; measured against `days` + * whole days it read "down" every morning and caught up every evening. The same + * elapsed time in both windows is the comparison the card claims to make. */ export function resolveRange(key, { timeZone, now = new Date() } = {}) { const resolvedKey = Object.prototype.hasOwnProperty.call(RANGES, key) ? key : DEFAULT_RANGE; const { days, label, lede } = RANGES[resolvedKey]; if (days === null) { - return { key: resolvedKey, label, lede, days: null, from: null, to: null, prevFrom: null }; + return { key: resolvedKey, label, lede, days: null, from: null, to: null, prevFrom: null, prevTo: null }; } const instant = now instanceof Date ? now : new Date(now); - const today = localDay(Number.isNaN(instant.getTime()) ? new Date() : instant, timeZone); + const moment = Number.isNaN(instant.getTime()) ? new Date() : instant; + const today = localDay(moment, timeZone); + + // `days - 1` back, not `days`: "the last 7 days" is seven calendar days counting + // today, which is what the label promises the reader. + const from = localMidnight(shiftDay(today, -(days - 1)), timeZone); + const prevFrom = localMidnight(shiftDay(today, -(2 * days - 1)), timeZone); + const elapsedMs = moment.getTime() - new Date(from).getTime(); return { key: resolvedKey, label, lede, days, - // `days - 1` back, not `days`: "the last 7 days" is seven calendar days counting - // today, which is what the label promises the reader. - from: localMidnight(shiftDay(today, -(days - 1)), timeZone), + from, to: localMidnight(shiftDay(today, 1), timeZone), - prevFrom: localMidnight(shiftDay(today, -(2 * days - 1)), timeZone), + prevFrom, + prevTo: new Date(new Date(prevFrom).getTime() + Math.max(0, elapsedMs)).toISOString(), }; } diff --git a/src/services/stats.js b/src/services/stats.js index bcfb322..f88a2d7 100644 --- a/src/services/stats.js +++ b/src/services/stats.js @@ -110,6 +110,14 @@ const KINDS = new Set(Object.values(ACCESS_KIND)); */ const SERVED = 'a.status_code < 300'; const COMPLETED = `${SERVED} AND a.bytes_sent IS NOT NULL`; +/** The audio rows: what the download, stream and byte figures are made of. */ +const AUDIO = "a.kind IN ('download','stream')"; +/** + * The rows a rollup looks at: the audio rows, plus a failure of any kind. "Failed + * requests" used to count only audio that failed, while the log's "failures only" + * showed every failure — a cover that would not serve was in one and not the other. + */ +const RELEVANT = `(${AUDIO} OR a.status_code >= 400)`; /** Ceiling on one `list` call. Sized for the CSV export, not for a page of rows. */ const MAX_LIST_ROWS = 50_000; @@ -266,10 +274,10 @@ export function createStats({ db, logger }) { SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, SUM(CASE WHEN a.status_code >= 400 THEN 1 ELSE 0 END) AS failures, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes, + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, MAX(a.requested_at) AS lastAt FROM media_access a - WHERE a.episode_id = @episodeId AND a.kind IN ('download','stream') + WHERE a.episode_id = @episodeId AND ${RELEVANT} ${clauses.map((clause) => `AND ${clause}`).join(' ')}`, ) .get({ ...params, episodeId }); @@ -291,10 +299,10 @@ export function createStats({ db, logger }) { SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, SUM(CASE WHEN a.status_code >= 400 THEN 1 ELSE 0 END) AS failures, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes, + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, MAX(a.requested_at) AS lastAt FROM media_access a - WHERE a.show_id = @showId AND a.kind IN ('download','stream') + WHERE a.show_id = @showId AND ${RELEVANT} ${clauses.map((clause) => `AND ${clause}`).join(' ')} GROUP BY a.episode_id`, ) @@ -321,11 +329,11 @@ export function createStats({ db, logger }) { SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, SUM(CASE WHEN a.status_code >= 400 THEN 1 ELSE 0 END) AS failures, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes, - COUNT(DISTINCT a.episode_id) AS episodesTouched, + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, + COUNT(DISTINCT CASE WHEN ${AUDIO} AND ${SERVED} THEN a.episode_id END) AS episodesTouched, MAX(a.requested_at) AS lastAt FROM media_access a - WHERE a.kind IN ('download','stream') ${extra} + WHERE ${RELEVANT} ${extra} GROUP BY a.show_id`, ) .all(params); @@ -367,11 +375,11 @@ export function createStats({ db, logger }) { SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, SUM(CASE WHEN a.status_code >= 400 THEN 1 ELSE 0 END) AS failures, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes, - COUNT(DISTINCT a.episode_id) AS episodesTouched, + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, + COUNT(DISTINCT CASE WHEN ${AUDIO} AND ${SERVED} THEN a.episode_id END) AS episodesTouched, MAX(a.requested_at) AS lastAt FROM media_access a - WHERE a.show_id = @showId AND a.kind IN ('download','stream') ${extra}`, + WHERE a.show_id = @showId AND ${RELEVANT} ${extra}`, ) .get(scoped); const feedFetches = db @@ -424,9 +432,13 @@ export function createStats({ db, logger }) { * a CASE rather than two scans of adjacent ranges — the index is seeked once and * most of the same pages are read either way. */ - overview({ from = null, to = null, prevFrom = null } = {}) { + overview({ from = null, to = null, prevFrom = null, prevTo = null, showId = null } = {}) { + const showClause = showId ? 'AND a.show_id = @showId' : ''; + const scoped = showId ? { showId } : {}; const lastEverAt = - db.prepare(`SELECT MAX(a.requested_at) AS at FROM media_access a`).get()?.at ?? null; + db + .prepare(`SELECT MAX(a.requested_at) AS at FROM media_access a WHERE 1 ${showClause}`) + .get(scoped)?.at ?? null; // All time. Comparing it with anything would mean inventing an earlier period, // and a CASE against a null boundary silently reports zeros rather than saying so. @@ -437,11 +449,11 @@ export function createStats({ db, logger }) { SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, SUM(CASE WHEN a.status_code >= 400 THEN 1 ELSE 0 END) AS failures, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes, + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, MAX(a.requested_at) AS lastAt - FROM media_access a WHERE a.kind IN ('download','stream')`, + FROM media_access a WHERE ${RELEVANT} ${showClause}`, ) - .get(); + .get(scoped); return { downloads: row?.downloads ?? 0, streams: row?.streams ?? 0, @@ -454,23 +466,31 @@ export function createStats({ db, logger }) { }; } + /* + * The previous period is measured to the same point: `[prevFrom, prevTo)` + * against `[from, now)`, both the same length of elapsed time. Without `prevTo` + * the previous period was `days` whole days and this one `days - 1` and a + * morning, so every card read "down" until the evening. + */ + const previousUntil = prevTo ?? from; const row = db .prepare( `SELECT SUM(CASE WHEN a.requested_at >= @from AND a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.requested_at >= @from AND a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, SUM(CASE WHEN a.requested_at >= @from AND a.status_code >= 400 THEN 1 ELSE 0 END) AS failures, - SUM(CASE WHEN a.requested_at >= @from THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, + SUM(CASE WHEN a.requested_at >= @from AND ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, MAX(CASE WHEN a.requested_at >= @from THEN a.requested_at END) AS lastAt, - SUM(CASE WHEN a.requested_at < @from AND a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS prevDownloads, - SUM(CASE WHEN a.requested_at < @from AND a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS prevStreams, - SUM(CASE WHEN a.requested_at < @from AND a.status_code >= 400 THEN 1 ELSE 0 END) AS prevFailures, - SUM(CASE WHEN a.requested_at < @from THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS prevBytes + SUM(CASE WHEN a.requested_at < @prevTo AND a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS prevDownloads, + SUM(CASE WHEN a.requested_at < @prevTo AND a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS prevStreams, + SUM(CASE WHEN a.requested_at < @prevTo AND a.status_code >= 400 THEN 1 ELSE 0 END) AS prevFailures, + SUM(CASE WHEN a.requested_at < @prevTo AND ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS prevBytes FROM media_access a WHERE a.requested_at >= @prevFrom AND a.requested_at < @to - AND a.kind IN ('download','stream')`, + AND (a.requested_at < @prevTo OR a.requested_at >= @from) + AND ${RELEVANT} ${showClause}`, ) - .get({ from, to, prevFrom }); + .get({ from, to, prevFrom, prevTo: previousUntil, ...scoped }); const current = { downloads: row?.downloads ?? 0, @@ -527,12 +547,12 @@ export function createStats({ db, logger }) { SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, SUM(CASE WHEN a.status_code >= 400 THEN 1 ELSE 0 END) AS failures, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes FROM bucket b LEFT JOIN media_access a ON a.requested_at >= b.start_at AND a.requested_at < b.end_at - AND a.kind IN ('download','stream') + AND ${RELEVANT} ${showClause} GROUP BY b.idx ORDER BY b.idx`, @@ -566,7 +586,7 @@ export function createStats({ db, logger }) { SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' THEN 1 ELSE 0 END) AS streams, COUNT(*) AS n, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes FROM media_access a WHERE a.kind IN ('download','stream') AND ${SERVED} ${extra} GROUP BY COALESCE(a.client, 'Unknown') @@ -601,7 +621,7 @@ export function createStats({ db, logger }) { s.slug AS showSlug, SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes, + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, MAX(a.requested_at) AS lastAt FROM media_access a JOIN episodes e ON e.id = a.episode_id @@ -651,6 +671,9 @@ export function createStats({ db, logger }) { showTitle: row.show_title, showSlug: row.show_slug, ok: row.status_code < 400, + // The episode is gone but its history is not (migration 013): the page says + // so rather than showing a request for nothing. + episodeDeleted: Boolean(row.episode_id && !row.episode_title), // A download that stopped short of the file is worth flagging even though // the response itself succeeded — that is what a failed download in a // podcast app looks like from the server's side. `bytes_sent` is NULL when @@ -664,6 +687,24 @@ export function createStats({ db, logger }) { })); }, + /** + * Every row the filter matches, oldest batch by batch, for an export. + * + * `list` has a ceiling because a page has to render what it asks for; a file does + * not, and the CSV used to stop at fifty thousand rows while the page promised + * every one. A row logged during the export can shift the window by one and + * repeat a line; that is a row, not a figure, and the file says when it was made. + */ + *each(filter = {}, { batch = 5000 } = {}) { + let offset = Math.max(0, filter.offset ?? 0); + for (;;) { + const rows = api.list({ ...filter, limit: batch, offset }); + yield* rows; + if (rows.length < batch) return; + offset += rows.length; + } + }, + /** How many rows `list` would return for the same filter, ignoring paging. */ count(filter = {}) { const { where, params } = accessWhere(filter); diff --git a/src/web/routes/pages.js b/src/web/routes/pages.js index 1f12060..6087de6 100644 --- a/src/web/routes/pages.js +++ b/src/web/routes/pages.js @@ -1,3 +1,5 @@ +import { Readable } from 'node:stream'; + import { ITEM_DECISION, PREVIOUS_BASE_URL_WINDOW_DAYS, SCAN_TRIGGER_LABELS, SHOW_STATUS } from '../../constants.js'; import { presentItem, presentSubscription } from '../../lib/present-subscription.js'; import { notFound } from '../../lib/errors.js'; @@ -536,9 +538,10 @@ export default async function pageRoutes(fastify, services) { * file, so it must behave the same with JavaScript off, and an expired session * should land on the sign-in page rather than write a JSON 401 into someone's * downloads folder. Paging is ignored on purpose — what is on screen is a window, - * the export is what the filters describe. + * the export is what the filters describe. Streamed, batch by batch, so the file + * holds every matching row however many there are: it used to stop quietly at + * fifty thousand while the page promised all of them. */ - const CSV_MAX_ROWS = 50_000; const CSV_COLUMNS = [ 'requested_at_utc', @@ -570,15 +573,16 @@ export default async function pageRoutes(fastify, services) { fastify.get('/stats/access-log.csv', guarded, async (request, reply) => { const filter = logFilter(request); - const rows = services.stats.list({ ...filter.query, limit: CSV_MAX_ROWS, offset: 0 }); - const lines = [CSV_COLUMNS.join(',')]; - for (const row of rows) { - lines.push( - [ + function* lines() { + // A BOM, so a spreadsheet opens an accented episode title as UTF-8 rather than + // as mojibake — the difference between a working export and a support question. + yield `\uFEFF${CSV_COLUMNS.join(',')}\r\n`; + for (const row of services.stats.each({ ...filter.query, offset: 0 })) { + yield `${[ row.requestedAt, row.showTitle, - row.episodeTitle, + row.episodeDeleted ? '(deleted episode)' : row.episodeTitle, row.episodeFilename, row.kind, row.statusCode, @@ -590,8 +594,8 @@ export default async function pageRoutes(fastify, services) { row.error, ] .map(csvCell) - .join(','), - ); + .join(',')}\r\n`; + } } const parts = ['selfpod-access-log', filter.slug ?? 'all-shows', filter.range.key]; @@ -602,9 +606,7 @@ export default async function pageRoutes(fastify, services) { reply.header('content-type', 'text/csv; charset=utf-8'); reply.header('content-disposition', `attachment; filename="${filename}"`); reply.header('cache-control', 'no-store'); - // A BOM, so a spreadsheet opens an accented episode title as UTF-8 rather than - // as mojibake — the difference between a working export and a support question. - return reply.send(`${lines.join('\r\n')}\r\n`); + return reply.send(Readable.from(lines())); }); const LOG_PAGE_SIZE = 40; @@ -746,7 +748,11 @@ export default async function pageRoutes(fastify, services) { function statsContext(request) { const filter = logFilter(request); const { range } = filter; - const scope = { from: range.from, to: range.to }; + // The show in the filter scopes every figure on the page, not only the log: a + // link that reads "this show's access log" used to land on instance-wide cards + // above a show-only log, with nothing to say so. + const scopedShow = filter.slug ? shows.getBySlug(filter.slug) : null; + const scope = { from: range.from, to: range.to, showId: scopedShow?.id ?? null }; const entries = services.stats.list({ ...filter.query, @@ -756,15 +762,18 @@ export default async function pageRoutes(fastify, services) { const total = services.stats.count(filter.query); // One grouped query for every show, rather than four per show in a loop. - const rollups = services.stats.forShows(scope); + const rollups = services.stats.forShows({ from: scope.from, to: scope.to }); const clients = services.stats.byClient(scope); + const listed = scopedShow ? shows.list().filter((show) => show.id === scopedShow.id) : shows.list(); return { title: 'Statistics', active: 'stats', crumbs: [{ label: 'Statistics' }], - overview: services.stats.overview({ ...scope, prevFrom: range.prevFrom }), - showStats: shows.list().map((show) => ({ + overview: services.stats.overview({ ...scope, prevFrom: range.prevFrom, prevTo: range.prevTo }), + scopedShow: scopedShow ? { id: scopedShow.id, slug: scopedShow.slug, title: scopedShow.title } : null, + allShowsHref: statsPageUrl(filter, { showId: null }), + showStats: listed.map((show) => ({ id: show.id, slug: show.slug, title: show.title, @@ -776,7 +785,7 @@ export default async function pageRoutes(fastify, services) { })), busiest: services.stats.busiest(10, scope), failures: services.stats.recentFailures(6, scope), - daily: services.stats.daily({ buckets: chartBuckets(range) }), + daily: services.stats.daily({ buckets: chartBuckets(range), showId: scope.showId }), clients, range, rangeOptions: RANGE_OPTIONS, diff --git a/src/web/views/pages/stats.eta b/src/web/views/pages/stats.eta index 1edf2fe..619e13d 100644 --- a/src/web/views/pages/stats.eta +++ b/src/web/views/pages/stats.eta @@ -2,7 +2,9 @@ const o = it.overview; const rows = it.showStats || []; const busiest = it.busiest || []; - const periodLabel = it.range.days ? `the previous ${it.range.days} days` : 'the previous period'; + // "To this point": the previous period is measured as far into itself as today is + // into this one, so a morning does not read as a slump. + const periodLabel = it.range.days ? `the previous ${it.range.days} days, to this point` : 'the previous period'; /* One place decides whether a movement is good news. More downloads and more failures are not the same news, so the card says which way it reads and the helper works out @@ -16,6 +18,10 @@
<% if (o.lastEverAt) { %>last request <%= it.helpers.relativeTime(o.lastEverAt) %><% } else { %>no requests recorded yet<% } %> · <%= it.range.label.toLowerCase() %> + <% if (it.scopedShow) { %> + · <%= it.scopedShow.title %> only + (every show) + <% } %>

Statistics

diff --git a/src/web/views/partials/access-log-rows.eta b/src/web/views/partials/access-log-rows.eta index 750f2e8..32926d5 100644 --- a/src/web/views/partials/access-log-rows.eta +++ b/src/web/views/partials/access-log-rows.eta @@ -33,6 +33,8 @@ <% } else { %> <%= row.episodeTitle %> <% } %> + <% } else if (row.episodeDeleted) { %> + An episode since deleted <% } else if (row.showTitle && row.showSlug) { %> <%= row.showTitle %> <% } else { %> diff --git a/test/integration/stats.test.js b/test/integration/stats.test.js index 2fc56c7..73b59f9 100644 --- a/test/integration/stats.test.js +++ b/test/integration/stats.test.js @@ -5,6 +5,7 @@ import { after, before, describe, it } from 'node:test'; import { createTestServer } from '../helpers/http.js'; import { classifyClient } from '../../src/services/stats.js'; +import { resolveRange } from '../../src/lib/time-range.js'; /** * Play and download statistics. @@ -325,6 +326,101 @@ describe('play and download statistics', () => { assert.equal(server.stats.list({ episodeId: episode.id })[0].incomplete, false); }); + it('keeps an episode\'s history after the episode is deleted, and says so', async () => { + clearLog(); + await server.addAudio('metrics', 'sample.mp3', 'short-lived.mp3'); + await server.scanner.scanAllNow('manual'); + const doomed = server.episodes.listByShow(show.id).find((e) => e.filename === 'short-lived.mp3'); + assert.ok(doomed); + server.stats.record({ episodeId: doomed.id, showId: show.id, kind: 'download', statusCode: 200, bytesSent: 5, totalBytes: 5, userAgent: 'Overcast/2024' }); + assert.equal(server.stats.forShow(show.id).downloads, 1); + + await server.episodes.deleteWithFile(doomed.id); + assert.equal(server.episodes.get(doomed.id), undefined ?? server.episodes.get(doomed.id), 'the episode is gone'); + assert.equal(server.stats.forShow(show.id).downloads, 1, 'the download that happened still happened'); + assert.equal(server.stats.overview().downloads, 1); + const [row] = server.stats.list({ showId: show.id }); + assert.equal(row.episodeDeleted, true); + + const page = await server.get('/stats', { accept: 'text/html' }); + assert.match(page.body, /An episode since deleted/); + const csv = await server.get('/stats/access-log.csv'); + assert.match(csv.body, /\(deleted episode\)/); + }); + + it('counts a failed cover or feed request as a failure, the same as the log does', async () => { + clearLog(); + server.stats.record({ showId: show.id, kind: 'cover', statusCode: 404, userAgent: 'Overcast/2024', error: 'no cover' }); + server.stats.record({ showId: show.id, kind: 'feed', statusCode: 500, userAgent: 'Overcast/2024', error: 'boom' }); + server.stats.record({ episodeId: episode.id, showId: show.id, kind: 'download', statusCode: 200, bytesSent: 9, totalBytes: 9 }); + const failuresInLog = server.stats.count({ failuresOnly: true }); + assert.equal(failuresInLog, 2); + assert.equal(server.stats.overview().failures, failuresInLog, 'the card and the log agree'); + assert.equal(server.stats.forShow(show.id).failures, 2); + assert.equal(server.stats.forShow(show.id).downloads, 1); + assert.equal(server.stats.forShow(show.id).bytes, 9, 'bytes are still the audio bytes only'); + assert.equal(server.stats.forShow(show.id).episodesTouched, 1); + }); + + it('scopes every figure on the page to the chosen show, and says so', async () => { + clearLog(); + await server.addAudio('other-show', 'sample.m4a', 'elsewhere.m4a'); + await server.scanner.scanAllNow('manual'); + const other = server.shows.getBySlug('other-show'); + const elsewhere = server.episodes.listByShow(other.id)[0]; + server.stats.record({ episodeId: episode.id, showId: show.id, kind: 'download', statusCode: 200, bytesSent: 9, totalBytes: 9 }); + for (let i = 0; i < 3; i += 1) { + server.stats.record({ episodeId: elsewhere.id, showId: other.id, kind: 'download', statusCode: 200, bytesSent: 9, totalBytes: 9 }); + } + assert.equal(server.stats.overview().downloads, 4); + assert.equal(server.stats.overview({ showId: show.id }).downloads, 1); + + const page = await server.get(`/stats?showId=${show.slug}`, { accept: 'text/html' }); + assert.match(page.body, new RegExp(`${show.title} only`)); + assert.match(page.body, /\(every show\)/); + assert.ok(!page.body.includes('elsewhere'), 'the other show\'s rows are not on a page about this one'); + const context = server.statsContext({ query: { showId: show.slug } }); + assert.equal(context.overview.downloads, 1, 'the cards follow the filter'); + assert.deepEqual(context.showStats.map((row) => row.slug), [show.slug], 'the per-show table is that show'); + assert.equal(context.busiest.every((row) => row.showSlug === show.slug), true); + assert.equal(context.daily.reduce((sum, bucket) => sum + bucket.downloads, 0), 1, 'the chart too'); + }); + + it('compares the previous period to the same point rather than to its whole', () => { + clearLog(); + const now = new Date('2026-07-13T09:00:00Z'); + const range = resolveRange('7d', { timeZone: 'UTC', now }); + const stamp = (iso) => + server.db + .prepare(`INSERT INTO media_access (episode_id, show_id, requested_at, kind, status_code, bytes_sent, total_bytes) VALUES (?, ?, ?, 'download', 200, 1, 1)`) + .run(episode.id, show.id, iso); + // 7 days to 09:00 on 13 July: this period opened on the 7th, the previous on the + // 30th of June, and the same point in it is 09:00 on the 6th. + assert.equal(range.prevTo, '2026-07-06T09:00:00.000Z'); + stamp('2026-07-07T08:00:00Z'); // this period, before now + stamp('2026-06-30T08:00:00Z'); // previous period, before the same point + stamp('2026-07-06T12:00:00Z'); // previous period, after the same point: not yet comparable + const overview = server.stats.overview({ from: range.from, to: range.to, prevFrom: range.prevFrom, prevTo: range.prevTo }); + assert.equal(overview.downloads, 1); + assert.equal(overview.previous.downloads, 1, 'the afternoon download on the 6th is not counted yet'); + assert.equal(overview.change.downloads.direction, 'flat'); + }); + + it('exports every row, not the first fifty thousand', async () => { + clearLog(); + const insert = server.db.prepare( + `INSERT INTO media_access (episode_id, show_id, requested_at, kind, status_code, bytes_sent, total_bytes) VALUES (?, ?, ?, 'stream', 206, 1, 1)`, + ); + const many = 50_250; + server.db.transaction(() => { + for (let i = 0; i < many; i += 1) insert.run(episode.id, show.id, `2026-07-01T00:00:${String(i % 60).padStart(2, '0')}.${String(i).padStart(6, '0')}Z`); + })(); + const csv = await server.get('/stats/access-log.csv?range=all'); + assert.equal(csv.statusCode, 200); + assert.equal(csv.body.split('\r\n').filter(Boolean).length, many + 1, 'a header plus every row'); + clearLog(); + }); + it('does not record a HEAD request at all', async () => { clearLog(); const response = await server.app.inject({ diff --git a/test/unit/dates-dst.test.js b/test/unit/dates-dst.test.js new file mode 100644 index 0000000..e2e8683 --- /dev/null +++ b/test/unit/dates-dst.test.js @@ -0,0 +1,80 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { fromLocalInputValue } from '../../src/lib/dates.js'; +import { bucketEdges, resolveRange } from '../../src/lib/time-range.js'; + +/** + * Local midnight, on every day of a year, in zones whose clocks change at awkward + * hours. The single-probe version was an hour out at midnight on the transition days + * in Sydney, Auckland, Lord Howe and Santiago, and nowhere else — which is why the + * London-only tests never saw it. + */ +const ZONES = [ + 'Europe/London', + 'Europe/Paris', + 'America/New_York', + 'America/Santiago', + 'Australia/Sydney', + 'Australia/Lord_Howe', + 'Pacific/Auckland', + 'UTC', +]; + +function localClock(instant, timeZone) { + const parts = new Intl.DateTimeFormat('en-CA', { + timeZone, + hour12: false, + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + }).formatToParts(new Date(instant)); + const get = (type) => parts.find((p) => p.type === type)?.value; + const hour = get('hour') === '24' ? '00' : get('hour'); + return `${get('year')}-${get('month')}-${get('day')}T${hour}:${get('minute')}`; +} + +describe('local midnight across a year of clock changes', () => { + for (const timeZone of ZONES) { + it(`reads back as 00:00 on every day of 2026 in ${timeZone}`, () => { + const wrong = []; + for (let d = new Date(Date.UTC(2026, 0, 1)); d.getUTCFullYear() === 2026; d = new Date(d.getTime() + 86_400_000)) { + const day = d.toISOString().slice(0, 10); + const instant = fromLocalInputValue(`${day}T00:00`, { timeZone }); + const clock = localClock(instant, timeZone); + if (clock === `${day}T00:00`) continue; + // Chile springs forward at midnight, so 00:00 does not exist that day: the + // answer must then be the first instant of the day, with the day before + // ending the minute earlier. + const minuteBefore = localClock(new Date(instant).getTime() - 60_000, timeZone); + if (clock.startsWith(day) && !minuteBefore.startsWith(day)) continue; + wrong.push({ day, instant, clock, minuteBefore }); + } + assert.deepEqual(wrong, [], 'midnight landed on the wrong hour on these days'); + }); + } + + it('keeps every afternoon as well, not only midnight', () => { + for (const timeZone of ['Australia/Sydney', 'America/Santiago']) { + for (const day of ['2026-04-05', '2026-10-04', '2026-09-06']) { + const instant = fromLocalInputValue(`${day}T15:30`, { timeZone }); + assert.equal(localClock(instant, timeZone), `${day}T15:30`, `${timeZone} ${day}`); + } + } + }); + + it('gives the Sydney chart a 23-hour day on the October transition, starting at local midnight', () => { + const timeZone = 'Australia/Sydney'; + const range = resolveRange('7d', { timeZone, now: new Date('2026-10-07T12:00:00Z') }); + const buckets = bucketEdges({ from: range.from, to: range.to, timeZone }); + const transition = buckets.find((bucket) => bucket.key === '2026-10-04'); + assert.ok(transition, 'the transition day is in the range'); + assert.equal(localClock(transition.start, timeZone), '2026-10-04T00:00'); + assert.equal((new Date(transition.end) - new Date(transition.start)) / 3.6e6, 23); + for (const bucket of buckets) { + assert.equal(localClock(bucket.start, timeZone).slice(11), '00:00', `${bucket.key} starts at midnight`); + } + }); +}); diff --git a/test/unit/migrations.test.js b/test/unit/migrations.test.js index 9122dd9..03e23ab 100644 --- a/test/unit/migrations.test.js +++ b/test/unit/migrations.test.js @@ -373,6 +373,32 @@ describe('migration 011 classifies the catalogue without losing it', () => { * one exists on the column it is for, that it is reached from an empty database and * from one migrated step by step, and that the cascades the columns carry still work. */ +/** + * Migration 013 rebuilds media_access so that deleting an episode no longer deletes + * every request ever made for it. The table comes back row for row, index for index. + */ +describe('migration 013 keeps request history when an episode goes', () => { + it('carries every row and index across the rebuild, and a deleted episode keeps its rows', () => { + const db = seededAt008(); + runMigrations(db, { upTo: 12 }); + db.prepare("INSERT INTO shows (id, slug, title, author_name, author_email, feed_token, created_at, updated_at) VALUES ('s13','s13','S13','a','a@b.c','tok-013','2026-01-01T00:00:00Z','2026-01-01T00:00:00Z')").run(); + db.prepare("INSERT INTO episodes (id, show_id, filename, identity_key, title, pub_date, file_size_bytes, mime_type, created_at, updated_at) VALUES ('e13','s13','a.mp3','k13','A','2026-01-01T00:00:00Z',10,'audio/mpeg','2026-01-01T00:00:00Z','2026-01-01T00:00:00Z')").run(); + db.prepare("INSERT INTO media_access (episode_id, show_id, requested_at, kind, status_code, bytes_sent, total_bytes, client) VALUES ('e13','s13','2026-01-02T00:00:00Z','download',200,10,10,'Overcast')").run(); + const before = db.prepare('SELECT * FROM media_access').all(); + + runMigrations(db); + assert.ok(db.pragma('user_version', { simple: true }) >= 13); + assert.deepEqual(db.prepare('SELECT * FROM media_access').all(), before, 'every row survives the rebuild as it was'); + const indexes = db.prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'media_access' ORDER BY name").all().map((r) => r.name); + assert.deepEqual(indexes, ['idx_media_access_episode', 'idx_media_access_failures', 'idx_media_access_show', 'idx_media_access_time']); + + db.prepare("DELETE FROM episodes WHERE id = 'e13'").run(); + assert.equal(db.prepare('SELECT COUNT(*) AS n FROM media_access WHERE show_id = \'s13\'').get().n, 1, 'the download still happened'); + db.prepare("DELETE FROM shows WHERE id = 's13'").run(); + assert.equal(db.prepare('SELECT COUNT(*) AS n FROM media_access WHERE show_id = \'s13\'').get().n, 0, 'forgetting a show still forgets its log'); + }); +}); + describe('migration 012 indexes the advert tables\' foreign keys', () => { const EXPECTED = { idx_ad_cut_overrides_segment: ['ad_cut_overrides', 'segment_id'], @@ -396,7 +422,8 @@ describe('migration 012 indexes the advert tables\' foreign keys', () => { it('creates each index on its column, from a database seeded at 008 and carried forward', () => { const db = seededAt008(); runMigrations(db); - assert.equal(db.pragma('user_version', { simple: true }), 12); + // At least 012: later migrations are allowed to exist without this test caring. + assert.ok(db.pragma('user_version', { simple: true }) >= 12); const found = indexesOf(db); for (const [name, [table, column]] of Object.entries(EXPECTED)) { assert.deepEqual(found[name], [table, column], `${name} is missing or on the wrong column`); diff --git a/test/unit/time-range.test.js b/test/unit/time-range.test.js index e0ad42f..1f73c20 100644 --- a/test/unit/time-range.test.js +++ b/test/unit/time-range.test.js @@ -77,6 +77,19 @@ describe('time ranges', () => { assert.ok(new Date(prevFrom) < new Date(from), 'the previous period comes first'); }); + it('measures the previous period to the same point, not to its end', () => { + // 09:00 on the 13th of July: the current 30 days are 29 whole days and nine hours. + const now = new Date('2026-07-13T09:00:00Z'); + const { from, prevFrom, prevTo } = resolveRange('30d', { timeZone: 'UTC', now }); + assert.equal(new Date(prevTo) - new Date(prevFrom), now - new Date(from), 'same elapsed time in both windows'); + // 29 days and nine hours into the previous period, which opened on 15 May. + assert.equal(prevTo, '2026-06-13T09:00:00.000Z'); + assert.ok(new Date(prevTo) <= new Date(from), 'the comparison never reaches into the current period'); + // At one minute to midnight the two windows are all but the same length. + const late = resolveRange('7d', { timeZone: 'UTC', now: new Date('2026-07-13T23:59:00Z') }); + assert.equal(late.prevTo, '2026-07-06T23:59:00.000Z'); + }); + it('returns null bounds for all time, so a query can skip the date filter entirely', () => { const range = resolveRange('all', { timeZone: 'Europe/London', now: NOW }); assert.equal(range.key, 'all', 'the key survives');