diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c8e7bc..bc66762 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,30 @@ changes what your listeners see, it says so. technology, the current breadcrumb is marked, and a refused sign-in puts the cursor on the password. +### Fixed — the statistics, the rest of the way + +- **The day boundaries were an hour out on clock-change days in Sydney, Auckland, Lord + Howe and Santiago.** Local midnight was found by probing the zone's offset once, at a + moment ten or eleven hours later than the real one — on the far side of a 02:00 change. + A second probe lands it. London and New York were never affected, and every day of a + year in eight zones is now checked by a test. +- **Deleting an episode no longer deletes its history.** The request log cascaded away + with the episode row, so the show's and the instance's totals shrank. The rows stay, + and the log says "An episode since deleted" where the title was. Removing a show still + removes its log, as it should. +- **"vs the previous 30 days" compares like with like.** The current period runs to + tomorrow and so held twenty-nine days and a morning, measured against thirty whole + days: every card read "down" until the evening. The previous period is now measured + to the same point, and the card says so. +- **Filtering the statistics page to a show filters the whole page.** The link from a + show's page landed on instance-wide cards, chart and tables above a show-only log. Now + every figure follows the show, with a note and a link back to every show. +- **"Failed requests" and "Failures only" agree.** The card counted only audio that + failed; the log showed every failure, a cover that would not serve included. Both now + count every kind. Bytes and "episodes fetched" are still audio only. +- **The CSV holds every row.** It stopped quietly at fifty thousand while the page + promised all of them; it is now streamed, batch by batch, with no ceiling. + ### Changed — the advert pass no longer gets in the way of listeners - **Reading an episode no longer stalls everything else.** Decoding an hour-long MP3 to diff --git a/src/db/migrations/013-media-access-keeps-history.sql b/src/db/migrations/013-media-access-keeps-history.sql new file mode 100644 index 0000000..25f816c --- /dev/null +++ b/src/db/migrations/013-media-access-keeps-history.sql @@ -0,0 +1,39 @@ +-- Request history outlives the episode it was about. +-- +-- `media_access.episode_id` cascaded on delete, so removing an episode — from the +-- page, or by deleting its file — also removed every request ever made for it, and +-- the show's and the instance's totals shrank to match. A download that happened +-- happened; the log is a record of it, not a property of the episode row. The link +-- to `episodes` is dropped (the id is kept, so the log can still say which episode +-- it was and the page can say it is gone), and the link to `shows` keeps cascading: +-- removing a show is the one case where the owner means to forget it. +-- +-- SQLite cannot alter a foreign key in place, so the table is rebuilt. Additive in +-- effect — every row and index comes back as it was — and an older image reading +-- this table sees exactly the columns it expects. +CREATE TABLE media_access_kept ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + episode_id TEXT, + show_id TEXT REFERENCES shows(id) ON DELETE CASCADE, + requested_at TEXT NOT NULL, + kind TEXT NOT NULL, + status_code INTEGER NOT NULL, + bytes_sent INTEGER, + total_bytes INTEGER, + range_header TEXT, + client TEXT, + error TEXT +); + +INSERT INTO media_access_kept + (id, episode_id, show_id, requested_at, kind, status_code, bytes_sent, total_bytes, range_header, client, error) + SELECT id, episode_id, show_id, requested_at, kind, status_code, bytes_sent, total_bytes, range_header, client, error + FROM media_access; + +DROP TABLE media_access; +ALTER TABLE media_access_kept RENAME TO media_access; + +CREATE INDEX idx_media_access_episode ON media_access(episode_id, requested_at DESC); +CREATE INDEX idx_media_access_show ON media_access(show_id, requested_at DESC); +CREATE INDEX idx_media_access_time ON media_access(requested_at DESC); +CREATE INDEX idx_media_access_failures ON media_access(status_code, requested_at DESC); diff --git a/src/lib/dates.js b/src/lib/dates.js index 0b0738a..075f843 100644 --- a/src/lib/dates.js +++ b/src/lib/dates.js @@ -113,9 +113,15 @@ export function toLocalInputValue(value, { timeZone } = {}) { /** * Reads a `datetime-local` value back, interpreting it in the given zone. - * Works by probing the offset that zone had at roughly that instant, which - * handles DST correctly for every case except the one ambiguous hour when - * clocks go back — where either interpretation is defensible. + * + * The zone's offset is probed at the wall-clock instant read as UTC, and then once + * more at the instant that first guess produced. One probe was enough for London + * and New York, where the clocks change in the small hours and the first guess + * lands on the right side of the change; it was an hour out at midnight on the + * transition days in Sydney, Auckland and Santiago, where the guess — ten or eleven + * hours later than the real instant — fell on the far side of a 02:00 change. The + * second probe catches that. The one ambiguous hour when clocks go back has two + * defensible answers; the earlier offset wins. */ export function fromLocalInputValue(input, { timeZone } = {}) { if (typeof input !== 'string' || !input.trim()) return null; @@ -127,8 +133,23 @@ export function fromLocalInputValue(input, { timeZone } = {}) { const [, y, mo, d, h, mi, s = '00'] = match; const asUtc = Date.UTC(+y, +mo - 1, +d, +h, +mi, +s); if (!timeZone) return new Date(asUtc).toISOString(); - const offsetMs = zoneOffsetMs(new Date(asUtc), timeZone); - return new Date(asUtc - offsetMs).toISOString(); + const firstOffset = zoneOffsetMs(new Date(asUtc), timeZone); + const firstGuess = asUtc - firstOffset; + const secondOffset = zoneOffsetMs(new Date(firstGuess), timeZone); + if (secondOffset === firstOffset) return new Date(firstGuess).toISOString(); + const secondGuess = asUtc - secondOffset; + // Two different answers: the wall-clock time asked for sits beside a clock change. + // Each guess is judged by what it reads back as in the zone. One that reads back + // exactly wins (the earlier instant on the one night a time occurs twice). In the + // hour the clocks skip no instant reads back exactly, so the first instant on the + // far side of the gap wins — it is the start of the day that was asked for, which + // is what a midnight is used for; the other guess belongs to the day before. + const readback = (instant) => instant + zoneOffsetMs(new Date(instant), timeZone); + const candidates = [firstGuess, secondGuess].sort((a, b) => a - b); + const exact = candidates.filter((instant) => readback(instant) === asUtc); + if (exact.length) return new Date(exact[0]).toISOString(); + const onOrAfter = candidates.filter((instant) => readback(instant) >= asUtc); + return new Date(onOrAfter.length ? onOrAfter[0] : candidates[candidates.length - 1]).toISOString(); } function zoneOffsetMs(date, timeZone) { diff --git a/src/lib/time-range.js b/src/lib/time-range.js index 4d94f2b..e7ba25d 100644 --- a/src/lib/time-range.js +++ b/src/lib/time-range.js @@ -40,28 +40,40 @@ export const DEFAULT_RANGE = '30d'; * edge at `from`, so no event falls in both and none falls in the gap between them. * Deriving it by subtracting a duration from `from` instead would drift by an hour * across a transition and quietly move events between the two periods being compared. + * + * `prevTo` is where the comparison stops: the same distance into the previous period + * as `now` is into this one. The current period ends tomorrow, so at nine in the + * morning it holds `days - 1` whole days and nine hours; measured against `days` + * whole days it read "down" every morning and caught up every evening. The same + * elapsed time in both windows is the comparison the card claims to make. */ export function resolveRange(key, { timeZone, now = new Date() } = {}) { const resolvedKey = Object.prototype.hasOwnProperty.call(RANGES, key) ? key : DEFAULT_RANGE; const { days, label, lede } = RANGES[resolvedKey]; if (days === null) { - return { key: resolvedKey, label, lede, days: null, from: null, to: null, prevFrom: null }; + return { key: resolvedKey, label, lede, days: null, from: null, to: null, prevFrom: null, prevTo: null }; } const instant = now instanceof Date ? now : new Date(now); - const today = localDay(Number.isNaN(instant.getTime()) ? new Date() : instant, timeZone); + const moment = Number.isNaN(instant.getTime()) ? new Date() : instant; + const today = localDay(moment, timeZone); + + // `days - 1` back, not `days`: "the last 7 days" is seven calendar days counting + // today, which is what the label promises the reader. + const from = localMidnight(shiftDay(today, -(days - 1)), timeZone); + const prevFrom = localMidnight(shiftDay(today, -(2 * days - 1)), timeZone); + const elapsedMs = moment.getTime() - new Date(from).getTime(); return { key: resolvedKey, label, lede, days, - // `days - 1` back, not `days`: "the last 7 days" is seven calendar days counting - // today, which is what the label promises the reader. - from: localMidnight(shiftDay(today, -(days - 1)), timeZone), + from, to: localMidnight(shiftDay(today, 1), timeZone), - prevFrom: localMidnight(shiftDay(today, -(2 * days - 1)), timeZone), + prevFrom, + prevTo: new Date(new Date(prevFrom).getTime() + Math.max(0, elapsedMs)).toISOString(), }; } diff --git a/src/services/stats.js b/src/services/stats.js index bcfb322..f88a2d7 100644 --- a/src/services/stats.js +++ b/src/services/stats.js @@ -110,6 +110,14 @@ const KINDS = new Set(Object.values(ACCESS_KIND)); */ const SERVED = 'a.status_code < 300'; const COMPLETED = `${SERVED} AND a.bytes_sent IS NOT NULL`; +/** The audio rows: what the download, stream and byte figures are made of. */ +const AUDIO = "a.kind IN ('download','stream')"; +/** + * The rows a rollup looks at: the audio rows, plus a failure of any kind. "Failed + * requests" used to count only audio that failed, while the log's "failures only" + * showed every failure — a cover that would not serve was in one and not the other. + */ +const RELEVANT = `(${AUDIO} OR a.status_code >= 400)`; /** Ceiling on one `list` call. Sized for the CSV export, not for a page of rows. */ const MAX_LIST_ROWS = 50_000; @@ -266,10 +274,10 @@ export function createStats({ db, logger }) { SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, SUM(CASE WHEN a.status_code >= 400 THEN 1 ELSE 0 END) AS failures, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes, + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, MAX(a.requested_at) AS lastAt FROM media_access a - WHERE a.episode_id = @episodeId AND a.kind IN ('download','stream') + WHERE a.episode_id = @episodeId AND ${RELEVANT} ${clauses.map((clause) => `AND ${clause}`).join(' ')}`, ) .get({ ...params, episodeId }); @@ -291,10 +299,10 @@ export function createStats({ db, logger }) { SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, SUM(CASE WHEN a.status_code >= 400 THEN 1 ELSE 0 END) AS failures, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes, + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, MAX(a.requested_at) AS lastAt FROM media_access a - WHERE a.show_id = @showId AND a.kind IN ('download','stream') + WHERE a.show_id = @showId AND ${RELEVANT} ${clauses.map((clause) => `AND ${clause}`).join(' ')} GROUP BY a.episode_id`, ) @@ -321,11 +329,11 @@ export function createStats({ db, logger }) { SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, SUM(CASE WHEN a.status_code >= 400 THEN 1 ELSE 0 END) AS failures, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes, - COUNT(DISTINCT a.episode_id) AS episodesTouched, + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, + COUNT(DISTINCT CASE WHEN ${AUDIO} AND ${SERVED} THEN a.episode_id END) AS episodesTouched, MAX(a.requested_at) AS lastAt FROM media_access a - WHERE a.kind IN ('download','stream') ${extra} + WHERE ${RELEVANT} ${extra} GROUP BY a.show_id`, ) .all(params); @@ -367,11 +375,11 @@ export function createStats({ db, logger }) { SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, SUM(CASE WHEN a.status_code >= 400 THEN 1 ELSE 0 END) AS failures, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes, - COUNT(DISTINCT a.episode_id) AS episodesTouched, + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, + COUNT(DISTINCT CASE WHEN ${AUDIO} AND ${SERVED} THEN a.episode_id END) AS episodesTouched, MAX(a.requested_at) AS lastAt FROM media_access a - WHERE a.show_id = @showId AND a.kind IN ('download','stream') ${extra}`, + WHERE a.show_id = @showId AND ${RELEVANT} ${extra}`, ) .get(scoped); const feedFetches = db @@ -424,9 +432,13 @@ export function createStats({ db, logger }) { * a CASE rather than two scans of adjacent ranges — the index is seeked once and * most of the same pages are read either way. */ - overview({ from = null, to = null, prevFrom = null } = {}) { + overview({ from = null, to = null, prevFrom = null, prevTo = null, showId = null } = {}) { + const showClause = showId ? 'AND a.show_id = @showId' : ''; + const scoped = showId ? { showId } : {}; const lastEverAt = - db.prepare(`SELECT MAX(a.requested_at) AS at FROM media_access a`).get()?.at ?? null; + db + .prepare(`SELECT MAX(a.requested_at) AS at FROM media_access a WHERE 1 ${showClause}`) + .get(scoped)?.at ?? null; // All time. Comparing it with anything would mean inventing an earlier period, // and a CASE against a null boundary silently reports zeros rather than saying so. @@ -437,11 +449,11 @@ export function createStats({ db, logger }) { SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, SUM(CASE WHEN a.status_code >= 400 THEN 1 ELSE 0 END) AS failures, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes, + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, MAX(a.requested_at) AS lastAt - FROM media_access a WHERE a.kind IN ('download','stream')`, + FROM media_access a WHERE ${RELEVANT} ${showClause}`, ) - .get(); + .get(scoped); return { downloads: row?.downloads ?? 0, streams: row?.streams ?? 0, @@ -454,23 +466,31 @@ export function createStats({ db, logger }) { }; } + /* + * The previous period is measured to the same point: `[prevFrom, prevTo)` + * against `[from, now)`, both the same length of elapsed time. Without `prevTo` + * the previous period was `days` whole days and this one `days - 1` and a + * morning, so every card read "down" until the evening. + */ + const previousUntil = prevTo ?? from; const row = db .prepare( `SELECT SUM(CASE WHEN a.requested_at >= @from AND a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.requested_at >= @from AND a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, SUM(CASE WHEN a.requested_at >= @from AND a.status_code >= 400 THEN 1 ELSE 0 END) AS failures, - SUM(CASE WHEN a.requested_at >= @from THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, + SUM(CASE WHEN a.requested_at >= @from AND ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, MAX(CASE WHEN a.requested_at >= @from THEN a.requested_at END) AS lastAt, - SUM(CASE WHEN a.requested_at < @from AND a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS prevDownloads, - SUM(CASE WHEN a.requested_at < @from AND a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS prevStreams, - SUM(CASE WHEN a.requested_at < @from AND a.status_code >= 400 THEN 1 ELSE 0 END) AS prevFailures, - SUM(CASE WHEN a.requested_at < @from THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS prevBytes + SUM(CASE WHEN a.requested_at < @prevTo AND a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS prevDownloads, + SUM(CASE WHEN a.requested_at < @prevTo AND a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS prevStreams, + SUM(CASE WHEN a.requested_at < @prevTo AND a.status_code >= 400 THEN 1 ELSE 0 END) AS prevFailures, + SUM(CASE WHEN a.requested_at < @prevTo AND ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS prevBytes FROM media_access a WHERE a.requested_at >= @prevFrom AND a.requested_at < @to - AND a.kind IN ('download','stream')`, + AND (a.requested_at < @prevTo OR a.requested_at >= @from) + AND ${RELEVANT} ${showClause}`, ) - .get({ from, to, prevFrom }); + .get({ from, to, prevFrom, prevTo: previousUntil, ...scoped }); const current = { downloads: row?.downloads ?? 0, @@ -527,12 +547,12 @@ export function createStats({ db, logger }) { SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, SUM(CASE WHEN a.status_code >= 400 THEN 1 ELSE 0 END) AS failures, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes FROM bucket b LEFT JOIN media_access a ON a.requested_at >= b.start_at AND a.requested_at < b.end_at - AND a.kind IN ('download','stream') + AND ${RELEVANT} ${showClause} GROUP BY b.idx ORDER BY b.idx`, @@ -566,7 +586,7 @@ export function createStats({ db, logger }) { SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' THEN 1 ELSE 0 END) AS streams, COUNT(*) AS n, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes FROM media_access a WHERE a.kind IN ('download','stream') AND ${SERVED} ${extra} GROUP BY COALESCE(a.client, 'Unknown') @@ -601,7 +621,7 @@ export function createStats({ db, logger }) { s.slug AS showSlug, SUM(CASE WHEN a.kind = 'download' AND ${COMPLETED} THEN 1 ELSE 0 END) AS downloads, SUM(CASE WHEN a.kind = 'stream' AND ${SERVED} THEN 1 ELSE 0 END) AS streams, - SUM(COALESCE(a.bytes_sent, 0)) AS bytes, + SUM(CASE WHEN ${AUDIO} THEN COALESCE(a.bytes_sent, 0) ELSE 0 END) AS bytes, MAX(a.requested_at) AS lastAt FROM media_access a JOIN episodes e ON e.id = a.episode_id @@ -651,6 +671,9 @@ export function createStats({ db, logger }) { showTitle: row.show_title, showSlug: row.show_slug, ok: row.status_code < 400, + // The episode is gone but its history is not (migration 013): the page says + // so rather than showing a request for nothing. + episodeDeleted: Boolean(row.episode_id && !row.episode_title), // A download that stopped short of the file is worth flagging even though // the response itself succeeded — that is what a failed download in a // podcast app looks like from the server's side. `bytes_sent` is NULL when @@ -664,6 +687,24 @@ export function createStats({ db, logger }) { })); }, + /** + * Every row the filter matches, oldest batch by batch, for an export. + * + * `list` has a ceiling because a page has to render what it asks for; a file does + * not, and the CSV used to stop at fifty thousand rows while the page promised + * every one. A row logged during the export can shift the window by one and + * repeat a line; that is a row, not a figure, and the file says when it was made. + */ + *each(filter = {}, { batch = 5000 } = {}) { + let offset = Math.max(0, filter.offset ?? 0); + for (;;) { + const rows = api.list({ ...filter, limit: batch, offset }); + yield* rows; + if (rows.length < batch) return; + offset += rows.length; + } + }, + /** How many rows `list` would return for the same filter, ignoring paging. */ count(filter = {}) { const { where, params } = accessWhere(filter); diff --git a/src/web/routes/pages.js b/src/web/routes/pages.js index 1f12060..6087de6 100644 --- a/src/web/routes/pages.js +++ b/src/web/routes/pages.js @@ -1,3 +1,5 @@ +import { Readable } from 'node:stream'; + import { ITEM_DECISION, PREVIOUS_BASE_URL_WINDOW_DAYS, SCAN_TRIGGER_LABELS, SHOW_STATUS } from '../../constants.js'; import { presentItem, presentSubscription } from '../../lib/present-subscription.js'; import { notFound } from '../../lib/errors.js'; @@ -536,9 +538,10 @@ export default async function pageRoutes(fastify, services) { * file, so it must behave the same with JavaScript off, and an expired session * should land on the sign-in page rather than write a JSON 401 into someone's * downloads folder. Paging is ignored on purpose — what is on screen is a window, - * the export is what the filters describe. + * the export is what the filters describe. Streamed, batch by batch, so the file + * holds every matching row however many there are: it used to stop quietly at + * fifty thousand while the page promised all of them. */ - const CSV_MAX_ROWS = 50_000; const CSV_COLUMNS = [ 'requested_at_utc', @@ -570,15 +573,16 @@ export default async function pageRoutes(fastify, services) { fastify.get('/stats/access-log.csv', guarded, async (request, reply) => { const filter = logFilter(request); - const rows = services.stats.list({ ...filter.query, limit: CSV_MAX_ROWS, offset: 0 }); - const lines = [CSV_COLUMNS.join(',')]; - for (const row of rows) { - lines.push( - [ + function* lines() { + // A BOM, so a spreadsheet opens an accented episode title as UTF-8 rather than + // as mojibake — the difference between a working export and a support question. + yield `\uFEFF${CSV_COLUMNS.join(',')}\r\n`; + for (const row of services.stats.each({ ...filter.query, offset: 0 })) { + yield `${[ row.requestedAt, row.showTitle, - row.episodeTitle, + row.episodeDeleted ? '(deleted episode)' : row.episodeTitle, row.episodeFilename, row.kind, row.statusCode, @@ -590,8 +594,8 @@ export default async function pageRoutes(fastify, services) { row.error, ] .map(csvCell) - .join(','), - ); + .join(',')}\r\n`; + } } const parts = ['selfpod-access-log', filter.slug ?? 'all-shows', filter.range.key]; @@ -602,9 +606,7 @@ export default async function pageRoutes(fastify, services) { reply.header('content-type', 'text/csv; charset=utf-8'); reply.header('content-disposition', `attachment; filename="${filename}"`); reply.header('cache-control', 'no-store'); - // A BOM, so a spreadsheet opens an accented episode title as UTF-8 rather than - // as mojibake — the difference between a working export and a support question. - return reply.send(`${lines.join('\r\n')}\r\n`); + return reply.send(Readable.from(lines())); }); const LOG_PAGE_SIZE = 40; @@ -746,7 +748,11 @@ export default async function pageRoutes(fastify, services) { function statsContext(request) { const filter = logFilter(request); const { range } = filter; - const scope = { from: range.from, to: range.to }; + // The show in the filter scopes every figure on the page, not only the log: a + // link that reads "this show's access log" used to land on instance-wide cards + // above a show-only log, with nothing to say so. + const scopedShow = filter.slug ? shows.getBySlug(filter.slug) : null; + const scope = { from: range.from, to: range.to, showId: scopedShow?.id ?? null }; const entries = services.stats.list({ ...filter.query, @@ -756,15 +762,18 @@ export default async function pageRoutes(fastify, services) { const total = services.stats.count(filter.query); // One grouped query for every show, rather than four per show in a loop. - const rollups = services.stats.forShows(scope); + const rollups = services.stats.forShows({ from: scope.from, to: scope.to }); const clients = services.stats.byClient(scope); + const listed = scopedShow ? shows.list().filter((show) => show.id === scopedShow.id) : shows.list(); return { title: 'Statistics', active: 'stats', crumbs: [{ label: 'Statistics' }], - overview: services.stats.overview({ ...scope, prevFrom: range.prevFrom }), - showStats: shows.list().map((show) => ({ + overview: services.stats.overview({ ...scope, prevFrom: range.prevFrom, prevTo: range.prevTo }), + scopedShow: scopedShow ? { id: scopedShow.id, slug: scopedShow.slug, title: scopedShow.title } : null, + allShowsHref: statsPageUrl(filter, { showId: null }), + showStats: listed.map((show) => ({ id: show.id, slug: show.slug, title: show.title, @@ -776,7 +785,7 @@ export default async function pageRoutes(fastify, services) { })), busiest: services.stats.busiest(10, scope), failures: services.stats.recentFailures(6, scope), - daily: services.stats.daily({ buckets: chartBuckets(range) }), + daily: services.stats.daily({ buckets: chartBuckets(range), showId: scope.showId }), clients, range, rangeOptions: RANGE_OPTIONS, diff --git a/src/web/views/pages/stats.eta b/src/web/views/pages/stats.eta index 1edf2fe..619e13d 100644 --- a/src/web/views/pages/stats.eta +++ b/src/web/views/pages/stats.eta @@ -2,7 +2,9 @@ const o = it.overview; const rows = it.showStats || []; const busiest = it.busiest || []; - const periodLabel = it.range.days ? `the previous ${it.range.days} days` : 'the previous period'; + // "To this point": the previous period is measured as far into itself as today is + // into this one, so a morning does not read as a slump. + const periodLabel = it.range.days ? `the previous ${it.range.days} days, to this point` : 'the previous period'; /* One place decides whether a movement is good news. More downloads and more failures are not the same news, so the card says which way it reads and the helper works out @@ -16,6 +18,10 @@
<% if (o.lastEverAt) { %>last request <%= it.helpers.relativeTime(o.lastEverAt) %><% } else { %>no requests recorded yet<% } %> · <%= it.range.label.toLowerCase() %> + <% if (it.scopedShow) { %> + · <%= it.scopedShow.title %> only + (every show) + <% } %>

Statistics

diff --git a/src/web/views/partials/access-log-rows.eta b/src/web/views/partials/access-log-rows.eta index 750f2e8..32926d5 100644 --- a/src/web/views/partials/access-log-rows.eta +++ b/src/web/views/partials/access-log-rows.eta @@ -33,6 +33,8 @@ <% } else { %> <%= row.episodeTitle %> <% } %> + <% } else if (row.episodeDeleted) { %> + An episode since deleted <% } else if (row.showTitle && row.showSlug) { %> <%= row.showTitle %> <% } else { %> diff --git a/test/integration/stats.test.js b/test/integration/stats.test.js index 2fc56c7..73b59f9 100644 --- a/test/integration/stats.test.js +++ b/test/integration/stats.test.js @@ -5,6 +5,7 @@ import { after, before, describe, it } from 'node:test'; import { createTestServer } from '../helpers/http.js'; import { classifyClient } from '../../src/services/stats.js'; +import { resolveRange } from '../../src/lib/time-range.js'; /** * Play and download statistics. @@ -325,6 +326,101 @@ describe('play and download statistics', () => { assert.equal(server.stats.list({ episodeId: episode.id })[0].incomplete, false); }); + it('keeps an episode\'s history after the episode is deleted, and says so', async () => { + clearLog(); + await server.addAudio('metrics', 'sample.mp3', 'short-lived.mp3'); + await server.scanner.scanAllNow('manual'); + const doomed = server.episodes.listByShow(show.id).find((e) => e.filename === 'short-lived.mp3'); + assert.ok(doomed); + server.stats.record({ episodeId: doomed.id, showId: show.id, kind: 'download', statusCode: 200, bytesSent: 5, totalBytes: 5, userAgent: 'Overcast/2024' }); + assert.equal(server.stats.forShow(show.id).downloads, 1); + + await server.episodes.deleteWithFile(doomed.id); + assert.equal(server.episodes.get(doomed.id), undefined ?? server.episodes.get(doomed.id), 'the episode is gone'); + assert.equal(server.stats.forShow(show.id).downloads, 1, 'the download that happened still happened'); + assert.equal(server.stats.overview().downloads, 1); + const [row] = server.stats.list({ showId: show.id }); + assert.equal(row.episodeDeleted, true); + + const page = await server.get('/stats', { accept: 'text/html' }); + assert.match(page.body, /An episode since deleted/); + const csv = await server.get('/stats/access-log.csv'); + assert.match(csv.body, /\(deleted episode\)/); + }); + + it('counts a failed cover or feed request as a failure, the same as the log does', async () => { + clearLog(); + server.stats.record({ showId: show.id, kind: 'cover', statusCode: 404, userAgent: 'Overcast/2024', error: 'no cover' }); + server.stats.record({ showId: show.id, kind: 'feed', statusCode: 500, userAgent: 'Overcast/2024', error: 'boom' }); + server.stats.record({ episodeId: episode.id, showId: show.id, kind: 'download', statusCode: 200, bytesSent: 9, totalBytes: 9 }); + const failuresInLog = server.stats.count({ failuresOnly: true }); + assert.equal(failuresInLog, 2); + assert.equal(server.stats.overview().failures, failuresInLog, 'the card and the log agree'); + assert.equal(server.stats.forShow(show.id).failures, 2); + assert.equal(server.stats.forShow(show.id).downloads, 1); + assert.equal(server.stats.forShow(show.id).bytes, 9, 'bytes are still the audio bytes only'); + assert.equal(server.stats.forShow(show.id).episodesTouched, 1); + }); + + it('scopes every figure on the page to the chosen show, and says so', async () => { + clearLog(); + await server.addAudio('other-show', 'sample.m4a', 'elsewhere.m4a'); + await server.scanner.scanAllNow('manual'); + const other = server.shows.getBySlug('other-show'); + const elsewhere = server.episodes.listByShow(other.id)[0]; + server.stats.record({ episodeId: episode.id, showId: show.id, kind: 'download', statusCode: 200, bytesSent: 9, totalBytes: 9 }); + for (let i = 0; i < 3; i += 1) { + server.stats.record({ episodeId: elsewhere.id, showId: other.id, kind: 'download', statusCode: 200, bytesSent: 9, totalBytes: 9 }); + } + assert.equal(server.stats.overview().downloads, 4); + assert.equal(server.stats.overview({ showId: show.id }).downloads, 1); + + const page = await server.get(`/stats?showId=${show.slug}`, { accept: 'text/html' }); + assert.match(page.body, new RegExp(`${show.title} only`)); + assert.match(page.body, /\(every show\)/); + assert.ok(!page.body.includes('elsewhere'), 'the other show\'s rows are not on a page about this one'); + const context = server.statsContext({ query: { showId: show.slug } }); + assert.equal(context.overview.downloads, 1, 'the cards follow the filter'); + assert.deepEqual(context.showStats.map((row) => row.slug), [show.slug], 'the per-show table is that show'); + assert.equal(context.busiest.every((row) => row.showSlug === show.slug), true); + assert.equal(context.daily.reduce((sum, bucket) => sum + bucket.downloads, 0), 1, 'the chart too'); + }); + + it('compares the previous period to the same point rather than to its whole', () => { + clearLog(); + const now = new Date('2026-07-13T09:00:00Z'); + const range = resolveRange('7d', { timeZone: 'UTC', now }); + const stamp = (iso) => + server.db + .prepare(`INSERT INTO media_access (episode_id, show_id, requested_at, kind, status_code, bytes_sent, total_bytes) VALUES (?, ?, ?, 'download', 200, 1, 1)`) + .run(episode.id, show.id, iso); + // 7 days to 09:00 on 13 July: this period opened on the 7th, the previous on the + // 30th of June, and the same point in it is 09:00 on the 6th. + assert.equal(range.prevTo, '2026-07-06T09:00:00.000Z'); + stamp('2026-07-07T08:00:00Z'); // this period, before now + stamp('2026-06-30T08:00:00Z'); // previous period, before the same point + stamp('2026-07-06T12:00:00Z'); // previous period, after the same point: not yet comparable + const overview = server.stats.overview({ from: range.from, to: range.to, prevFrom: range.prevFrom, prevTo: range.prevTo }); + assert.equal(overview.downloads, 1); + assert.equal(overview.previous.downloads, 1, 'the afternoon download on the 6th is not counted yet'); + assert.equal(overview.change.downloads.direction, 'flat'); + }); + + it('exports every row, not the first fifty thousand', async () => { + clearLog(); + const insert = server.db.prepare( + `INSERT INTO media_access (episode_id, show_id, requested_at, kind, status_code, bytes_sent, total_bytes) VALUES (?, ?, ?, 'stream', 206, 1, 1)`, + ); + const many = 50_250; + server.db.transaction(() => { + for (let i = 0; i < many; i += 1) insert.run(episode.id, show.id, `2026-07-01T00:00:${String(i % 60).padStart(2, '0')}.${String(i).padStart(6, '0')}Z`); + })(); + const csv = await server.get('/stats/access-log.csv?range=all'); + assert.equal(csv.statusCode, 200); + assert.equal(csv.body.split('\r\n').filter(Boolean).length, many + 1, 'a header plus every row'); + clearLog(); + }); + it('does not record a HEAD request at all', async () => { clearLog(); const response = await server.app.inject({ diff --git a/test/unit/dates-dst.test.js b/test/unit/dates-dst.test.js new file mode 100644 index 0000000..e2e8683 --- /dev/null +++ b/test/unit/dates-dst.test.js @@ -0,0 +1,80 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { fromLocalInputValue } from '../../src/lib/dates.js'; +import { bucketEdges, resolveRange } from '../../src/lib/time-range.js'; + +/** + * Local midnight, on every day of a year, in zones whose clocks change at awkward + * hours. The single-probe version was an hour out at midnight on the transition days + * in Sydney, Auckland, Lord Howe and Santiago, and nowhere else — which is why the + * London-only tests never saw it. + */ +const ZONES = [ + 'Europe/London', + 'Europe/Paris', + 'America/New_York', + 'America/Santiago', + 'Australia/Sydney', + 'Australia/Lord_Howe', + 'Pacific/Auckland', + 'UTC', +]; + +function localClock(instant, timeZone) { + const parts = new Intl.DateTimeFormat('en-CA', { + timeZone, + hour12: false, + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + }).formatToParts(new Date(instant)); + const get = (type) => parts.find((p) => p.type === type)?.value; + const hour = get('hour') === '24' ? '00' : get('hour'); + return `${get('year')}-${get('month')}-${get('day')}T${hour}:${get('minute')}`; +} + +describe('local midnight across a year of clock changes', () => { + for (const timeZone of ZONES) { + it(`reads back as 00:00 on every day of 2026 in ${timeZone}`, () => { + const wrong = []; + for (let d = new Date(Date.UTC(2026, 0, 1)); d.getUTCFullYear() === 2026; d = new Date(d.getTime() + 86_400_000)) { + const day = d.toISOString().slice(0, 10); + const instant = fromLocalInputValue(`${day}T00:00`, { timeZone }); + const clock = localClock(instant, timeZone); + if (clock === `${day}T00:00`) continue; + // Chile springs forward at midnight, so 00:00 does not exist that day: the + // answer must then be the first instant of the day, with the day before + // ending the minute earlier. + const minuteBefore = localClock(new Date(instant).getTime() - 60_000, timeZone); + if (clock.startsWith(day) && !minuteBefore.startsWith(day)) continue; + wrong.push({ day, instant, clock, minuteBefore }); + } + assert.deepEqual(wrong, [], 'midnight landed on the wrong hour on these days'); + }); + } + + it('keeps every afternoon as well, not only midnight', () => { + for (const timeZone of ['Australia/Sydney', 'America/Santiago']) { + for (const day of ['2026-04-05', '2026-10-04', '2026-09-06']) { + const instant = fromLocalInputValue(`${day}T15:30`, { timeZone }); + assert.equal(localClock(instant, timeZone), `${day}T15:30`, `${timeZone} ${day}`); + } + } + }); + + it('gives the Sydney chart a 23-hour day on the October transition, starting at local midnight', () => { + const timeZone = 'Australia/Sydney'; + const range = resolveRange('7d', { timeZone, now: new Date('2026-10-07T12:00:00Z') }); + const buckets = bucketEdges({ from: range.from, to: range.to, timeZone }); + const transition = buckets.find((bucket) => bucket.key === '2026-10-04'); + assert.ok(transition, 'the transition day is in the range'); + assert.equal(localClock(transition.start, timeZone), '2026-10-04T00:00'); + assert.equal((new Date(transition.end) - new Date(transition.start)) / 3.6e6, 23); + for (const bucket of buckets) { + assert.equal(localClock(bucket.start, timeZone).slice(11), '00:00', `${bucket.key} starts at midnight`); + } + }); +}); diff --git a/test/unit/migrations.test.js b/test/unit/migrations.test.js index 9122dd9..03e23ab 100644 --- a/test/unit/migrations.test.js +++ b/test/unit/migrations.test.js @@ -373,6 +373,32 @@ describe('migration 011 classifies the catalogue without losing it', () => { * one exists on the column it is for, that it is reached from an empty database and * from one migrated step by step, and that the cascades the columns carry still work. */ +/** + * Migration 013 rebuilds media_access so that deleting an episode no longer deletes + * every request ever made for it. The table comes back row for row, index for index. + */ +describe('migration 013 keeps request history when an episode goes', () => { + it('carries every row and index across the rebuild, and a deleted episode keeps its rows', () => { + const db = seededAt008(); + runMigrations(db, { upTo: 12 }); + db.prepare("INSERT INTO shows (id, slug, title, author_name, author_email, feed_token, created_at, updated_at) VALUES ('s13','s13','S13','a','a@b.c','tok-013','2026-01-01T00:00:00Z','2026-01-01T00:00:00Z')").run(); + db.prepare("INSERT INTO episodes (id, show_id, filename, identity_key, title, pub_date, file_size_bytes, mime_type, created_at, updated_at) VALUES ('e13','s13','a.mp3','k13','A','2026-01-01T00:00:00Z',10,'audio/mpeg','2026-01-01T00:00:00Z','2026-01-01T00:00:00Z')").run(); + db.prepare("INSERT INTO media_access (episode_id, show_id, requested_at, kind, status_code, bytes_sent, total_bytes, client) VALUES ('e13','s13','2026-01-02T00:00:00Z','download',200,10,10,'Overcast')").run(); + const before = db.prepare('SELECT * FROM media_access').all(); + + runMigrations(db); + assert.ok(db.pragma('user_version', { simple: true }) >= 13); + assert.deepEqual(db.prepare('SELECT * FROM media_access').all(), before, 'every row survives the rebuild as it was'); + const indexes = db.prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'media_access' ORDER BY name").all().map((r) => r.name); + assert.deepEqual(indexes, ['idx_media_access_episode', 'idx_media_access_failures', 'idx_media_access_show', 'idx_media_access_time']); + + db.prepare("DELETE FROM episodes WHERE id = 'e13'").run(); + assert.equal(db.prepare('SELECT COUNT(*) AS n FROM media_access WHERE show_id = \'s13\'').get().n, 1, 'the download still happened'); + db.prepare("DELETE FROM shows WHERE id = 's13'").run(); + assert.equal(db.prepare('SELECT COUNT(*) AS n FROM media_access WHERE show_id = \'s13\'').get().n, 0, 'forgetting a show still forgets its log'); + }); +}); + describe('migration 012 indexes the advert tables\' foreign keys', () => { const EXPECTED = { idx_ad_cut_overrides_segment: ['ad_cut_overrides', 'segment_id'], @@ -396,7 +422,8 @@ describe('migration 012 indexes the advert tables\' foreign keys', () => { it('creates each index on its column, from a database seeded at 008 and carried forward', () => { const db = seededAt008(); runMigrations(db); - assert.equal(db.pragma('user_version', { simple: true }), 12); + // At least 012: later migrations are allowed to exist without this test caring. + assert.ok(db.pragma('user_version', { simple: true }) >= 12); const found = indexesOf(db); for (const [name, [table, column]] of Object.entries(EXPECTED)) { assert.deepEqual(found[name], [table, column], `${name} is missing or on the wrong column`); diff --git a/test/unit/time-range.test.js b/test/unit/time-range.test.js index e0ad42f..1f73c20 100644 --- a/test/unit/time-range.test.js +++ b/test/unit/time-range.test.js @@ -77,6 +77,19 @@ describe('time ranges', () => { assert.ok(new Date(prevFrom) < new Date(from), 'the previous period comes first'); }); + it('measures the previous period to the same point, not to its end', () => { + // 09:00 on the 13th of July: the current 30 days are 29 whole days and nine hours. + const now = new Date('2026-07-13T09:00:00Z'); + const { from, prevFrom, prevTo } = resolveRange('30d', { timeZone: 'UTC', now }); + assert.equal(new Date(prevTo) - new Date(prevFrom), now - new Date(from), 'same elapsed time in both windows'); + // 29 days and nine hours into the previous period, which opened on 15 May. + assert.equal(prevTo, '2026-06-13T09:00:00.000Z'); + assert.ok(new Date(prevTo) <= new Date(from), 'the comparison never reaches into the current period'); + // At one minute to midnight the two windows are all but the same length. + const late = resolveRange('7d', { timeZone: 'UTC', now: new Date('2026-07-13T23:59:00Z') }); + assert.equal(late.prevTo, '2026-07-06T23:59:00.000Z'); + }); + it('returns null bounds for all time, so a query can skip the date filter entirely', () => { const range = resolveRange('all', { timeZone: 'Europe/London', now: NOW }); assert.equal(range.key, 'all', 'the key survives');