From 9a7ade2b61b731d9de0b7eb0e1d67bacee23a464 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 12:31:10 +0100 Subject: [PATCH 1/3] =?UTF-8?q?fix(spanish-poller):=20never=20report=20a?= =?UTF-8?q?=20DEH=C3=BA=20inbox=20it=20did=20not=20read=20as=20=C2=AB0=20p?= =?UTF-8?q?ending=C2=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An accepted list that read fewer rows than its tab counter threw, which dropped the pending list already read. The run then printed the kept snapshot of an earlier poll as if it were new: «0 pending», «no changes», while the inbox held two pending notifications. Each DEHú list now carries its own read state (read, partial, not read), checked against its tab counter. A short accepted or communications list is reported in the summary and no longer throws the pending list away; a pending list read short is an error. A side or list that was not read prints as NOT READ with the time of the snapshot kept, and the last line names what was not read. A click on the pending tab that lands on a connection reset is reloaded once. A sede search that answers «Página no encontrada» is named as such. --- CHANGELOG.md | 1 + docs/spanish-poller.md | 10 ++ spanish-poller/gwt_spanish_poller.js | 199 +++++++++++++++++---- spanish-poller/test/spanish-poller.test.js | 165 +++++++++++++++++ 4 files changed, 337 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 48568af7..7e2247a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Every release has a **Change lists** heading. It names each change that makes a ### Fixed +- `gwt-spanish-poller` no longer prints `0 pending notifications` or `no changes` for a DEHú inbox it did not read. Before, an accepted list that read fewer rows than its tab counter failed the whole DEHú read, and the summary then showed the kept snapshot from an earlier poll as if it were new, while the inbox had two pending notifications. Now each DEHú list is checked against its tab counter on its own, a short accepted or communications list no longer throws the pending list away, and a side or list that was not read says `NOT READ` in the summary. A sede search that answers «Página no encontrada» is named as such. - A second `gwt-apply --apply` of a list with chained merges no longer makes a person or family the list already merged away, and no longer stops at the first merge. A merge whose winner is gone is read on the one object that holds its merge note now (#263). ## [1.5.0] - 2026-11-10 diff --git a/docs/spanish-poller.md b/docs/spanish-poller.md index c26421ae..db53e284 100644 --- a/docs/spanish-poller.md +++ b/docs/spanish-poller.md @@ -29,6 +29,13 @@ Per person it writes `/.json` (the snapshot plus the events), `/-errors/` even without `--dump`. + ## Config Keep the config, the state and the certificates outside this repo. @@ -77,6 +84,9 @@ Add the same entry for `pasarela-ident.clave.gob.es`, `se-pasarela.clave.gob.es` - **DEHú is built from web components.** Its tables live inside shadow roots and its buttons inside closed components, so rows are read through every shadow root and «Detalles» is clicked by position. - **DEHú shows a CAPTCHA after many logins in a short time** (we hit it after about a dozen test logins in one afternoon). The poller stops with a clear error. Poll once or twice a day, not in a tight loop. - **Only the first page of each DEHú list is read** (10 rows). If there are more the summary says so instead of dropping them silently. +- **DEHú's accepted and communications lists show the last 30 days only.** Older entries need the search form, which the poller does not use. +- **The click on the pending tab sometimes lands on a connection-reset page.** The poller goes back to the inbox home once and clicks again. +- **The sede sometimes answers a search with its own «Página no encontrada» page** instead of the results. The error says so; the next poll usually reads fine. - **Every browser is closed by PID** when the run ends, on SIGTERM and SIGINT, and when the time budget runs out. ## Schedule it diff --git a/spanish-poller/gwt_spanish_poller.js b/spanish-poller/gwt_spanish_poller.js index eb4648fd..685cca9b 100644 --- a/spanish-poller/gwt_spanish_poller.js +++ b/spanish-poller/gwt_spanish_poller.js @@ -8,7 +8,7 @@ DEHú inbox for one or more people, each logging in with their own client certif Per person it snapshots the case list and the DEHú notifications and communications, diffs them against the previous snapshot, writes /.json and /.txt, and prints the summary. Exit 0: nothing changed. Exit 10: something changed. Exit 2: a login or a page failed and nothing -else changed (the failed part keeps its previous snapshot). +else changed (the failed part keeps its previous snapshot, and the summary says it was not read). Read-only toward the administration: it never submits a form and never opens, accepts or rejects a notification. Communications that are already readable are downloaded only when their sender matches @@ -83,6 +83,8 @@ function parseCases(rows) { const DEHU_ID = /^[0-9a-f]{16,}$/i; const DEHU_ROW_ID = /^(?=.*\d)[A-Z0-9][A-Z0-9-]{5,}$/i; +// The three DEHú lists: snapshot key and the word the summary uses. +const DEHU_TABS = [['pending', 'pending'], ['done', 'accepted'], ['communications', 'communications']]; const DETAIL_LABELS = ['Concepto', 'Organismo raíz', 'Fecha de estado final', 'Fecha leída', 'Fecha de caducidad', 'Fecha caducidad', 'Fecha de expiración']; // Where the notification id sits in a data row, or -1. @@ -213,25 +215,57 @@ function describe(e) { } } +// One DEHú tab as a summary phrase. A tab that was not read in full says so; it never shows a count +// it did not read. +function dehuTabPhrase(label, list, tab) { + if (!tab || tab.state === 'read') return `${list.length} ${label}`; + if (tab.state === 'not read') return `${label} NOT READ (${tab.note})`; + return `${label} NOT FULLY READ (${tab.note})`; +} + +// What this poll could not read: whole sides kept from an earlier poll, and DEHú tabs read short. +function notRead(snap) { + const out = []; + if (snap.sede_stale) out.push('sede'); + if (snap.dehu_stale) out.push('dehu'); + else if (snap.dehu) { + for (const [key, label] of DEHU_TABS) { + const t = (snap.dehu.tabs || {})[key]; + if (t && t.state !== 'read') out.push(`dehu ${label}`); + } + } + return out; +} + function summarize(id, snap, events, errors) { const L = [`gwt-spanish-poller ${id} ${snap.polled_at}`]; - if (snap.sede) { + const kept = (at) => `the snapshot ${at ? 'read at ' + at : 'of an earlier poll'} is kept`; + if (snap.sede_stale) { + L.push(`sede: NOT READ this run; ${kept(snap.sede_read_at)}`); + } else if (snap.sede) { const n = Object.values(snap.sede.cases); const by = {}; for (const c of n) by[c.status] = (by[c.status] || 0) + 1; L.push(`sede: ${n.length} cases (${Object.entries(by).map(([k, v]) => `${v} ${k}`).join(', ') || 'none'})`); } - if (snap.dehu) { - const d = snap.dehu; - L.push(`dehu: ${d.pending.length} pending notifications, ${d.done.length} accepted, ${d.communications.length} communications`); + if (snap.dehu_stale) { + L.push(`dehu: NOT READ this run; ${kept(snap.dehu_read_at)}`); + } else if (snap.dehu) { + const d = snap.dehu, tabs = d.tabs || {}; + const fresh = (key) => d[key].length - ((tabs[key] && tabs[key].kept) || 0); + L.push('dehu: ' + [['pending', 'pending notifications'], ['done', 'accepted'], ['communications', 'communications']] + .map(([key, label]) => dehuTabPhrase(label, d[key].slice(0, fresh(key)), tabs[key])).join(', ')); for (const w of d.warnings || []) L.push(` warning: ${w}`); - for (const n of d.pending) L.push(` pending: ${[n.sender, n.subject, n.expires && 'caduca ' + n.expires].filter(Boolean).join(' · ') || n.cells.join(' | ')}`); + for (const n of d.pending.slice(0, fresh('pending'))) L.push(` pending: ${[n.sender, n.subject, n.expires && 'caduca ' + n.expires].filter(Boolean).join(' · ') || n.cells.join(' | ')}`); } for (const f of snap.downloaded || []) L.push(`downloaded: ${f}`); for (const e of errors) L.push(`ERROR: ${e}`); const real = events.filter(e => e.kind !== 'baseline' || events.length === 1); - L.push(events.length ? 'changes:' : 'no changes'); + const missing = notRead(snap); + if (events.length) L.push('changes:'); + else L.push(missing.length ? 'no changes in what was read' : 'no changes'); for (const e of real) L.push(' ' + describe(e)); + if (missing.length) L.push(`not read this run: ${missing.join(', ')}`); return L.join('\n') + '\n'; } @@ -376,6 +410,15 @@ async function sedeForm(page, dumpDir) { } } +// The sede sometimes answers a search with its own «Página no encontrada» page instead of the results +// (seen on several days, on the first or a later slice; the next poll usually reads fine). Name that +// case so the summary does not read like a parser fault. +function sedeNoCountError(body, from, to) { + const range = `${dmy(from)}-${dmy(to)}`; + if (/P[aá]gina no encontrada/i.test(body)) return `sede: the sede answered «Página no encontrada» to the search ${range}`; + return `sede: search ${range} returned no result count`; +} + // One search over [from, to] (Date objects). Returns the cases, or null when the sede answers // «demasiados registros» for that range. async function sedeSearch(page, dumpDir, role, from, to) { @@ -398,7 +441,7 @@ async function sedeSearch(page, dumpDir, role, from, to) { if (!found) { if (/no se encuentran datos de expedientes|no se han encontrado/i.test(body)) return {}; await dump(page, dumpDir, 'sede-err'); - throw new Error('sede: search returned no result count'); + throw new Error(sedeNoCountError(body, from, to)); } const total = Number(found[1]); const cases = {}; @@ -509,26 +552,46 @@ function dehuLoadState(items, empty) { return empty ? 'empty' : 'did not load'; } -async function dehuList(page, dumpDir, name, expected) { - // A single-page app: wait until the table shows rows, or the page says the list is empty. - let got = { rows: [], total: null }, items = [], empty = false; +const DEHU_PAGE_ROWS = 10; + +// How completely one DEHú tab was read: {state, expected, rows, note}. state is «read», «partial» (some +// rows are missing) or «not read». `counter` is the number on the tab («Notificaciones pendientes 2»), +// `total` the list's own «1-10 de N elementos» line. The tab counter wins when both are known: it is +// the number the person sees. Only the first page of a list is read. +function dehuTabRead(items, { counter = null, total = null, empty = false } = {}) { + const m = String(total || '').match(/de (\d+) elementos/); + const listed = m ? Number(m[1]) : null; + const expected = counter != null ? counter : listed; + const rows = items.length; + const out = (state, note, paged) => ({ state, expected, rows, note, ...(paged ? { paged } : {}) }); + if (!rows) { + if (expected === 0 || (expected == null && dehuLoadState(items, empty) === 'empty')) return out('read', null); + return out('not read', expected ? `tab counter says ${expected}, the list did not load` : 'the list did not load'); + } + if (expected == null || rows >= expected) return out('read', null); + if (rows < Math.min(expected, DEHU_PAGE_ROWS)) return out('partial', `tab counter says ${expected}, ${rows} rows read`); + // A full first page and more behind it: the page was read, the later pages were not. + return out('partial', `${expected - rows} more on later pages, not read`, true); +} + +async function dehuList(page, dumpDir, name, counter) { + // A single-page app: wait until the table shows every row the counter promises (one page at most), + // or the page says the list is empty. + let got = { rows: [], total: null }, items = [], read = null; for (let i = 0; i < 30; i++) { await wait(page, 1500); got = await deepRows(page); items = parseDehuRows(got.rows); - empty = /no dispones/i.test(await page.innerText('body').catch(() => '')); - if (items.length && (expected == null || items.length >= Math.min(expected, 10))) break; - if (empty && !items.length) break; + const empty = /no dispones/i.test(await page.innerText('body').catch(() => '')); + read = dehuTabRead(items, { counter, total: got.total, empty }); + if (read.state === 'read' || read.paged) break; } await dump(page, dumpDir, name); if (dumpDir) fs.writeFileSync(path.join(dumpDir, name + '.rows.json'), JSON.stringify(got, null, 1)); - if (dehuLoadState(items, empty) === 'did not load') throw new Error(`dehu: ${name} did not load`); - if (expected != null && items.length < Math.min(expected, 10)) throw new Error(`dehu: ${name} counter says ${expected}, parsed ${items.length}`); - // «1-10 de 14 elementos»: only the first page is read; say so rather than drop the rest silently. - const m = String(got.total || '').match(/de (\d+) elementos/); - const total = m ? Number(m[1]) : expected; - if (total != null && total > items.length) items.more = total - items.length; - return items; + // A list that did not match its counter is saved even without --dump, so the next time it happens + // the page is there to look at. + if (read.state !== 'read' && !read.paged) await dump(page, null, name + '-err'); + return { items, ...read }; } async function dehuCounters(page) { @@ -537,23 +600,59 @@ async function dehuCounters(page) { return { pending: n(/Notificaciones pendientes\s*\n\s*(\d+)/), done: n(/Notificaciones realizadas\s*\n\s*(\d+)/) }; } +// Read the three DEHú lists. One list that fails or reads short never throws the others away: each +// tab carries its own {state, expected, rows, note} in `tabs`. async function pollDehu(page, dumpDir) { await dehuLogin(page, dumpDir); - await page.getByText('Notificaciones pendientes', { exact: true }).first().click(); let counters = { pending: null, done: null }; - for (let i = 0; i < 20 && counters.pending == null; i++) { await wait(page, 1000); counters = await dehuCounters(page); } - if (counters.pending == null) throw new Error('dehu: notification counters not found'); + for (let attempt = 0; attempt < 2 && counters.pending == null; attempt++) { + // The click on the tab sometimes lands on a connection-reset page; one reload of the home fixes it. + if (attempt) { await goto(page, DEHU_HOME); await wait(page, 4000); } + await page.getByText('Notificaciones pendientes', { exact: true }).first().click().catch(() => {}); + for (let i = 0; i < 20 && counters.pending == null; i++) { await wait(page, 1000); counters = await dehuCounters(page).catch(() => counters); } + } + if (counters.pending == null) { + await dump(page, dumpDir, 'dehu-err-counters'); + throw new Error('dehu: notification counters not found'); + } + const lists = {}; + const read = async (key, name, open, counter) => { + try { + await open(); + lists[key] = await dehuList(page, dumpDir, name, counter); + } catch (e) { + lists[key] = { items: [], state: 'not read', expected: counter, rows: 0, note: String(e.message || e) }; + } + }; // Listing only: the pending rows are read, never clicked, so nothing is accepted or rejected. - const pending = await dehuList(page, dumpDir, 'dehu-1-pending', counters.pending); - await page.locator('#pane-button-1').click({ timeout: 15000 }); - const done = await dehuList(page, dumpDir, 'dehu-2-done', counters.done); - await goto(page, DEHU_HOME); - await wait(page, 4000); - await page.getByText('Comunicaciones', { exact: true }).first().click(); - const communications = await dehuList(page, dumpDir, 'dehu-3-communications', null); - const warnings = [['pending', pending], ['done', done], ['communications', communications]] - .filter(([, l]) => l.more).map(([k, l]) => `${k}: ${l.more} more on later pages, not read`); - return { pending: [...pending], done: [...done], communications: [...communications], warnings }; + await read('pending', 'dehu-1-pending', async () => {}, counters.pending); + await read('done', 'dehu-2-done', () => page.locator('#pane-button-1').click({ timeout: 15000 }), counters.done); + await read('communications', 'dehu-3-communications', async () => { + await goto(page, DEHU_HOME); + await wait(page, 4000); + await page.getByText('Comunicaciones', { exact: true }).first().click(); + }, null); + const out = { counters, tabs: {}, warnings: [] }; + for (const [key] of DEHU_TABS) { + const { items, ...state } = lists[key]; + out[key] = [...items]; + out.tabs[key] = state; + } + return out; +} + +// A tab that was not read in full keeps the previous snapshot's rows next to the ones it did read, so +// the next poll does not report them as new. Returns a new dehu snapshot. +function keepUnreadRows(prevDehu, dehu) { + const out = { ...dehu }; + for (const [key] of DEHU_TABS) { + const t = (dehu.tabs || {})[key]; + if (!t || t.state === 'read' || !prevDehu) continue; + const seen = new Set((dehu[key] || []).map(x => x.id)); + out[key] = [...(dehu[key] || []), ...(prevDehu[key] || []).filter(x => !seen.has(x.id))]; + out.tabs = { ...out.tabs, [key]: { ...t, kept: out[key].length - (dehu[key] || []).length } }; + } + return out; } // Centre of the first element matching `rx` by its own text, searched through shadow roots. With @@ -622,6 +721,26 @@ async function downloadCommunications(page, items, senders, dir, already, notes) return got; } +// When a kept side of a snapshot was last read for real (snapshots older than this field: their poll +// time, unless that poll had already kept it from an earlier one). +function readAt(snap, side) { + if (snap[side + '_read_at']) return snap[side + '_read_at']; + return snap[side + '_stale'] ? null : snap.polled_at; +} + +// A pending list that was not read in full is an error: it is the list someone has to act on. The +// other two lists only count as errors when they did not load at all; a short read there is reported +// in the summary and does not throw the pending result away. +function dehuTabErrors(tabs) { + const errs = []; + for (const [key, label] of DEHU_TABS) { + const t = (tabs || {})[key]; + if (!t || t.state === 'read') continue; + if (t.state === 'not read' || (key === 'pending' && !t.paged)) errs.push(`dehu: ${label} ${t.state === 'not read' ? 'not read' : 'not fully read'}: ${t.note}`); + } + return errs; +} + async function pollPerson(cfg, person, dumpDir) { const stateFile = path.join(cfg.state_dir, person.id + '.json'); const prev = fs.existsSync(stateFile) ? JSON.parse(fs.readFileSync(stateFile, 'utf8')) : null; @@ -643,13 +762,16 @@ async function pollPerson(cfg, person, dumpDir) { if (person.sede) try { session = await open(); snap.sede = await pollSede(session.page, d, person.sede_role, person.sede_days, person.sede_slice_days); + snap.sede_read_at = snap.polled_at; } catch (e) { errors.push(String(e.message || e)); } // DEHú gets its own empty browser: with the Cl@ve session the sede left behind, the DEHú login // in chrome mode kept landing back on the public page. if (session) { await session.close(); session = null; } if (person.dehu) try { session = await open(); - snap.dehu = await pollDehu(session.page, d); + snap.dehu = keepUnreadRows(prev && prev.dehu, await pollDehu(session.page, d)); + snap.dehu_read_at = snap.polled_at; + errors.push(...dehuTabErrors(snap.dehu.tabs)); const already = (prev && prev.downloaded_ids) || []; if (cfg.download_senders.length) { snap.downloaded = await downloadCommunications(session.page, snap.dehu.communications, cfg.download_senders, @@ -665,8 +787,9 @@ async function pollPerson(cfg, person, dumpDir) { } if (errors.length && errorDumpDir && fs.existsSync(errorDumpDir)) errors.push('failed pages saved in ' + errorDumpDir); // A failed part keeps its previous snapshot, so a broken login never reads as «every case disappeared». - if (!snap.sede && prev && prev.sede) snap.sede_stale = true, snap.sede = prev.sede; - if (!snap.dehu && prev && prev.dehu) snap.dehu_stale = true, snap.dehu = prev.dehu; + // The summary says so and names when the kept snapshot was read. + if (!snap.sede && prev && prev.sede) Object.assign(snap, { sede_stale: true, sede: prev.sede, sede_read_at: readAt(prev, 'sede') }); + if (!snap.dehu && prev && prev.dehu) Object.assign(snap, { dehu_stale: true, dehu: prev.dehu, dehu_read_at: readAt(prev, 'dehu') }); if (!snap.downloaded_ids) snap.downloaded_ids = (prev && prev.downloaded_ids) || []; const events = diff(prev, snap).filter(e => !(snap.sede_stale && /^case_/.test(e.kind)) && !(snap.dehu_stale && /notification|communication/.test(e.kind))); const text = summarize(person.id, snap, events, errors); @@ -714,4 +837,4 @@ async function main(argv) { return changed ? EXIT_CHANGED : failed ? EXIT_ERROR : EXIT_SAME; } -module.exports = { dump, setErrorDumpDir, isoDate, openPkcs12, openChrome, dehuLogin, loadConfig, parseCases, parseDehuRows, dehuLoadState, diff, senderAllowed, summarize, describe, main, EXIT_SAME, EXIT_CHANGED, EXIT_ERROR }; +module.exports = { dump, setErrorDumpDir, isoDate, openPkcs12, openChrome, dehuLogin, pollDehu, loadConfig, parseCases, parseDehuRows, dehuLoadState, dehuTabRead, dehuTabErrors, keepUnreadRows, readAt, sedeNoCountError, diff, senderAllowed, summarize, describe, main, EXIT_SAME, EXIT_CHANGED, EXIT_ERROR }; diff --git a/spanish-poller/test/spanish-poller.test.js b/spanish-poller/test/spanish-poller.test.js index f277263a..d0fa1075 100644 --- a/spanish-poller/test/spanish-poller.test.js +++ b/spanish-poller/test/spanish-poller.test.js @@ -116,3 +116,168 @@ test('dump saves error pages without --dump, and nothing else', async () => { setErrorDumpDir(null); fs.rmSync(dir, { recursive: true, force: true }); }); + +// ---------- DEHú tabs read in full, or said not to be ---------- +// Made-up rows in the shape DEHú renders: calendar rows, a header, a data row per notification with +// empty cells in front, and an expandable detail row after each. +const CAL = [['26', '27', '28', '29', '30', '31', '1'], ['2', '3', '4', '5', '6', '7', '8']]; +const PENDING_HEAD = ['Columna ampliable', 'Identificador', 'Concepto', 'Organismo emisor', 'Titular', 'Fecha disposición', 'Tipo de notificación', '', 'Acciones']; +const DONE_HEAD = ['Columna ampliable', 'Identificador', 'Organismo emisor', 'Titular', 'Vínculo', 'Fecha disposición', 'Tipo de notificación', '', 'Estado', 'Acciones']; +const COMM_HEAD = ['Columna ampliable', 'Identificador', 'Concepto', 'Organismo emisor', 'Titular', 'Vínculo', 'Fecha disposición', 'Estado', 'Acciones']; +const pendingRow = (id, sender, date) => [ + ['', '', id, 'Requerimiento', sender, '00000000T', date, 'Electrónica', '', 'Aceptar\nRechazar'], + ['', `Organismo raíz Organismo raíz de prueba Fecha caducidad ${date} `], +]; +const doneRow = (id, sender, date) => [ + ['', '', id, sender, '00000000T', 'Titular', date, 'Postal', '', 'Aceptada', 'Detalle'], + ['', `Concepto Notificación de prueba Organismo raíz Ministerio de prueba Fecha de estado final ${date} `], +]; +const commRow = (id, date) => [['', '', id, 'Comunicación de prueba', 'Organismo C', '00000000T', 'Titular', date, 'Leída', 'Detalles']]; +const HEX = (n) => n.toString(16).padStart(20, '0'); + +// A fake DEHú page: the inbox is already open, tabs switch on the same clicks the poller makes, and +// each tab answers with its rows. Waits return at once. +function fakeDehu({ counters, pending, done, comm, resetOnce = false }) { + let tab = 'home', resets = resetOnce ? 1 : 0; + const lists = { pending, done, comm }; + const total = (rows) => { const n = rows ? rows.filter(r => r.length > 2 && r[2]).length : 0; return n ? `1-${n} de ${n} elementos` : ''; }; + const body = () => (tab === 'reset' ? 'This site can’t be reached\nERR_CONNECTION_RESET\n' + : `Notificaciones pendientes\n${counters.pending}\nNotificaciones realizadas\n${counters.done}\n`); + const clickable = (fn) => ({ first: () => clickable(fn), click: async () => fn(), count: async () => 0, getByText: () => clickable(() => {}) }); + return { + goto: async () => { tab = 'home'; }, + waitForTimeout: async () => {}, + innerText: async () => body(), + url: () => 'https://dehu.example/es/home-view', + content: async () => '', + screenshot: async () => {}, + locator: (sel) => clickable(() => { if (sel === '#pane-button-1') tab = 'done'; }), + getByText: (t) => clickable(() => { + if (/pendientes/.test(t) && resets) { resets--; tab = 'reset'; return; } + tab = /pendientes/.test(t) ? 'pending' : /Comunicaciones/.test(t) ? 'comm' : tab; + }), + evaluate: async () => { + const l = lists[tab]; + return { rows: l ? [...CAL, ...l.rows] : [], total: l ? (l.total != null ? l.total : total(l.rows)) : null }; + }, + }; +} + +// The 20-11 run: two pending rows on the page, and an accepted tab whose counter says 6 with only 3 rows shown. +const SHORT_DONE = { + counters: { pending: 2, done: 6 }, + pending: { rows: [PENDING_HEAD, ...pendingRow('0a00000000000000aa01', 'Ayuntamiento de prueba', '20/11/2026'), ...pendingRow('0a00000000000000aa02', 'Ayuntamiento de prueba', '19/11/2026')] }, + done: { rows: [DONE_HEAD, ...doneRow(HEX(1), 'Organismo A', '09/11/2026'), ...doneRow('11112222333344', 'Organismo B', '27/10/2026'), ...doneRow(HEX(2), 'Organismo C', '04/11/2026')], total: '1-6 de 6 elementos' }, + comm: { rows: [COMM_HEAD, ...commRow(HEX(3), '13/11/2026')] }, +}; + +test('pollDehu: an accepted tab that reads short does not throw the pending list away', async () => { + const { pollDehu } = require('../gwt_spanish_poller.js'); + const d = await pollDehu(fakeDehu(SHORT_DONE), null); + assert.deepStrictEqual(d.pending.map(n => n.id), ['0a00000000000000aa01', '0a00000000000000aa02']); + assert.strictEqual(d.tabs.pending.state, 'read'); + assert.strictEqual(d.tabs.pending.expected, 2); + assert.strictEqual(d.done.length, 3); + assert.strictEqual(d.tabs.done.state, 'partial'); + assert.strictEqual(d.communications.length, 1); +}); + +test('pollDehu: a pending list that does not parse is «not read», never 0 pending', async () => { + const { pollDehu, dehuTabErrors } = require('../gwt_spanish_poller.js'); + const broken = { ...SHORT_DONE, pending: { rows: [PENDING_HEAD] } }; + const d = await pollDehu(fakeDehu(broken), null); + assert.strictEqual(d.tabs.pending.state, 'not read'); + assert.match(d.tabs.pending.note, /tab counter says 2/); + assert.strictEqual(d.done.length, 3); + assert.match(dehuTabErrors(d.tabs).join('\n'), /pending not read: tab counter says 2/); +}); + +test('pollDehu: a connection reset after the pending-tab click is reloaded once, not a lost read', async () => { + const { pollDehu } = require('../gwt_spanish_poller.js'); + const d = await pollDehu(fakeDehu({ ...SHORT_DONE, resetOnce: true }), null); + assert.deepStrictEqual(d.counters, { pending: 2, done: 6 }); + assert.strictEqual(d.pending.length, 2); +}); + +test('dehuTabRead: the pending count comes from the tab counter and is checked against the rows', () => { + const { dehuTabRead } = require('../gwt_spanish_poller.js'); + const two = parseDehuRows(SHORT_DONE.pending.rows); + assert.strictEqual(dehuTabRead(two, { counter: 2 }).state, 'read'); + assert.strictEqual(dehuTabRead(two.slice(0, 1), { counter: 2 }).state, 'partial'); + assert.strictEqual(dehuTabRead([], { counter: 2 }).state, 'not read'); + assert.strictEqual(dehuTabRead([], { counter: 0 }).state, 'read'); + assert.strictEqual(dehuTabRead([], { counter: null, empty: true }).state, 'read'); + assert.strictEqual(dehuTabRead([], { counter: null }).state, 'not read'); + // The tab counter wins over the list's own total line. + assert.strictEqual(dehuTabRead(two, { counter: 3, total: '1-2 de 2 elementos' }).state, 'partial'); + // A full first page with more behind it: the page was read, the rest was not. + const ten = Array.from({ length: 10 }, (_, k) => ({ id: HEX(k) })); + const paged = dehuTabRead(ten, { counter: 14, total: '1-10 de 14 elementos' }); + assert.deepStrictEqual([paged.state, paged.paged, paged.note], ['partial', true, '4 more on later pages, not read']); +}); + +test('dehuTabErrors: a short pending list is an error, a short accepted list is not', () => { + const { dehuTabErrors } = require('../gwt_spanish_poller.js'); + const read = { state: 'read' }; + assert.deepStrictEqual(dehuTabErrors({ pending: read, done: { state: 'partial', note: 'x' }, communications: read }), []); + assert.strictEqual(dehuTabErrors({ pending: { state: 'partial', note: 'x' }, done: read, communications: read }).length, 1); + assert.deepStrictEqual(dehuTabErrors({ pending: { state: 'partial', paged: true, note: 'x' }, done: read, communications: read }), []); + assert.strictEqual(dehuTabErrors({ pending: read, done: { state: 'not read', note: 'x' }, communications: read }).length, 1); +}); + +test('summarize: DEHú kept from an earlier poll says NOT READ, never «0 pending» or a plain «no changes»', () => { + const snap = { + polled_at: '2026-11-20T10:04:10Z', dehu_stale: true, dehu_read_at: '2026-11-17T11:00:40Z', + sede_stale: true, sede_read_at: '2026-11-17T11:00:40Z', sede: { cases: {} }, + dehu: { pending: [], done: [{ id: HEX(1), cells: [] }], communications: [] }, + }; + const text = summarize('p', snap, [], ['dehu: login did not reach the inbox']); + assert.doesNotMatch(text, /0 pending/); + assert.doesNotMatch(text, /^no changes$/m); + assert.match(text, /dehu: NOT READ this run; the snapshot read at 2026-11-17T11:00:40Z is kept/); + assert.match(text, /sede: NOT READ this run/); + assert.match(text, /not read this run: sede, dehu/); +}); + +test('summarize: a tab read short says so, and the pending count is the one read now', () => { + const snap = { + polled_at: 't', sede: { cases: {} }, + dehu: { + pending: [{ id: 'a', sender: 'X', subject: 'Y', cells: [] }], done: [{ id: 'b', cells: [] }, { id: 'old', cells: [] }], communications: [], + tabs: { pending: { state: 'read', expected: 1 }, done: { state: 'partial', note: 'tab counter says 6, 1 rows read', kept: 1 }, communications: { state: 'read' } }, + }, + }; + const text = summarize('p', snap, [], []); + assert.match(text, /dehu: 1 pending notifications, accepted NOT FULLY READ \(tab counter says 6, 1 rows read\), 0 communications/); + assert.match(text, /no changes in what was read/); + assert.match(text, /not read this run: dehu accepted/); +}); + +test('keepUnreadRows: rows of a tab read short stay in the snapshot and are not reported again', () => { + const { keepUnreadRows } = require('../gwt_spanish_poller.js'); + const prev = { pending: [], done: [{ id: 'old1' }, { id: 'old2' }, { id: 'n1' }], communications: [{ id: 'c1' }] }; + const cur = { + pending: [{ id: 'p1' }], done: [{ id: 'n1' }, { id: 'n2' }], communications: [], + tabs: { pending: { state: 'read' }, done: { state: 'partial' }, communications: { state: 'not read' } }, + }; + const kept = keepUnreadRows(prev, cur); + assert.deepStrictEqual(kept.done.map(x => x.id), ['n1', 'n2', 'old1', 'old2']); + assert.deepStrictEqual(kept.communications.map(x => x.id), ['c1']); + assert.strictEqual(kept.tabs.done.kept, 2); + const ev = diff({ dehu: prev }, { dehu: kept }); + assert.deepStrictEqual(ev.map(e => [e.kind, e.item.id]), [['notification_pending', 'p1'], ['notification_done', 'n2']]); +}); + +test('readAt: a kept side names when it was last read for real', () => { + const { readAt } = require('../gwt_spanish_poller.js'); + assert.strictEqual(readAt({ polled_at: 'a' }, 'dehu'), 'a'); + assert.strictEqual(readAt({ polled_at: 'b', dehu_stale: true, dehu_read_at: 'a' }, 'dehu'), 'a'); + assert.strictEqual(readAt({ polled_at: 'b', dehu_stale: true }, 'dehu'), null); +}); + +test('sedeNoCountError names the sede\'s own «Página no encontrada» page', () => { + const { sedeNoCountError } = require('../gwt_spanish_poller.js'); + const from = new Date(2026, 9, 22), to = new Date(2026, 10, 20); + assert.strictEqual(sedeNoCountError('Trámites\nPágina no encontrada\n', from, to), 'sede: the sede answered «Página no encontrada» to the search 22/10/2026-20/11/2026'); + assert.strictEqual(sedeNoCountError('x', from, to), 'sede: search 22/10/2026-20/11/2026 returned no result count'); +}); From 42026f93c61d603386281008bd24da54621cfb75 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 12:43:15 +0100 Subject: [PATCH 2/3] =?UTF-8?q?fix(spanish-poller):=20say=20DEH=C3=BA=20wa?= =?UTF-8?q?s=20not=20read=20even=20with=20no=20earlier=20snapshot=20of=20i?= =?UTF-8?q?t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When DEHú failed and the previous snapshot had no DEHú side, the summary still ended in a plain «no changes», and the first DEHú read that worked after that reported no events for the pending notifications it found. - A side that fails is marked not read even with nothing to keep, and the summary says there is no earlier snapshot of it. - The first DEHú read after that reports each pending notification. - Any list read shorter than its counter is an error again (exit 2 when nothing else changed); a list longer than one page is the exception. - The DEHú read time moves only when the pending list was read. - The error dump of a short list now saves the rows the poller read, which sit in shadow roots the .html and .txt do not hold. - Only rows read in this run are offered for download. - The work after the browser closes is one pure function, finishPoll, so tests cover how the parts fit together. --- CHANGELOG.md | 2 +- docs/spanish-poller.md | 9 +- spanish-poller/gwt_spanish_poller.js | 78 ++++++++--- spanish-poller/test/spanish-poller.test.js | 154 ++++++++++++++++++++- 4 files changed, 210 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e2247a6..7f69b053 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ Every release has a **Change lists** heading. It names each change that makes a ### Fixed -- `gwt-spanish-poller` no longer prints `0 pending notifications` or `no changes` for a DEHú inbox it did not read. Before, an accepted list that read fewer rows than its tab counter failed the whole DEHú read, and the summary then showed the kept snapshot from an earlier poll as if it were new, while the inbox had two pending notifications. Now each DEHú list is checked against its tab counter on its own, a short accepted or communications list no longer throws the pending list away, and a side or list that was not read says `NOT READ` in the summary. A sede search that answers «Página no encontrada» is named as such. +- `gwt-spanish-poller` no longer prints `0 pending notifications` or `no changes` for a DEHú inbox it did not read. Before, an accepted list that read fewer rows than its tab counter failed the whole DEHú read, and the summary then showed the kept snapshot from an earlier poll as if it were new, while the inbox had two pending notifications. Now each DEHú list is checked against its tab counter on its own, a short accepted or communications list no longer throws the pending list away, and a side or list that was not read says `NOT READ` in the summary, also when there is no earlier snapshot of it. A short list of any kind still exits 2. The first DEHú read that works after failed ones reports its pending notifications. A sede search that answers «Página no encontrada» is named as such. - A second `gwt-apply --apply` of a list with chained merges no longer makes a person or family the list already merged away, and no longer stops at the first merge. A merge whose winner is gone is read on the one object that holds its merge note now (#263). ## [1.5.0] - 2026-11-10 diff --git a/docs/spanish-poller.md b/docs/spanish-poller.md index db53e284..d7f25675 100644 --- a/docs/spanish-poller.md +++ b/docs/spanish-poller.md @@ -29,12 +29,13 @@ Per person it writes `/.json` (the snapshot plus the events), `/-errors/` even without `--dump`. +- A list read short keeps the rows of the previous snapshot next to the ones it read, so they are not reported as new on the next poll. Its page is saved in `/-errors/` even without `--dump`, with a `.rows.json` that holds the rows the poller read. DEHú draws its rows inside shadow roots, so the `.html` and `.txt` do not hold them. ## Config diff --git a/spanish-poller/gwt_spanish_poller.js b/spanish-poller/gwt_spanish_poller.js index 685cca9b..cfad2a7e 100644 --- a/spanish-poller/gwt_spanish_poller.js +++ b/spanish-poller/gwt_spanish_poller.js @@ -191,6 +191,10 @@ function diff(prev, cur) { const a = byId(prev.dehu[key]); for (const x of cur.dehu[key] || []) if (!a[x.id]) ev.push({ kind, item: x }); } + } else if (cur.dehu) { + // The first DEHú read after polls that never read it: like the baseline, every pending + // notification is reported. + for (const n of cur.dehu.pending || []) ev.push({ kind: 'notification_pending', item: n }); } return ev; } @@ -239,9 +243,10 @@ function notRead(snap) { function summarize(id, snap, events, errors) { const L = [`gwt-spanish-poller ${id} ${snap.polled_at}`]; - const kept = (at) => `the snapshot ${at ? 'read at ' + at : 'of an earlier poll'} is kept`; + const kept = (side) => (snap[side] ? `the snapshot ${snap[side + '_read_at'] ? 'read at ' + snap[side + '_read_at'] : 'of an earlier poll'} is kept` + : 'no earlier snapshot of it to keep'); if (snap.sede_stale) { - L.push(`sede: NOT READ this run; ${kept(snap.sede_read_at)}`); + L.push(`sede: NOT READ this run; ${kept('sede')}`); } else if (snap.sede) { const n = Object.values(snap.sede.cases); const by = {}; @@ -249,7 +254,7 @@ function summarize(id, snap, events, errors) { L.push(`sede: ${n.length} cases (${Object.entries(by).map(([k, v]) => `${v} ${k}`).join(', ') || 'none'})`); } if (snap.dehu_stale) { - L.push(`dehu: NOT READ this run; ${kept(snap.dehu_read_at)}`); + L.push(`dehu: NOT READ this run; ${kept('dehu')}`); } else if (snap.dehu) { const d = snap.dehu, tabs = d.tabs || {}; const fresh = (key) => d[key].length - ((tabs[key] && tabs[key].kept) || 0); @@ -589,8 +594,15 @@ async function dehuList(page, dumpDir, name, counter) { await dump(page, dumpDir, name); if (dumpDir) fs.writeFileSync(path.join(dumpDir, name + '.rows.json'), JSON.stringify(got, null, 1)); // A list that did not match its counter is saved even without --dump, so the next time it happens - // the page is there to look at. - if (read.state !== 'read' && !read.paged) await dump(page, null, name + '-err'); + // the page is there to look at. The rows sit in shadow roots that the .html and .txt do not hold, so + // what deepRows returned is saved next to them. + if (read.state !== 'read' && !read.paged) { + await dump(page, null, name + '-err'); + if (errorDumpDir) { + fs.mkdirSync(errorDumpDir, { recursive: true }); + fs.writeFileSync(path.join(errorDumpDir, name + '-err.rows.json'), JSON.stringify({ counter, read: { ...read }, ...got }, null, 1)); + } + } return { items, ...read }; } @@ -728,15 +740,15 @@ function readAt(snap, side) { return snap[side + '_stale'] ? null : snap.polled_at; } -// A pending list that was not read in full is an error: it is the list someone has to act on. The -// other two lists only count as errors when they did not load at all; a short read there is reported -// in the summary and does not throw the pending result away. +// A list that did not load, or showed fewer rows than its counter, is an error (exit 2 when nothing +// else changed). It is reported next to the lists that did read, never instead of them. A list longer +// than one page is not an error: its first page was read, and the summary says the rest was not. function dehuTabErrors(tabs) { const errs = []; for (const [key, label] of DEHU_TABS) { const t = (tabs || {})[key]; - if (!t || t.state === 'read') continue; - if (t.state === 'not read' || (key === 'pending' && !t.paged)) errs.push(`dehu: ${label} ${t.state === 'not read' ? 'not read' : 'not fully read'}: ${t.note}`); + if (!t || t.state === 'read' || t.paged) continue; + errs.push(`dehu: ${label} ${t.state === 'not read' ? 'not read' : 'not fully read'}: ${t.note}`); } return errs; } @@ -769,9 +781,8 @@ async function pollPerson(cfg, person, dumpDir) { if (session) { await session.close(); session = null; } if (person.dehu) try { session = await open(); - snap.dehu = keepUnreadRows(prev && prev.dehu, await pollDehu(session.page, d)); - snap.dehu_read_at = snap.polled_at; - errors.push(...dehuTabErrors(snap.dehu.tabs)); + snap.dehu = await pollDehu(session.page, d); + // Only rows read in this run are downloaded; rows kept from an earlier poll are not on the page. const already = (prev && prev.downloaded_ids) || []; if (cfg.download_senders.length) { snap.downloaded = await downloadCommunications(session.page, snap.dehu.communications, cfg.download_senders, @@ -786,18 +797,41 @@ async function pollPerson(cfg, person, dumpDir) { if (session) await session.close(); } if (errors.length && errorDumpDir && fs.existsSync(errorDumpDir)) errors.push('failed pages saved in ' + errorDumpDir); + const r = finishPoll(person, prev, snap, errors); + fs.mkdirSync(cfg.state_dir, { recursive: true }); + if (prev) fs.writeFileSync(path.join(cfg.state_dir, person.id + '.prev.json'), JSON.stringify(prev, null, 1)); + fs.writeFileSync(stateFile, JSON.stringify({ ...r.snap, events: r.events }, null, 1)); + fs.writeFileSync(path.join(cfg.state_dir, person.id + '.txt'), r.text); + return { text: r.text, changed: r.changed, failed: r.failed }; +} + +// Everything after the browser: merge what was read with the previous snapshot, decide what was not +// read, diff and summarize. `snap` holds the sides read in this run (sede, and dehu as pollDehu returned +// it); a side missing from it failed. Returns {snap, events, text, changed, failed}. +function finishPoll(person, prev, snap, errors) { + snap = { ...snap }; + errors = [...errors]; + const prevDehu = prev && prev.dehu; + if (snap.dehu) { + snap.dehu = keepUnreadRows(prevDehu, snap.dehu); + errors.push(...dehuTabErrors(snap.dehu.tabs)); + // The pending list is the one someone acts on, so the read time is the time it was last read. + const pending = (snap.dehu.tabs || {}).pending; + snap.dehu_read_at = !pending || pending.state === 'read' || !prevDehu ? snap.polled_at : readAt(prev, 'dehu'); + } // A failed part keeps its previous snapshot, so a broken login never reads as «every case disappeared». - // The summary says so and names when the kept snapshot was read. - if (!snap.sede && prev && prev.sede) Object.assign(snap, { sede_stale: true, sede: prev.sede, sede_read_at: readAt(prev, 'sede') }); - if (!snap.dehu && prev && prev.dehu) Object.assign(snap, { dehu_stale: true, dehu: prev.dehu, dehu_read_at: readAt(prev, 'dehu') }); + // The summary says so and names when the kept snapshot was read. A part that failed with no earlier + // snapshot to keep is still marked stale, so the summary never ends in a plain «no changes». + if (person.sede !== false && !snap.sede) { + Object.assign(snap, { sede_stale: true }, prev && prev.sede ? { sede: prev.sede, sede_read_at: readAt(prev, 'sede') } : {}); + } + if (person.dehu !== false && !snap.dehu) { + Object.assign(snap, { dehu_stale: true }, prevDehu ? { dehu: prevDehu, dehu_read_at: readAt(prev, 'dehu') } : {}); + } if (!snap.downloaded_ids) snap.downloaded_ids = (prev && prev.downloaded_ids) || []; const events = diff(prev, snap).filter(e => !(snap.sede_stale && /^case_/.test(e.kind)) && !(snap.dehu_stale && /notification|communication/.test(e.kind))); const text = summarize(person.id, snap, events, errors); - fs.mkdirSync(cfg.state_dir, { recursive: true }); - if (prev) fs.writeFileSync(path.join(cfg.state_dir, person.id + '.prev.json'), JSON.stringify(prev, null, 1)); - fs.writeFileSync(stateFile, JSON.stringify({ ...snap, events }, null, 1)); - fs.writeFileSync(path.join(cfg.state_dir, person.id + '.txt'), text); - return { text, changed: events.length > 0, failed: errors.length > 0 }; + return { snap, events, text, changed: events.length > 0, failed: errors.length > 0 }; } function parseArgs(argv) { @@ -837,4 +871,4 @@ async function main(argv) { return changed ? EXIT_CHANGED : failed ? EXIT_ERROR : EXIT_SAME; } -module.exports = { dump, setErrorDumpDir, isoDate, openPkcs12, openChrome, dehuLogin, pollDehu, loadConfig, parseCases, parseDehuRows, dehuLoadState, dehuTabRead, dehuTabErrors, keepUnreadRows, readAt, sedeNoCountError, diff, senderAllowed, summarize, describe, main, EXIT_SAME, EXIT_CHANGED, EXIT_ERROR }; +module.exports = { dump, setErrorDumpDir, isoDate, openPkcs12, openChrome, dehuLogin, pollDehu, loadConfig, parseCases, parseDehuRows, dehuLoadState, dehuTabRead, dehuTabErrors, keepUnreadRows, readAt, finishPoll, dehuList, sedeNoCountError, diff, senderAllowed, summarize, describe, main, EXIT_SAME, EXIT_CHANGED, EXIT_ERROR }; diff --git a/spanish-poller/test/spanish-poller.test.js b/spanish-poller/test/spanish-poller.test.js index d0fa1075..3297dfa8 100644 --- a/spanish-poller/test/spanish-poller.test.js +++ b/spanish-poller/test/spanish-poller.test.js @@ -137,10 +137,10 @@ const HEX = (n) => n.toString(16).padStart(20, '0'); // A fake DEHú page: the inbox is already open, tabs switch on the same clicks the poller makes, and // each tab answers with its rows. Waits return at once. -function fakeDehu({ counters, pending, done, comm, resetOnce = false }) { +function fakeDehu({ counters, pending, done, comm, resetOnce = false, doneThrows = false }) { let tab = 'home', resets = resetOnce ? 1 : 0; const lists = { pending, done, comm }; - const total = (rows) => { const n = rows ? rows.filter(r => r.length > 2 && r[2]).length : 0; return n ? `1-${n} de ${n} elementos` : ''; }; + const total = (rows) => { const n = rows ? rows.filter(r => r.length > 2 && r[2] && r[0] !== 'Columna ampliable').length : 0; return n ? `1-${n} de ${n} elementos` : ''; }; const body = () => (tab === 'reset' ? 'This site can’t be reached\nERR_CONNECTION_RESET\n' : `Notificaciones pendientes\n${counters.pending}\nNotificaciones realizadas\n${counters.done}\n`); const clickable = (fn) => ({ first: () => clickable(fn), click: async () => fn(), count: async () => 0, getByText: () => clickable(() => {}) }); @@ -151,7 +151,11 @@ function fakeDehu({ counters, pending, done, comm, resetOnce = false }) { url: () => 'https://dehu.example/es/home-view', content: async () => '', screenshot: async () => {}, - locator: (sel) => clickable(() => { if (sel === '#pane-button-1') tab = 'done'; }), + locator: (sel) => clickable(() => { + if (sel !== '#pane-button-1') return; + if (doneThrows) throw new Error('locator.click: Timeout 15000ms exceeded'); + tab = 'done'; + }), getByText: (t) => clickable(() => { if (/pendientes/.test(t) && resets) { resets--; tab = 'reset'; return; } tab = /pendientes/.test(t) ? 'pending' : /Comunicaciones/.test(t) ? 'comm' : tab; @@ -180,6 +184,7 @@ test('pollDehu: an accepted tab that reads short does not throw the pending list assert.strictEqual(d.done.length, 3); assert.strictEqual(d.tabs.done.state, 'partial'); assert.strictEqual(d.communications.length, 1); + assert.strictEqual(d.tabs.communications.state, 'read'); }); test('pollDehu: a pending list that does not parse is «not read», never 0 pending', async () => { @@ -216,15 +221,152 @@ test('dehuTabRead: the pending count comes from the tab counter and is checked a assert.deepStrictEqual([paged.state, paged.paged, paged.note], ['partial', true, '4 more on later pages, not read']); }); -test('dehuTabErrors: a short pending list is an error, a short accepted list is not', () => { +test('dehuTabErrors: any list read short is an error, a list longer than one page is not', () => { const { dehuTabErrors } = require('../gwt_spanish_poller.js'); const read = { state: 'read' }; - assert.deepStrictEqual(dehuTabErrors({ pending: read, done: { state: 'partial', note: 'x' }, communications: read }), []); + assert.deepStrictEqual(dehuTabErrors({ pending: read, done: { state: 'partial', note: 'tab counter says 6, 3 rows read' }, communications: read }), + ['dehu: accepted not fully read: tab counter says 6, 3 rows read']); assert.strictEqual(dehuTabErrors({ pending: { state: 'partial', note: 'x' }, done: read, communications: read }).length, 1); - assert.deepStrictEqual(dehuTabErrors({ pending: { state: 'partial', paged: true, note: 'x' }, done: read, communications: read }), []); + assert.deepStrictEqual(dehuTabErrors({ pending: { state: 'partial', paged: true, note: 'x' }, done: { state: 'partial', paged: true, note: 'x' }, communications: read }), []); assert.strictEqual(dehuTabErrors({ pending: read, done: { state: 'not read', note: 'x' }, communications: read }).length, 1); }); +test('pollDehu: a throw while opening the accepted tab keeps the pending list', async () => { + const { pollDehu } = require('../gwt_spanish_poller.js'); + const d = await pollDehu(fakeDehu({ ...SHORT_DONE, doneThrows: true }), null); + assert.deepStrictEqual(d.pending.map(n => n.id), ['0a00000000000000aa01', '0a00000000000000aa02']); + assert.strictEqual(d.tabs.pending.state, 'read'); + assert.strictEqual(d.tabs.done.state, 'not read'); + assert.match(d.tabs.done.note, /Timeout/); +}); + +test('summarize: pending rows kept from an earlier poll are not printed as pending now', () => { + const snap = { + polled_at: 't', sede: { cases: {} }, + dehu: { + pending: [{ id: 'old', sender: 'Kept sender', subject: 'Kept subject', cells: [] }], done: [], communications: [], + tabs: { pending: { state: 'not read', note: 'tab counter says 1, the list did not load', kept: 1 }, done: { state: 'read' }, communications: { state: 'read' } }, + }, + }; + const text = summarize('p', snap, [], []); + assert.doesNotMatch(text, /Kept sender/); + assert.match(text, /pending notifications NOT READ \(tab counter says 1, the list did not load\)/); + assert.match(text, /not read this run: dehu pending/); +}); + +test('dehuList: a list read short saves its rows next to the error page, and a paged list stops at once', async () => { + const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); + const { dehuList, setErrorDumpDir } = require('../gwt_spanish_poller.js'); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'poller-err-')); + setErrorDumpDir(dir); + try { + const page = fakeDehu(SHORT_DONE); + await page.locator('#pane-button-1').click(); + const r = await dehuList(page, null, 'dehu-2-done', 6); + assert.strictEqual(r.state, 'partial'); + const saved = JSON.parse(fs.readFileSync(path.join(dir, 'dehu-2-done-err.rows.json'), 'utf8')); + assert.strictEqual(saved.counter, 6); + assert.strictEqual(saved.total, '1-6 de 6 elementos'); + assert.ok(JSON.stringify(saved.rows).includes('11112222333344')); + // Ten rows of fourteen: the first page is all there is to read, so one look is enough. + const ten = Array.from({ length: 10 }, (_, k) => doneRow(HEX(100 + k), 'Organismo A', '01/11/2026')).flat(); + const paged = fakeDehu({ ...SHORT_DONE, counters: { pending: 2, done: 14 }, done: { rows: [DONE_HEAD, ...ten], total: '1-10 de 14 elementos' } }); + await paged.locator('#pane-button-1').click(); + let looks = 0; + const ev = paged.evaluate; + paged.evaluate = async () => { looks++; return ev(); }; + const p = await dehuList(paged, null, 'dehu-2-done', 14); + assert.deepStrictEqual([p.state, p.paged, looks], ['partial', true, 1]); + } finally { + setErrorDumpDir(null); + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('diff: the first DEHú read after polls that never read it reports the pending notifications', () => { + const ev = diff({ sede: { cases: {} } }, { sede: { cases: {} }, dehu: { pending: [{ id: 'p1' }], done: [{ id: 'd1' }], communications: [] } }); + assert.deepStrictEqual(ev.map(e => [e.kind, e.item.id]), [['notification_pending', 'p1']]); +}); + +// ---------- after the browser: what pollPerson does with what was read ---------- +const PERSON = { id: 'p', sede: true, dehu: true }; +const SEDE = { since: '01/05/2026', cases: { '20261019/000001': CASE('20261019/000001', 'En trámite') } }; +const tabsRead = { pending: { state: 'read' }, done: { state: 'read' }, communications: { state: 'read' } }; +const item = (id) => ({ id, sender: 'Organismo ' + id, subject: 'Asunto', cells: [] }); + +test('finishPoll: DEHú failed with no earlier DEHú snapshot says NOT READ, never a plain «no changes»', () => { + const { finishPoll } = require('../gwt_spanish_poller.js'); + const prev = { polled_at: 'a', sede: SEDE }; + const r = finishPoll(PERSON, prev, { polled_at: 'b', sede: SEDE }, ['dehu: login did not reach the inbox']); + assert.match(r.text, /^dehu: NOT READ this run; no earlier snapshot of it to keep$/m); + assert.doesNotMatch(r.text, /^no changes$/m); + assert.match(r.text, /not read this run: dehu/); + assert.deepStrictEqual([r.changed, r.failed], [false, true]); + // The very first poll, with DEHú failed, says the same next to the baseline. + const first = finishPoll(PERSON, null, { polled_at: 'b', sede: SEDE }, ['dehu: login did not reach the inbox']); + assert.match(first.text, /dehu: NOT READ this run/); + assert.match(first.text, /not read this run: dehu/); + // A side switched off in the config is not «not read». + const off = finishPoll({ ...PERSON, dehu: false }, prev, { polled_at: 'b', sede: SEDE }, []); + assert.match(off.text, /^no changes$/m); +}); + +test('finishPoll: the first good DEHú read after failed ones reports its pending notifications', () => { + const { finishPoll } = require('../gwt_spanish_poller.js'); + const prev = { polled_at: 'a', sede: SEDE, dehu_stale: true }; + const dehu = { pending: [item('p1')], done: [], communications: [], tabs: tabsRead, warnings: [] }; + const r = finishPoll(PERSON, prev, { polled_at: 'b', sede: SEDE, dehu }, []); + assert.deepStrictEqual(r.events.map(e => [e.kind, e.item.id]), [['notification_pending', 'p1']]); + assert.deepStrictEqual([r.changed, r.failed], [true, false]); + assert.strictEqual(r.snap.dehu_stale, undefined); + assert.strictEqual(r.snap.dehu_read_at, 'b'); +}); + +test('finishPoll: a pending list read short fails the run and never prints «0 pending»', () => { + const { finishPoll } = require('../gwt_spanish_poller.js'); + const prev = { polled_at: 'a', sede: SEDE, dehu: { pending: [], done: [], communications: [] } }; + const dehu = { pending: [], done: [], communications: [], warnings: [], + tabs: { ...tabsRead, pending: { state: 'not read', expected: 2, rows: 0, note: 'tab counter says 2, the list did not load' } } }; + const r = finishPoll(PERSON, prev, { polled_at: 'b', sede: SEDE, dehu }, []); + assert.strictEqual(r.failed, true); + assert.doesNotMatch(r.text, /0 pending/); + assert.match(r.text, /ERROR: dehu: pending not read: tab counter says 2/); + assert.match(r.text, /pending notifications NOT READ/); +}); + +test('finishPoll: a short accepted list keeps the old rows, keeps the pending result and fails the run', () => { + const { finishPoll } = require('../gwt_spanish_poller.js'); + const prev = { polled_at: 'a', sede: SEDE, dehu: { pending: [], done: [item('d1'), item('d2'), item('d3')], communications: [] } }; + const dehu = { pending: [item('p1')], done: [item('d1')], communications: [], warnings: [], + tabs: { ...tabsRead, done: { state: 'partial', expected: 3, rows: 1, note: 'tab counter says 3, 1 rows read' } } }; + const r = finishPoll(PERSON, prev, { polled_at: 'b', sede: SEDE, dehu }, []); + assert.deepStrictEqual(r.snap.dehu.done.map(x => x.id), ['d1', 'd2', 'd3']); + assert.deepStrictEqual(r.events.map(e => [e.kind, e.item.id]), [['notification_pending', 'p1']]); + assert.strictEqual(r.failed, true); + assert.match(r.text, /ERROR: dehu: accepted not fully read/); + // The same short read with nothing new: exit 2, not 0. + const same = finishPoll(PERSON, { ...prev, dehu: { ...prev.dehu, pending: [item('p1')] } }, { polled_at: 'b', sede: SEDE, dehu }, []); + assert.deepStrictEqual([same.changed, same.failed], [false, true]); +}); + +test('finishPoll: the DEHú read time is the time the pending list was last read', () => { + const { finishPoll } = require('../gwt_spanish_poller.js'); + const prev = { polled_at: 'b', dehu_read_at: 'a', sede: SEDE, dehu: { pending: [item('p0')], done: [], communications: [] } }; + const unread = { pending: [], done: [], communications: [], warnings: [], + tabs: { ...tabsRead, pending: { state: 'not read', note: 'the list did not load' } } }; + const r = finishPoll(PERSON, prev, { polled_at: 'c', sede: SEDE, dehu: unread }, []); + assert.strictEqual(r.snap.dehu_read_at, 'a'); + assert.deepStrictEqual(r.snap.dehu.pending.map(x => x.id), ['p0']); + // DEHú failing next time names that same time, not the poll that kept the old rows. + const next = finishPoll(PERSON, r.snap, { polled_at: 'd', sede: SEDE }, ['dehu: login did not reach the inbox']); + assert.match(next.text, /dehu: NOT READ this run; the snapshot read at a is kept/); + assert.strictEqual(next.snap.dehu_read_at, 'a'); + // A sede that failed keeps its snapshot and names when it was read. + const sedeFail = finishPoll({ ...PERSON, dehu: false }, { polled_at: 'a', sede: SEDE }, { polled_at: 'b' }, ['sede: x']); + assert.match(sedeFail.text, /sede: NOT READ this run; the snapshot read at a is kept/); + assert.deepStrictEqual(sedeFail.events, []); +}); + test('summarize: DEHú kept from an earlier poll says NOT READ, never «0 pending» or a plain «no changes»', () => { const snap = { polled_at: '2026-11-20T10:04:10Z', dehu_stale: true, dehu_read_at: '2026-11-17T11:00:40Z', From e50f22723b2abded6cba1b0da54174ea5f6f528e Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Fri, 20 Nov 2026 12:50:48 +0100 Subject: [PATCH 3/3] =?UTF-8?q?fix(spanish-poller):=20keep=20the=20DEH?= =?UTF-8?q?=C3=BA=20read=20time=20unknown=20when=20the=20pending=20list=20?= =?UTF-8?q?was=20never=20read?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first DEHú poll that could not read the pending list stamped its own time as the read time, so a later failed login named a read that never happened. The time is now null in that case, and readAt keeps a null instead of falling back to the poll time. --- spanish-poller/gwt_spanish_poller.js | 9 +++++---- spanish-poller/test/spanish-poller.test.js | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/spanish-poller/gwt_spanish_poller.js b/spanish-poller/gwt_spanish_poller.js index cfad2a7e..bc2f97d7 100644 --- a/spanish-poller/gwt_spanish_poller.js +++ b/spanish-poller/gwt_spanish_poller.js @@ -734,9 +734,9 @@ async function downloadCommunications(page, items, senders, dir, already, notes) } // When a kept side of a snapshot was last read for real (snapshots older than this field: their poll -// time, unless that poll had already kept it from an earlier one). +// time, unless that poll had already kept it from an earlier one). A null field means unknown. function readAt(snap, side) { - if (snap[side + '_read_at']) return snap[side + '_read_at']; + if ((side + '_read_at') in snap) return snap[side + '_read_at']; return snap[side + '_stale'] ? null : snap.polled_at; } @@ -815,9 +815,10 @@ function finishPoll(person, prev, snap, errors) { if (snap.dehu) { snap.dehu = keepUnreadRows(prevDehu, snap.dehu); errors.push(...dehuTabErrors(snap.dehu.tabs)); - // The pending list is the one someone acts on, so the read time is the time it was last read. + // The pending list is the one someone acts on, so the read time is the time it was last read + // (null when it never was). const pending = (snap.dehu.tabs || {}).pending; - snap.dehu_read_at = !pending || pending.state === 'read' || !prevDehu ? snap.polled_at : readAt(prev, 'dehu'); + snap.dehu_read_at = !pending || pending.state === 'read' ? snap.polled_at : prevDehu ? readAt(prev, 'dehu') : null; } // A failed part keeps its previous snapshot, so a broken login never reads as «every case disappeared». // The summary says so and names when the kept snapshot was read. A part that failed with no earlier diff --git a/spanish-poller/test/spanish-poller.test.js b/spanish-poller/test/spanish-poller.test.js index 3297dfa8..d8616fbd 100644 --- a/spanish-poller/test/spanish-poller.test.js +++ b/spanish-poller/test/spanish-poller.test.js @@ -367,6 +367,20 @@ test('finishPoll: the DEHú read time is the time the pending list was last read assert.deepStrictEqual(sedeFail.events, []); }); +test('finishPoll: a pending list never read leaves the DEHú read time unknown', () => { + const { finishPoll } = require('../gwt_spanish_poller.js'); + const unread = { pending: [], done: [], communications: [], warnings: [], + tabs: { ...tabsRead, pending: { state: 'not read', note: 'the list did not load' } } }; + // The first DEHú poll could not read the pending list: there is no read time to give it. + const r = finishPoll(PERSON, { polled_at: 'a', sede: SEDE }, { polled_at: 'b', sede: SEDE, dehu: unread }, []); + assert.strictEqual(r.snap.dehu_read_at, null); + // A login that fails next time keeps it unknown, never the time of the poll that did not read it. + const next = finishPoll(PERSON, r.snap, { polled_at: 'c', sede: SEDE }, ['dehu: login did not reach the inbox']); + assert.strictEqual(next.snap.dehu_read_at, null); + assert.match(next.text, /dehu: NOT READ this run; the snapshot of an earlier poll is kept/); + assert.doesNotMatch(next.text, /read at b/); +}); + test('summarize: DEHú kept from an earlier poll says NOT READ, never «0 pending» or a plain «no changes»', () => { const snap = { polled_at: '2026-11-20T10:04:10Z', dehu_stale: true, dehu_read_at: '2026-11-17T11:00:40Z', @@ -415,6 +429,7 @@ test('readAt: a kept side names when it was last read for real', () => { assert.strictEqual(readAt({ polled_at: 'a' }, 'dehu'), 'a'); assert.strictEqual(readAt({ polled_at: 'b', dehu_stale: true, dehu_read_at: 'a' }, 'dehu'), 'a'); assert.strictEqual(readAt({ polled_at: 'b', dehu_stale: true }, 'dehu'), null); + assert.strictEqual(readAt({ polled_at: 'b', dehu_read_at: null }, 'dehu'), null); }); test('sedeNoCountError names the sede\'s own «Página no encontrada» page', () => {