diff --git a/controller/app/src/main/java/org/appdevforall/k2go/redesign/FqrController.java b/controller/app/src/main/java/org/appdevforall/k2go/redesign/FqrController.java index 24d78a816..e3cfab378 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/redesign/FqrController.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/redesign/FqrController.java @@ -113,6 +113,11 @@ public final class FqrController { private View overlay; private CircularProgressIndicator overlayRing; // the ring = the percent private TextView overlayEta, overlayPct; // inside the ring: ETA + percent + // K2GO-317: the highest percent shown so far (client high-water mark) and the blink used while a resume + // re-derives progress. A resume relaunches the runner, which re-counts completed layers from 0, so it + // briefly reports LESS than we already showed. We hold the peak and blink, never fall back to 0 and climb. + private int overlayPeakPct = -1; + private android.animation.ObjectAnimator overlayBlink; private TextView overlayStatus, overlaySize; // header action word + the "size : speed" line private View overlayHeader, overlayMeta; // chrome, hidden when minimized private String overlaySizeStr = ""; // the size half of the "size : speed" line @@ -401,18 +406,22 @@ private void showConsent(String name, String box, long transfer, long archive, l // (startDownload is invoked from the positive button above.) pendingBox = box; pendingArchive = archive; + pendingTransfer = transfer; } private String pendingBox; // box paired with the name at consent time private long pendingArchive; // on-disk size (bytes) to show in the overlay header + private long pendingTransfer; // K2GO-317: estimated download bytes (consent "transfer"), sent to the server + private long dlEstBytes; // the estimate in effect for the current download (reused on Retry) private void startDownload(String name) { final String box = pendingBox; if (box == null) return; dlName = name; dlBox = box; + dlEstBytes = pendingTransfer; showOverlay(name, pendingArchive); - client.download(name, box, downloadListener()); + client.download(name, box, dlEstBytes, downloadListener()); } /** K2GO-541: one reusable listener, so Retry can re-invoke the download on the same overlay. The poll @@ -440,12 +449,19 @@ private MapsRegionClient.DownloadListener downloadListener() { overlayStopped = true; setStopIcon(true); setStopEnabled(true); + stopPctBlink(); // K2GO-317: a paused ring is steady amber, not blinking if (overlayStatus != null) overlayStatus.setText(R.string.k2go_dl_paused_notif); if (overlayEta != null) overlayEta.setText(""); // a paused job must not advertise a time-to-finish - if (overlayRing != null && percent >= 0) overlayRing.setProgressCompat(percent, false); + // K2GO-317: never show less than the peak already reached (a late re-derivation can report lower). + int shown = Math.max(percent, overlayPeakPct); + if (shown >= 0) { + if (overlayRing != null) overlayRing.setProgressCompat(shown, false); + if (overlayPct != null) overlayPct.setText(str(R.string.k2go_fqr_percent, shown)); + } setRingState(cAmber, false); // amber + dimmed: a paused job must not look live } @Override public void onDone() { + stopPctBlink(); // K2GO-317: every exit path stops the blink (done, like pause/fail/hide) setRingState(cLeaf, true); // done: a full leaf ring reads as success updateOverlay(100, 0, -1); if (overlayStatus != null) overlayStatus.setText(R.string.k2go_fqr_region_added); // may be gone if minimized @@ -550,7 +566,10 @@ private void showOverlay(String name, long sizeBytes) { overlayEta = new TextView(themed); overlayEta.setGravity(Gravity.CENTER); overlayEta.setMaxLines(2); - overlayEta.setMaxWidth(dp(84)); + // K2GO-541: narrow enough that "6 s restantes" wraps to two short lines inside the ring : the magnitude + // ("6 s") over the remaining-word ("restantes") : instead of one wide line. Tuned for es/en; a locale + // with a longer remaining-word (de "verbleibend") may need a wider value or per-locale stacked strings. + overlayEta.setMaxWidth(dp(64)); applyOverlayType(overlayEta, R.style.TextAppearance_K2Go_Caption, cOnSurfaceVariant); overlayEta.setText(R.string.k2go_fqr_starting); // ETA present from the start, held until done overlayPct = ringText(R.style.TextAppearance_K2Go_Title, cOnSurface); @@ -690,6 +709,7 @@ private void showFailed() { if (overlayStopBtn != null) overlayStopBtn.setContentDescription(str(R.string.k2go_dl_retry)); if (overlayStatus != null) overlayStatus.setText(R.string.k2go_fqr_download_failed); if (overlayEta != null) overlayEta.setText(""); // no time-to-finish on a stopped job + stopPctBlink(); // K2GO-317: a failed panel must not keep the percent blinking setRingState(cError, false); // red + dimmed: the job is not running } @@ -697,7 +717,7 @@ private void retryDownload() { setStopEnabled(false); // guard a double tap until the first poll or error confirms clearFailed(); if (overlayEta != null) overlayEta.setText(R.string.k2go_fqr_starting); - client.download(dlName, dlBox, downloadListener()); + client.download(dlName, dlBox, dlEstBytes, downloadListener()); } /** Restore the running look (live teal ring, Pause icon, "Downloading" word) out of the failed state. */ @@ -786,17 +806,44 @@ private int[] ringCenterOnScreen() { private void updateOverlay(int percent, long speed, int etaSeconds) { if (overlayRing == null) return; if (overlaySize != null) overlaySize.setText(sizeLine(speed)); // "708 MB : 1.3 MB/s" - if (overlayEta != null) { + // K2GO-317: a resume relaunches the runner, which re-derives the overall percent from 0 (it re-counts + // completed layers), so it momentarily reports LESS than we already showed. Do NOT drop the ring back + // to that lower value (it reads as "restarting"): hold the peak and blink the percent to show it is + // still working. When the server climbs back past the peak, resume advancing from there. The size/speed + // line keeps updating throughout, so the card still shows live activity while the number holds. + boolean regressing = (percent >= 0 && percent < overlayPeakPct); + if (overlayEta != null && !regressing) { String eta = etaPlain(etaSeconds); if (!eta.isEmpty()) overlayEta.setText(eta); // hold the last known ETA; never blank mid-download else if (percent >= 100) overlayEta.setText(""); // done: clear the estimate } - if (percent >= 0) { // hold the ring + percent on an unknown poll - overlayRing.setProgressCompat(percent, true); - overlayPct.setText(str(R.string.k2go_fqr_percent, percent)); + if (percent >= 0) { + if (regressing) { + startPctBlink(); // hold peak, blink: a resume is re-deriving progress + } else { + stopPctBlink(); + overlayPeakPct = percent; + overlayRing.setProgressCompat(percent, true); + overlayPct.setText(str(R.string.k2go_fqr_percent, percent)); + } } } + /** K2GO-317: pulse the percent while a resume re-derives progress (the server dips below the peak). It + * signals work in flight without dropping the ring; stopped once progress passes the peak again. */ + private void startPctBlink() { + if (overlayBlink != null || overlayPct == null) return; + overlayBlink = android.animation.ObjectAnimator.ofFloat(overlayPct, "alpha", 1f, 0.2f); + overlayBlink.setDuration(450); + overlayBlink.setRepeatMode(android.animation.ValueAnimator.REVERSE); + overlayBlink.setRepeatCount(android.animation.ValueAnimator.INFINITE); + overlayBlink.start(); + } + private void stopPctBlink() { + if (overlayBlink != null) { overlayBlink.cancel(); overlayBlink = null; } + if (overlayPct != null) overlayPct.setAlpha(1f); + } + /** The "size : speed" line under the name ("708 MB : 1.3 MB/s"); size alone until the speed arrives. */ private String sizeLine(long speed) { String r = humanRate(speed); @@ -896,6 +943,8 @@ private static String humanRate(long bps) { private void hideOverlay() { if (overlay == null) return; + stopPctBlink(); // K2GO-317: cancel the blink before overlayPct is cleared + overlayPeakPct = -1; // the high-water mark is per download; a new overlay starts fresh ViewGroup parent = (ViewGroup) overlay.getParent(); if (parent != null) parent.removeView(overlay); overlay = null; overlayRing = null; overlayEta = null; overlayPct = null; diff --git a/controller/app/src/main/java/org/appdevforall/k2go/redesign/MapsRegionClient.java b/controller/app/src/main/java/org/appdevforall/k2go/redesign/MapsRegionClient.java index 106717137..c9ddb67b1 100644 --- a/controller/app/src/main/java/org/appdevforall/k2go/redesign/MapsRegionClient.java +++ b/controller/app/src/main/java/org/appdevforall/k2go/redesign/MapsRegionClient.java @@ -82,14 +82,16 @@ public void estimate(@NonNull String box, @NonNull EstimateListener l) { }); } - /** Start the region download and poll it to completion. */ - public void download(@NonNull String name, @NonNull String box, @NonNull DownloadListener l) { + /** Start the region download and poll it to completion. estBytes is the consent-step size estimate; the + * server uses it to drive the overall percent and ETA from cumulative bytes (K2GO-317). 0 = unknown. */ + public void download(@NonNull String name, @NonNull String box, long estBytes, @NonNull DownloadListener l) { this.dl = l; this.finished = false; this.pollErrors = 0; AppExecutors.get().io().execute(() -> { try { JSONObject item = new JSONObject().put("name", name).put("box", box); + if (estBytes > 0) item.put("est", estBytes); // K2GO-317: total-bytes estimate for overall %/ETA JSONObject body = new JSONObject().put("items", new JSONArray().put(item)); JSONObject resp = httpJson("POST", BASE + "/download", body); String id = resp.optString("id", ""); diff --git a/static/dashboard/CHANGELOG.md b/static/dashboard/CHANGELOG.md index ed909904d..fd1f17e8f 100644 --- a/static/dashboard/CHANGELOG.md +++ b/static/dashboard/CHANGELOG.md @@ -4,6 +4,7 @@ One line per version, newest first. Every REST-facing change bumps the version i (the app surfaces it via `/system/dashboard/update-check` and the "Update available" pill), so this file is the human record of what each bump enables. Keep entries short: `version - change (TICKET)`. +- **1.5.0** - FQR map download progress and resume resilience (K2GO-317). (1) Overall percent and a smoothed total ETA: the ring and ETA no longer come from go-pmtiles' per-layer numbers. The device sends the consent-step size estimate (`est` on the download item); the maps runner drives the percent from CUMULATIVE bytes vs that total, so each layer takes its true byte share and the ring no longer sticks on the big layer or jumps at layer boundaries (it was `(layersDone + frac) / 3`, equal thirds regardless of layer size). It also computes one whole-region ETA from a SMOOTHED speed (EMA over ~4 s, with a short warm-up) instead of go-pmtiles' raw `eta_s`, which was instantaneous and reset every layer (so the time jumped 1 s to 2 min to 15 s and read "1 s left" at 65%). The reported speed is the smoothed value too. An old job with no estimate falls back to the per-layer math. (2) Pause grace: the SIGTERM grace before a fallback SIGKILL rises from 2.5 s to 60 s, because go-pmtiles' journal flush (an fsync of up to a checkpoint interval of data) is not time-bounded and a short grace could SIGKILL mid-flush and lose progress; the child normally exits on its own right after the flush, so the grace is only a last-resort cap (engine-wide: affects every job's pause). (3) Stall watchdog: it rises from 7 s to 30 s and now counts a change in go-pmtiles' `retries` as activity, so it fires only on a true wedge (no bytes AND no retries) and no longer false-pauses during a plan, a finalize, the next layer's start, or a retry wait : a false pause would re-plan the layer and, before the go-pmtiles skip fix, re-download every completed layer. (4) Frozen-process watchdog: go-pmtiles emits one JSON line per second in every phase, so if no line arrives for 30 s the process is frozen (e.g. storage stopped responding during plan or finalize, which the byte watchdog does not see); the runner then SIGTERMs for the same lossless pause. Both watchdogs turn a hang into a resumable pause, not a lost download. (5) A transient network error in tile-extract.py's own mirror or catalog request (its `requests.head` between layers, which produces no go-pmtiles exit_code 2) is now treated as resumable, not a hard error: the runner keeps every completed layer and marks the job paused, so a resume skips the completed layers (r2) instead of re-downloading them. Phone links blip often, and before this a late-layer blip purged every earlier layer. Localhost-only. (K2GO-317) - **1.4.12** - FQR downloads retain progress on a network drop, and the resume percent is corrected (K2GO-317). (1) Progress retention: go-pmtiles' own network give-up checkpoints a STALE journal and loses the bytes since its last periodic checkpoint (measured ~2/3 of a large region). The maps runner now watches the progress stream and, after ~7 s with no byte progress, SIGTERMs the download itself so go-pmtiles flushes the CURRENT bytes losslessly, then marks the job `paused` (resumable); a SIGTERM flush resumes from exactly where it stopped (measured), and go-pmtiles' own lossy give-up stays as a fallback. (2) Revert the 1.4.11 percent change: `bytes_done` already includes `resumed_bytes`, so adding it double-counted and clamped a resumed layer to 100%; back to `bytes_done / bytes_total`. Localhost-only. (K2GO-317) - **1.4.11** - FQR resume progress shows where it resumed, not from zero (K2GO-317). The job percent now counts go-pmtiles' `resumed_bytes` (the bytes continued from the `.resume` journal) in each layer's fraction, so a resumed download picks the ring up near where it stopped instead of appearing to restart at 0%. Display only: the download already resumed from the journal; this stops the ring from misreporting it as a restart. Localhost-only. (K2GO-317) - **1.4.10** - FQR map downloads resume after a network drop (K2GO-317). A mid-download Wi-Fi drop no longer restarts from zero. The custom go-pmtiles build keeps its `.resume` journal and reports `exit_code` 2 ("network gave up") on its last json line; the maps runner now reads that `exit_code` and, on a stop that is not an engine pause or cancel, keeps the partial and the journal and marks the job `paused` (resumable) instead of purging, so the app's Resume continues from the journal. A hard failure (any other exit) still purges. No new job phase (reuses `paused`) and no rootfs rebake (the signal was already on stdout). Resume needs the box rootfs to carry the m16 pmtiles binary. Localhost-only. (K2GO-317) diff --git a/static/dashboard/package.json b/static/dashboard/package.json index 5f3ce6446..5d0ead89b 100644 --- a/static/dashboard/package.json +++ b/static/dashboard/package.json @@ -1,6 +1,6 @@ { "name": "dashboard-console", - "version": "1.4.12", + "version": "1.5.0", "description": "", "main": "index.js", "scripts": { diff --git a/static/dashboard/sockets/jobs.ts b/static/dashboard/sockets/jobs.ts index 9f25e3dc5..fdd5ee8b2 100644 --- a/static/dashboard/sockets/jobs.ts +++ b/static/dashboard/sockets/jobs.ts @@ -105,9 +105,15 @@ export function throwIfStopped(ctx: RunnerContext): void { const DB_PATH = process.env.K2GO_JOBS_DB || '/library/dashboard/jobs.db'; const ACTIVE: JobPhase[] = ['queued', 'downloading', 'indexing', 'processing']; -// K2GO-317: grace a paused child gets to flush its resume checkpoint on SIGTERM before killAll -// SIGKILLs a survivor that ignored it. -const PAUSE_GRACE_MS = 2500; +// K2GO-317: time a paused child gets to flush its resume checkpoint on SIGTERM before killAll +// SIGKILLs a survivor that ignored it. The flush is not bounded: go-pmtiles fsyncs the output (up to +// the periodic-checkpoint interval of unsynced data) before it writes the journal, so a fast link onto +// slow storage can take many seconds. So this is a last-resort cap, not the expected path: the child +// normally exits on its own right after the flush (go-pmtiles ends with phase:paused, exit_code:3) and +// the exit handler resolves before this fires. The job is already shown 'paused' the moment we SIGTERM +// (patch below), so a long cap has no user-visible cost. A SIGKILL during the save does not corrupt: the +// journal is replaced atomically and lists only on-disk data. +const PAUSE_GRACE_MS = 60_000; class JobManager { private db: Database.Database; diff --git a/static/dashboard/sockets/maps.exec.ts b/static/dashboard/sockets/maps.exec.ts index a72488411..d94e4f7d5 100644 --- a/static/dashboard/sockets/maps.exec.ts +++ b/static/dashboard/sockets/maps.exec.ts @@ -19,7 +19,7 @@ const MAPS_TMP_DIR = '/library/downloads/maps'; // name: letters/digits/hyphen/underscore, 1..34 (same rule as the socket handler). const NAME_RE = /^[A-Za-z0-9_-]{1,34}$/; -interface MapItem { name?: string; box?: string; noninteractive?: boolean; } +interface MapItem { name?: string; box?: string; noninteractive?: boolean; est?: number; } /** ADFA-4896: run a tile-extract maintenance command to completion, best-effort. Used for the * stop/cancel cleanup, which must run while the job's own context is being torn down, so it does @@ -91,12 +91,31 @@ const mapsRunner: (ctx: RunnerContext) => Promise = async (ctx) => { const TOTAL_LAYERS = 3; let layersDone = 0; let overall = 0; // monotonic across the whole FQR + // K2GO-317: the device passes the total-bytes estimate (the consent "transfer" figure) so the ring and ETA + // run off CUMULATIVE bytes vs that total : not 1/3-per-layer buckets (which stick on the big layer and jump + // at layer boundaries) : and off a SMOOTHED speed, not go-pmtiles' raw per-layer eta_s (instantaneous, and + // reset every layer). estBytes 0 (an old job) falls back to the layer-bucket math. + const estBytes = Math.max(0, Number(item.est) || 0); + let completedBytes = 0; // bytes of the layers already finished this run + let curLayerTotal = 0; // the in-progress layer's byte total + let emaBps = -1; // EMA of speed_bps; smooths the bursty per-request rate that made the ETA swing + let speedSamples = 0; // warm-up guard: hold the ETA until the smoothed speed has settled + const SPEED_ALPHA = 0.25; // ~4 s window at one JSON line per second + const WARMUP_SAMPLES = 3; // K2GO-317 follow-up: go-pmtiles prints exit_code on each layer's last JSON line. On a stop we read it // to tell a resumable network drop (2) from a hard failure. The last value seen is the relevant one. let lastExitCode: number | null = null; // K2GO-317: set true when WE proactively SIGTERM a stalled download (watchdog below), so the catch // keeps the journal and surfaces PAUSED just like an engine pause. let stalled = false; + // K2GO-317: tile-extract.py's own mirror/catalog HEAD request (between layers) can fail with a transient + // network error when the phone link blips. go-pmtiles never ran for that layer, so there is no exit_code 2; + // without this the catch would purge every completed layer, and the r2 skip could not then save them on a + // retry. Detect the signature in stderr and treat it as resumable (keep partials, surface PAUSED). The + // rolling tail guards against the signature splitting across stderr chunks. + let sawNetError = false; + let stderrTail = ''; + const NET_ERR_RE = /ConnectionError|Max retries exceeded|HTTPSConnectionPool|Failed to establish a new connection|Connection (refused|reset)|Temporary failure in name resolution|Name or service not known/i; await new Promise((resolve, reject) => { // ADFA-4896: detached so `sudo` is a process-group leader and the engine's group-kill takes @@ -104,25 +123,45 @@ const mapsRunner: (ctx: RunnerContext) => Promise = async (ctx) => { // extract finishes in the background). const p = ctx.spawn('sudo', args, { env: { ...process.env }, detached: true }); // K2GO-317 (progress retention): go-pmtiles' own network give-up checkpoints a STALE journal and - // loses the bytes since its last periodic checkpoint (measured ~2/3 of a large region). A SIGTERM - // makes it flush the CURRENT bytes losslessly (measured: it resumes from exactly where it stopped). - // So this watchdog SIGTERMs the download itself once it makes no progress for STALL_MS, instead of - // waiting ~15-20 s for the lossy give-up. Armed on byte progress; cleared when nothing is in flight - // (finalize/done) or the child exits. An engine pause/cancel still wins: it fires first and the - // catch checks classifyStop before `stalled`. - const STALL_MS = 7000; + // loses the bytes since its last periodic checkpoint. A SIGTERM makes it flush the CURRENT bytes + // losslessly (measured: it resumes from exactly where it stopped), so BOTH watchdogs below SIGTERM + // the download group to turn a hang into a lossless, resumable pause. + // STALL (byte): fires when the transfer is truly WEDGED : no byte progress AND no retry progress for + // STALL_MS, in phase:download. A normal network drop is NOT a wedge : go-pmtiles retries (bumping + // `retries`), which we count as activity and re-arm, so this does not preempt its retry; that drop + // takes go-pmtiles' own retry-then-exit_code-2 path. STALL_MS is long (30 s) because several normal + // phases make no bytes for a while (plan, finalize/fsync, the next layer's start, a 5 s retry wait + // plus a slow new connection); a shorter window false-pauses, which re-plans the layer and, before + // the go-pmtiles skip fix, re-downloads every completed layer : worse than no watchdog. + // IDLE (line): go-pmtiles emits one JSON line per second in EVERY phase, so NO line for IDLE_MS means + // the process is frozen (e.g. storage stopped responding in plan or finalize, which the byte + // watchdog, armed only in download, would miss). Re-armed on every progress line; disarmed once all + // layers are done, so the final move (which emits no lines) does not false-fire. An engine + // pause/cancel still wins: it fires first and the catch checks classifyStop before `stalled`. + const STALL_MS = 30_000; + const IDLE_MS = 30_000; let lastBytes = -1; + let lastRetries = -1; // a change in go-pmtiles' retry count is activity, not a stall let stallTimer: ReturnType | null = null; + let idleTimer: ReturnType | null = null; const clearStall = () => { if (stallTimer) { clearTimeout(stallTimer); stallTimer = null; } }; + const clearIdle = () => { if (idleTimer) { clearTimeout(idleTimer); idleTimer = null; } }; + // Turn a stall or freeze into a lossless, resumable pause: SIGTERM the group so go-pmtiles flushes a + // complete journal. The `stalled` guard makes either watchdog fire at most once. + const sigtermPause = (msg: string): void => { + if (stalled) return; + stalled = true; + ctx.log(msg); + try { if (p.pid) process.kill(-p.pid, 'SIGTERM'); } catch { /* group already gone */ } + try { p.kill('SIGTERM'); } catch { /* already gone */ } + }; const armStall = () => { clearStall(); - stallTimer = setTimeout(() => { - if (stalled) return; - stalled = true; - ctx.log('[maps] download stalled; flushing the resume journal and pausing (no progress lost)'); - try { if (p.pid) process.kill(-p.pid, 'SIGTERM'); } catch { /* group already gone */ } - try { p.kill('SIGTERM'); } catch { /* already gone */ } - }, STALL_MS); + stallTimer = setTimeout(() => sigtermPause('[maps] download stalled; flushing the resume journal and pausing (no progress lost)'), STALL_MS); + }; + const armIdle = () => { + clearIdle(); + idleTimer = setTimeout(() => sigtermPause('[maps] no progress line for 30 s (extract frozen); flushing the resume journal and pausing'), IDLE_MS); }; // pmtiles writes the JSON progress on stdout, one line per second. A 'data' chunk can split a // line across two events, so buffer stdout and act only on complete '\n'-terminated lines: an @@ -133,28 +172,58 @@ const mapsRunner: (ctx: RunnerContext) => Promise = async (ctx) => { if (!s) return; ctx.log(s); if (!s.startsWith('{') || s.indexOf('"phase"') < 0) return; - let j: { phase?: string; bytes_done?: number; bytes_total?: number; speed_bps?: number; eta_s?: number | null; exit_code?: number }; + let j: { phase?: string; bytes_done?: number; bytes_total?: number; speed_bps?: number; exit_code?: number; retries?: number }; try { j = JSON.parse(s); } catch { return; } if (typeof j.exit_code === 'number') lastExitCode = j.exit_code; // present on each layer's last line if (typeof j.phase !== 'string') return; - let frac = 0; + armIdle(); // any progress line is a heartbeat: re-arm the frozen-process watchdog + let etaOut = -1; // seconds remaining, -1 = unknown (jobs.ts holds the previous value) if (j.phase === 'download' && (j.bytes_total ?? 0) > 0) { - // bytes_done ALREADY includes resumed_bytes (on a resume it starts at the resumed amount - // and climbs to bytes_total), so this is the true progress of a resumed layer too. Do NOT - // add resumed_bytes here: that double-counts and clamps a resumed layer to 100%. - frac = Math.min(1, (j.bytes_done ?? 0) / (j.bytes_total as number)); - const bd = j.bytes_done ?? 0; // reset the stall watchdog whenever bytes advance - if (bd !== lastBytes) { lastBytes = bd; armStall(); } + curLayerTotal = j.bytes_total as number; + // bytes_done ALREADY includes resumed_bytes (a resumed layer starts at the resumed amount and + // climbs to bytes_total), so cumulative progress is just completedBytes + bytes_done. + const bd = j.bytes_done ?? 0; + // Re-arm on byte progress OR a rise in the retry count (go-pmtiles is retrying, not wedged), + // so a retry cycle with no new bytes does not trip the stall watchdog. + const rt = j.retries ?? 0; + if (bd !== lastBytes || rt !== lastRetries) { lastBytes = bd; lastRetries = rt; armStall(); } + const doneBytes = completedBytes + bd; + const sp = (typeof j.speed_bps === 'number' && j.speed_bps > 0) ? j.speed_bps : 0; + if (sp > 0) { emaBps = emaBps < 0 ? sp : SPEED_ALPHA * sp + (1 - SPEED_ALPHA) * emaBps; speedSamples++; } + if (estBytes > 0) { + // On the last layer every size is known, so the exact sum is the true total (this corrects + // an over/under estimate); on earlier layers trust the whole-region estimate (it covers the + // layers not seen yet). max() keeps a low estimate from reading over 100 before then. + const known = completedBytes + curLayerTotal; + const total = (layersDone >= TOTAL_LAYERS - 1) ? known : Math.max(estBytes, known); + overall = Math.max(overall, Math.min(99, Math.round((doneBytes / total) * 100))); + if (speedSamples >= WARMUP_SAMPLES && emaBps > 0) etaOut = Math.max(0, Math.round((total - doneBytes) / emaBps)); + } else { + const frac = Math.min(1, bd / (j.bytes_total as number)); // fallback: per-layer bucket + overall = Math.max(overall, Math.min(99, Math.round(((layersDone + frac) / TOTAL_LAYERS) * 100))); + if (speedSamples >= WARMUP_SAMPLES && emaBps > 0 && curLayerTotal > bd) etaOut = Math.max(0, Math.round((curLayerTotal - bd) / emaBps)); + } } else if (j.phase === 'finalize' || j.phase === 'done') { - frac = 1; clearStall(); // nothing downloading now: do not let a stale watchdog fire + if ((j.bytes_total ?? 0) > 0) curLayerTotal = j.bytes_total as number; // a layer with no download line + const doneBytes = completedBytes + curLayerTotal; + if (estBytes > 0) { + const total = (layersDone >= TOTAL_LAYERS - 1) ? doneBytes : Math.max(estBytes, doneBytes); + overall = Math.max(overall, Math.min(99, Math.round((doneBytes / total) * 100))); + } else { + overall = Math.max(overall, Math.min(99, Math.round(((layersDone + 1) / TOTAL_LAYERS) * 100))); + } } - overall = Math.max(overall, Math.min(99, Math.round(((layersDone + frac) / TOTAL_LAYERS) * 100))); const patch: JobUpdate = { phase: 'processing', percent: overall, detail: name }; - if (typeof j.speed_bps === 'number' && j.speed_bps > 0) patch.speed = j.speed_bps; - if (typeof j.eta_s === 'number' && j.eta_s >= 0) patch.eta = Math.round(j.eta_s); + if (emaBps > 0) patch.speed = Math.round(emaBps); // smoothed, so the "size : speed" line stops jumping + if (etaOut >= 0) patch.eta = etaOut; ctx.update(patch); - if (j.phase === 'done') layersDone = Math.min(TOTAL_LAYERS, layersDone + 1); + if (j.phase === 'done') { + layersDone = Math.min(TOTAL_LAYERS, layersDone + 1); + completedBytes += curLayerTotal; // roll the finished layer into the cumulative base + curLayerTotal = 0; + if (layersDone >= TOTAL_LAYERS) clearIdle(); // all layers done: the final move emits no lines + } }; let stdoutBuf = ''; p.stdout?.on('data', (buf: Buffer) => { @@ -165,10 +234,17 @@ const mapsRunner: (ctx: RunnerContext) => Promise = async (ctx) => { stdoutBuf = stdoutBuf.slice(nl + 1); } }); - p.stderr?.on('data', (buf: Buffer) => { const t = buf.toString().trim(); if (t) ctx.log(t); }); + p.stderr?.on('data', (buf: Buffer) => { + const t = buf.toString().trim(); + if (!t) return; + ctx.log(t); + stderrTail = (stderrTail + '\n' + t).slice(-4000); // bounded tail so a split signature still matches + if (!sawNetError && NET_ERR_RE.test(stderrTail)) sawNetError = true; + }); p.on('error', reject); p.on('exit', (code, signal) => { clearStall(); + clearIdle(); if (code === 0) return resolve(); // Non-zero or killed: an engine pause/cancel (group-killed) or tile-extract's own exit. // The stop kind is read from the engine's job flags (classifyStop) in the catch, not from @@ -184,13 +260,17 @@ const mapsRunner: (ctx: RunnerContext) => Promise = async (ctx) => { // 2. A network drop: our watchdog SIGTERMs the stalled download (stalled=true) so go-pmtiles // flushes the current bytes; if the watchdog did not catch it first, go-pmtiles gives up on its // own and reports exit_code 2 on its last JSON line (tile-extract.py masks the process exit to - // 1, so we read the JSON line). Either way we keep the journal. A cancel, or any other exit, is - // a hard error: purge. + // 1, so we read the JSON line). Either way we keep the journal. + // 3. tile-extract.py's OWN mirror/catalog request hit a transient network error (sawNetError, from + // its stderr). go-pmtiles never ran for that layer, so there is no exit_code 2. We keep every + // completed layer so the resume skips it (r2) instead of re-downloading. + // A cancel, or any other exit, is a hard error: purge. const stop = classifyStop(ctx); if (stop === 'canceled') { await purgeRegion(name, ctx.log); throw new CanceledError(); } if (stop === 'paused') throw new PausedError(); // keep partial: resume continues if (stalled) throw new PausedError(); // our stall-pause flushed a full journal if (lastExitCode === 2) throw new PausedError(); // go-pmtiles gave up on its own: still resumable + if (sawNetError) throw new PausedError(); // tile-extract's own mirror request blipped: keep partials, resume skips completed layers await purgeRegion(name, ctx.log); // hard error: purge and surface throw e; }); diff --git a/tools/rebuild-dashboard.sh b/tools/rebuild-dashboard.sh index 5418ae1e1..2711947d0 100755 --- a/tools/rebuild-dashboard.sh +++ b/tools/rebuild-dashboard.sh @@ -134,6 +134,8 @@ log "rebuild start (branch=$BRANCH, clone=$CLONE_DIR)" log "git fetch + reset --hard origin/$BRANCH" git -C "$CLONE_DIR" fetch origin "$BRANCH" >>"$LOG" 2>&1 || fail "git fetch (offline?)" git -C "$CLONE_DIR" reset --hard "origin/$BRANCH" >>"$LOG" 2>&1 || fail "git reset" +# K2GO: name the exact commit being built, so the operator can confirm it is the intended one. +log "building commit $(git -C "$CLONE_DIR" log -1 --format='%h %s' 2>/dev/null)" # 2) build in staging. Reuse live node_modules to speed the install; build fresh dist. log "staging build" diff --git a/tools/upstream-patches/patches/0004-maps-pmtiles-r2-skip-redownload.patch b/tools/upstream-patches/patches/0004-maps-pmtiles-r2-skip-redownload.patch new file mode 100644 index 000000000..ce186ad6b --- /dev/null +++ b/tools/upstream-patches/patches/0004-maps-pmtiles-r2-skip-redownload.patch @@ -0,0 +1,30 @@ +Upstream-PR: WIP: pending the IIAB maps-role update that pins the r2 go-pmtiles build +Upstream-Status: open +Applies-to: roles/maps/defaults/main.yml +Summary: Pin the pmtiles binary to the 1.31.2-r2 build (armhf, arm64, amd64). r2 adds the skip-already-complete-layer check, so a resume no longer re-downloads a layer that already finished (K2GO-317). Interim carry for our bake, ahead of the upstream maps-role update; drop it when that update lands (as 0003 was dropped for iiab#4565). The device-side companion is dash-node 1.5.0 (pause grace, stall and idle watchdogs). + +diff --git a/roles/maps/defaults/main.yml b/roles/maps/defaults/main.yml +index 661fa87..4eb753c 100644 +--- a/roles/maps/defaults/main.yml ++++ b/roles/maps/defaults/main.yml +@@ -32,14 +32,14 @@ maps_tools: + # in README.md. + releases: + armhf: +- src_url: https://iiab.switnet.org/binaries/pmtiles-1.31.2-m16/go-pmtiles_1.31.2-m16_Linux_armv7.tar.gz +- sha256sum: 3ca57c093c915906546bff251b02627682c134385f7cde2ff101b2aa360efaf3 ++ src_url: https://iiab.switnet.org/binaries/pmtiles-1.31.2-r2/go-pmtiles_1.31.2-r2_Linux_armv7.tar.gz ++ sha256sum: fc09956de173ca13e8d7af6d1eb1c0b67b1a2b02fb54e5ee42ab1207ee5e4481 + arm64: +- src_url: https://iiab.switnet.org/binaries/pmtiles-1.31.2-m16/go-pmtiles_1.31.2-m16_Linux_arm64.tar.gz +- sha256sum: c43bcdf275ac6eaf63504c36a9e0c674ff7b22396d6affa9bc76e84432c6a757 ++ src_url: https://iiab.switnet.org/binaries/pmtiles-1.31.2-r2/go-pmtiles_1.31.2-r2_Linux_arm64.tar.gz ++ sha256sum: e3234a33297145f58686b381554b4eeae66a0ba4e6a3b91e13a9700119643306 + amd64: +- src_url: https://iiab.switnet.org/binaries/pmtiles-1.31.2-m16/go-pmtiles_1.31.2-m16_Linux_x86_64.tar.gz +- sha256sum: b2f2e522e624d7ffc4a1079398635e323a41332c4b8124bdd1b243eb76cf2c55 ++ src_url: https://iiab.switnet.org/binaries/pmtiles-1.31.2-r2/go-pmtiles_1.31.2-r2_Linux_x86_64.tar.gz ++ sha256sum: 065c820e802528ba4009902ced49355538a938fc56602571e5f163494c618faf + archive_tmp_dir: "/opt/iiab/downloads/maps/pmtiles" + archive_tmp_file: "/opt/iiab/downloads/maps/pmtiles/pmtiles.tar.gz" + executable_path: "{{ maps_tools.base_dir }}/pmtiles"