From 82501252ab3d167f4bda79ea2530d641e0a366fd Mon Sep 17 00:00:00 2001 From: luen Date: Tue, 3 Nov 2026 08:26:23 +1000 Subject: [PATCH] Import: keep serving when DATA_DIR (or part of it) is unreadable listWorkbooks() now skips folders and files it can't read and reports them, so an EACCES on the mounted data folder shows up on the Data quality page instead of failing every request. Unreadable paths are part of the fingerprint, so fixing the permissions triggers a rebuild without a restart. Fixes #83 Co-Authored-By: Claude Opus 5.5 --- docs/DEPLOY.md | 2 +- src/lib/server/import/index.ts | 44 ++++++++++++++++++++---- src/lib/server/import/unreadable.test.ts | 27 +++++++++++++++ 3 files changed, 65 insertions(+), 8 deletions(-) create mode 100644 src/lib/server/import/unreadable.test.ts diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 9c01b9a..5a954e7 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -38,4 +38,4 @@ Set `ORIGIN` in `.env` to the public URL if the proxy does not send `X-Forwarded - `docker logs physioshark-data` shows `[import] rebuilt in … ms: {...}` after each import, or the error that stopped it. - The cache is disposable: `docker compose -f docker-compose.ghcr.yml down && docker volume rm physioshark-data_cache` forces a clean rebuild. -- The image is distroless and runs as uid 65532 with no shell. The data folder must be readable by that user (default `644` files are fine). +- The image is distroless and runs as uid 65532 with no shell. The data folder must be readable by that user (default `644` files are fine). If the host umask makes new folders private (e.g. `drwxr-x---`), the Data quality page lists "Could not read the data folder (EACCES)" and the site shows no survey data. Fix it on the host with `chmod -R o+rX data`. The next check (within a minute) rebuilds without a restart. diff --git a/src/lib/server/import/index.ts b/src/lib/server/import/index.ts index 930252d..2576e83 100644 --- a/src/lib/server/import/index.ts +++ b/src/lib/server/import/index.ts @@ -64,7 +64,7 @@ import { SITE_DEFS } from './sites'; import { readWorkbook } from './xlsx'; /** Bump when import logic changes so existing caches rebuild on the next start. */ -export const IMPORTER_VERSION = '2'; +export const IMPORTER_VERSION = '3'; const MIGRATIONS = import.meta.glob('/drizzle/*.sql', { query: '?raw', @@ -95,16 +95,35 @@ export interface SourceFile { mtime: number; } -export function listWorkbooks(dir: string): SourceFile[] { +/** + * Every workbook under `dir`. Folders or files the app can't read (e.g. EACCES when the host + * folder isn't readable by the container user) are skipped and reported through `onError`, so a + * permissions slip shows up on the Data quality page instead of taking the site down (#83). + */ +export function listWorkbooks( + dir: string, + onError?: (path: string, err: NodeJS.ErrnoException) => void +): SourceFile[] { if (!existsSync(dir)) return []; const out: SourceFile[] = []; const walk = (d: string) => { - for (const ent of readdirSync(d, { withFileTypes: true })) { + let entries; + try { + entries = readdirSync(d, { withFileTypes: true }); + } catch (err) { + onError?.(d, err as NodeJS.ErrnoException); + return; + } + for (const ent of entries) { const p = join(d, ent.name); if (ent.isDirectory()) walk(p); else if (/\.xlsx$/i.test(ent.name) && !ent.name.startsWith('~$')) { - const st = statSync(p); - out.push({ path: p, size: st.size, mtime: Math.round(st.mtimeMs) }); + try { + const st = statSync(p); + out.push({ path: p, size: st.size, mtime: Math.round(st.mtimeMs) }); + } catch (err) { + onError?.(p, err as NodeJS.ErrnoException); + } } } }; @@ -118,7 +137,10 @@ export function importHash(opts: ImportOptions): string { h.update(`importer:${IMPORTER_VERSION}\n`); for (const [name, sql] of Object.entries(MIGRATIONS).sort()) h.update(`${name}:${sql.length}:${sql}\n`); - for (const f of listWorkbooks(opts.dataDir)) + // Unreadable paths are part of the fingerprint, so fixing their permissions triggers a rebuild. + const unreadable = (p: string, err: NodeJS.ErrnoException) => + h.update(`!${relative(opts.dataDir, p)}:${err.code}\n`); + for (const f of listWorkbooks(opts.dataDir, unreadable)) h.update(`${relative(opts.dataDir, f.path)}:${f.size}:${f.mtime}\n`); for (const name of EXTERNAL_FILES) { for (const p of [join(opts.externalDir, name), join(opts.externalDir, `${name}.gz`)]) { @@ -173,7 +195,15 @@ function parseAll(dataDir: string, log: IssueLog): Parsed { adultEffort: [], files: [] }; - for (const f of listWorkbooks(dataDir)) { + const unreadable = (path: string, err: NodeJS.ErrnoException) => + log.add( + 'error', + 'files', + `Could not read ${path === dataDir ? 'the data folder' : 'this path'} (${err.code ?? err.message}). ` + + 'Make it readable by the app (in Docker, uid 65532), e.g. chmod -R o+rX on the host folder.', + { file: relative(dataDir, path).replace(/\\/g, '/') || '.', sheet: '' } + ); + for (const f of listWorkbooks(dataDir, unreadable)) { const rel = relative(dataDir, f.path).replace(/\\/g, '/'); const base = basename(f.path); let wb; diff --git a/src/lib/server/import/unreadable.test.ts b/src/lib/server/import/unreadable.test.ts new file mode 100644 index 0000000..6c397af --- /dev/null +++ b/src/lib/server/import/unreadable.test.ts @@ -0,0 +1,27 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, describe, expect, it } from 'vitest'; +import { importHash, listWorkbooks } from '.'; + +// A folder the app can't list (EACCES in Docker) must not throw (#83). A file passed as the data +// folder fails readdir the same way (ENOTDIR) on every OS, including Windows CI. +describe('unreadable DATA_DIR', () => { + const dir = mkdtempSync(join(tmpdir(), 'physioshark-')); + const notADir = join(dir, 'data'); + writeFileSync(notADir, 'not a folder'); + afterAll(() => rmSync(dir, { recursive: true, force: true })); + + it('skips the path and reports it', () => { + const errors: string[] = []; + expect(listWorkbooks(notADir, (p, err) => errors.push(`${p}:${err.code}`))).toEqual([]); + expect(errors).toEqual([`${notADir}:ENOTDIR`]); + }); + + it('still fingerprints, and differently from an empty folder', () => { + const empty = join(dir, 'empty'); + const opts = { dataDir: notADir, externalDir: join(dir, 'none') }; + expect(() => importHash(opts)).not.toThrow(); + expect(importHash(opts)).not.toBe(importHash({ ...opts, dataDir: empty })); + }); +});