From 4fdbc5c280aea7816aff90e795077975ca45fe48 Mon Sep 17 00:00:00 2001 From: AI Dev Date: Thu, 8 Oct 2026 15:18:25 +0000 Subject: [PATCH 1/2] Collect the kernel crashes cameras recover from, rank them, and pay Club stars for them A camera on OpenIPC firmware keeps the kernel's log across a crash (pstore), and its WebUI offers the owner a crash log -- which nobody upstream ever sees. The lab's gk7205v300 + imx335 had one since 2026-09-28: a NULL dereference in RGN_PutRegion called from the VENC interrupt, which reads as an ISP bug because hi_isp_run was the task it interrupted. - POST /api/v1/crashes takes the WebUI's crashlog_*.tar.gz (service/internal/ crashes, CRASH.md is the contract); a member can send the one they downloaded from /club. The records are parsed -- ARM, MIPS and arm64 backtraces, oops, BUG, WARNING, panic, a failsafe boot loop -- and filed under a signature, the top five frames without offsets, so one bug from many cameras is one row. pstore's Oops#1 and Panic#2 of one crash are one crash, and the same records sent again are one event. - Signatures are ranked by how bad a crash is and how many cameras it hits, more when today's firmware still has it. /crashes lists them publicly: signatures, chips and status, never a log, a camera or a member. - Maintainers triage at /club/crashes or with `openipc crashes status`, and see what reproducing one takes: the chips, sensors and builds it was seen on, the firmware builds matching its kernel, the task and what it interrupted, the warnings and i2c/MMZ trouble before it, and each log with MACs and IP addresses replaced by keyed hashes. The bundle as sent is kept in PostgreSQL and never served; events and bundles are append-only. - Stars (crash_stars, migration 030) are paid by the nightly settlement only: 1 per camera and bug, 3 more to the first reporter once confirmed, 5 when fixed, at most 10 a month. A crash asked for with sysrq pays nothing, and marking a signature bogus takes back what it paid. A camera's crashes are its owner's through the Open Wall's link. The leaderboard and the member's total count the third ledger. - nginx proxies /api/v1/crashes on both vhosts. --- CLAUDE.md | 18 + data/locales/boards.en.yml | 77 +++ data/locales/boards.ru.yml | 81 +++ deploy/nginx/sites-available/org.openipc | 20 + deploy/nginx/sites-available/org.openipc.dev | 21 + .../apps/site/src/components/club/Club.tsx | 4 + .../site/src/components/club/CrashTriage.tsx | 153 +++++ .../site/src/components/club/Crashes.test.tsx | 44 ++ .../apps/site/src/components/club/Crashes.tsx | 97 +++ .../site/src/components/club/Leaderboard.tsx | 6 +- .../site/src/components/crashes/CrashList.tsx | 121 ++++ .../src/components/pages/ClubCrashes.astro | 19 + .../site/src/components/pages/Crashes.astro | 18 + frontend/apps/site/src/i18n/boards.en.json | 72 +++ frontend/apps/site/src/i18n/boards.ru.json | 76 +++ frontend/apps/site/src/i18n/en.json | 10 + frontend/apps/site/src/i18n/ru.json | 10 + frontend/apps/site/src/lib/club.ts | 5 +- frontend/apps/site/src/lib/crashes.ts | 162 +++++ frontend/apps/site/src/lib/page-paths.ts | 4 + frontend/apps/site/src/lib/pages.ts | 4 + service/README.md | 3 +- service/cmd/openipc/club.go | 3 +- service/cmd/openipc/crashes.go | 110 ++++ service/cmd/openipc/main.go | 24 +- service/internal/club/api.go | 10 +- service/internal/club/crashes.go | 149 +++++ service/internal/club/crashes_test.go | 101 +++ service/internal/club/members.go | 8 +- service/internal/club/wall.go | 8 +- service/internal/crashes/CRASH.md | 69 +++ service/internal/crashes/api.go | 307 ++++++++++ service/internal/crashes/parse.go | 577 ++++++++++++++++++ service/internal/crashes/parse_test.go | 174 ++++++ service/internal/crashes/settle.go | 119 ++++ service/internal/crashes/store.go | 570 +++++++++++++++++ service/internal/crashes/store_test.go | 337 ++++++++++ .../testdata/gk7205v300-imx335-rgn.tar.gz | Bin 0 -> 8245 bytes .../internal/db/migrations/030_crashes.sql | 165 +++++ service/internal/wallstars/wallstars.go | 17 +- service/routes.json | 40 ++ 41 files changed, 3796 insertions(+), 17 deletions(-) create mode 100644 frontend/apps/site/src/components/club/CrashTriage.tsx create mode 100644 frontend/apps/site/src/components/club/Crashes.test.tsx create mode 100644 frontend/apps/site/src/components/club/Crashes.tsx create mode 100644 frontend/apps/site/src/components/crashes/CrashList.tsx create mode 100644 frontend/apps/site/src/components/pages/ClubCrashes.astro create mode 100644 frontend/apps/site/src/components/pages/Crashes.astro create mode 100644 frontend/apps/site/src/lib/crashes.ts create mode 100644 service/cmd/openipc/crashes.go create mode 100644 service/internal/club/crashes.go create mode 100644 service/internal/club/crashes_test.go create mode 100644 service/internal/crashes/CRASH.md create mode 100644 service/internal/crashes/api.go create mode 100644 service/internal/crashes/parse.go create mode 100644 service/internal/crashes/parse_test.go create mode 100644 service/internal/crashes/settle.go create mode 100644 service/internal/crashes/store.go create mode 100644 service/internal/crashes/store_test.go create mode 100644 service/internal/crashes/testdata/gk7205v300-imx335-rgn.tar.gz create mode 100644 service/internal/db/migrations/030_crashes.sql diff --git a/CLAUDE.md b/CLAUDE.md index 1e162d8d..3e660310 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -208,6 +208,24 @@ restores the image but never the schema, so keep migrations additive. `wall_notices` for the next run. `openipc club wall-revoke` takes a faked camera's stars back. The owner's name on the camera's wall page is opt-in; the leaderboard (`/club/leaderboard`) is opt-out (migration 027). +- `internal/crashes` -- **kernel crashes cameras recovered from**: the + bundle the firmware's `S98crashlog` makes of pstore's records after an + oops or a panic, which the WebUI offers its owner. Sent by the WebUI to + `POST /api/v1/crashes` (the contract is `internal/crashes/CRASH.md`), or + downloaded and sent by a member from `/club`. Filed under a **signature** + -- the top five backtrace frames, `fn [module]` without offsets -- so one + bug from many cameras is one row (migration 030); the records pstore keeps + twice (`Oops#1`, `Panic#2`) are one crash, and the same records sent again + are one event. Ranked by kind, how many cameras and whether today's + firmware still has it; public at `/crashes` (signatures only), triaged by + maintainers at `/club/crashes` or `openipc crashes status`. Logs are + redacted (MACs and IPs keyed-hashed) and maintainers' only; bundles are + kept as sent in PostgreSQL (`crash_bundles`), never served; events and + bundles are guarded like owner reports. Stars go into `crash_stars`, the + third ledger, paid by the nightly `settle` only: 1 per camera and bug, 3 to + the first reporter once confirmed, 5 more when fixed, at most 10 a month; + a crash asked for (sysrq) pays nothing and `bogus` takes everything back. + A camera's crash is its owner's through the Open Wall's link. - `internal/tools`, `internal/nfsro` — **ipctool for stock firmware**, which has no curl and no TLS. ipctool's release job pushes each build to `PUT /api/v1/tools/{name}` (OIDC, `internal/tools/PUSH.md`); nginx serves diff --git a/data/locales/boards.en.yml b/data/locales/boards.en.yml index 5caedc67..084b4ae5 100644 --- a/data/locales/boards.en.yml +++ b/data/locales/boards.en.yml @@ -46,6 +46,14 @@ en: eyebrow: 'OpenIPC Club' title: 'Who helps the most' lede: 'Stars for documenting camera boards and for keeping cameras on the Open Wall, from the members who chose to be listed.' + club_crashes: + eyebrow: 'OpenIPC Club' + title: 'Crash triage' + lede: 'Kernel crashes cameras sent, worst first, with what it takes to reproduce each one.' + crashes: + eyebrow: 'OpenIPC firmware' + title: 'Kernel crashes from the field' + lede: 'When a camera running OpenIPC crashes, it keeps the kernel''s last words and offers them to its owner. Sent here, the same bug from many cameras becomes one row, ranked by how badly and how widely it hurts, so it can be fixed.' nav: boards: Camera boards cameras: @@ -605,3 +613,72 @@ en: lb_empty_30d: 'Nobody on the leaderboard earned stars in the last 30 days.' lb_note: 'Every member with stars appears here unless they opted out on their club page. Ranked by stars, never by how many cameras someone has.' lb_join: 'Join the OpenIPC Club' + side_from_crashes: 'From crash reports' + lb_crashes: 'Crashes' + crashes: + list_label: 'Kernel crashes, worst first' + empty: 'No crash has been sent yet.' + kind_bootloop: 'Boot loop' + kind_panic: 'Kernel panic' + kind_oops: 'Oops' + kind_bug: 'BUG' + kind_warning: 'Warning' + in_irq: 'in an interrupt' + status_open: 'Not yet looked at' + status_confirmed: 'Confirmed' + status_fixed: 'Fixed' + status_wontfix: 'Will not be fixed' + status_bogus: 'Not a real crash' + current: 'In current firmware' + cameras: + one: '{count} camera' + other: '{count} cameras' + events: + one: '{count} crash' + other: '{count} crashes' + last_seen: 'last {date}' + issue: 'The issue tracking it' + col_soc: 'SoC' + col_sensor: 'Sensor' + col_build: 'Kernel · firmware · majestic' + col_cameras: 'Cameras' + signature: 'Signature {id}' + firmware: 'firmware {list}' + how_title: 'Your camera crashed?' + how_1: 'After a crash, the camera''s web interface shows "This camera recovered from a crash" on every page. Open the crash report.' + how_2: 'Press Send to OpenIPC, or download the crash log and send it from' + how_club: 'your OpenIPC Club page' + how_3: 'Link the camera to the Club on the Open Wall, and the crashes it sends earn you stars.' + how_privacy: 'What is sent is the kernel''s log of the crash, the firmware version and the chip. The camera''s MAC and every IP address in it are replaced with hashes; the logs are read only by OpenIPC''s maintainers. Nothing is sent unless you send it.' + mine_title: 'My crash reports' + all_link: 'All crashes' + mine_lede: 'Each bug a camera of yours hits earns {report} ★ once settled; if you were the first to send it, {first} ★ more when a maintainer confirms it and {fixed} ★ when it is fixed. At most {cap} ★ a month from crashes; a crash you caused on purpose earns nothing.' + file_label: 'Crash log from the camera''s web interface (crashlog_*.tar.gz)' + mac_label: 'The camera''s MAC, if it is linked on the Open Wall (optional)' + send: 'Send the crash log' + sent: 'Received, filed under' + sent_again: 'This crash was already received, under' + sent_self: 'Received, but this crash was asked for (sysrq), so it earns nothing:' + self_inflicted: 'caused on purpose' + mine_empty: 'No crash sent yet, from you or your linked cameras.' + triage_link: 'Crash triage' + triage_forbidden: 'Only OpenIPC''s maintainers triage crashes.' + merged_into: 'merged into {id}' + taken_back: 'Saved; {n} ★ taken back.' + saved: 'Saved.' + f_status: 'Status' + f_fixed_in: 'Fixed in (firmware version)' + f_issue: 'Issue' + f_merge: 'Same bug as signature' + f_note: 'Note' + f_save: 'Save' + d_hardware: 'Hardware' + d_firmware: 'Firmware' + d_kernel: 'Kernel' + d_builds: 'Matching builds' + d_task: 'Task' + d_interrupted: 'interrupted' + d_uptime: 'Uptime' + d_cmdline: 'Command line' + d_anomalies: 'Before it' + d_before: 'Warnings' diff --git a/data/locales/boards.ru.yml b/data/locales/boards.ru.yml index 2a1bd4ca..3bf8863b 100644 --- a/data/locales/boards.ru.yml +++ b/data/locales/boards.ru.yml @@ -46,6 +46,14 @@ ru: eyebrow: 'Клуб OpenIPC' title: 'Кто помогает больше всех' lede: 'Звёзды за описанные платы камер и за камеры на Open Wall — у участников, которые согласились попасть в список.' + club_crashes: + eyebrow: 'Клуб OpenIPC' + title: 'Разбор падений' + lede: 'Падения ядра, присланные камерами, — сначала самые тяжёлые, с тем, что нужно, чтобы воспроизвести каждое.' + crashes: + eyebrow: 'Прошивка OpenIPC' + title: 'Падения ядра с реальных камер' + lede: 'Когда камера с OpenIPC падает, она сохраняет последние слова ядра и предлагает их владельцу. Присланные сюда, одинаковые падения со многих камер становятся одной строкой, отсортированной по тому, насколько тяжело и насколько широко они бьют, — чтобы их можно было исправить.' nav: boards: Платы камер cameras: @@ -646,3 +654,76 @@ ru: lb_empty_30d: 'За последние 30 дней никто в рейтинге не получил звёзд.' lb_note: 'Здесь каждый участник со звёздами, если он не отказался от этого на своей странице клуба. Порядок — по звёздам, а не по числу камер.' lb_join: 'Вступить в клуб OpenIPC' + side_from_crashes: 'За отчёты о падениях' + lb_crashes: 'Падения' + crashes: + list_label: 'Падения ядра, сначала самые тяжёлые' + empty: 'Пока никто не прислал ни одного падения.' + kind_bootloop: 'Циклическая перезагрузка' + kind_panic: 'Kernel panic' + kind_oops: 'Oops' + kind_bug: 'BUG' + kind_warning: 'Предупреждение' + in_irq: 'в прерывании' + status_open: 'Ещё не разобрано' + status_confirmed: 'Подтверждено' + status_fixed: 'Исправлено' + status_wontfix: 'Исправлено не будет' + status_bogus: 'Не настоящее падение' + current: 'В текущей прошивке' + cameras: + one: '{count} камера' + few: '{count} камеры' + many: '{count} камер' + other: '{count} камеры' + events: + one: '{count} падение' + few: '{count} падения' + many: '{count} падений' + other: '{count} падения' + last_seen: 'последнее {date}' + issue: 'Задача, где это чинят' + col_soc: 'SoC' + col_sensor: 'Сенсор' + col_build: 'Ядро · прошивка · majestic' + col_cameras: 'Камер' + signature: 'Сигнатура {id}' + firmware: 'прошивка {list}' + how_title: 'Ваша камера упала?' + how_1: 'После падения веб-интерфейс камеры на каждой странице пишет «This camera recovered from a crash». Откройте отчёт о падении.' + how_2: 'Нажмите «Send to OpenIPC» или скачайте лог падения и отправьте его со' + how_club: 'своей страницы в клубе OpenIPC' + how_3: 'Привяжите камеру к клубу на Open Wall — и падения, которые она присылает, принесут вам звёзды.' + how_privacy: 'Отправляется журнал ядра о падении, версия прошивки и чип. MAC камеры и все IP-адреса в нём заменяются хешами; журналы читают только мейнтейнеры OpenIPC. Без вашего действия ничего не отправляется.' + mine_title: 'Мои отчёты о падениях' + all_link: 'Все падения' + mine_lede: 'Каждая ошибка, на которую попала ваша камера, приносит {report} ★ после расчёта; если вы прислали её первым — ещё {first} ★, когда мейнтейнер подтвердит, и {fixed} ★, когда её исправят. Не больше {cap} ★ в месяц за падения; падение, вызванное нарочно, ничего не приносит.' + file_label: 'Лог падения из веб-интерфейса камеры (crashlog_*.tar.gz)' + mac_label: 'MAC камеры, если она привязана на Open Wall (необязательно)' + send: 'Отправить лог падения' + sent: 'Получено, отнесено к' + sent_again: 'Это падение уже было получено:' + sent_self: 'Получено, но это падение вызвано нарочно (sysrq), поэтому ничего не приносит:' + self_inflicted: 'вызвано нарочно' + mine_empty: 'Пока ни одного падения — ни от вас, ни от привязанных камер.' + triage_link: 'Разбор падений' + triage_forbidden: 'Разбирать падения могут только мейнтейнеры OpenIPC.' + merged_into: 'объединено с {id}' + taken_back: 'Сохранено; списано {n} ★.' + saved: 'Сохранено.' + f_status: 'Состояние' + f_fixed_in: 'Исправлено в (версия прошивки)' + f_issue: 'Задача' + f_merge: 'Та же ошибка, что сигнатура' + f_note: 'Заметка' + f_save: 'Сохранить' + d_hardware: 'Железо' + d_firmware: 'Прошивка' + d_kernel: 'Ядро' + d_builds: 'Подходящие сборки' + d_task: 'Задача ядра' + d_interrupted: 'прервана' + d_uptime: 'Время работы' + d_cmdline: 'Командная строка' + d_anomalies: 'Перед этим' + d_before: 'Предупреждения' diff --git a/deploy/nginx/sites-available/org.openipc b/deploy/nginx/sites-available/org.openipc index a9d453cb..0516e0e3 100644 --- a/deploy/nginx/sites-available/org.openipc +++ b/deploy/nginx/sites-available/org.openipc @@ -455,6 +455,26 @@ server { add_header X-Served-By go always; } + # Kernel crashes cameras recovered from (service/internal/crashes): the + # crash log a camera's WebUI sends (a tar.gz of pstore's records, at most + # 256 KB), and the public list of signatures, which the web role lets + # caches keep for five minutes. The upload is limited per address and per + # camera by the web role itself. + location ^~ /api/v1/crashes { + client_max_body_size 512k; + limit_req zone=boards_search burst=20 nodelay; + limit_req_status 429; + gzip on; + gzip_proxied any; + gzip_types application/json; + proxy_pass http://127.0.0.1:3002; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Request-Id $request_id; + add_header Strict-Transport-Security max-age=15768000; + add_header X-Served-By go always; + } + # The OpenIPC Club (service/internal/club): signing in, the send form, # members' own reports and the maintainers' review. Its session cookie's # path is this prefix, so nothing else on the site ever receives it, and diff --git a/deploy/nginx/sites-available/org.openipc.dev b/deploy/nginx/sites-available/org.openipc.dev index e21c5650..42e0c43f 100644 --- a/deploy/nginx/sites-available/org.openipc.dev +++ b/deploy/nginx/sites-available/org.openipc.dev @@ -368,6 +368,27 @@ server { add_header X-Robots-Tag "noindex, nofollow, noarchive" always; } + # Kernel crashes cameras recovered from (service/internal/crashes): the + # crash log a camera's WebUI sends (a tar.gz of pstore's records, at most + # 256 KB), and the public list of signatures, which the web role lets + # caches keep for five minutes. The upload is limited per address and per + # camera by the web role itself. + location ^~ /api/v1/crashes { + client_max_body_size 512k; + limit_req zone=boards_search burst=20 nodelay; + limit_req_status 429; + gzip on; + gzip_proxied any; + gzip_types application/json; + proxy_pass http://127.0.0.1:3012; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Request-Id $request_id; + add_header Strict-Transport-Security max-age=15768000; + add_header X-Served-By go always; + add_header X-Robots-Tag "noindex, nofollow, noarchive" always; + } + # The OpenIPC Club (service/internal/club): signing in, the send form, # members' own reports and the maintainers' review. Its session cookie's # path is this prefix, so nothing else on the site ever receives it, and diff --git a/frontend/apps/site/src/components/club/Club.tsx b/frontend/apps/site/src/components/club/Club.tsx index 2902f901..f7be1762 100644 --- a/frontend/apps/site/src/components/club/Club.tsx +++ b/frontend/apps/site/src/components/club/Club.tsx @@ -17,6 +17,7 @@ import { } from '../../lib/club'; import { size } from '../../lib/reports'; import Cameras from './Cameras'; +import Crashes from './Crashes'; import IpctoolCode from './IpctoolCode'; import { STATUS_TONE, Stars } from './parts'; @@ -273,6 +274,7 @@ function MemberPage({ member, ways, locale, t, onChange }: {
{t('club.side_from_reports')}
{t('club.side_from_wall')}
+
{t('club.side_from_crashes')}
{cams && ( @@ -304,6 +306,7 @@ function MemberPage({ member, ways, locale, t, onChange }: { ) : } {member.maintainer && {t('club.review_link')}} + {member.maintainer && {t('crashes.triage_link')}} {telegram && (
{member.quiet ? t('club.bot_muted') : t('club.bot_on')} @@ -323,6 +326,7 @@ function MemberPage({ member, ways, locale, t, onChange }: { {cams ? { void loadCams(); onChange(); }} /> : !error &&
} +

{t('club.mine_title')}

diff --git a/frontend/apps/site/src/components/club/CrashTriage.tsx b/frontend/apps/site/src/components/club/CrashTriage.tsx new file mode 100644 index 00000000..da2ea8a8 --- /dev/null +++ b/frontend/apps/site/src/components/club/CrashTriage.tsx @@ -0,0 +1,153 @@ +/** + * /club/crashes: the maintainers' triage of kernel crashes. Every signature, + * worst first; one opens to what it takes to reproduce it -- the chips, + * sensors and builds it was seen on, the firmware builds whose time matches + * its kernel, and each crash's redacted log, warnings and lead-up. + * + * Confirming a bug pays its first reporter, fixing it pays them again; + * bogus takes back every star it paid. The same is `openipc crashes status`. + */ +import { useEffect, useState } from 'preact/hooks'; +import { useBoardsTranslations, type BoardsT } from '../../lib/boards-i18n'; +import { pathFor, type Locale } from '../../lib/i18n'; +import { ClubError, fetchMe } from '../../lib/club'; +import { + decideCrash, fetchCrashDetail, fetchTriage, frame, KIND_TONE, STATUS_TONE, type Combo, type Detail, type Signature, type Status, +} from '../../lib/crashes'; + +type Load = { state: 'loading' } | { state: 'ok'; list: Signature[] } | { state: 'forbidden' } | { state: 'error' }; +const STATUSES: Status[] = ['open', 'confirmed', 'fixed', 'wontfix', 'bogus']; + +export default function CrashTriage({ locale }: { locale: Locale }) { + const t = useBoardsTranslations(locale); + const [load, setLoad] = useState({ state: 'loading' }); + const [open, setOpen] = useState(null); + + const reload = () => fetchTriage() + .then((r) => setLoad({ state: 'ok', list: r.signatures })) + .catch((e) => setLoad(e instanceof ClubError && (e.status === 401 || e.status === 403) ? { state: 'forbidden' } : { state: 'error' })); + useEffect(() => { void fetchMe().catch(() => {}); void reload(); }, []); + + if (load.state === 'forbidden') { + return ( + + ); + } + return ( +
+ {load.state === 'loading' &&
} + {load.state === 'error' && } + {load.state === 'ok' && load.list.length === 0 &&

{t('crashes.empty')}

} + {load.state === 'ok' && load.list.map((g) => ( +
+ + {open === g.id && { void reload(); }} />} +
+ ))} +
+ ); +} + +function Open({ g, t, onDone }: { g: Signature; t: BoardsT; onDone: () => void }) { + const [data, setData] = useState<{ seen_on: Combo[]; crashes: Detail[] } | null>(null); + const [status, setStatus] = useState(g.status); + const [fixedIn, setFixedIn] = useState(g.fixed_in ?? ''); + const [issue, setIssue] = useState(g.issue_url ?? ''); + const [merge, setMerge] = useState(''); + const [note, setNote] = useState(g.note ?? ''); + const [msg, setMsg] = useState(null); + useEffect(() => { fetchCrashDetail(g.id).then(setData).catch((e: Error) => setMsg(e.message)); }, [g.id]); + + const save = (e: Event) => { + e.preventDefault(); + setMsg(null); + decideCrash(g.id, { status, fixed_in: fixedIn, issue_url: issue, merge_into: merge, note }) + .then((r) => { setMsg(r.stars_taken_back ? t('crashes.taken_back', { n: r.stars_taken_back }) : t('crashes.saved')); onDone(); }) + .catch((err: Error) => setMsg(err.message)); + }; + const field = 'min-w-0 rounded-md border border-hairline px-2 py-1 text-sm'; + return ( +
+ 
+        {g.frames.slice(0, 10).map((f, i) => `${i === 0 ? '' : '← '}${frame(f)}`).join('\n')}
+
+
+ + + + +