diff --git a/CLAUDE.md b/CLAUDE.md
index 1e162d8d..29e5ae0c 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -208,6 +208,26 @@ restores the image but never the schema, so keep migrations additive.
`wall_notices` for the next run. `openipc club wall-revoke` takes a faked
camera's stars back. The owner's name on the camera's wall page is opt-in;
the leaderboard (`/club/leaderboard`) is opt-out (migration 027).
+- `internal/crashes` -- **kernel crashes cameras recovered from**: the
+ bundle the firmware's `S98crashlog` makes of pstore's records after an
+ oops or a panic, which the WebUI offers its owner. Sent by the WebUI to
+ `POST /api/v1/crashes` (the contract is `internal/crashes/CRASH.md`), or
+ downloaded and sent by a member from `/club`. Filed under a **signature**
+ -- the top five backtrace frames, `fn [module]` without offsets -- so one
+ bug from many cameras is one row (migration 030); the records pstore keeps
+ twice (`Oops#1`, `Panic#2`) are one crash, and the same records sent again
+ by one camera are one event. Ranked by kind, how many cameras and whether today's
+ firmware still has it; public at `/crashes` (signatures only), triaged by
+ maintainers at `/club/crashes` or `openipc crashes status`. Logs are
+ redacted (MACs and IPs keyed-hashed) and maintainers' only; bundles are
+ kept as sent in PostgreSQL (`crash_bundles`), never served; events and
+ bundles are guarded like owner reports. Stars go into `crash_stars`, the
+ third ledger, paid by the nightly `settle` only: 1 per camera and bug, 3 to
+ the first reporter once confirmed, 5 more when fixed, at most 10 a month;
+ a crash asked for (sysrq) pays nothing. The ledger is net per bug, through
+ merges: `bogus` takes back what a bug and the signatures merged into it
+ paid, and undoing it lets the settlement pay again. A crash with a MAC is
+ the camera's linked owner's, whoever sent it.
- `internal/tools`, `internal/nfsro` — **ipctool for stock firmware**, which
has no curl and no TLS. ipctool's release job pushes each build to
`PUT /api/v1/tools/{name}` (OIDC, `internal/tools/PUSH.md`); nginx serves
diff --git a/data/locales/boards.en.yml b/data/locales/boards.en.yml
index 5caedc67..084b4ae5 100644
--- a/data/locales/boards.en.yml
+++ b/data/locales/boards.en.yml
@@ -46,6 +46,14 @@ en:
eyebrow: 'OpenIPC Club'
title: 'Who helps the most'
lede: 'Stars for documenting camera boards and for keeping cameras on the Open Wall, from the members who chose to be listed.'
+ club_crashes:
+ eyebrow: 'OpenIPC Club'
+ title: 'Crash triage'
+ lede: 'Kernel crashes cameras sent, worst first, with what it takes to reproduce each one.'
+ crashes:
+ eyebrow: 'OpenIPC firmware'
+ title: 'Kernel crashes from the field'
+ lede: 'When a camera running OpenIPC crashes, it keeps the kernel''s last words and offers them to its owner. Sent here, the same bug from many cameras becomes one row, ranked by how badly and how widely it hurts, so it can be fixed.'
nav:
boards: Camera boards
cameras:
@@ -605,3 +613,72 @@ en:
lb_empty_30d: 'Nobody on the leaderboard earned stars in the last 30 days.'
lb_note: 'Every member with stars appears here unless they opted out on their club page. Ranked by stars, never by how many cameras someone has.'
lb_join: 'Join the OpenIPC Club'
+ side_from_crashes: 'From crash reports'
+ lb_crashes: 'Crashes'
+ crashes:
+ list_label: 'Kernel crashes, worst first'
+ empty: 'No crash has been sent yet.'
+ kind_bootloop: 'Boot loop'
+ kind_panic: 'Kernel panic'
+ kind_oops: 'Oops'
+ kind_bug: 'BUG'
+ kind_warning: 'Warning'
+ in_irq: 'in an interrupt'
+ status_open: 'Not yet looked at'
+ status_confirmed: 'Confirmed'
+ status_fixed: 'Fixed'
+ status_wontfix: 'Will not be fixed'
+ status_bogus: 'Not a real crash'
+ current: 'In current firmware'
+ cameras:
+ one: '{count} camera'
+ other: '{count} cameras'
+ events:
+ one: '{count} crash'
+ other: '{count} crashes'
+ last_seen: 'last {date}'
+ issue: 'The issue tracking it'
+ col_soc: 'SoC'
+ col_sensor: 'Sensor'
+ col_build: 'Kernel · firmware · majestic'
+ col_cameras: 'Cameras'
+ signature: 'Signature {id}'
+ firmware: 'firmware {list}'
+ how_title: 'Your camera crashed?'
+ how_1: 'After a crash, the camera''s web interface shows "This camera recovered from a crash" on every page. Open the crash report.'
+ how_2: 'Press Send to OpenIPC, or download the crash log and send it from'
+ how_club: 'your OpenIPC Club page'
+ how_3: 'Link the camera to the Club on the Open Wall, and the crashes it sends earn you stars.'
+ how_privacy: 'What is sent is the kernel''s log of the crash, the firmware version and the chip. The camera''s MAC and every IP address in it are replaced with hashes; the logs are read only by OpenIPC''s maintainers. Nothing is sent unless you send it.'
+ mine_title: 'My crash reports'
+ all_link: 'All crashes'
+ mine_lede: 'Each bug a camera of yours hits earns {report} ★ once settled; if you were the first to send it, {first} ★ more when a maintainer confirms it and {fixed} ★ when it is fixed. At most {cap} ★ a month from crashes; a crash you caused on purpose earns nothing.'
+ file_label: 'Crash log from the camera''s web interface (crashlog_*.tar.gz)'
+ mac_label: 'The camera''s MAC, if it is linked on the Open Wall (optional)'
+ send: 'Send the crash log'
+ sent: 'Received, filed under'
+ sent_again: 'This crash was already received, under'
+ sent_self: 'Received, but this crash was asked for (sysrq), so it earns nothing:'
+ self_inflicted: 'caused on purpose'
+ mine_empty: 'No crash sent yet, from you or your linked cameras.'
+ triage_link: 'Crash triage'
+ triage_forbidden: 'Only OpenIPC''s maintainers triage crashes.'
+ merged_into: 'merged into {id}'
+ taken_back: 'Saved; {n} ★ taken back.'
+ saved: 'Saved.'
+ f_status: 'Status'
+ f_fixed_in: 'Fixed in (firmware version)'
+ f_issue: 'Issue'
+ f_merge: 'Same bug as signature'
+ f_note: 'Note'
+ f_save: 'Save'
+ d_hardware: 'Hardware'
+ d_firmware: 'Firmware'
+ d_kernel: 'Kernel'
+ d_builds: 'Matching builds'
+ d_task: 'Task'
+ d_interrupted: 'interrupted'
+ d_uptime: 'Uptime'
+ d_cmdline: 'Command line'
+ d_anomalies: 'Before it'
+ d_before: 'Warnings'
diff --git a/data/locales/boards.ru.yml b/data/locales/boards.ru.yml
index 2a1bd4ca..3bf8863b 100644
--- a/data/locales/boards.ru.yml
+++ b/data/locales/boards.ru.yml
@@ -46,6 +46,14 @@ ru:
eyebrow: 'Клуб OpenIPC'
title: 'Кто помогает больше всех'
lede: 'Звёзды за описанные платы камер и за камеры на Open Wall — у участников, которые согласились попасть в список.'
+ club_crashes:
+ eyebrow: 'Клуб OpenIPC'
+ title: 'Разбор падений'
+ lede: 'Падения ядра, присланные камерами, — сначала самые тяжёлые, с тем, что нужно, чтобы воспроизвести каждое.'
+ crashes:
+ eyebrow: 'Прошивка OpenIPC'
+ title: 'Падения ядра с реальных камер'
+ lede: 'Когда камера с OpenIPC падает, она сохраняет последние слова ядра и предлагает их владельцу. Присланные сюда, одинаковые падения со многих камер становятся одной строкой, отсортированной по тому, насколько тяжело и насколько широко они бьют, — чтобы их можно было исправить.'
nav:
boards: Платы камер
cameras:
@@ -646,3 +654,76 @@ ru:
lb_empty_30d: 'За последние 30 дней никто в рейтинге не получил звёзд.'
lb_note: 'Здесь каждый участник со звёздами, если он не отказался от этого на своей странице клуба. Порядок — по звёздам, а не по числу камер.'
lb_join: 'Вступить в клуб OpenIPC'
+ side_from_crashes: 'За отчёты о падениях'
+ lb_crashes: 'Падения'
+ crashes:
+ list_label: 'Падения ядра, сначала самые тяжёлые'
+ empty: 'Пока никто не прислал ни одного падения.'
+ kind_bootloop: 'Циклическая перезагрузка'
+ kind_panic: 'Kernel panic'
+ kind_oops: 'Oops'
+ kind_bug: 'BUG'
+ kind_warning: 'Предупреждение'
+ in_irq: 'в прерывании'
+ status_open: 'Ещё не разобрано'
+ status_confirmed: 'Подтверждено'
+ status_fixed: 'Исправлено'
+ status_wontfix: 'Исправлено не будет'
+ status_bogus: 'Не настоящее падение'
+ current: 'В текущей прошивке'
+ cameras:
+ one: '{count} камера'
+ few: '{count} камеры'
+ many: '{count} камер'
+ other: '{count} камеры'
+ events:
+ one: '{count} падение'
+ few: '{count} падения'
+ many: '{count} падений'
+ other: '{count} падения'
+ last_seen: 'последнее {date}'
+ issue: 'Задача, где это чинят'
+ col_soc: 'SoC'
+ col_sensor: 'Сенсор'
+ col_build: 'Ядро · прошивка · majestic'
+ col_cameras: 'Камер'
+ signature: 'Сигнатура {id}'
+ firmware: 'прошивка {list}'
+ how_title: 'Ваша камера упала?'
+ how_1: 'После падения веб-интерфейс камеры на каждой странице пишет «This camera recovered from a crash». Откройте отчёт о падении.'
+ how_2: 'Нажмите «Send to OpenIPC» или скачайте лог падения и отправьте его со'
+ how_club: 'своей страницы в клубе OpenIPC'
+ how_3: 'Привяжите камеру к клубу на Open Wall — и падения, которые она присылает, принесут вам звёзды.'
+ how_privacy: 'Отправляется журнал ядра о падении, версия прошивки и чип. MAC камеры и все IP-адреса в нём заменяются хешами; журналы читают только мейнтейнеры OpenIPC. Без вашего действия ничего не отправляется.'
+ mine_title: 'Мои отчёты о падениях'
+ all_link: 'Все падения'
+ mine_lede: 'Каждая ошибка, на которую попала ваша камера, приносит {report} ★ после расчёта; если вы прислали её первым — ещё {first} ★, когда мейнтейнер подтвердит, и {fixed} ★, когда её исправят. Не больше {cap} ★ в месяц за падения; падение, вызванное нарочно, ничего не приносит.'
+ file_label: 'Лог падения из веб-интерфейса камеры (crashlog_*.tar.gz)'
+ mac_label: 'MAC камеры, если она привязана на Open Wall (необязательно)'
+ send: 'Отправить лог падения'
+ sent: 'Получено, отнесено к'
+ sent_again: 'Это падение уже было получено:'
+ sent_self: 'Получено, но это падение вызвано нарочно (sysrq), поэтому ничего не приносит:'
+ self_inflicted: 'вызвано нарочно'
+ mine_empty: 'Пока ни одного падения — ни от вас, ни от привязанных камер.'
+ triage_link: 'Разбор падений'
+ triage_forbidden: 'Разбирать падения могут только мейнтейнеры OpenIPC.'
+ merged_into: 'объединено с {id}'
+ taken_back: 'Сохранено; списано {n} ★.'
+ saved: 'Сохранено.'
+ f_status: 'Состояние'
+ f_fixed_in: 'Исправлено в (версия прошивки)'
+ f_issue: 'Задача'
+ f_merge: 'Та же ошибка, что сигнатура'
+ f_note: 'Заметка'
+ f_save: 'Сохранить'
+ d_hardware: 'Железо'
+ d_firmware: 'Прошивка'
+ d_kernel: 'Ядро'
+ d_builds: 'Подходящие сборки'
+ d_task: 'Задача ядра'
+ d_interrupted: 'прервана'
+ d_uptime: 'Время работы'
+ d_cmdline: 'Командная строка'
+ d_anomalies: 'Перед этим'
+ d_before: 'Предупреждения'
diff --git a/deploy/nginx/sites-available/org.openipc b/deploy/nginx/sites-available/org.openipc
index a9d453cb..0516e0e3 100644
--- a/deploy/nginx/sites-available/org.openipc
+++ b/deploy/nginx/sites-available/org.openipc
@@ -455,6 +455,26 @@ server {
add_header X-Served-By go always;
}
+ # Kernel crashes cameras recovered from (service/internal/crashes): the
+ # crash log a camera's WebUI sends (a tar.gz of pstore's records, at most
+ # 256 KB), and the public list of signatures, which the web role lets
+ # caches keep for five minutes. The upload is limited per address and per
+ # camera by the web role itself.
+ location ^~ /api/v1/crashes {
+ client_max_body_size 512k;
+ limit_req zone=boards_search burst=20 nodelay;
+ limit_req_status 429;
+ gzip on;
+ gzip_proxied any;
+ gzip_types application/json;
+ proxy_pass http://127.0.0.1:3002;
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Request-Id $request_id;
+ add_header Strict-Transport-Security max-age=15768000;
+ add_header X-Served-By go always;
+ }
+
# The OpenIPC Club (service/internal/club): signing in, the send form,
# members' own reports and the maintainers' review. Its session cookie's
# path is this prefix, so nothing else on the site ever receives it, and
diff --git a/deploy/nginx/sites-available/org.openipc.dev b/deploy/nginx/sites-available/org.openipc.dev
index e21c5650..42e0c43f 100644
--- a/deploy/nginx/sites-available/org.openipc.dev
+++ b/deploy/nginx/sites-available/org.openipc.dev
@@ -368,6 +368,27 @@ server {
add_header X-Robots-Tag "noindex, nofollow, noarchive" always;
}
+ # Kernel crashes cameras recovered from (service/internal/crashes): the
+ # crash log a camera's WebUI sends (a tar.gz of pstore's records, at most
+ # 256 KB), and the public list of signatures, which the web role lets
+ # caches keep for five minutes. The upload is limited per address and per
+ # camera by the web role itself.
+ location ^~ /api/v1/crashes {
+ client_max_body_size 512k;
+ limit_req zone=boards_search burst=20 nodelay;
+ limit_req_status 429;
+ gzip on;
+ gzip_proxied any;
+ gzip_types application/json;
+ proxy_pass http://127.0.0.1:3012;
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Request-Id $request_id;
+ add_header Strict-Transport-Security max-age=15768000;
+ add_header X-Served-By go always;
+ add_header X-Robots-Tag "noindex, nofollow, noarchive" always;
+ }
+
# The OpenIPC Club (service/internal/club): signing in, the send form,
# members' own reports and the maintainers' review. Its session cookie's
# path is this prefix, so nothing else on the site ever receives it, and
diff --git a/frontend/apps/site/src/components/club/Club.tsx b/frontend/apps/site/src/components/club/Club.tsx
index 2902f901..f7be1762 100644
--- a/frontend/apps/site/src/components/club/Club.tsx
+++ b/frontend/apps/site/src/components/club/Club.tsx
@@ -17,6 +17,7 @@ import {
} from '../../lib/club';
import { size } from '../../lib/reports';
import Cameras from './Cameras';
+import Crashes from './Crashes';
import IpctoolCode from './IpctoolCode';
import { STATUS_TONE, Stars } from './parts';
@@ -273,6 +274,7 @@ function MemberPage({ member, ways, locale, t, onChange }: {
+ )}
+
+ );
+}
diff --git a/frontend/apps/site/src/components/club/Leaderboard.tsx b/frontend/apps/site/src/components/club/Leaderboard.tsx
index 74ae8fcb..11c3bbc9 100644
--- a/frontend/apps/site/src/components/club/Leaderboard.tsx
+++ b/frontend/apps/site/src/components/club/Leaderboard.tsx
@@ -1,5 +1,5 @@
/**
- * /club/leaderboard: members ranked by stars, from both ledgers, all time or
+ * /club/leaderboard: members ranked by stars, from every ledger, all time or
* the last 30 days. Every member is in the answer unless they untick "Show me
* on the leaderboard" on /club; the signed-in member's own row is marked.
*/
@@ -42,13 +42,14 @@ export default function Leaderboard({ locale }: { locale: Locale }) {
{rows && rows.length === 0 &&
))}
diff --git a/frontend/apps/site/src/components/crashes/CrashList.tsx b/frontend/apps/site/src/components/crashes/CrashList.tsx
new file mode 100644
index 00000000..d6a93963
--- /dev/null
+++ b/frontend/apps/site/src/components/crashes/CrashList.tsx
@@ -0,0 +1,121 @@
+/**
+ * /crashes: the kernel crashes cameras recovered from, one row per bug,
+ * worst first (service/internal/crashes). Public: the signature, where it was
+ * seen and what is being done about it -- never a log, a camera or a member.
+ * A row opens to the chips and builds it was seen on.
+ */
+import { useEffect, useState } from 'preact/hooks';
+import { useBoardsTranslations, type BoardsT } from '../../lib/boards-i18n';
+import { pathFor, type Locale } from '../../lib/i18n';
+import { fetchSignature, fetchSignatures, frame, KIND_TONE, STATUS_TONE, type Combo, type Signature } from '../../lib/crashes';
+
+export default function CrashList({ locale }: { locale: Locale }) {
+ const t = useBoardsTranslations(locale);
+ const [list, setList] = useState(null);
+ const [error, setError] = useState(false);
+ const [open, setOpen] = useState(null);
+
+ useEffect(() => {
+ fetchSignatures().then((r) => {
+ setList(r.signatures);
+ const hash = location.hash.slice(1);
+ if (/^[0-9a-f]{12}$/.test(hash)) setOpen(hash);
+ }).catch(() => setError(true));
+ }, []);
+ useEffect(() => {
+ if (open && list) document.getElementById(open)?.scrollIntoView({ block: 'nearest' });
+ }, [open, list !== null]);
+
+ return (
+
+ {error &&
+ );
+}
+
+function HowToSend({ locale, t }: { locale: Locale; t: BoardsT }) {
+ return (
+
+ );
+}
diff --git a/frontend/apps/site/src/components/pages/ClubCrashes.astro b/frontend/apps/site/src/components/pages/ClubCrashes.astro
new file mode 100644
index 00000000..c2f9d420
--- /dev/null
+++ b/frontend/apps/site/src/components/pages/ClubCrashes.astro
@@ -0,0 +1,19 @@
+---
+/**
+ * The maintainers' triage of kernel crashes (service/internal/crashes). The
+ * page is a shell: the signatures, logs and builds are only ever in the
+ * answer /api/v1/club/crashes/triage gives a maintainer.
+ */
+import CrashTriage from '../club/CrashTriage.tsx';
+import { useTranslations, type Locale } from '../../lib/i18n';
+
+interface Props { locale: Locale }
+const { locale } = Astro.props;
+const t = useTranslations(locale);
+---
+
+
{t('pages.club_crashes.eyebrow')}
+
{t('pages.club_crashes.title')}
+
{t('pages.club_crashes.lede')}
+
+
diff --git a/frontend/apps/site/src/components/pages/Crashes.astro b/frontend/apps/site/src/components/pages/Crashes.astro
new file mode 100644
index 00000000..d7bb0913
--- /dev/null
+++ b/frontend/apps/site/src/components/pages/Crashes.astro
@@ -0,0 +1,18 @@
+---
+/**
+ * The kernel crashes cameras recovered from (service/internal/crashes), one
+ * row per bug, worst first. Static: the island asks /api/v1/crashes.
+ */
+import CrashList from '../crashes/CrashList.tsx';
+import { useTranslations, type Locale } from '../../lib/i18n';
+
+interface Props { locale: Locale }
+const { locale } = Astro.props;
+const t = useTranslations(locale);
+---
+
+
{t('pages.crashes.eyebrow')}
+
{t('pages.crashes.title')}
+
{t('pages.crashes.lede')}
+
+
diff --git a/frontend/apps/site/src/i18n/boards.en.json b/frontend/apps/site/src/i18n/boards.en.json
index eb5ce212..4fcd56ae 100644
--- a/frontend/apps/site/src/i18n/boards.en.json
+++ b/frontend/apps/site/src/i18n/boards.en.json
@@ -85,6 +85,7 @@
"kind_yaml": "ipctool report",
"lb_30d": "Last 30 days",
"lb_all": "All time",
+ "lb_crashes": "Crashes",
"lb_empty": "Nobody on the leaderboard has stars yet. A linked camera earns its first 5 stars no sooner than 30 days after it was linked, once it has had at least 20 days of lit, changing pictures since; a report earns stars when a maintainer publishes it.",
"lb_empty_30d": "Nobody on the leaderboard earned stars in the last 30 days.",
"lb_join": "Join the OpenIPC Club",
@@ -187,6 +188,7 @@
"sent_guest": "Received as {id}. A maintainer reviews it before it is shown.",
"sent_member": "Received as {id}. A maintainer reviews it before it is shown; follow it in My submissions.",
"show_yaml": "ipctool output",
+ "side_from_crashes": "From crash reports",
"side_from_reports": "From reports",
"side_from_wall": "From the Open Wall",
"sign_in_link": "Sign in",
@@ -228,6 +230,76 @@
"cov_photos": "Photos",
"cov_pinout": "Pinout",
"cov_uboot_env": "U-Boot console",
+ "crashes": {
+ "all_link": "All crashes",
+ "cameras": {
+ "one": "{count} camera",
+ "other": "{count} cameras"
+ },
+ "col_build": "Kernel · firmware · majestic",
+ "col_cameras": "Cameras",
+ "col_sensor": "Sensor",
+ "col_soc": "SoC",
+ "current": "In current firmware",
+ "d_anomalies": "Before it",
+ "d_before": "Warnings",
+ "d_builds": "Matching builds",
+ "d_cmdline": "Command line",
+ "d_firmware": "Firmware",
+ "d_hardware": "Hardware",
+ "d_interrupted": "interrupted",
+ "d_kernel": "Kernel",
+ "d_task": "Task",
+ "d_uptime": "Uptime",
+ "empty": "No crash has been sent yet.",
+ "events": {
+ "one": "{count} crash",
+ "other": "{count} crashes"
+ },
+ "f_fixed_in": "Fixed in (firmware version)",
+ "f_issue": "Issue",
+ "f_merge": "Same bug as signature",
+ "f_note": "Note",
+ "f_save": "Save",
+ "f_status": "Status",
+ "file_label": "Crash log from the camera's web interface (crashlog_*.tar.gz)",
+ "firmware": "firmware {list}",
+ "how_1": "After a crash, the camera's web interface shows \"This camera recovered from a crash\" on every page. Open the crash report.",
+ "how_2": "Press Send to OpenIPC, or download the crash log and send it from",
+ "how_3": "Link the camera to the Club on the Open Wall, and the crashes it sends earn you stars.",
+ "how_club": "your OpenIPC Club page",
+ "how_privacy": "What is sent is the kernel's log of the crash, the firmware version and the chip. The camera's MAC and every IP address in it are replaced with hashes; the logs are read only by OpenIPC's maintainers. Nothing is sent unless you send it.",
+ "how_title": "Your camera crashed?",
+ "in_irq": "in an interrupt",
+ "issue": "The issue tracking it",
+ "kind_bootloop": "Boot loop",
+ "kind_bug": "BUG",
+ "kind_oops": "Oops",
+ "kind_panic": "Kernel panic",
+ "kind_warning": "Warning",
+ "last_seen": "last {date}",
+ "list_label": "Kernel crashes, worst first",
+ "mac_label": "The camera's MAC, if it is linked on the Open Wall (optional)",
+ "merged_into": "merged into {id}",
+ "mine_empty": "No crash sent yet, from you or your linked cameras.",
+ "mine_lede": "Each bug a camera of yours hits earns {report} ★ once settled; if you were the first to send it, {first} ★ more when a maintainer confirms it and {fixed} ★ when it is fixed. At most {cap} ★ a month from crashes; a crash you caused on purpose earns nothing.",
+ "mine_title": "My crash reports",
+ "saved": "Saved.",
+ "self_inflicted": "caused on purpose",
+ "send": "Send the crash log",
+ "sent": "Received, filed under",
+ "sent_again": "This crash was already received, under",
+ "sent_self": "Received, but this crash was asked for (sysrq), so it earns nothing:",
+ "signature": "Signature {id}",
+ "status_bogus": "Not a real crash",
+ "status_confirmed": "Confirmed",
+ "status_fixed": "Fixed",
+ "status_open": "Not yet looked at",
+ "status_wontfix": "Will not be fixed",
+ "taken_back": "Saved; {n} ★ taken back.",
+ "triage_forbidden": "Only OpenIPC's maintainers triage crashes.",
+ "triage_link": "Crash triage"
+ },
"details": "Details",
"details_of": "Details of {board}",
"device_count": {
diff --git a/frontend/apps/site/src/i18n/boards.ru.json b/frontend/apps/site/src/i18n/boards.ru.json
index 2174fdb0..97f7154a 100644
--- a/frontend/apps/site/src/i18n/boards.ru.json
+++ b/frontend/apps/site/src/i18n/boards.ru.json
@@ -87,6 +87,7 @@
"kind_yaml": "Отчёт ipctool",
"lb_30d": "За 30 дней",
"lb_all": "За всё время",
+ "lb_crashes": "Падения",
"lb_empty": "Ни у кого в рейтинге пока нет звёзд. Привязанная камера получает первые 5 звёзд не раньше чем через 30 дней после привязки, когда с тех пор наберёт хотя бы 20 дней светлых и меняющихся кадров; отчёт приносит звёзды, когда его публикует мейнтейнер.",
"lb_empty_30d": "За последние 30 дней никто в рейтинге не получил звёзд.",
"lb_join": "Вступить в клуб OpenIPC",
@@ -189,6 +190,7 @@
"sent_guest": "Получено, номер {id}. Мейнтейнер проверит перед публикацией.",
"sent_member": "Получено, номер {id}. Мейнтейнер проверит перед публикацией; следите в разделе «Мои материалы».",
"show_yaml": "Вывод ipctool",
+ "side_from_crashes": "За отчёты о падениях",
"side_from_reports": "За отчёты",
"side_from_wall": "За Open Wall",
"sign_in_link": "Войти",
@@ -230,6 +232,80 @@
"cov_photos": "Фото",
"cov_pinout": "Распиновка",
"cov_uboot_env": "Консоль U-Boot",
+ "crashes": {
+ "all_link": "Все падения",
+ "cameras": {
+ "few": "{count} камеры",
+ "many": "{count} камер",
+ "one": "{count} камера",
+ "other": "{count} камеры"
+ },
+ "col_build": "Ядро · прошивка · majestic",
+ "col_cameras": "Камер",
+ "col_sensor": "Сенсор",
+ "col_soc": "SoC",
+ "current": "В текущей прошивке",
+ "d_anomalies": "Перед этим",
+ "d_before": "Предупреждения",
+ "d_builds": "Подходящие сборки",
+ "d_cmdline": "Командная строка",
+ "d_firmware": "Прошивка",
+ "d_hardware": "Железо",
+ "d_interrupted": "прервана",
+ "d_kernel": "Ядро",
+ "d_task": "Задача ядра",
+ "d_uptime": "Время работы",
+ "empty": "Пока никто не прислал ни одного падения.",
+ "events": {
+ "few": "{count} падения",
+ "many": "{count} падений",
+ "one": "{count} падение",
+ "other": "{count} падения"
+ },
+ "f_fixed_in": "Исправлено в (версия прошивки)",
+ "f_issue": "Задача",
+ "f_merge": "Та же ошибка, что сигнатура",
+ "f_note": "Заметка",
+ "f_save": "Сохранить",
+ "f_status": "Состояние",
+ "file_label": "Лог падения из веб-интерфейса камеры (crashlog_*.tar.gz)",
+ "firmware": "прошивка {list}",
+ "how_1": "После падения веб-интерфейс камеры на каждой странице пишет «This camera recovered from a crash». Откройте отчёт о падении.",
+ "how_2": "Нажмите «Send to OpenIPC» или скачайте лог падения и отправьте его со",
+ "how_3": "Привяжите камеру к клубу на Open Wall — и падения, которые она присылает, принесут вам звёзды.",
+ "how_club": "своей страницы в клубе OpenIPC",
+ "how_privacy": "Отправляется журнал ядра о падении, версия прошивки и чип. MAC камеры и все IP-адреса в нём заменяются хешами; журналы читают только мейнтейнеры OpenIPC. Без вашего действия ничего не отправляется.",
+ "how_title": "Ваша камера упала?",
+ "in_irq": "в прерывании",
+ "issue": "Задача, где это чинят",
+ "kind_bootloop": "Циклическая перезагрузка",
+ "kind_bug": "BUG",
+ "kind_oops": "Oops",
+ "kind_panic": "Kernel panic",
+ "kind_warning": "Предупреждение",
+ "last_seen": "последнее {date}",
+ "list_label": "Падения ядра, сначала самые тяжёлые",
+ "mac_label": "MAC камеры, если она привязана на Open Wall (необязательно)",
+ "merged_into": "объединено с {id}",
+ "mine_empty": "Пока ни одного падения — ни от вас, ни от привязанных камер.",
+ "mine_lede": "Каждая ошибка, на которую попала ваша камера, приносит {report} ★ после расчёта; если вы прислали её первым — ещё {first} ★, когда мейнтейнер подтвердит, и {fixed} ★, когда её исправят. Не больше {cap} ★ в месяц за падения; падение, вызванное нарочно, ничего не приносит.",
+ "mine_title": "Мои отчёты о падениях",
+ "saved": "Сохранено.",
+ "self_inflicted": "вызвано нарочно",
+ "send": "Отправить лог падения",
+ "sent": "Получено, отнесено к",
+ "sent_again": "Это падение уже было получено:",
+ "sent_self": "Получено, но это падение вызвано нарочно (sysrq), поэтому ничего не приносит:",
+ "signature": "Сигнатура {id}",
+ "status_bogus": "Не настоящее падение",
+ "status_confirmed": "Подтверждено",
+ "status_fixed": "Исправлено",
+ "status_open": "Ещё не разобрано",
+ "status_wontfix": "Исправлено не будет",
+ "taken_back": "Сохранено; списано {n} ★.",
+ "triage_forbidden": "Разбирать падения могут только мейнтейнеры OpenIPC.",
+ "triage_link": "Разбор падений"
+ },
"details": "Подробнее",
"details_of": "Подробности: {board}",
"device_count": {
diff --git a/frontend/apps/site/src/i18n/en.json b/frontend/apps/site/src/i18n/en.json
index 038dea81..e7e4c5bb 100644
--- a/frontend/apps/site/src/i18n/en.json
+++ b/frontend/apps/site/src/i18n/en.json
@@ -162,6 +162,11 @@
"lede": "Sign in to follow what you sent to the board catalogue, keep your flash dumps private, link your cameras on the Open Wall, and collect stars for both.",
"title": "OpenIPC Club"
},
+ "club_crashes": {
+ "eyebrow": "OpenIPC Club",
+ "lede": "Kernel crashes cameras sent, worst first, with what it takes to reproduce each one.",
+ "title": "Crash triage"
+ },
"club_leaderboard": {
"eyebrow": "OpenIPC Club",
"lede": "Stars for documenting camera boards and for keeping cameras on the Open Wall, from the members who chose to be listed.",
@@ -206,6 +211,11 @@
"reach_wiki_html": "The wiki — installation, FPV and troubleshooting guides.",
"title": "Community"
},
+ "crashes": {
+ "eyebrow": "OpenIPC firmware",
+ "lede": "When a camera running OpenIPC crashes, it keeps the kernel's last words and offers them to its owner. Sent here, the same bug from many cameras becomes one row, ranked by how badly and how widely it hurts, so it can be fixed.",
+ "title": "Kernel crashes from the field"
+ },
"digital_twins": {
"bring1_text": "Your production image boots in the model — the same bytes you ship. No test build, no stubbed drivers, nothing to argue about later.",
"bring1_title": "Unmodified firmware",
diff --git a/frontend/apps/site/src/i18n/ru.json b/frontend/apps/site/src/i18n/ru.json
index 9193b6bd..dd14b61c 100644
--- a/frontend/apps/site/src/i18n/ru.json
+++ b/frontend/apps/site/src/i18n/ru.json
@@ -162,6 +162,11 @@
"lede": "Войдите, чтобы следить за тем, что вы прислали в каталог плат, держать дампы флеш-памяти закрытыми, привязать свои камеры на Open Wall и получать звёзды за то и другое.",
"title": "Клуб OpenIPC"
},
+ "club_crashes": {
+ "eyebrow": "Клуб OpenIPC",
+ "lede": "Падения ядра, присланные камерами, — сначала самые тяжёлые, с тем, что нужно, чтобы воспроизвести каждое.",
+ "title": "Разбор падений"
+ },
"club_leaderboard": {
"eyebrow": "Клуб OpenIPC",
"lede": "Звёзды за описанные платы камер и за камеры на Open Wall — у участников, которые согласились попасть в список.",
@@ -206,6 +211,11 @@
"reach_wiki_html": "Вики — гайды по установке, FPV и решению проблем.",
"title": "Сообщество"
},
+ "crashes": {
+ "eyebrow": "Прошивка OpenIPC",
+ "lede": "Когда камера с OpenIPC падает, она сохраняет последние слова ядра и предлагает их владельцу. Присланные сюда, одинаковые падения со многих камер становятся одной строкой, отсортированной по тому, насколько тяжело и насколько широко они бьют, — чтобы их можно было исправить.",
+ "title": "Падения ядра с реальных камер"
+ },
"digital_twins": {
"bring1_text": "Ваш серийный образ загружается в модели — те же байты, что вы отгружаете. Ни тестовой сборки, ни заглушенных драйверов, ни поводов для споров потом.",
"bring1_title": "Немодифицированная прошивка",
diff --git a/frontend/apps/site/src/lib/club.ts b/frontend/apps/site/src/lib/club.ts
index 9dc61728..eb06de4a 100644
--- a/frontend/apps/site/src/lib/club.ts
+++ b/frontend/apps/site/src/lib/club.ts
@@ -19,9 +19,10 @@ export interface Member {
quiet: boolean;
identities: Identity[];
stars: number;
- /** The stars' two sources: accepted reports, and cameras on the Open Wall. */
+ /** The stars' sources: accepted reports, cameras on the Open Wall, and kernel crashes. */
report_stars?: number;
wall_stars?: number;
+ crash_stars?: number;
pending: number;
}
@@ -246,7 +247,7 @@ export const showOwner = (token: string, show: boolean) => post<{ show: boolean
export const setListed = (listed: boolean) => post<{ listed: boolean }>('/listed', { listed });
-export interface Leader { rank: number; name: string; reports: number; wall: number; stars: number; you?: boolean }
+export interface Leader { rank: number; name: string; reports: number; wall: number; crashes: number; stars: number; you?: boolean }
export const fetchLeaderboard = (period: 'all' | '30d') =>
call<{ period: string; members: Leader[] }>(`/leaderboard?${new URLSearchParams({ period })}`);
diff --git a/frontend/apps/site/src/lib/crashes.ts b/frontend/apps/site/src/lib/crashes.ts
new file mode 100644
index 00000000..250e8fdd
--- /dev/null
+++ b/frontend/apps/site/src/lib/crashes.ts
@@ -0,0 +1,162 @@
+/**
+ * Kernel crashes cameras recovered from, as the service answers them
+ * (service/internal/crashes): the public list of signatures at
+ * /api/v1/crashes, and under /api/v1/club the member's own crashes and the
+ * maintainers' triage. A signature is the top of a crash's backtrace: one bug,
+ * however many cameras sent it.
+ */
+import { ClubError } from './club';
+
+export type Kind = 'bootloop' | 'panic' | 'oops' | 'bug' | 'warning';
+export type Status = 'open' | 'confirmed' | 'fixed' | 'wontfix' | 'bogus';
+
+export interface Frame { fn: string; module?: string }
+
+export interface Signature {
+ id: string;
+ class: 'fatal' | 'warning';
+ kind: Kind;
+ title: string;
+ frames: Frame[];
+ status: Status;
+ fixed_in?: string;
+ issue_url?: string;
+ in_irq: boolean;
+ events: number;
+ cameras: number;
+ socs: string[];
+ sensors: string[];
+ firmware: string[];
+ first_seen: string;
+ last_seen: string;
+ /** Seen on a kernel built within two weeks of the newest firmware. */
+ current: boolean;
+ score: number;
+ merged_into?: string;
+ note?: string;
+}
+
+export interface Combo {
+ soc: string;
+ sensor: string;
+ kernel?: string;
+ kernel_built?: string;
+ firmware?: string;
+ majestic?: string;
+ events: number;
+ cameras: number;
+}
+
+export interface Trace {
+ kind: Kind;
+ reason: string;
+ comm?: string;
+ pc?: string;
+ in_irq?: boolean;
+ frames: Frame[];
+ interrupted?: Frame[];
+}
+
+/** One crash, as a maintainer reads it to reproduce it. */
+export interface Detail {
+ id: string;
+ signature: string;
+ received_at: string;
+ channel: string;
+ kind: Kind;
+ self_inflicted: boolean;
+ firmware: string;
+ majestic: string;
+ soc: string;
+ sensor: string;
+ board: string;
+ machine: string;
+ kernel: string;
+ kernel_build: string;
+ kernel_built?: string;
+ cmdline: string;
+ uptime?: number;
+ modules: string[];
+ fatal: Trace;
+ before: Trace[];
+ anomalies: Record;
+ leadup: string[];
+ meta?: unknown;
+ log?: string;
+ builds?: { id: string; release: string; sha: string; built_at: string }[];
+}
+
+export interface Mine {
+ id: string;
+ signature: string;
+ title: string;
+ kind: Kind;
+ status: Status;
+ soc: string;
+ sensor: string;
+ received_at: string;
+ self_inflicted: boolean;
+ camera: boolean;
+}
+
+export interface Rules { report: number; first: number; fixed: number; month_cap: number }
+
+/** What a sent crash answers. */
+export interface Sent {
+ id: string;
+ signature: string;
+ title: string;
+ kind: Kind;
+ duplicate: boolean;
+ self_inflicted: boolean;
+ url: string;
+}
+
+async function call(url: string, init?: RequestInit): Promise {
+ const r = await fetch(url, { credentials: 'same-origin', ...init, headers: { Accept: 'application/json', ...(init?.headers ?? {}) } });
+ let body: unknown = null;
+ try { body = await r.json(); } catch { /* not JSON */ }
+ if (!r.ok) throw new ClubError(r.status, (body as { error?: string } | null)?.error ?? `HTTP ${r.status}`);
+ return body as T;
+}
+
+export const fetchSignatures = () => call<{ signatures: Signature[] }>('/api/v1/crashes');
+
+export const fetchSignature = (id: string) =>
+ call<{ signature: Signature; seen_on: Combo[] }>(`/api/v1/crashes/${encodeURIComponent(id)}`);
+
+export const fetchMyCrashes = () => call<{ crashes: Mine[]; stars: number; rules: Rules }>('/api/v1/club/crashes');
+
+/** The member's upload of the crash log their WebUI downloaded. */
+export const sendCrash = (form: FormData) => call('/api/v1/club/crashes', { method: 'POST', body: form });
+
+export const fetchTriage = () => call<{ signatures: Signature[] }>('/api/v1/club/crashes/triage');
+
+export const fetchCrashDetail = (id: string) =>
+ call<{ signature: Signature; seen_on: Combo[]; crashes: Detail[] }>(`/api/v1/club/crashes/${encodeURIComponent(id)}`);
+
+export interface Triage { status: Status; fixed_in?: string; issue_url?: string; merge_into?: string; note?: string }
+
+export const decideCrash = (id: string, t: Triage) =>
+ call<{ status: Status; stars_taken_back: number }>(`/api/v1/club/crashes/${encodeURIComponent(id)}`, {
+ method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(t),
+ });
+
+/** A frame as a backtrace prints it: fn [module]. */
+export const frame = (f: Frame) => (f.module ? `${f.fn} [${f.module}]` : f.fn);
+
+export const KIND_TONE: Record = {
+ bootloop: 'bg-[#fbe9ea] text-[#a3262e]',
+ panic: 'bg-[#fbe9ea] text-[#a3262e]',
+ oops: 'bg-[#fff4e2] text-[#8a4b00]',
+ bug: 'bg-[#fff4e2] text-[#8a4b00]',
+ warning: 'bg-surface-alt text-body-secondary',
+};
+
+export const STATUS_TONE: Record = {
+ open: 'border border-dashed border-hairline bg-surface-alt text-body-secondary',
+ confirmed: 'bg-[#eef0fc] text-[#3d4dad]',
+ fixed: 'bg-[#e3f5ec] text-[#146c3c]',
+ wontfix: 'bg-surface-alt text-body-secondary',
+ bogus: 'bg-[#fbe9ea] text-[#a3262e]',
+};
diff --git a/frontend/apps/site/src/lib/page-paths.ts b/frontend/apps/site/src/lib/page-paths.ts
index f4645121..b942fba7 100644
--- a/frontend/apps/site/src/lib/page-paths.ts
+++ b/frontend/apps/site/src/lib/page-paths.ts
@@ -109,6 +109,10 @@ export const PAGE_PATHS: PagePath[] = [
{ path: '/club/review', titleKey: 'pages.club_review.title', descriptionKey: 'pages.club_review.lede' },
// The leaderboard: members who chose to be listed, by stars (service/internal/wallstars).
{ path: '/club/leaderboard', titleKey: 'pages.club_leaderboard.title', descriptionKey: 'pages.club_leaderboard.lede' },
+ // The maintainers' triage of kernel crashes; a shell, as the review queue is.
+ { path: '/club/crashes', titleKey: 'pages.club_crashes.title', descriptionKey: 'pages.club_crashes.lede' },
+ // Kernel crashes cameras recovered from, one row per bug (service/internal/crashes).
+ { path: '/crashes', titleKey: 'pages.crashes.title', descriptionKey: 'pages.crashes.lede' },
{ path: '/business', titleKey: 'pages.business.title' },
{ path: '/community', titleKey: 'pages.community.title' },
diff --git a/frontend/apps/site/src/lib/pages.ts b/frontend/apps/site/src/lib/pages.ts
index 0a97cd63..085696c8 100644
--- a/frontend/apps/site/src/lib/pages.ts
+++ b/frontend/apps/site/src/lib/pages.ts
@@ -22,6 +22,8 @@ import Report from '../components/pages/Report.astro';
import Club from '../components/pages/Club.astro';
import ClubReview from '../components/pages/ClubReview.astro';
import ClubLeaderboard from '../components/pages/ClubLeaderboard.astro';
+import ClubCrashes from '../components/pages/ClubCrashes.astro';
+import Crashes from '../components/pages/Crashes.astro';
import Business from '../components/pages/Business.astro';
import Community from '../components/pages/Community.astro';
import Donate from '../components/pages/Donate.astro';
@@ -99,6 +101,8 @@ const COMPONENTS: Record = {
'/club': { component: Club },
'/club/review': { component: ClubReview },
'/club/leaderboard': { component: ClubLeaderboard },
+ '/club/crashes': { component: ClubCrashes },
+ '/crashes': { component: Crashes },
'/business': { component: Business },
'/community': { component: Community },
diff --git a/service/README.md b/service/README.md
index c3c6dc2a..7facccab 100644
--- a/service/README.md
+++ b/service/README.md
@@ -24,12 +24,13 @@ key the wall JSON signs them with.
|---|---|
| `serve --role web\|firmware` | the two HTTP processes above |
| `migrate` | applies the embedded SQL migrations, under an advisory lock |
-| `purge [--snapshots] [--firmware]` | snapshots past two days with their images, orphan wall directories, and firmware of any release but the current one; with `--snapshots`, the Open Wall's stars are settled afterwards |
+| `purge [--snapshots] [--firmware]` | snapshots past two days with their images, orphan wall directories, and firmware of any release but the current one; with `--snapshots`, the Open Wall's and the crashes' stars are settled afterwards |
| `probe` | the numbers only a probe sees: all-digit `public_id`s, HEIF uploads, a stuck variant queue |
| `builds import-history [--keep 90] [--kconfig-all] [--skip-builder]` | once per environment: the builds GitHub still holds, into PostgreSQL |
| `boards import-openhisiipcam [--from ]` | once per environment: the OpenHisiIpCam board archive (firmware#659, pinned commit) into the board catalogue, its files under `BOARDS_ROOT`; a second run adds nothing. Run it in the web container, which mounts that directory |
| `reports list\|show\|publish\|reject\|link\|unlink\|takedown\|verify` | the owner reports' review queue (`internal/reports`): nothing a camera owner or an agent sends is public until `publish`; `takedown` is the one way a report's content is ever removed; `verify` re-hashes every stored file and runs nightly |
| `club settle-wall\|wall-revoke --reason text\|wall-unlink ` | the Open Wall's stars (`internal/wallstars`): `settle-wall` pays linked cameras what they earned and tells their owners through the bot (the nightly purge runs it too); `wall-revoke` takes a faked camera's stars back, by its public name or MAC; `wall-unlink` frees a camera someone else linked first |
+| `crashes list\|show \|status \|settle\|takedown ` | kernel crashes cameras sent (`internal/crashes`, `CRASH.md`): `list` ranks every signature, `show` is what reproducing one takes (where it was seen, the matching builds, the redacted logs), `status` is the maintainers' triage (also on `/club/crashes`; `bogus` takes its stars back), `settle` pays the crash stars (the nightly purge runs it too) |
| `serve --role nfs` | ipctool's builds from `TOOLS_ROOT`, read-only over NFS (portmapper :111, MOUNT and NFS :2049, UDP and TCP), for cameras on stock firmware; no database |
| `routes --json` | the routes table |
| `version` | the commit the binary was built from |
diff --git a/service/cmd/openipc/club.go b/service/cmd/openipc/club.go
index 86391937..5b9e23fc 100644
--- a/service/cmd/openipc/club.go
+++ b/service/cmd/openipc/club.go
@@ -15,6 +15,7 @@ import (
"github.com/OpenIPC/website/service/internal/boards"
"github.com/OpenIPC/website/service/internal/club"
"github.com/OpenIPC/website/service/internal/config"
+ "github.com/OpenIPC/website/service/internal/crashes"
"github.com/OpenIPC/website/service/internal/reports"
"github.com/OpenIPC/website/service/internal/snapshots"
"github.com/OpenIPC/website/service/internal/wallstars"
@@ -152,7 +153,7 @@ func clubCommand(ctx context.Context, cfg *config.Config, log *slog.Logger, args
func settleWall(ctx context.Context, cfg *config.Config, log *slog.Logger, pool *pgxpool.Pool) error {
st := &wallstars.Store{DB: pool, Token: (&snapshots.Store{TokenKey: cfg.CameraTokenKey}).CameraToken}
api := &club.API{DB: pool, Log: log, Reports: &reports.API{DB: pool}, Wall: st,
- Cfg: club.Config{SiteURL: cfg.ClubSiteURL}}
+ Crashes: &crashes.API{DB: pool, Log: log}, Cfg: club.Config{SiteURL: cfg.ClubSiteURL}}
if cfg.TelegramBotToken != "" {
api.Telegram = &club.Telegram{Token: cfg.TelegramBotToken, API: "https://api.telegram.org",
HTTP: &http.Client{Timeout: 20 * time.Second}, Log: log}
diff --git a/service/cmd/openipc/crashes.go b/service/cmd/openipc/crashes.go
new file mode 100644
index 00000000..e768f807
--- /dev/null
+++ b/service/cmd/openipc/crashes.go
@@ -0,0 +1,124 @@
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "flag"
+ "log/slog"
+ "os"
+ "time"
+
+ "github.com/jackc/pgx/v5/pgxpool"
+
+ "github.com/OpenIPC/website/service/internal/config"
+ "github.com/OpenIPC/website/service/internal/crashes"
+)
+
+const crashesUsage = `usage: openipc crashes
+ list every signature, worst first, bogus and merged too (JSON)
+ show where it was seen, on which builds, and its crashes' redacted logs (JSON)
+ status open|confirmed|fixed|wontfix|bogus [--fixed-in v] [--issue url] [--merge-into sig] [--by who] [--note text]
+ triage; a flag left out keeps its value, --merge-into '' unmerges;
+ bogus takes back every star the bug paid, any other status lets them be paid again
+ settle pay what crashes earned (also run by purge)
+ takedown delete one crash and, when nothing else came in it, its bundle`
+
+// crashesCommand is `openipc crashes ...`, run in the web container; the
+// same triage is on /club for maintainers.
+func crashesCommand(ctx context.Context, cfg *config.Config, log *slog.Logger, args []string) error {
+ if len(args) == 0 {
+ return errors.New(crashesUsage)
+ }
+ pool, err := open(ctx, cfg)
+ if err != nil {
+ return err
+ }
+ defer pool.Close()
+ st := &crashes.Store{DB: pool}
+
+ fs := flag.NewFlagSet("crashes "+args[0], flag.ExitOnError)
+ by := fs.String("by", whoami(), "who decided")
+ note := fs.String("note", "", "why, for the record")
+ fixedIn := fs.String("fixed-in", "", "the firmware version the fix is in")
+ issue := fs.String("issue", "", "the issue tracking it (https://)")
+ mergeInto := fs.String("merge-into", "", "the signature this one is the same bug as")
+ pos, err := parseInterleaved(fs, args[1:])
+ if err != nil {
+ return err
+ }
+ out := json.NewEncoder(os.Stdout)
+ out.SetIndent("", " ")
+ now := time.Now()
+ switch args[0] {
+ case "list":
+ list, err := st.Ranked(ctx, true, now)
+ if err != nil {
+ return err
+ }
+ return out.Encode(list)
+ case "show":
+ if len(pos) != 1 {
+ return errors.New(crashesUsage)
+ }
+ g, err := st.Get(ctx, pos[0], true, now)
+ if err != nil {
+ return err
+ }
+ combos, err := st.Combos(ctx, pos[0], true)
+ if err != nil {
+ return err
+ }
+ details, err := st.Details(ctx, pos[0])
+ if err != nil {
+ return err
+ }
+ return out.Encode(map[string]any{"signature": g, "seen_on": combos, "crashes": details})
+ case "status":
+ if len(pos) != 2 {
+ return errors.New(crashesUsage)
+ }
+ // Only the flags given change the signature; --merge-into '' unmerges.
+ t := crashes.Triage{Status: pos[1]}
+ fs.Visit(func(f *flag.Flag) {
+ switch f.Name {
+ case "fixed-in":
+ t.FixedIn = fixedIn
+ case "issue":
+ t.IssueURL = issue
+ case "merge-into":
+ t.MergeInto = mergeInto
+ case "note":
+ t.Note = note
+ }
+ })
+ taken, err := st.Decide(ctx, pos[0], *by, t)
+ if err != nil {
+ return err
+ }
+ log.Info("crashes: triaged", "signature", pos[0], "status", pos[1], "by", *by, "stars_taken_back", taken)
+ return nil
+ case "settle":
+ return settleCrashes(ctx, log, pool)
+ case "takedown":
+ if len(pos) != 1 {
+ return errors.New(crashesUsage)
+ }
+ if err := st.Takedown(ctx, pos[0]); err != nil {
+ return err
+ }
+ log.Info("crashes: taken down", "id", pos[0], "by", *by)
+ return nil
+ }
+ return errors.New(crashesUsage)
+}
+
+// settleCrashes is the nightly settlement of the crashes' stars.
+func settleCrashes(ctx context.Context, log *slog.Logger, pool *pgxpool.Pool) error {
+ res, err := (&crashes.Store{DB: pool}).Settle(ctx, time.Now())
+ if err != nil {
+ return err
+ }
+ log.Info("crashes: settled", "members", res.Members, "stars", res.Awarded, "held_over_cap", res.Held)
+ return nil
+}
diff --git a/service/cmd/openipc/main.go b/service/cmd/openipc/main.go
index 0cac7c56..93d07b47 100644
--- a/service/cmd/openipc/main.go
+++ b/service/cmd/openipc/main.go
@@ -40,6 +40,7 @@ import (
"github.com/OpenIPC/website/service/internal/catalogue"
"github.com/OpenIPC/website/service/internal/club"
"github.com/OpenIPC/website/service/internal/config"
+ "github.com/OpenIPC/website/service/internal/crashes"
"github.com/OpenIPC/website/service/internal/db"
"github.com/OpenIPC/website/service/internal/downloads"
"github.com/OpenIPC/website/service/internal/drift"
@@ -93,6 +94,8 @@ func main() {
err = reportsCommand(ctx, cfg, log, args)
case "club":
err = clubCommand(ctx, cfg, log, args)
+ case "crashes":
+ err = crashesCommand(ctx, cfg, log, args)
case "routes":
err = printRoutes()
case "version":
@@ -106,7 +109,7 @@ func main() {
}
func usage() {
- fmt.Fprintln(os.Stderr, "usage: openipc serve --role web|firmware | migrate | purge [--snapshots] [--firmware] [--builds] | probe | builds import-history|import-uboot | boards import-openhisiipcam | boards import-snapshot | reports list|show|publish|reject|link|unlink|takedown|verify | club settle-wall|wall-revoke|wall-unlink | vendor-firmware import-history | routes --json | version")
+ fmt.Fprintln(os.Stderr, "usage: openipc serve --role web|firmware | migrate | purge [--snapshots] [--firmware] [--builds] | probe | builds import-history|import-uboot | boards import-openhisiipcam | boards import-snapshot | reports list|show|publish|reject|link|unlink|takedown|verify | club settle-wall|wall-revoke|wall-unlink | crashes list|show|status|settle|takedown | vendor-firmware import-history | routes --json | version")
os.Exit(2)
}
@@ -205,6 +208,14 @@ var routes = []Route{
{"web", "POST", "/api/v1/club/cameras/{camera}/owner"},
{"web", "POST", "/api/v1/club/listed"},
{"web", "GET", "/api/v1/club/leaderboard"},
+ {"web", "POST", "/api/v1/club/crashes"},
+ {"web", "GET", "/api/v1/club/crashes"},
+ {"web", "GET", "/api/v1/club/crashes/triage"},
+ {"web", "GET", "/api/v1/club/crashes/{id}"},
+ {"web", "POST", "/api/v1/club/crashes/{id}"},
+ {"web", "POST", "/api/v1/crashes"},
+ {"web", "GET", "/api/v1/crashes"},
+ {"web", "GET", "/api/v1/crashes/{id}"},
{"web", "PUT", "/api/v1/tools/{name}"},
{"web", "GET", "/api/v1/tools"},
{"share", "GET", "/up"},
@@ -382,6 +393,10 @@ func web(ctx context.Context, cfg *config.Config, log *slog.Logger, pool *pgxpoo
// frame may link a camera and the club tells its member.
wallStars := &wallstars.Store{DB: pool, Token: store.CameraToken}
clubAPI := newClub(bg, cfg, log, pool, ownerReports, wallStars)
+ // Kernel crashes cameras recovered from (internal/crashes): sent by the
+ // WebUI or from /club, triaged by the maintainers in the club.
+ crashAPI := &crashes.API{DB: pool, Log: log, SiteURL: cfg.ClubSiteURL}
+ clubAPI.Crashes = crashAPI
proc := &variants.Processor{Wall: wallFS, Store: store, Log: log, FFmpeg: cfg.FFmpegBin,
Workers: cfg.VariantWorkers,
Published: func(ctx context.Context, id string) {
@@ -485,6 +500,9 @@ func web(ctx context.Context, cfg *config.Config, log *slog.Logger, pool *pgxpoo
for k, h := range clubAPI.Handlers() {
handlers[k] = h
}
+ for k, h := range crashAPI.Handlers() {
+ handlers[k] = h
+ }
// ipctool's builds, pushed by its release job (tools/PUSH.md).
for k, h := range (&tools.API{Verifier: &builds.LazyVerifier{Issuer: builds.GitHubIssuer},
DB: pool, Root: cfg.ToolsRoot, Log: log}).Handlers() {
@@ -679,6 +697,10 @@ func runPurge(ctx context.Context, cfg *config.Config, log *slog.Logger, args []
if err := settleWall(ctx, cfg, log, pool); err != nil {
return err
}
+ // And what the crashes cameras sent have earned (internal/crashes).
+ if err := settleCrashes(ctx, log, pool); err != nil {
+ return err
+ }
}
return nil
}
diff --git a/service/internal/club/api.go b/service/internal/club/api.go
index c13f94e1..65944dd7 100644
--- a/service/internal/club/api.go
+++ b/service/internal/club/api.go
@@ -15,6 +15,7 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"github.com/OpenIPC/website/service/internal/boards"
+ "github.com/OpenIPC/website/service/internal/crashes"
"github.com/OpenIPC/website/service/internal/reports"
"github.com/OpenIPC/website/service/internal/wallstars"
)
@@ -53,7 +54,9 @@ type API struct {
Cfg Config
Reports *reports.API
// Wall is the members' cameras on the Open Wall and their stars.
- Wall *wallstars.Store
+ Wall *wallstars.Store
+ // Crashes is the kernel crashes cameras sent, and their stars.
+ Crashes *crashes.API
Telegram *Telegram
GitHub *GitHub
Mail Mailer
@@ -98,6 +101,11 @@ func (a *API) Handlers() map[string]http.Handler {
h[k] = v
}
}
+ if a.Crashes != nil {
+ for k, v := range a.crashHandlers() {
+ h[k] = v
+ }
+ }
return h
}
diff --git a/service/internal/club/crashes.go b/service/internal/club/crashes.go
new file mode 100644
index 00000000..abac3849
--- /dev/null
+++ b/service/internal/club/crashes.go
@@ -0,0 +1,149 @@
+package club
+
+import (
+ "encoding/json"
+ "errors"
+ "net/http"
+
+ "github.com/OpenIPC/website/service/internal/crashes"
+)
+
+// Kernel crashes cameras recovered from (internal/crashes): a member sending
+// the crash log their WebUI downloaded, their own crashes and their stars,
+// and the maintainers' triage. The storing, ranking and settling are the
+// crashes package's; the public list is its own API.
+
+func (a *API) crashHandlers() map[string]http.Handler {
+ return map[string]http.Handler{
+ "POST /api/v1/club/crashes": a.post(a.sendCrash),
+ "GET /api/v1/club/crashes": http.HandlerFunc(a.myCrashes),
+ "GET /api/v1/club/crashes/triage": http.HandlerFunc(a.crashQueue),
+ "GET /api/v1/club/crashes/{id}": http.HandlerFunc(a.crashDetail),
+ "POST /api/v1/club/crashes/{id}": a.post(a.decideCrash),
+ }
+}
+
+// sendCrash is POST /api/v1/club/crashes: the member's own upload of a
+// crash log, as the WebUI's Download button gave it. Signing in is what
+// makes it theirs, so it is required.
+func (a *API) sendCrash(w http.ResponseWriter, r *http.Request) {
+ m, ok := a.signedIn(w, r)
+ if !ok {
+ return
+ }
+ a.Crashes.Submit(w, r, m.ID, "club")
+}
+
+// myCrashes is GET /api/v1/club/crashes: the crashes the member sent or
+// their linked cameras did, and the stars they earned.
+func (a *API) myCrashes(w http.ResponseWriter, r *http.Request) {
+ m, ok := a.signedIn(w, r)
+ if !ok {
+ return
+ }
+ st := a.Crashes.Store()
+ list, err := st.Mine(r.Context(), m.ID)
+ if err != nil {
+ a.fail(w, "the crashes", err)
+ return
+ }
+ stars, err := st.StarsOf(r.Context(), m.ID)
+ if err != nil {
+ a.fail(w, "the stars", err)
+ return
+ }
+ if list == nil {
+ list = []crashes.Mine{}
+ }
+ writeJSON(w, http.StatusOK, map[string]any{"crashes": list, "stars": stars,
+ "rules": map[string]int{"report": crashes.ReportStars, "first": crashes.FirstStars, "fixed": crashes.FixedStars, "month_cap": crashes.MonthCap}})
+}
+
+func (a *API) maintainer(w http.ResponseWriter, r *http.Request) (*Member, bool) {
+ m, ok := a.signedIn(w, r)
+ if !ok {
+ return nil, false
+ }
+ if !m.Maintainer {
+ a.refuse(w, http.StatusForbidden, "only OpenIPC's maintainers triage crashes")
+ return nil, false
+ }
+ return m, true
+}
+
+// crashQueue is GET /api/v1/club/crashes/triage: every signature, worst
+// first, the bogus and merged ones too.
+func (a *API) crashQueue(w http.ResponseWriter, r *http.Request) {
+ if _, ok := a.maintainer(w, r); !ok {
+ return
+ }
+ list, err := a.Crashes.Store().Ranked(r.Context(), true, a.now())
+ if err != nil {
+ a.fail(w, "the crashes", err)
+ return
+ }
+ if list == nil {
+ list = []*crashes.Signature{}
+ }
+ writeJSON(w, http.StatusOK, map[string]any{"signatures": list})
+}
+
+// crashDetail is GET /api/v1/club/crashes/{id}: what it takes to reproduce
+// a signature -- where it was seen, on which builds, and its crashes with
+// their redacted logs.
+func (a *API) crashDetail(w http.ResponseWriter, r *http.Request) {
+ if _, ok := a.maintainer(w, r); !ok {
+ return
+ }
+ st := a.Crashes.Store()
+ id := r.PathValue("id")
+ g, err := st.Get(r.Context(), id, true, a.now())
+ if errors.Is(err, crashes.ErrNoSignature) {
+ a.refuse(w, http.StatusNotFound, err.Error())
+ return
+ }
+ if err != nil {
+ a.fail(w, "the crash", err)
+ return
+ }
+ combos, err := st.Combos(r.Context(), id, true)
+ if err != nil {
+ a.fail(w, "the crash", err)
+ return
+ }
+ details, err := st.Details(r.Context(), id)
+ if err != nil {
+ a.fail(w, "the crash", err)
+ return
+ }
+ writeJSON(w, http.StatusOK, map[string]any{"signature": g, "seen_on": combos, "crashes": details})
+}
+
+// decideCrash is POST /api/v1/club/crashes/{id} {"status", "fixed_in",
+// "issue_url", "merge_into", "note"}: a maintainer's triage. Marking it
+// bogus takes back every star it paid.
+func (a *API) decideCrash(w http.ResponseWriter, r *http.Request) {
+ m, ok := a.maintainer(w, r)
+ if !ok {
+ return
+ }
+ var t crashes.Triage
+ if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 16<<10)).Decode(&t); err != nil {
+ a.refuse(w, http.StatusBadRequest, "send JSON: status, fixed_in, issue_url, merge_into, note")
+ return
+ }
+ taken, err := a.Crashes.Store().Decide(r.Context(), r.PathValue("id"), m.ID, t)
+ if errors.Is(err, crashes.ErrNoSignature) {
+ a.refuse(w, http.StatusNotFound, err.Error())
+ return
+ }
+ if errors.Is(err, crashes.ErrInvalid) {
+ a.refuse(w, http.StatusBadRequest, err.Error())
+ return
+ }
+ if err != nil {
+ a.fail(w, "the triage", err)
+ return
+ }
+ writeJSON(w, http.StatusOK, map[string]any{"status": t.Status, "stars_taken_back": taken})
+}
diff --git a/service/internal/club/crashes_test.go b/service/internal/club/crashes_test.go
new file mode 100644
index 00000000..da42db68
--- /dev/null
+++ b/service/internal/club/crashes_test.go
@@ -0,0 +1,108 @@
+package club
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "mime/multipart"
+ "net/http"
+ "os"
+ "strings"
+ "testing"
+
+ "github.com/OpenIPC/website/service/internal/crashes"
+)
+
+// withCrashes gives the env's club the crashes, as main does.
+func (e *env) withCrashes(t *testing.T) {
+ t.Helper()
+ e.api.Crashes = &crashes.API{DB: e.pool, Log: quiet(), SiteURL: site}
+ e.mux = http.NewServeMux()
+ for k, h := range e.api.Handlers() {
+ e.mux.Handle(k, h)
+ }
+}
+
+func (b *browser) sendCrash(t *testing.T, bundle []byte, fields ...string) (int, map[string]any) {
+ t.Helper()
+ var body bytes.Buffer
+ mw := multipart.NewWriter(&body)
+ for i := 0; i+1 < len(fields); i += 2 {
+ _ = mw.WriteField(fields[i], fields[i+1])
+ }
+ w, _ := mw.CreateFormFile("bundle", "crashlog.tar.gz")
+ _, _ = w.Write(bundle)
+ _ = mw.Close()
+ rec := b.do(t, "POST", "/api/v1/club/crashes", &body, mw.FormDataContentType())
+ var out map[string]any
+ _ = json.Unmarshal(rec.Body.Bytes(), &out)
+ return rec.Code, out
+}
+
+// A member sends the crash log their WebUI downloaded; it is theirs, and
+// pays a star once settled. A maintainer triages it; nobody else can.
+func TestAMemberSendsACrashAndAMaintainerTriagesIt(t *testing.T) {
+ e := newEnv(t)
+ e.withWall(t) // the leaderboard is the wall's
+ e.withCrashes(t)
+ ctx := context.Background()
+ lab, err := os.ReadFile("../crashes/testdata/gk7205v300-imx335-rgn.tar.gz")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if code, _ := e.browser("203.0.113.9").sendCrash(t, lab); code != http.StatusUnauthorized {
+ t.Fatalf("anonymous send from /club: %d", code)
+ }
+ ivan := e.signedIn(t, 6001, "ivan", "203.0.113.5")
+ // A camera that is not his: its crashes pay its owner, so he cannot name it.
+ if code, out := ivan.sendCrash(t, lab, "mac", "02:00:00:00:0a:99"); code != http.StatusBadRequest || !strings.Contains(out["error"].(string), "not linked to you") {
+ t.Fatalf("someone else's camera: %d %v", code, out)
+ }
+ code, out := ivan.sendCrash(t, lab)
+ if code != http.StatusCreated {
+ t.Fatalf("send: %d %v", code, out)
+ }
+ sig := out["signature"].(string)
+ _, mine := ivan.json(t, "GET", "/api/v1/club/crashes", nil)
+ if list := mine["crashes"].([]any); len(list) != 1 || list[0].(map[string]any)["signature"] != sig {
+ t.Fatalf("mine: %v", mine)
+ }
+ if _, err := e.api.Crashes.Store().Settle(ctx, e.clock); err != nil {
+ t.Fatal(err)
+ }
+ if m := ivan.me(t); m["crash_stars"].(float64) != crashes.ReportStars || m["stars"].(float64) != crashes.ReportStars {
+ t.Fatalf("stars: %v", m)
+ }
+
+ if code, _ := ivan.json(t, "GET", "/api/v1/club/crashes/triage", nil); code != http.StatusForbidden {
+ t.Fatalf("a member saw the triage: %d", code)
+ }
+ if code, _ := ivan.json(t, "POST", "/api/v1/club/crashes/"+sig, map[string]string{"status": "fixed"}); code != http.StatusForbidden {
+ t.Fatalf("a member triaged: %d", code)
+ }
+ maint := e.signedIn(t, 6002, "maint", "203.0.113.6")
+ e.api.Cfg.Maintainers = []string{maint.me(t)["id"].(string)}
+ code, detail := maint.json(t, "GET", "/api/v1/club/crashes/"+sig, nil)
+ if code != 200 {
+ t.Fatalf("detail: %d %v", code, detail)
+ }
+ logs := detail["crashes"].([]any)
+ if len(logs) != 1 || !strings.Contains(logs[0].(map[string]any)["log"].(string), "RGN_PutRegion") {
+ t.Fatalf("detail crashes: %v", logs)
+ }
+ if code, out := maint.json(t, "POST", "/api/v1/club/crashes/"+sig, map[string]string{"status": "sorted"}); code != http.StatusBadRequest {
+ t.Fatalf("a bad status: %d %v", code, out)
+ }
+ if code, out := maint.json(t, "POST", "/api/v1/club/crashes/"+sig,
+ map[string]string{"status": "confirmed", "issue_url": "https://github.com/OpenIPC/firmware/issues/1"}); code != 200 {
+ t.Fatalf("triage: %d %v", code, out)
+ }
+ if _, err := e.api.Crashes.Store().Settle(ctx, e.clock); err != nil {
+ t.Fatal(err)
+ }
+ _, board := e.browser("203.0.113.8").json(t, "GET", "/api/v1/club/leaderboard", nil)
+ rows := board["members"].([]any)
+ if len(rows) != 1 || rows[0].(map[string]any)["crashes"].(float64) != crashes.ReportStars+crashes.FirstStars {
+ t.Fatalf("leaderboard: %v", rows)
+ }
+}
diff --git a/service/internal/club/members.go b/service/internal/club/members.go
index 59d6fca5..4204a4ef 100644
--- a/service/internal/club/members.go
+++ b/service/internal/club/members.go
@@ -51,6 +51,7 @@ type Member struct {
// The stars' two sources: accepted reports, and cameras on the Open Wall.
ReportStars int `json:"report_stars"`
WallStars int `json:"wall_stars"`
+ CrashStars int `json:"crash_stars"`
// Pending is filled where the member's reports are read anyway
// (GET /api/v1/club/reports); /me leaves it 0.
Pending int `json:"pending"`
@@ -190,7 +191,12 @@ func (a *API) member(ctx context.Context, id string) (*Member, error) {
return nil, err
}
}
- m.Stars = m.ReportStars + m.WallStars
+ if a.Crashes != nil {
+ if m.CrashStars, err = a.Crashes.Store().StarsOf(ctx, id); err != nil {
+ return nil, err
+ }
+ }
+ m.Stars = m.ReportStars + m.WallStars + m.CrashStars
return m, nil
}
diff --git a/service/internal/club/wall.go b/service/internal/club/wall.go
index 81f38fe1..fa795013 100644
--- a/service/internal/club/wall.go
+++ b/service/internal/club/wall.go
@@ -228,7 +228,7 @@ func cameraName(loc, name string) string {
return "“" + html.EscapeString(name) + "”"
}
-// totalStars is a member's stars from both ledgers.
+// totalStars is a member's stars from every ledger.
func (a *API) totalStars(ctx context.Context, member string) (int, error) {
reports, err := a.Reports.Store().StarsOf(ctx, member)
if err != nil {
@@ -238,5 +238,9 @@ func (a *API) totalStars(ctx context.Context, member string) (int, error) {
return reports, nil
}
wall, err := a.Wall.StarsOf(ctx, member)
- return reports + wall, err
+ if err != nil || a.Crashes == nil {
+ return reports + wall, err
+ }
+ crash, err := a.Crashes.Store().StarsOf(ctx, member)
+ return reports + wall + crash, err
}
diff --git a/service/internal/crashes/CRASH.md b/service/internal/crashes/CRASH.md
new file mode 100644
index 00000000..467f920b
--- /dev/null
+++ b/service/internal/crashes/CRASH.md
@@ -0,0 +1,69 @@
+# Sending a kernel crash to openipc.org
+
+The contract between a camera (the WebUI's crash report page, or the
+firmware itself) and `POST https://openipc.org/api/v1/crashes`.
+
+## What a camera has
+
+After a kernel oops or panic, pstore keeps the kernel's log across the
+reboot. On the next boot `S98crashlog` packs `/sys/fs/pstore/dmesg-*` into
+`/etc/crash/crash.tar.gz`, writes `/etc/crash/pending`, and the WebUI shows
+"This camera recovered from a crash" with Download and Dismiss. A boot loop
+that never reached a record leaves `/etc/crash/failsafe` (`reason=bootlimit`)
+instead; include it in the tar as `failsafe` and it is filed as a boot loop.
+
+## The request
+
+`multipart/form-data`, or the bundle alone as the body.
+
+| part | | |
+|---|---|---|
+| `bundle` | required | the tar.gz as `S98crashlog` made it (a plain tar, or one record as text, also works); at most 256 KB, 1 MB unpacked |
+| `mac` | optional | the camera's MAC, as the Open Wall upload sends it. A crash with a MAC is the camera's linked owner's, for stars, whoever sent it; a member sending from `/club` may name only a camera linked to them |
+| `firmware` | optional | `/etc/os-release`'s version, e.g. `2.6.10.05-lite` |
+| `majestic` | optional | `majestic -v` |
+| `soc`, `sensor` | optional | as `ipcinfo` says; the log's own `sensor=… chip=…` line is used when absent |
+| `meta` | optional | JSON, at most 64 KB: anything else that helps reproduce it -- `/proc/cmdline`, `lsmod`, the flash size, uptime, and `majestic.yaml` **with passwords, keys, Wi-Fi and URLs removed** |
+
+```sh
+curl -sS -L --post301 --max-time 30 \
+ -F bundle=@/etc/crash/crash.tar.gz -F mac="$(cat /sys/class/net/eth0/address)" \
+ -F firmware=2.6.10.05-lite -F majestic="$(majestic -v 2>/dev/null | head -1)" \
+ https://openipc.org/api/v1/crashes
+```
+
+`-L --post301`: a camera in a network the TSPU blocks is sent to openipc.ru
+with a 301, and curl repeats a POST across a 301 only when told to.
+
+## The answer
+
+`201` for a new crash, `200` for one this camera already sent (the same
+records, however packed), both:
+
+```json
+{"id": "c-…", "signature": "6b1f0c2a9e44", "title": "NULL pointer dereference in __wake_up_common ← RGN_PutRegion [open_rgn]",
+ "kind": "panic", "duplicate": false, "self_inflicted": false, "url": "https://openipc.org/crashes/#6b1f0c2a9e44"}
+```
+
+Show the title and the link; write them to `/etc/crash/sent` so the page can
+say it was sent. `422` is a bundle with no crash in it, `413` one too large,
+`429` past the limits (20 a day from one address, 5 from one camera), with
+`Retry-After`.
+
+## What happens to it
+
+The records are parsed for the fatal trace, the warnings before it, the
+kernel's build, the command line, the SoC and sensor, and lines that often
+come before a crash (i2c aborts, failed MMZ allocations). Every MAC and IP
+address in the log and in `meta` is replaced by a keyed hash; only OpenIPC's
+maintainers read that copy. The bundle as sent is kept and never served.
+Public: the signature, the chips and sensors it was seen on, and its status.
+
+A crash asked for -- `echo c > /proc/sysrq-trigger` -- is kept but never
+listed and never pays: send real ones.
+
+## Sending automatically
+
+Only with the owner's consent: a setting the owner turns on, after which
+`S98crashlog` sends each crash it harvests. Without it, nothing leaves the
+camera until the owner presses Send.
diff --git a/service/internal/crashes/api.go b/service/internal/crashes/api.go
new file mode 100644
index 00000000..decd9bd2
--- /dev/null
+++ b/service/internal/crashes/api.go
@@ -0,0 +1,389 @@
+package crashes
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "mime"
+ "net/http"
+ "regexp"
+ "strings"
+ "time"
+
+ "github.com/jackc/pgx/v5/pgxpool"
+
+ "github.com/OpenIPC/website/service/internal/httpx"
+ "github.com/OpenIPC/website/service/internal/reports"
+ "github.com/OpenIPC/website/service/internal/snapshots"
+)
+
+// Limits on sending.
+const (
+ DailyPerClient = 20 // crashes per client address per day
+ DailyPerCamera = 5 // per camera (MAC) per day
+ maxMeta = 64 << 10
+ maxField = 200
+)
+
+// API is the crashes' public addresses on the web role. The members' and
+// the maintainers' are the club's (internal/club), under its cookie's path.
+type API struct {
+ DB *pgxpool.Pool
+ Log *slog.Logger
+ // SiteURL is where a sent crash's link points.
+ SiteURL string
+ Now func() time.Time
+}
+
+func (a *API) Handlers() map[string]http.Handler {
+ return map[string]http.Handler{
+ "POST /api/v1/crashes": http.HandlerFunc(a.upload),
+ "GET /api/v1/crashes": http.HandlerFunc(a.list),
+ "GET /api/v1/crashes/{id}": http.HandlerFunc(a.get),
+ }
+}
+
+// Store is the crashes' rows.
+func (a *API) Store() *Store { return &Store{DB: a.DB} }
+
+func (a *API) now() time.Time {
+ if a.Now != nil {
+ return a.Now()
+ }
+ return time.Now()
+}
+
+// upload is POST /api/v1/crashes, from the camera's WebUI.
+func (a *API) upload(w http.ResponseWriter, r *http.Request) {
+ a.Submit(w, r, "", "webui")
+}
+
+// Submit is a crash sent: from a camera (member "") or from /club by a
+// signed-in member. The body is multipart -- the part bundle (the WebUI's
+// crashlog_*.tar.gz), and the fields mac, firmware, majestic, soc, sensor
+// and meta (the firmware's meta.json) -- or the bundle alone.
+func (a *API) Submit(w http.ResponseWriter, r *http.Request, member, channel string) {
+ ctx := r.Context()
+ key, err := (&reports.Store{DB: a.DB}).Key(ctx)
+ if err != nil {
+ a.fail(w, "the report key", err)
+ return
+ }
+ in, status, err := read(r)
+ if err != nil {
+ a.refuse(w, status, err.Error())
+ return
+ }
+ mac := strings.ToLower(strings.TrimSpace(in.fields["mac"]))
+ macKey := ""
+ if mac != "" {
+ if !macRe.MatchString(mac) {
+ a.refuse(w, http.StatusBadRequest, "mac: six pairs of hex digits, aa:bb:cc:dd:ee:ff")
+ return
+ }
+ if macKey = snapshots.MACKey(mac); strings.Trim(macKey, "0") == "" || strings.Trim(macKey, "f") == "" {
+ macKey = ""
+ }
+ }
+ soc, sensor := strings.ToLower(in.fields["soc"]), strings.ToLower(in.fields["sensor"])
+ for name, v := range map[string]string{"soc": soc, "sensor": sensor} {
+ if v != "" && !hardwareRe.MatchString(v) {
+ a.refuse(w, http.StatusBadRequest, name+": a chip's name as ipcinfo prints it, like gk7205v300 or imx335")
+ return
+ }
+ }
+ // A member names only a camera of theirs: the MAC is what pays a
+ // crash's stars to the camera's owner.
+ if member != "" && macKey != "" {
+ mine, err := a.Store().LinkedTo(ctx, macKey, member)
+ if err != nil {
+ a.fail(w, "the camera", err)
+ return
+ }
+ if !mine {
+ a.refuse(w, http.StatusBadRequest, "mac: that camera is not linked to you on the Open Wall; link it from /club, or send the crash without it")
+ return
+ }
+ }
+ client := reports.Keyed(key, "client", httpx.ClientIP(r))
+
+ files, err := Unpack(in.bundle)
+ if err != nil {
+ a.refuse(w, http.StatusUnprocessableEntity, err.Error())
+ return
+ }
+ c, err := Parse(files)
+ if err != nil {
+ a.refuse(w, http.StatusUnprocessableEntity, err.Error())
+ return
+ }
+ redactCrash(c, mac, key)
+ sum := sha256.Sum256(in.bundle)
+ e := &Event{
+ ID: NewID(), Channel: channel, MACKey: macKey, Member: member, ClientKey: client,
+ Firmware: in.fields["firmware"], Majestic: in.fields["majestic"],
+ SoC: orDefault(soc, c.SoC), Sensor: orDefault(sensor, c.Sensor),
+ Crash: c, Bundle: in.bundle, BundleSHA256: hex.EncodeToString(sum[:]),
+ SignatureID: c.Fatal.Signature(),
+ Redacted: Redact(c.Text, mac, key),
+ }
+ if in.meta != "" {
+ red := Redact(in.meta, mac, key)
+ if !json.Valid([]byte(red)) {
+ a.refuse(w, http.StatusBadRequest, "meta is not JSON")
+ return
+ }
+ e.Meta = json.RawMessage(red)
+ }
+ id, sig, dup, err := a.Store().Insert(ctx, e, a.now().Add(-24*time.Hour))
+ if errors.Is(err, ErrQuota) {
+ w.Header().Set("Retry-After", "3600")
+ a.refuse(w, http.StatusTooManyRequests, fmt.Sprintf("%d crashes a day from one address, %d from one camera, is the limit", DailyPerClient, DailyPerCamera))
+ return
+ }
+ if err != nil {
+ a.fail(w, "the crash", err)
+ return
+ }
+ a.Log.Info("crashes: received", "id", id, "signature", sig, "kind", c.Kind, "duplicate", dup, "channel", channel,
+ "soc", e.SoC, "self_inflicted", c.SelfInflicted)
+ code := http.StatusCreated
+ if dup {
+ code = http.StatusOK
+ }
+ writeJSON(w, code, map[string]any{
+ "id": id, "signature": sig, "title": c.Fatal.Title(), "kind": c.Kind, "duplicate": dup,
+ "self_inflicted": c.SelfInflicted,
+ "url": strings.TrimRight(a.SiteURL, "/") + "/crashes/#" + sig,
+ })
+}
+
+var (
+ macRe = regexp.MustCompile(`^[0-9a-f]{2}([:-]?[0-9a-f]{2}){5}$`)
+ hardwareRe = regexp.MustCompile(`^[a-z0-9][a-z0-9._+-]{0,39}$`)
+)
+
+// redactCrash replaces identifiers in every part of a parsed crash that is
+// stored apart from the log -- the command line, the lines before it, each
+// trace's reason (a panic's message is free text) -- and drops a SoC or
+// sensor the log names in any shape but a chip's. The title and the
+// signature are made from what is left.
+func redactCrash(c *Crash, mac, key string) {
+ c.Cmdline = Redact(c.Cmdline, mac, key)
+ for i := range c.Leadup {
+ c.Leadup[i] = Redact(c.Leadup[i], mac, key)
+ }
+ for _, t := range append([]*Trace{c.Fatal}, c.Before...) {
+ t.Reason = Redact(t.Reason, mac, key)
+ }
+ if !hardwareRe.MatchString(c.SoC) {
+ c.SoC = ""
+ }
+ if !hardwareRe.MatchString(c.Sensor) {
+ c.Sensor = ""
+ }
+}
+
+type upload struct {
+ bundle []byte
+ meta string
+ fields map[string]string
+}
+
+func read(r *http.Request) (*upload, int, error) {
+ r.Body = http.MaxBytesReader(nil, r.Body, MaxBundle+maxMeta+64<<10)
+ out := &upload{fields: map[string]string{}}
+ ct, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
+ if ct != "multipart/form-data" {
+ b, err := io.ReadAll(r.Body)
+ if err != nil {
+ return nil, http.StatusRequestEntityTooLarge, fmt.Errorf("the bundle is larger than %d bytes", MaxBundle)
+ }
+ out.bundle = b
+ return out, 0, nil
+ }
+ mr, err := r.MultipartReader()
+ if err != nil {
+ return nil, http.StatusBadRequest, err
+ }
+ for {
+ p, err := mr.NextPart()
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ var tooBig *http.MaxBytesError
+ if errors.As(err, &tooBig) {
+ return nil, http.StatusRequestEntityTooLarge, fmt.Errorf("the bundle is larger than %d bytes", MaxBundle)
+ }
+ return nil, http.StatusBadRequest, err
+ }
+ name := p.FormName()
+ switch name {
+ case "bundle":
+ b, err := io.ReadAll(io.LimitReader(p, MaxBundle+1))
+ if err != nil {
+ return nil, http.StatusBadRequest, err
+ }
+ if len(b) > MaxBundle {
+ return nil, http.StatusRequestEntityTooLarge, fmt.Errorf("the bundle is larger than %d bytes", MaxBundle)
+ }
+ out.bundle = b
+ case "meta":
+ b, err := io.ReadAll(io.LimitReader(p, maxMeta+1))
+ if err != nil {
+ return nil, http.StatusBadRequest, err
+ }
+ if len(b) > maxMeta {
+ return nil, http.StatusRequestEntityTooLarge, fmt.Errorf("meta is larger than %d bytes", maxMeta)
+ }
+ out.meta = strings.TrimSpace(string(b))
+ case "mac", "firmware", "majestic", "soc", "sensor":
+ b, err := io.ReadAll(io.LimitReader(p, maxField+1))
+ if err != nil {
+ return nil, http.StatusBadRequest, err
+ }
+ if len(b) > maxField {
+ return nil, http.StatusBadRequest, fmt.Errorf("%s: at most %d characters", name, maxField)
+ }
+ out.fields[name] = strings.TrimSpace(string(b))
+ }
+ }
+ if len(out.bundle) == 0 {
+ return nil, http.StatusBadRequest, errors.New("no bundle: send the crash log as the part bundle")
+ }
+ return out, 0, nil
+}
+
+func orDefault(s, d string) string {
+ if s == "" {
+ return d
+ }
+ return s
+}
+
+// Redact replaces the camera's MAC (and any other, in any of its spellings)
+// and every IP address in a crash's text with a keyed hash: two crashes of
+// one camera still show they are one camera's, and neither says which.
+func Redact(text, mac, key string) string {
+ text = reports.Redact(text, reports.Facts{MAC: mac}, key)
+ text = replace(text, dottedMAC, func(m string) string { return "" },
+ func(prev, next byte) bool { return !hexOrDot(prev) && !hexOrDot(next) })
+ text = replace(text, ipv4, func(m string) string {
+ if m == "0.0.0.0" || strings.HasPrefix(m, "127.") || strings.HasPrefix(m, "255.") {
+ return m
+ }
+ return ""
+ }, func(prev, next byte) bool { return !digitOrDot(prev) && !digitOrDot(next) })
+ return replace(text, ipv6, func(m string) string {
+ if m == "::" || m == "::1" {
+ return m
+ }
+ return ""
+ }, func(prev, next byte) bool { return !hexOrColon(prev) && !hexOrColon(next) })
+}
+
+// replace rewrites each match of re whose neighbours ok accepts. The
+// neighbours are looked at, never matched, so two addresses one separator
+// apart -- ip=10.0.0.2:10.0.0.1 -- are both found.
+func replace(text string, re *regexp.Regexp, with func(string) string, ok func(prev, next byte) bool) string {
+ var b strings.Builder
+ last := 0
+ for _, m := range re.FindAllStringIndex(text, -1) {
+ var prev, next byte
+ if m[0] > 0 {
+ prev = text[m[0]-1]
+ }
+ if m[1] < len(text) {
+ next = text[m[1]]
+ }
+ if !ok(prev, next) {
+ continue
+ }
+ b.WriteString(text[last:m[0]])
+ b.WriteString(with(text[m[0]:m[1]]))
+ last = m[1]
+ }
+ b.WriteString(text[last:])
+ return b.String()
+}
+
+func digitOrDot(c byte) bool { return c == '.' || c >= '0' && c <= '9' }
+func hexOrDot(c byte) bool { return c == '.' || isHex(c) }
+func hexOrColon(c byte) bool {
+ return c == ':' || isHex(c)
+}
+func isHex(c byte) bool {
+ return c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F'
+}
+
+var (
+ // ipv4 is a dotted quad; replace refuses one inside a longer dotted
+ // number (a version, 4.9.37.1.2).
+ ipv4 = regexp.MustCompile(`(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}`)
+ // ipv6: eight groups, or fewer with "::". Never a time (17:57:19) or a
+ // register dump (9dc0:), which have neither.
+ ipv6 = regexp.MustCompile(`(?i)(?:[0-9a-f]{1,4}:){7}[0-9a-f]{1,4}|(?:[0-9a-f]{1,4}:){1,7}:(?:[0-9a-f]{1,4}(?::[0-9a-f]{1,4}){0,6})?|::(?:[0-9a-f]{1,4}(?::[0-9a-f]{1,4}){0,6})?`)
+ // dottedMAC is Cisco's 0012.3456.789a.
+ dottedMAC = regexp.MustCompile(`(?i)[0-9a-f]{4}\.[0-9a-f]{4}\.[0-9a-f]{4}`)
+)
+
+// list is GET /api/v1/crashes: the signatures, worst first. No log, no
+// camera, no member.
+func (a *API) list(w http.ResponseWriter, r *http.Request) {
+ list, err := a.Store().Ranked(r.Context(), false, a.now())
+ if err != nil {
+ a.fail(w, "the crashes", err)
+ return
+ }
+ w.Header().Set("Cache-Control", "public, max-age=300")
+ writeJSONCached(w, http.StatusOK, map[string]any{"signatures": orEmpty(list)})
+}
+
+// get is GET /api/v1/crashes/{id}: a signature and where it was seen.
+func (a *API) get(w http.ResponseWriter, r *http.Request) {
+ id := r.PathValue("id")
+ g, err := a.Store().Get(r.Context(), id, false, a.now())
+ if errors.Is(err, ErrNoSignature) {
+ a.refuse(w, http.StatusNotFound, err.Error())
+ return
+ }
+ if err != nil {
+ a.fail(w, "the crash", err)
+ return
+ }
+ combos, err := a.Store().Combos(r.Context(), id, false)
+ if err != nil {
+ a.fail(w, "the crash", err)
+ return
+ }
+ w.Header().Set("Cache-Control", "public, max-age=300")
+ writeJSONCached(w, http.StatusOK, map[string]any{"signature": g, "seen_on": orEmpty(combos)})
+}
+
+func (a *API) refuse(w http.ResponseWriter, status int, reason string) {
+ a.Log.Info("crashes: refused", "status", status, "reason", reason)
+ writeJSON(w, status, map[string]string{"error": reason})
+}
+
+func (a *API) fail(w http.ResponseWriter, what string, err error) {
+ a.Log.Error("crashes: "+what+" failed", "err", err)
+ writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "the crash could not be stored; retry"})
+}
+
+func writeJSON(w http.ResponseWriter, status int, v any) {
+ w.Header().Set("Cache-Control", "no-store")
+ writeJSONCached(w, status, v)
+}
+
+func writeJSONCached(w http.ResponseWriter, status int, v any) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(status)
+ enc := json.NewEncoder(w)
+ enc.SetIndent("", " ")
+ _ = enc.Encode(v)
+}
diff --git a/service/internal/crashes/parse.go b/service/internal/crashes/parse.go
new file mode 100644
index 00000000..571d98e2
--- /dev/null
+++ b/service/internal/crashes/parse.go
@@ -0,0 +1,580 @@
+// Package crashes is the kernel crashes cameras recovered from: the bundle
+// the firmware's S98crashlog makes of pstore's records after a panic or an
+// oops (/etc/crash/crash.tar.gz), sent by the WebUI or downloaded from it
+// and sent from /club. A crash is filed under its signature -- the top of
+// its backtrace, without offsets -- so one bug sent by a hundred cameras is
+// one row, ranked by how bad it is and how many cameras it hits.
+//
+// The logs are the maintainers' (redacted); the signatures are public.
+// Stars are paid for them by the nightly settlement, never by an upload.
+package crashes
+
+import (
+ "archive/tar"
+ "bytes"
+ "compress/gzip"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "regexp"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// Limits on one bundle.
+const (
+ MaxBundle = 256 << 10 // compressed, as sent
+ MaxUnpacked = 1 << 20 // every record together
+ maxRecords = 16
+ sigFrames = 5 // frames a signature is made of
+)
+
+// Kinds, worst first.
+const (
+ KindBootloop = "bootloop" // the watchdog's bootlimit ran out: failsafe, no record
+ KindPanic = "panic"
+ KindOops = "oops"
+ KindBug = "bug"
+ KindWarning = "warning"
+)
+
+var kindRank = map[string]int{KindBootloop: 5, KindPanic: 4, KindOops: 3, KindBug: 2, KindWarning: 1}
+
+// Worse says whether kind a is worse than b.
+func Worse(a, b string) bool { return kindRank[a] > kindRank[b] }
+
+// Frame is one function in a backtrace.
+type Frame struct {
+ Fn string `json:"fn"`
+ Module string `json:"module,omitempty"`
+}
+
+func (f Frame) String() string {
+ if f.Module == "" {
+ return f.Fn
+ }
+ return f.Fn + " [" + f.Module + "]"
+}
+
+// Trace is one report the kernel printed: an oops, a BUG, a WARNING, a panic.
+type Trace struct {
+ Kind string `json:"kind"`
+ Reason string `json:"reason"` // "NULL pointer dereference", "WARNING at mm/page_alloc.c:7373", ...
+ Comm string `json:"comm,omitempty"`
+ PC string `json:"pc,omitempty"`
+ InIRQ bool `json:"in_irq,omitempty"`
+ // Frames: the backtrace, innermost first, without the frames every
+ // crash has (dump_stack, the exception and IRQ entry, the syscall).
+ Frames []Frame `json:"frames"`
+ // Interrupted: what the CPU was doing when the interrupt that crashed came.
+ Interrupted []Frame `json:"interrupted,omitempty"`
+ end string // the "end trace" id, to tell the same trace in two records
+}
+
+// Signature is the trace's identity: the same bug on any build gives the same.
+func (t *Trace) Signature() string {
+ class := "fatal"
+ if t.Kind == KindWarning {
+ class = "warning"
+ }
+ var b strings.Builder
+ b.WriteString(class)
+ for i, f := range t.Frames {
+ if i == sigFrames {
+ break
+ }
+ b.WriteString("\n" + f.String())
+ }
+ if len(t.Frames) == 0 {
+ b.WriteString("\n" + t.Reason)
+ }
+ sum := sha256.Sum256([]byte(b.String()))
+ return hex.EncodeToString(sum[:])[:12]
+}
+
+// Title names the trace for a person: why, and where in whose code.
+func (t *Trace) Title() string {
+ if len(t.Frames) == 0 {
+ return t.Reason
+ }
+ where := t.Frames[0].Fn
+ // The first frame in a vendor module says more than a kernel helper,
+ // unless it is the module everything calls through.
+ for _, f := range t.Frames[:min(len(t.Frames), sigFrames)] {
+ if f.Module != "" && !wrapperModule(f.Module) {
+ if f.Fn != where {
+ where += " ← " + f.String()
+ }
+ break
+ }
+ }
+ return t.Reason + " in " + where
+}
+
+func wrapperModule(m string) bool {
+ return strings.HasSuffix(m, "osal") || strings.HasSuffix(m, "_base")
+}
+
+// Crash is what one bundle says: one boot that ended in a crash.
+type Crash struct {
+ Kind string `json:"kind"`
+ Fatal *Trace `json:"fatal"`
+ // Before: the warnings the kernel printed on its way there.
+ Before []*Trace `json:"before,omitempty"`
+ // SelfInflicted: the crash was asked for (sysrq-trigger), not suffered.
+ SelfInflicted bool `json:"self_inflicted,omitempty"`
+
+ Kernel string `json:"kernel,omitempty"` // 4.9.37
+ KernelBuild string `json:"kernel_build,omitempty"` // #1 Thu Sep 24 17:57:19 UTC 2026
+ KernelBuilt time.Time `json:"kernel_built,omitempty"`
+ Toolchain string `json:"toolchain,omitempty"`
+ Machine string `json:"machine,omitempty"`
+ Cmdline string `json:"cmdline,omitempty"`
+ SoC string `json:"soc,omitempty"`
+ Sensor string `json:"sensor,omitempty"`
+ Board string `json:"board,omitempty"`
+ Modules []string `json:"modules,omitempty"`
+ // Anomalies: lines that often come before a crash, counted.
+ Anomalies map[string]int `json:"anomalies,omitempty"`
+ // Leadup: the last lines before the fatal trace.
+ Leadup []string `json:"leadup,omitempty"`
+ // Uptime: seconds from boot to the crash, when printk prints the time.
+ Uptime float64 `json:"uptime,omitempty"`
+ // Records: how many pstore records the bundle had; Failsafe the
+ // firmware's note that the bootlimit ran out.
+ Records int `json:"records"`
+ Failsafe bool `json:"failsafe,omitempty"`
+
+ // Text is every record, as sent; ContentSum identifies the crash
+ // whatever tar or gzip carried it.
+ Text string `json:"-"`
+ ContentSum string `json:"-"`
+}
+
+// ErrNotACrash is a bundle with nothing a crash leaves.
+var ErrNotACrash = errors.New("the bundle has no pstore record (dmesg-*) and no failsafe note: nothing crashed")
+
+// Unpack reads a crash bundle -- a tar.gz, a plain tar, or a single record
+// as text -- into its records by name.
+func Unpack(data []byte) (map[string]string, error) {
+ if len(data) > MaxBundle {
+ return nil, fmt.Errorf("the bundle is larger than %d bytes", MaxBundle)
+ }
+ r := io.Reader(bytes.NewReader(data))
+ if len(data) > 2 && data[0] == 0x1f && data[1] == 0x8b {
+ z, err := gzip.NewReader(r)
+ if err != nil {
+ return nil, fmt.Errorf("the bundle is not a valid gzip: %w", err)
+ }
+ r = z
+ }
+ raw, err := io.ReadAll(io.LimitReader(r, MaxUnpacked+1))
+ if err != nil {
+ return nil, fmt.Errorf("the bundle cannot be read: %w", err)
+ }
+ if len(raw) > MaxUnpacked {
+ return nil, fmt.Errorf("the bundle unpacks to more than %d bytes", MaxUnpacked)
+ }
+ out := map[string]string{}
+ if len(raw) >= 512 && bytes.Equal(raw[257:262], []byte("ustar")) {
+ tr := tar.NewReader(bytes.NewReader(raw))
+ for {
+ h, err := tr.Next()
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ return nil, fmt.Errorf("the bundle is not a valid tar: %w", err)
+ }
+ if h.Typeflag != tar.TypeReg {
+ continue
+ }
+ name := h.Name[strings.LastIndex(h.Name, "/")+1:]
+ if !strings.HasPrefix(name, "dmesg-") && name != "failsafe" && name != "pending" {
+ continue
+ }
+ if len(out) == maxRecords {
+ return nil, fmt.Errorf("more than %d records", maxRecords)
+ }
+ b, err := io.ReadAll(tr)
+ if err != nil {
+ return nil, err
+ }
+ out[name] = string(b)
+ }
+ } else if printable(raw) {
+ out["dmesg-0"] = string(raw)
+ } else {
+ return nil, errors.New("the bundle is not the crash log the WebUI downloads (crashlog_*.tar.gz)")
+ }
+ return out, nil
+}
+
+func printable(b []byte) bool {
+ if len(b) == 0 {
+ return false
+ }
+ bad := 0
+ for _, c := range b {
+ if c < 0x20 && c != '\n' && c != '\r' && c != '\t' {
+ bad++
+ }
+ }
+ return bad*100 < len(b)
+}
+
+// Parse reads the records a bundle unpacked to.
+func Parse(files map[string]string) (*Crash, error) {
+ var names []string
+ for n := range files {
+ if strings.HasPrefix(n, "dmesg-") {
+ names = append(names, n)
+ }
+ }
+ sort.Strings(names)
+ c := &Crash{Records: len(names), Anomalies: map[string]int{}}
+ if f, ok := files["failsafe"]; ok && strings.Contains(f, "bootlimit") {
+ c.Failsafe = true
+ }
+ if len(names) == 0 && !c.Failsafe {
+ return nil, ErrNotACrash
+ }
+
+ sum := sha256.New()
+ var texts []string
+ seen := map[string]bool{}
+ var traces []*Trace
+ panicked, panicMsg := false, ""
+ for _, n := range names {
+ body := files[n]
+ texts = append(texts, "==> "+n+" <==\n"+body)
+ sum.Write([]byte(body))
+ sum.Write([]byte{0})
+ lines := strings.Split(strings.ReplaceAll(body, "\r", ""), "\n")
+ if len(lines) > 0 && strings.HasPrefix(lines[0], "Panic#") {
+ panicked = true
+ }
+ c.context(lines)
+ ts, msg := traces1(lines)
+ if msg != "" {
+ panicked, panicMsg = true, msg
+ }
+ for _, t := range ts {
+ k := t.Kind + t.end + t.Signature()
+ if seen[k] {
+ continue
+ }
+ seen[k] = true
+ traces = append(traces, t)
+ }
+ if c.Leadup == nil {
+ c.Leadup, c.Uptime = leadup(lines)
+ }
+ }
+ c.Text = strings.Join(texts, "\n")
+ if c.Failsafe {
+ sum.Write([]byte("failsafe\x00" + files["failsafe"]))
+ }
+ c.ContentSum = hex.EncodeToString(sum.Sum(nil))
+
+ for _, t := range traces {
+ if t.Kind == KindWarning {
+ c.Before = append(c.Before, t)
+ continue
+ }
+ if c.Fatal == nil || Worse(t.Kind, c.Fatal.Kind) {
+ c.Fatal = t
+ }
+ }
+ switch {
+ case c.Fatal != nil:
+ c.Kind = c.Fatal.Kind
+ if panicked {
+ c.Kind = KindPanic
+ }
+ if strings.Contains(panicMsg, "in interrupt") {
+ c.Fatal.InIRQ = true
+ }
+ case panicked:
+ c.Fatal = &Trace{Kind: KindPanic, Reason: "Kernel panic: " + panicMsg}
+ c.Kind = KindPanic
+ case c.Failsafe:
+ c.Fatal = &Trace{Kind: KindBootloop, Reason: "Boot loop: the watchdog's boot limit ran out"}
+ c.Kind = KindBootloop
+ case len(c.Before) > 0:
+ // Only warnings: the record was written for something else (a
+ // reboot that dumped the log); the last warning is what there is.
+ c.Fatal = c.Before[len(c.Before)-1]
+ c.Before = c.Before[:len(c.Before)-1]
+ c.Kind = KindWarning
+ default:
+ return nil, errors.New("the records hold no oops, BUG, WARNING or panic")
+ }
+ for _, f := range c.Fatal.Frames {
+ if strings.Contains(f.Fn, "sysrq") {
+ c.SelfInflicted = true
+ }
+ }
+ if strings.Contains(c.Text, "SysRq : Trigger a crash") || strings.Contains(c.Text, "sysrq: Trigger a crash") {
+ c.SelfInflicted = true
+ }
+ for k, v := range c.Anomalies {
+ if v == 0 {
+ delete(c.Anomalies, k)
+ }
+ }
+ return c, nil
+}
+
+// A printk line, its prefix and [ timestamp] dropped.
+var linePrefix = regexp.MustCompile(`^(?:<\d>)?(?:\[\s*\d+\.\d+\]\s?)?`)
+
+func clean(l string) string { return linePrefix.ReplaceAllString(l, "") }
+
+var (
+ bannerRe = regexp.MustCompile(`^Linux version (\S+) \((.*?)\) \((.*?)\) (#\d+.*)$`)
+ sensorRe = regexp.MustCompile(`sensor=([\w.-]+), chip=([\w.-]+), board=([\w.-]+?)=*$`)
+ buildTime = regexp.MustCompile(`\w{3} \w{3} +\d+ \d\d:\d\d:\d\d \w+ \d{4}$`)
+)
+
+// anomalies are lines worth counting before a crash: a bus that stopped
+// answering, memory running out, a CPU stuck.
+var anomalies = []struct {
+ name string
+ re *regexp.Regexp
+}{
+ {"i2c_error", regexp.MustCompile(`(?i)i2c.*(abort|timeout|timed out|error|nack|no ack)`)},
+ {"spi_error", regexp.MustCompile(`(?i)\bspi\b.*(timeout|error)`)},
+ {"mmz_alloc_failed", regexp.MustCompile(`(?i)(mmz|mmb|cma).*(fail|no memory|not enough)`)},
+ {"oom", regexp.MustCompile(`(?i)out of memory|oom-killer|page allocation failure`)},
+ {"soft_lockup", regexp.MustCompile(`(?i)soft lockup|hung_task|rcu.*stall`)},
+ {"mtd_error", regexp.MustCompile(`(?i)(ubi|ubifs|jffs2|mtd|squashfs).*(error|err\b|corrupt)`)},
+ {"ethernet_down", regexp.MustCompile(`(?i)eth\d.*link is down`)},
+}
+
+func (c *Crash) context(lines []string) {
+ counts := map[string]int{}
+ for _, raw := range lines {
+ l := clean(raw)
+ if m := bannerRe.FindStringSubmatch(l); m != nil && c.Kernel == "" {
+ c.Kernel, c.Toolchain, c.KernelBuild = m[1], m[3], m[4]
+ if s := buildTime.FindString(m[4]); s != "" {
+ if t, err := time.Parse("Mon Jan _2 15:04:05 MST 2006", s); err == nil {
+ c.KernelBuilt = t.UTC()
+ }
+ }
+ }
+ if v, ok := strings.CutPrefix(l, "Kernel command line: "); ok && c.Cmdline == "" {
+ c.Cmdline = v
+ }
+ if _, v, ok := strings.Cut(l, "Machine model: "); ok && c.Machine == "" {
+ c.Machine = v
+ }
+ if m := sensorRe.FindStringSubmatch(l); m != nil && c.SoC == "" {
+ c.Sensor, c.SoC, c.Board = m[1], m[2], m[3]
+ }
+ if v, ok := strings.CutPrefix(l, "Modules linked in: "); ok && c.Modules == nil {
+ for _, m := range strings.Fields(v) {
+ c.Modules = append(c.Modules, strings.TrimSuffix(m, "(O)"))
+ }
+ }
+ for _, a := range anomalies {
+ if a.re.MatchString(l) {
+ counts[a.name]++
+ }
+ }
+ }
+ // Records of one crash overlap: the most any of them saw.
+ for k, v := range counts {
+ c.Anomalies[k] = max(c.Anomalies[k], v)
+ }
+}
+
+// Backtrace lines, by architecture:
+//
+// ARM: [] (__wake_up_common) from [] (__wake_up+0x24/0x30)
+// [] (osal_wakeup [open_osal]) from [] (RGN_PutRegion+0x2d0/0x31c [open_rgn])
+// MIPS: [<80123456>] foo+0x10/0x20 [module]
+// arm64: [] foo+0x10/0x20 [module] or foo+0x10/0x20 [module]
+var (
+ armFrame = regexp.MustCompile(`^\[<[0-9a-f]+>\] \(([^ ()+]+)(?:\+0x[0-9a-f]+/0x[0-9a-f]+)?(?: \[([\w-]+)\])?\) from \[<[0-9a-f]+>\] \(([^ ()+]+)`)
+ otherFrame = regexp.MustCompile(`^(?:\[<[0-9a-f]+>\] )?\s*([A-Za-z_][\w.$]*)\+0x[0-9a-f]+/0x[0-9a-f]+(?: \[([\w-]+)\])?\s*$`)
+ pcRe = regexp.MustCompile(`^(?:PC is at |pc : (?:\[<[0-9a-f]+>\] )?|epc\s*: [0-9a-f]+ )([A-Za-z_][\w.$]*)`)
+ commRe = regexp.MustCompile(`^CPU: \d+ PID: \d+ Comm: (\S+)`)
+ faultRe = regexp.MustCompile(`^(?:CPU \d+ )?Unable to handle kernel (NULL pointer dereference|paging request) at virtual address`)
+ // ARM's "Internal error: Oops: 5 [#1] ARM", MIPS's "Oops[#1]:" -- never
+ // pstore's own "Oops#1 Part1" record header.
+ oopsRe = regexp.MustCompile(`^(?:Internal error: )?Oops(?:\[#\d+\]|: )`)
+ bugRe = regexp.MustCompile(`^(?:kernel BUG at (\S+)!|BUG: (.+))$`)
+ warnRe = regexp.MustCompile(`^WARNING: (?:CPU: \d+ PID: \d+ )?at (\S+)`)
+ endRe = regexp.MustCompile(`^---\[ end trace ([0-9a-f]+) \]---`)
+ panicRe = regexp.MustCompile(`^Kernel panic - not syncing: (.*)$`)
+)
+
+// glue: frames every crash has. They say how the kernel got to the report,
+// never what went wrong.
+var glue = map[string]bool{
+ "dump_backtrace": true, "show_stack": true, "dump_stack": true, "__dump_stack": true,
+ "__warn": true, "warn_slowpath_fmt": true, "warn_slowpath_null": true, "warn_slowpath_common": true,
+ "panic": true, "die": true, "__die": true, "oops_end": true, "bug_handler": true,
+ "__do_kernel_fault": true, "do_page_fault": true, "do_translation_fault": true, "do_DataAbort": true,
+ "do_PrefetchAbort": true, "do_undefinstr": true, "__dabt_svc": true, "__dabt_usr": true, "__pabt_usr": true,
+ "__und_svc": true, "__irq_svc": true, "__irq_usr": true, "gic_handle_irq": true, "__handle_domain_irq": true,
+ "generic_handle_irq": true, "handle_fasteoi_irq": true, "handle_level_irq": true, "handle_edge_irq": true,
+ "handle_irq_event": true, "handle_irq_event_percpu": true, "__handle_irq_event_percpu": true,
+ "irq_exit": true, "__irq_exit_rcu": true, "do_IRQ": true, "plat_irq_dispatch": true, "ret_from_irq": true,
+ "ret_fast_syscall": true, "ret_from_fork": true, "ret_from_exception": true, "syscall_common": true,
+ "vfs_ioctl": true, "do_vfs_ioctl": true, "SyS_ioctl": true, "sys_ioctl": true, "ksys_ioctl": true,
+ "__arm64_sys_ioctl": true, "__se_sys_ioctl": true, "el0_svc_naked": true, "el0_svc": true, "el1_irq": true,
+ "el0_irq": true, "el1_da": true, "el1_sync": true, "osal_unlocked_ioctl": true, "osal_ioctl": true,
+ "__do_softirq": true,
+}
+
+// irqEntry: the frame where the interrupted code's frames begin.
+var irqEntry = map[string]bool{"__irq_svc": true, "__irq_usr": true, "el1_irq": true, "el0_irq": true, "ret_from_irq": true}
+
+// traces1 reads the traces in one record, and the panic message if it has one.
+func traces1(lines []string) ([]*Trace, string) {
+ var out []*Trace
+ var cur *Trace
+ panicMsg := ""
+ afterIRQ := false
+ start := func(kind, reason string) {
+ if cur != nil && len(cur.Frames) == 0 && cur.Kind == KindOops && kind == KindOops {
+ return // "Unable to handle..." then "Internal error: Oops": one oops
+ }
+ if cur != nil {
+ out = append(out, cur)
+ }
+ cur, afterIRQ = &Trace{Kind: kind, Reason: reason}, false
+ }
+ for _, raw := range lines {
+ l := clean(raw)
+ switch {
+ case faultRe.MatchString(l):
+ m := faultRe.FindStringSubmatch(l)
+ start(KindOops, strings.ToUpper(m[1][:1])+m[1][1:])
+ continue
+ case oopsRe.MatchString(l):
+ if cur == nil || cur.Kind != KindOops || len(cur.Frames) > 0 {
+ start(KindOops, "Oops")
+ }
+ continue
+ case bugRe.MatchString(l):
+ m := bugRe.FindStringSubmatch(l)
+ if m[1] != "" {
+ start(KindBug, "BUG at "+m[1])
+ } else {
+ start(KindBug, "BUG: "+strings.TrimSpace(m[2]))
+ }
+ continue
+ case warnRe.MatchString(l):
+ start(KindWarning, "WARNING at "+warnRe.FindStringSubmatch(l)[1])
+ continue
+ case panicRe.MatchString(l):
+ panicMsg = strings.TrimSpace(panicRe.FindStringSubmatch(l)[1])
+ if cur == nil && !fatalIn(out) {
+ // A panic of its own (init killed, a watchdog): its
+ // backtrace follows.
+ start(KindPanic, "Kernel panic: "+panicMsg)
+ }
+ continue
+ }
+ if cur == nil {
+ continue
+ }
+ if m := endRe.FindStringSubmatch(l); m != nil {
+ cur.end = m[1]
+ out = append(out, cur)
+ cur = nil
+ continue
+ }
+ if m := commRe.FindStringSubmatch(l); m != nil && cur.Comm == "" {
+ cur.Comm = m[1]
+ continue
+ }
+ if m := pcRe.FindStringSubmatch(l); m != nil && cur.PC == "" {
+ cur.PC = m[1]
+ continue
+ }
+ var f Frame
+ var next string
+ if m := armFrame.FindStringSubmatch(l); m != nil {
+ f, next = Frame{Fn: m[1], Module: m[2]}, m[3]
+ } else if m := otherFrame.FindStringSubmatch(l); m != nil {
+ f = Frame{Fn: m[1], Module: m[2]}
+ } else {
+ continue
+ }
+ if irqEntry[f.Fn] || irqEntry[next] {
+ cur.InIRQ = true
+ }
+ wasAfter := afterIRQ
+ if irqEntry[f.Fn] || irqEntry[next] {
+ afterIRQ = true
+ }
+ if irqEntry[f.Fn] || glue[f.Fn] {
+ continue
+ }
+ if wasAfter {
+ if len(cur.Interrupted) < sigFrames {
+ cur.Interrupted = append(cur.Interrupted, f)
+ }
+ } else {
+ cur.Frames = append(cur.Frames, f)
+ }
+ }
+ if cur != nil {
+ out = append(out, cur)
+ }
+ return out, panicMsg
+}
+
+func fatalIn(ts []*Trace) bool {
+ for _, t := range ts {
+ if t.Kind != KindWarning {
+ return true
+ }
+ }
+ return false
+}
+
+// leadup is the last lines before the first oops, BUG or panic -- what the
+// kernel said just before -- and the uptime it happened at, when printk
+// prints timestamps.
+func leadup(lines []string) ([]string, float64) {
+ for i, raw := range lines {
+ l := clean(raw)
+ if faultRe.MatchString(l) || oopsRe.MatchString(l) || bugRe.MatchString(l) || panicRe.MatchString(l) {
+ var out []string
+ for _, p := range lines[max(0, i-8):i] {
+ p = strings.TrimSpace(clean(p))
+ if p != "" && !strings.HasPrefix(p, "[<") && !registers.MatchString(p) {
+ out = append(out, p)
+ }
+ }
+ up, _ := uptime(raw)
+ return out, up
+ }
+ }
+ return nil, 0
+}
+
+var (
+ registers = regexp.MustCompile(`^(r\d+|[0-9a-f]{4}):`)
+ stamp = regexp.MustCompile(`^(?:<\d>)?\[\s*(\d+\.\d+)\]`)
+)
+
+// uptime is a printk timestamp's seconds.
+func uptime(l string) (float64, bool) {
+ m := stamp.FindStringSubmatch(l)
+ if m == nil {
+ return 0, false
+ }
+ v, err := strconv.ParseFloat(m[1], 64)
+ return v, err == nil
+}
diff --git a/service/internal/crashes/parse_test.go b/service/internal/crashes/parse_test.go
new file mode 100644
index 00000000..59e4c57a
--- /dev/null
+++ b/service/internal/crashes/parse_test.go
@@ -0,0 +1,184 @@
+package crashes
+
+import (
+ "archive/tar"
+ "bytes"
+ "compress/gzip"
+ "os"
+ "strings"
+ "testing"
+)
+
+func bundle(t *testing.T, name string) []byte {
+ t.Helper()
+ b, err := os.ReadFile("testdata/" + name)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return b
+}
+
+func parseBundle(t *testing.T, data []byte) *Crash {
+ t.Helper()
+ files, err := Unpack(data)
+ if err != nil {
+ t.Fatal(err)
+ }
+ c, err := Parse(files)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return c
+}
+
+// tgz packs records the way S98crashlog does.
+func tgz(t *testing.T, files map[string]string) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ z := gzip.NewWriter(&buf)
+ tw := tar.NewWriter(z)
+ for name, body := range files {
+ if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o444, Size: int64(len(body)), Typeflag: tar.TypeReg}); err != nil {
+ t.Fatal(err)
+ }
+ tw.Write([]byte(body))
+ }
+ tw.Close()
+ z.Close()
+ return buf.Bytes()
+}
+
+// The lab's gk7205v300 + imx335: the OSD region code oopsed in the VENC
+// interrupt while hi_isp_run was in the ISP driver, and the kernel panicked.
+// pstore kept it twice (Oops#1, Panic#2); it is one crash.
+func TestLabBundle(t *testing.T) {
+ c := parseBundle(t, bundle(t, "gk7205v300-imx335-rgn.tar.gz"))
+ if c.Records != 2 || c.Kind != KindPanic {
+ t.Fatalf("records %d kind %q, want 2 records, a panic", c.Records, c.Kind)
+ }
+ f := c.Fatal
+ if f.Kind != KindOops || f.Reason != "NULL pointer dereference" || f.Comm != "hi_isp_run" || f.PC != "__wake_up_common" || !f.InIRQ {
+ t.Fatalf("fatal %+v", f)
+ }
+ var got []string
+ for _, fr := range f.Frames[:5] {
+ got = append(got, fr.String())
+ }
+ want := "__wake_up_common|__wake_up|osal_wakeup [open_osal]|RGN_PutRegion [open_rgn]|VencPutOsd [open_venc]"
+ if strings.Join(got, "|") != want {
+ t.Fatalf("frames\n %s\nwant\n %s", strings.Join(got, "|"), want)
+ }
+ if len(f.Interrupted) == 0 || f.Interrupted[0].Fn != "osal_spin_unlock_irqrestore" || f.Interrupted[1].Fn != "ISP_DRV_BeBufCtl" {
+ t.Fatalf("interrupted %v", f.Interrupted)
+ }
+ if f.Title() != "NULL pointer dereference in __wake_up_common ← RGN_PutRegion [open_rgn]" {
+ t.Fatalf("title %q", f.Title())
+ }
+ if len(c.Before) != 1 || c.Before[0].Reason != "WARNING at mm/page_alloc.c:7373" || c.Before[0].Comm != "majestic" ||
+ c.Before[0].Frames[0].Fn != "free_contig_range" {
+ t.Fatalf("before %+v", c.Before)
+ }
+ if c.SoC != "gk7205v300" || c.Sensor != "imx335" || c.Board != "demo" || c.Kernel != "4.9.37" ||
+ c.KernelBuilt.Format("2006-01-02T15:04:05") != "2026-09-24T17:57:19" || !strings.HasPrefix(c.Cmdline, "mem=128M") ||
+ c.Machine != "Goke GK7205V300 DEMO Board" || len(c.Modules) == 0 || c.Modules[0] != "open_cipher" {
+ t.Fatalf("context %+v", c)
+ }
+ if c.Anomalies["i2c_error"] != 16 {
+ t.Fatalf("anomalies %v, want 16 i2c errors (one record's)", c.Anomalies)
+ }
+ if len(c.Leadup) == 0 || !strings.Contains(c.Leadup[len(c.Leadup)-1], "i2c") {
+ t.Fatalf("leadup %q", c.Leadup)
+ }
+ if c.SelfInflicted {
+ t.Fatal("a real crash flagged self-inflicted")
+ }
+ // The same records packed again are the same crash.
+ files, _ := Unpack(bundle(t, "gk7205v300-imx335-rgn.tar.gz"))
+ if again := parseBundle(t, tgz(t, files)); again.ContentSum != c.ContentSum || again.Fatal.Signature() != f.Signature() {
+ t.Fatal("repacked bundle is another crash")
+ }
+}
+
+const sysrq = `Panic#1 Part1
+<6>SysRq : Trigger a crash
+<1>Unable to handle kernel NULL pointer dereference at virtual address 00000000
+<0>Internal error: Oops: 817 [#1] ARM
+<0>CPU: 0 PID: 1203 Comm: sh Tainted: G O 4.9.37 #1
+<0>PC is at sysrq_handle_crash+0x24/0x2c
+<0>Backtrace:
+<0>[] (sysrq_handle_crash) from [] (__handle_sysrq+0xa4/0x154)
+<0>[] (__handle_sysrq) from [] (write_sysrq_trigger+0x4c/0x5c)
+<0>[] (write_sysrq_trigger) from [] (proc_reg_write+0x6c/0x90)
+<4>---[ end trace 0000000000000001 ]---
+<0>Kernel panic - not syncing: Fatal exception
+`
+
+func TestSysrqIsSelfInflicted(t *testing.T) {
+ c := parseBundle(t, tgz(t, map[string]string{"dmesg-ramoops-0": sysrq}))
+ if !c.SelfInflicted || c.Kind != KindPanic || c.Fatal.Frames[0].Fn != "sysrq_handle_crash" {
+ t.Fatalf("%+v", c)
+ }
+}
+
+// MIPS (Ingenic) prints "Call Trace:" with one function per line.
+const mips = `<1>CPU 0 Unable to handle kernel paging request at virtual address 00000010, epc == 80123456, ra == 80123400
+<4>Oops[#1]:
+<4>CPU: 0 PID: 712 Comm: majestic Tainted: G O 3.10.14 #1
+<4>epc : 80123456 tx_isp_frame_done+0x30/0x90 [tx-isp-t31]
+<4>Call Trace:
+<4>[<80123456>] tx_isp_frame_done+0x30/0x90 [tx-isp-t31]
+<4>[<80123500>] isp_irq_handle+0x120/0x200 [tx-isp-t31]
+<4>[<8004a000>] handle_irq_event_percpu+0x50/0x1a0
+<4>[<8004d000>] handle_level_irq+0xa0/0x120
+<4>[<80001000>] do_IRQ+0x20/0x30
+<4>---[ end trace 1111111111111111 ]---
+<0>Kernel panic - not syncing: Fatal exception in interrupt
+`
+
+func TestMIPS(t *testing.T) {
+ c := parseBundle(t, []byte(mips))
+ if c.Kind != KindPanic || c.Fatal.Comm != "majestic" || c.Fatal.PC != "tx_isp_frame_done" || !c.Fatal.InIRQ {
+ t.Fatalf("%+v", c.Fatal)
+ }
+ if len(c.Fatal.Frames) != 2 || c.Fatal.Frames[1].String() != "isp_irq_handle [tx-isp-t31]" {
+ t.Fatalf("frames %v", c.Fatal.Frames)
+ }
+ if c.Fatal.Reason != "Paging request" {
+ t.Fatalf("reason %q", c.Fatal.Reason)
+ }
+ // MIPS without the "Unable to handle" line: the Oops header alone opens it.
+ c = parseBundle(t, []byte(strings.Replace(mips, "<1>CPU 0 Unable to handle kernel paging request at virtual address 00000010, epc == 80123456, ra == 80123400\n", "", 1)))
+ if c.Fatal.Kind != KindOops || len(c.Fatal.Frames) != 2 {
+ t.Fatalf("header-only MIPS oops: %+v", c.Fatal)
+ }
+ // pstore's record header is not an oops.
+ if _, err := Parse(map[string]string{"dmesg-ramoops-0": "Oops#1 Part1\n<6>Booting Linux\n"}); err == nil {
+ t.Fatal("a record header alone is a crash")
+ }
+}
+
+func TestFailsafeOnly(t *testing.T) {
+ c := parseBundle(t, tgz(t, map[string]string{"failsafe": "reason=bootlimit\nutc=0\n"}))
+ if c.Kind != KindBootloop || c.Fatal.Signature() == "" {
+ t.Fatalf("%+v", c)
+ }
+}
+
+func TestNotACrash(t *testing.T) {
+ if _, err := Unpack(append([]byte{0x1f, 0x8b}, make([]byte, 50)...)); err == nil {
+ t.Fatal("a broken gzip unpacked")
+ }
+ files, err := Unpack(tgz(t, map[string]string{"pending": "utc=1\n"}))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := Parse(files); err != ErrNotACrash {
+ t.Fatalf("err %v", err)
+ }
+ if _, err := Parse(map[string]string{"dmesg-0": "<6>Booting Linux\n<6>all fine\n"}); err == nil {
+ t.Fatal("a clean log is a crash")
+ }
+ if _, err := Unpack(make([]byte, MaxBundle+1)); err == nil {
+ t.Fatal("an oversized bundle unpacked")
+ }
+}
diff --git a/service/internal/crashes/settle.go b/service/internal/crashes/settle.go
new file mode 100644
index 00000000..2c00a416
--- /dev/null
+++ b/service/internal/crashes/settle.go
@@ -0,0 +1,124 @@
+package crashes
+
+import (
+ "context"
+ "time"
+
+ "github.com/jackc/pgx/v5"
+)
+
+// What a crash earns. A crash is worth reporting once per camera: the same
+// bug sent again by the same camera pays nothing more. Reporting a bug nobody
+// had sent pays more once a maintainer has confirmed it is real, and again
+// when it is fixed.
+const (
+ ReportStars = 1
+ FirstStars = 3
+ FixedStars = 5
+ // MonthCap: the most crash stars a member is paid in 30 days. What is
+ // over it is not lost; a later run pays it.
+ MonthCap = 10
+)
+
+// Settled is what one run paid.
+type Settled struct {
+ Members int
+ Awarded int
+ Held int // over the cap, left for a later run
+}
+
+type due struct {
+ member, signature, macKey, reason string
+ points int
+ at time.Time
+}
+
+// Settle pays every member what their crashes earned and they do not hold.
+// A crash with a camera's MAC is that camera's linked owner's, whoever sent
+// it -- anyone can send any MAC, and a member can name only a camera of
+// theirs (Submit) -- so a crash from a camera linked later pays then. A crash
+// without one is the member's who sent it from /club. A self-inflicted
+// crash, a revoked camera's and a bogus bug's pay nothing.
+//
+// What is held is the net of the ledger's rows under the bug, through every
+// signature merged into it: running twice pays nothing twice, a merge does
+// not pay again what either signature paid, and a bug marked bogus and then
+// restored is paid again. A report is held per member and camera; the first
+// reporter's bonuses per bug.
+func (s *Store) Settle(ctx context.Context, now time.Time) (*Settled, error) {
+ out := &Settled{}
+ err := pgx.BeginFunc(ctx, s.DB, func(tx pgx.Tx) error {
+ if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended('crash-stars-settle', 0))`); err != nil {
+ return err
+ }
+ rows, err := tx.Query(ctx, `
+ WITH ev AS (
+ SELECT e.id, e.received_at, coalesce(sig.merged_into, sig.id) AS root, e.mac_key,
+ CASE WHEN e.mac_key <> '' THEN l.member_id ELSE e.member_id END AS member
+ FROM crash_events e
+ JOIN crash_signatures sig ON sig.id = e.signature_id
+ LEFT JOIN camera_links l ON e.mac_key <> '' AND l.mac_key = e.mac_key
+ WHERE NOT e.self_inflicted
+ AND NOT EXISTS (SELECT 1 FROM wall_revoked r WHERE e.mac_key <> '' AND r.mac_key = e.mac_key)
+ ), live AS (
+ SELECT ev.* FROM ev JOIN crash_signatures root ON root.id = ev.root
+ WHERE ev.member IS NOT NULL AND root.status <> 'bogus'
+ ), per_camera AS (
+ SELECT DISTINCT ON (member, root, mac_key) member, root, mac_key, 'report' AS reason, $1::int AS points, received_at
+ FROM live ORDER BY member, root, mac_key, received_at
+ ), firsts AS (
+ SELECT DISTINCT ON (root) member, root, ''::text AS mac_key, received_at FROM live
+ ORDER BY root, received_at, id
+ ), bonuses AS (
+ SELECT f.member, f.root, '', 'first', $2::int, f.received_at FROM firsts f
+ JOIN crash_signatures r ON r.id = f.root WHERE r.status IN ('confirmed', 'fixed')
+ UNION ALL
+ SELECT f.member, f.root, '', 'fixed', $3::int, f.received_at FROM firsts f
+ JOIN crash_signatures r ON r.id = f.root WHERE r.status = 'fixed'
+ ), owed AS (
+ SELECT * FROM per_camera UNION ALL SELECT * FROM bonuses
+ )
+ SELECT o.member, o.root, o.mac_key, o.reason, o.points, o.received_at FROM owed o
+ WHERE coalesce((SELECT sum(c.points) FROM crash_stars c JOIN crash_signatures s ON s.id = c.signature_id
+ WHERE coalesce(s.merged_into, s.id) = o.root AND c.reason = o.reason
+ AND (o.reason <> 'report' OR (c.member_id = o.member AND c.mac_key = o.mac_key))), 0) <= 0
+ ORDER BY o.member, o.received_at, o.root, o.reason`, ReportStars, FirstStars, FixedStars)
+ if err != nil {
+ return err
+ }
+ list, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (due, error) {
+ var d due
+ err := r.Scan(&d.member, &d.signature, &d.macKey, &d.reason, &d.points, &d.at)
+ return d, err
+ })
+ if err != nil {
+ return err
+ }
+ paid := map[string]int{}
+ for _, d := range list {
+ if _, ok := paid[d.member]; !ok {
+ var n int
+ // Net: stars taken back and paid again are not paid twice.
+ if err := tx.QueryRow(ctx, `SELECT coalesce(sum(points), 0) FROM crash_stars
+ WHERE member_id = $1 AND at > $2`, d.member, now.Add(-30*24*time.Hour)).Scan(&n); err != nil {
+ return err
+ }
+ paid[d.member] = n
+ out.Members++
+ }
+ if paid[d.member]+d.points > MonthCap {
+ out.Held += d.points
+ continue
+ }
+ if _, err := tx.Exec(ctx, `INSERT INTO crash_stars (member_id, signature_id, mac_key, kind, reason, points, at)
+ VALUES ($1, $2, $3, 'award', $4, $5, $6)`,
+ d.member, d.signature, d.macKey, d.reason, d.points, now); err != nil {
+ return err
+ }
+ paid[d.member] += d.points
+ out.Awarded += d.points
+ }
+ return nil
+ })
+ return out, err
+}
diff --git a/service/internal/crashes/store.go b/service/internal/crashes/store.go
new file mode 100644
index 00000000..fb35b6d3
--- /dev/null
+++ b/service/internal/crashes/store.go
@@ -0,0 +1,620 @@
+package crashes
+
+import (
+ "context"
+ "crypto/rand"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "math"
+ "sort"
+ "time"
+
+ "github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/pgxpool"
+)
+
+// Store is the crashes' rows.
+type Store struct {
+ DB *pgxpool.Pool
+}
+
+// Event is one crash, ready to store.
+type Event struct {
+ ID string
+ Channel string // webui, club, api
+ MACKey string
+ Member string
+ ClientKey string
+ Firmware string
+ Majestic string
+ SoC, Sensor string
+ Crash *Crash
+ Meta json.RawMessage // redacted, or nil
+ Redacted string
+ Bundle []byte
+ BundleSHA256 string
+ SignatureID string
+}
+
+// NewID is an event's id: c- and eight characters.
+func NewID() string {
+ const alphabet = "abcdefghijkmnpqrstuvwxyz23456789"
+ var b [8]byte
+ _, _ = rand.Read(b[:])
+ out := []byte("c-")
+ for _, c := range b {
+ out = append(out, alphabet[int(c)%len(alphabet)])
+ }
+ return string(out)
+}
+
+// kindOrder is the kinds, mildest first, for SQL's array_position.
+const kindOrder = `ARRAY['warning','bug','oops','panic','bootloop']`
+
+// ErrQuota is a client address or a camera past its daily limit.
+var ErrQuota = errors.New("the daily limit is reached")
+
+// Insert stores the event, its bundle and its signature. A crash already
+// stored from the same camera (the same records, sent again) is not stored
+// twice: dup is true and the stored event's id and signature are returned.
+// Otherwise the client's and the camera's crashes since are counted under
+// their locks, so uploads racing each other cannot pass the daily limits
+// together.
+func (s *Store) Insert(ctx context.Context, e *Event, since time.Time) (id, signature string, dup bool, err error) {
+ c := e.Crash
+ fatal, _ := json.Marshal(c.Fatal)
+ before, _ := json.Marshal(orEmpty(c.Before))
+ anomalies, _ := json.Marshal(c.Anomalies)
+ leadup, _ := json.Marshal(orEmpty(c.Leadup))
+ frames, _ := json.Marshal(orEmpty(c.Fatal.Frames))
+ class := "fatal"
+ if c.Fatal.Kind == KindWarning {
+ class = "warning"
+ }
+ var built *time.Time
+ if !c.KernelBuilt.IsZero() {
+ built = &c.KernelBuilt
+ }
+ var uptime *float64
+ if c.Uptime > 0 {
+ uptime = &c.Uptime
+ }
+ var meta any
+ if len(e.Meta) > 0 {
+ meta = string(e.Meta)
+ }
+ err = pgx.BeginFunc(ctx, s.DB, func(tx pgx.Tx) error {
+ // Locks in one order: the camera, then the client, then the crash.
+ for _, k := range []string{"crash-camera:" + e.MACKey, "crash-client:" + e.ClientKey, "crash:" + c.ContentSum + ":" + e.MACKey} {
+ if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, k); err != nil {
+ return err
+ }
+ }
+ err := tx.QueryRow(ctx, `SELECT id, signature_id FROM crash_events WHERE content_sum = $1 AND mac_key = $2`,
+ c.ContentSum, e.MACKey).Scan(&id, &signature)
+ if err == nil {
+ dup = true
+ return nil
+ }
+ if !errors.Is(err, pgx.ErrNoRows) {
+ return err
+ }
+ var byClient, byCamera int
+ if err := tx.QueryRow(ctx, `
+ SELECT count(*) FILTER (WHERE client_key = $1), count(*) FILTER (WHERE $2 <> '' AND mac_key = $2)
+ FROM crash_events WHERE received_at >= $3 AND (client_key = $1 OR ($2 <> '' AND mac_key = $2))`,
+ e.ClientKey, e.MACKey, since).Scan(&byClient, &byCamera); err != nil {
+ return err
+ }
+ if byClient >= DailyPerClient || byCamera >= DailyPerCamera {
+ return ErrQuota
+ }
+ if _, err := tx.Exec(ctx, `INSERT INTO crash_bundles (sha256, bytes) VALUES ($1, $2) ON CONFLICT DO NOTHING`,
+ e.BundleSHA256, e.Bundle); err != nil {
+ return err
+ }
+ if _, err := tx.Exec(ctx, `
+ INSERT INTO crash_signatures (id, class, kind, title, frames) VALUES ($1, $2, $3, $4, $5)
+ ON CONFLICT (id) DO UPDATE SET
+ kind = CASE WHEN array_position(`+kindOrder+`, EXCLUDED.kind) > array_position(`+kindOrder+`, crash_signatures.kind)
+ THEN EXCLUDED.kind ELSE crash_signatures.kind END`,
+ e.SignatureID, class, e.Crash.Kind, c.Fatal.Title(), frames); err != nil {
+ return err
+ }
+ id, signature = e.ID, e.SignatureID
+ _, err = tx.Exec(ctx, `
+ INSERT INTO crash_events (id, content_sum, signature_id, channel, mac_key, member_id, client_key, kind, in_irq,
+ self_inflicted, title, firmware, majestic, soc, sensor, board, machine, kernel, kernel_build, kernel_built,
+ cmdline, uptime, records, modules, fatal, before, anomalies, leadup, meta, redacted, bundle_sha256)
+ VALUES ($1, $2, $3, $4, $5, nullif($6, ''), $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20,
+ $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31)`,
+ e.ID, c.ContentSum, e.SignatureID, e.Channel, e.MACKey, e.Member, e.ClientKey, c.Kind, c.Fatal.InIRQ,
+ c.SelfInflicted, c.Fatal.Title(), e.Firmware, e.Majestic, e.SoC, e.Sensor, c.Board, c.Machine, c.Kernel,
+ c.KernelBuild, built, c.Cmdline, uptime, c.Records, orEmpty(c.Modules), fatal, before, anomalies, leadup,
+ meta, e.Redacted, e.BundleSHA256)
+ return err
+ })
+ return id, signature, dup, err
+}
+
+func orEmpty[T any](s []T) []T {
+ if s == nil {
+ return []T{}
+ }
+ return s
+}
+
+// LinkedTo says whether the camera is linked to the member on the Open Wall.
+func (s *Store) LinkedTo(ctx context.Context, macKey, member string) (bool, error) {
+ var ok bool
+ err := s.DB.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM camera_links WHERE mac_key = $1 AND member_id = $2)`,
+ macKey, member).Scan(&ok)
+ return ok, err
+}
+
+// camera is how the rankings count cameras: by MAC, and a crash sent
+// without one as its sender's address, so one person sending many crashes
+// without a MAC is one camera.
+const camera = `CASE WHEN %[1]smac_key <> '' THEN 'm:' || %[1]smac_key ELSE 'c:' || %[1]sclient_key END`
+
+// Signature is a bug as the public list and the maintainers see it.
+type Signature struct {
+ ID string `json:"id"`
+ Class string `json:"class"`
+ Kind string `json:"kind"`
+ Title string `json:"title"`
+ Frames []Frame `json:"frames"`
+ Status string `json:"status"`
+ FixedIn string `json:"fixed_in,omitempty"`
+ IssueURL string `json:"issue_url,omitempty"`
+ InIRQ bool `json:"in_irq"`
+ Events int `json:"events"`
+ Cameras int `json:"cameras"`
+ SoCs []string `json:"socs"`
+ Sensors []string `json:"sensors"`
+ Firmware []string `json:"firmware"`
+ FirstSeen time.Time `json:"first_seen"`
+ LastSeen time.Time `json:"last_seen"`
+ Current bool `json:"current"`
+ Score float64 `json:"score"`
+ MergedInto string `json:"merged_into,omitempty"`
+ Note string `json:"note,omitempty"`
+ newest *time.Time
+}
+
+// base is what a crash of the kind costs an owner: a camera that cannot
+// boot is worst, a panic in an interrupt next (no process to kill, the whole
+// camera goes), a warning least.
+func base(kind string, irq bool) float64 {
+ switch kind {
+ case KindBootloop:
+ return 50
+ case KindPanic:
+ if irq {
+ return 40
+ }
+ return 35
+ case KindOops:
+ return 25
+ case KindBug:
+ return 20
+ }
+ return 3
+}
+
+// CurrentWindow: a signature seen on a kernel built within this of the
+// newest firmware build is still in today's firmware, as far as anyone knows.
+const CurrentWindow = 14 * 24 * time.Hour
+
+// Ranked is the signatures, worst first. maintainers also see the bogus,
+// the merged and the self-inflicted (which public lists leave out).
+func (s *Store) Ranked(ctx context.Context, maintainers bool, now time.Time) ([]*Signature, error) {
+ return s.ranked(ctx, maintainers, "", now)
+}
+
+func (s *Store) ranked(ctx context.Context, maintainers bool, only string, now time.Time) ([]*Signature, error) {
+ rows, err := s.DB.Query(ctx, `
+ WITH ev AS (
+ SELECT coalesce(sig.merged_into, sig.id) AS root, e.*
+ FROM crash_events e JOIN crash_signatures sig ON sig.id = e.signature_id
+ WHERE $1 OR NOT e.self_inflicted
+ )
+ SELECT r.id, r.class,
+ coalesce((`+kindOrder+`)[max(array_position(`+kindOrder+`, ev.kind))], r.kind), r.title, r.frames, r.status, r.fixed_in, r.issue_url, r.first_seen,
+ coalesce(r.merged_into, ''), r.note,
+ count(ev.id)::int, count(DISTINCT `+fmt.Sprintf(camera, "ev.")+`)::int,
+ coalesce(bool_or(ev.in_irq), false), coalesce(max(ev.received_at), r.first_seen),
+ coalesce(array_agg(DISTINCT ev.soc) FILTER (WHERE ev.soc <> ''), '{}'),
+ coalesce(array_agg(DISTINCT ev.sensor) FILTER (WHERE ev.sensor <> ''), '{}'),
+ coalesce(array_agg(DISTINCT ev.firmware) FILTER (WHERE ev.firmware <> ''), '{}'),
+ max(ev.kernel_built)
+ FROM crash_signatures r LEFT JOIN ev ON ev.root = r.id
+ WHERE ($1 OR (r.merged_into IS NULL AND r.status <> 'bogus')) AND ($2 = '' OR r.id = $2)
+ GROUP BY r.id
+ HAVING $1 OR count(ev.id) > 0`, maintainers, only)
+ if err != nil {
+ return nil, err
+ }
+ var out []*Signature
+ for rows.Next() {
+ g := &Signature{}
+ var frames []byte
+ if err := rows.Scan(&g.ID, &g.Class, &g.Kind, &g.Title, &frames, &g.Status, &g.FixedIn, &g.IssueURL, &g.FirstSeen,
+ &g.MergedInto, &g.Note, &g.Events, &g.Cameras, &g.InIRQ, &g.LastSeen, &g.SoCs, &g.Sensors, &g.Firmware,
+ &g.newest); err != nil {
+ rows.Close()
+ return nil, err
+ }
+ _ = json.Unmarshal(frames, &g.Frames)
+ out = append(out, g)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ newest, err := s.newestKernel(ctx)
+ if err != nil {
+ return nil, err
+ }
+ for _, g := range out {
+ g.Current = g.newest != nil && !newest.IsZero() && !g.newest.Before(newest.Add(-CurrentWindow))
+ g.Score = score(g, now)
+ }
+ sort.SliceStable(out, func(i, j int) bool {
+ if out[i].Score != out[j].Score {
+ return out[i].Score > out[j].Score
+ }
+ return out[i].LastSeen.After(out[j].LastSeen)
+ })
+ return out, nil
+}
+
+// score ranks a signature: how bad a crash of it is, times how many cameras
+// it hits (a bug on forty cameras before one camera crashing forty times),
+// discounted when nothing sent it for a month or it is not in today's
+// firmware. A bug that is dealt with goes to the bottom.
+func score(g *Signature, now time.Time) float64 {
+ v := base(g.Kind, g.InIRQ) * (1 + math.Log2(float64(max(g.Cameras, 1))))
+ if now.Sub(g.LastSeen) > 30*24*time.Hour {
+ v *= 0.5
+ }
+ if g.Current {
+ v *= 1.5
+ }
+ switch g.Status {
+ case "fixed", "wontfix", "bogus":
+ v *= 0.01
+ }
+ if g.MergedInto != "" || g.Events == 0 {
+ v = 0
+ }
+ return math.Round(v*10) / 10
+}
+
+// newestKernel is when the newest firmware was built: the pushed builds' if
+// there are any, else the newest kernel any crash ran.
+func (s *Store) newestKernel(ctx context.Context) (time.Time, error) {
+ var t *time.Time
+ err := s.DB.QueryRow(ctx, `SELECT coalesce(
+ (SELECT max(built_at) FROM builds WHERE source = 'firmware'),
+ (SELECT max(kernel_built) FROM crash_events))`).Scan(&t)
+ if err != nil || t == nil {
+ return time.Time{}, err
+ }
+ return *t, nil
+}
+
+// ErrNoSignature is a signature nobody sent.
+var ErrNoSignature = errors.New("no crash has that signature")
+
+// Get is one signature, ranked as the list ranks it.
+func (s *Store) Get(ctx context.Context, id string, maintainers bool, now time.Time) (*Signature, error) {
+ list, err := s.ranked(ctx, maintainers, id, now)
+ if err != nil {
+ return nil, err
+ }
+ if len(list) == 0 {
+ return nil, ErrNoSignature
+ }
+ return list[0], nil
+}
+
+// Combo is one SoC, sensor and build a signature was seen on.
+type Combo struct {
+ SoC string `json:"soc"`
+ Sensor string `json:"sensor"`
+ Kernel string `json:"kernel,omitempty"`
+ KernelBuilt *time.Time `json:"kernel_built,omitempty"`
+ Firmware string `json:"firmware,omitempty"`
+ Majestic string `json:"majestic,omitempty"`
+ Events int `json:"events"`
+ Cameras int `json:"cameras"`
+}
+
+// Combos is where a signature was seen, most cameras first.
+func (s *Store) Combos(ctx context.Context, id string, detailed bool) ([]Combo, error) {
+ cols := `soc, sensor, '', NULL::timestamptz, '', ''`
+ if detailed {
+ cols = `soc, sensor, kernel, kernel_built, firmware, majestic`
+ }
+ rows, err := s.DB.Query(ctx, `
+ SELECT `+cols+`, count(*)::int, count(DISTINCT `+fmt.Sprintf(camera, "e.")+`)::int
+ FROM crash_events e JOIN crash_signatures sig ON sig.id = e.signature_id
+ WHERE coalesce(sig.merged_into, sig.id) = $1 AND NOT e.self_inflicted
+ GROUP BY 1, 2, 3, 4, 5, 6 ORDER BY 8 DESC, 7 DESC, 1, 2`, id)
+ if err != nil {
+ return nil, err
+ }
+ return pgx.CollectRows(rows, func(r pgx.CollectableRow) (Combo, error) {
+ var c Combo
+ err := r.Scan(&c.SoC, &c.Sensor, &c.Kernel, &c.KernelBuilt, &c.Firmware, &c.Majestic, &c.Events, &c.Cameras)
+ return c, err
+ })
+}
+
+// Detail is a crash as maintainers read it to reproduce it.
+type Detail struct {
+ ID string `json:"id"`
+ Signature string `json:"signature"`
+ ReceivedAt time.Time `json:"received_at"`
+ Channel string `json:"channel"`
+ Kind string `json:"kind"`
+ SelfInflicted bool `json:"self_inflicted"`
+ Firmware string `json:"firmware"`
+ Majestic string `json:"majestic"`
+ SoC string `json:"soc"`
+ Sensor string `json:"sensor"`
+ Board string `json:"board"`
+ Machine string `json:"machine"`
+ Kernel string `json:"kernel"`
+ KernelBuild string `json:"kernel_build"`
+ KernelBuilt *time.Time `json:"kernel_built,omitempty"`
+ Cmdline string `json:"cmdline"`
+ Uptime *float64 `json:"uptime,omitempty"`
+ Modules []string `json:"modules"`
+ Fatal json.RawMessage `json:"fatal"`
+ Before json.RawMessage `json:"before"`
+ Anomalies json.RawMessage `json:"anomalies"`
+ Leadup json.RawMessage `json:"leadup"`
+ Meta json.RawMessage `json:"meta,omitempty"`
+ Log string `json:"log,omitempty"`
+ // Builds: the firmware builds whose time matches the kernel's, newest
+ // guess first -- the build to flash to reproduce it.
+ Builds []Build `json:"builds,omitempty"`
+}
+
+// Build is a pushed firmware build.
+type Build struct {
+ ID string `json:"id"`
+ Release string `json:"release"`
+ SHA string `json:"sha"`
+ BuiltAt time.Time `json:"built_at"`
+}
+
+// MaxDetails is how many crashes of a signature its page shows.
+const MaxDetails = 25
+
+// Details is a signature's crashes, newest first, with their redacted logs.
+func (s *Store) Details(ctx context.Context, id string) ([]Detail, error) {
+ rows, err := s.DB.Query(ctx, `
+ SELECT e.id, e.signature_id, e.received_at, e.channel, e.kind, e.self_inflicted, e.firmware, e.majestic,
+ e.soc, e.sensor, e.board, e.machine, e.kernel, e.kernel_build, e.kernel_built, e.cmdline, e.uptime,
+ e.modules, e.fatal, e.before, e.anomalies, e.leadup, e.meta, e.redacted
+ FROM crash_events e JOIN crash_signatures sig ON sig.id = e.signature_id
+ WHERE coalesce(sig.merged_into, sig.id) = $1
+ ORDER BY e.received_at DESC LIMIT $2`, id, MaxDetails)
+ if err != nil {
+ return nil, err
+ }
+ out, err := pgx.CollectRows(rows, func(r pgx.CollectableRow) (Detail, error) {
+ var d Detail
+ var meta []byte
+ err := r.Scan(&d.ID, &d.Signature, &d.ReceivedAt, &d.Channel, &d.Kind, &d.SelfInflicted, &d.Firmware, &d.Majestic,
+ &d.SoC, &d.Sensor, &d.Board, &d.Machine, &d.Kernel, &d.KernelBuild, &d.KernelBuilt, &d.Cmdline, &d.Uptime,
+ &d.Modules, &d.Fatal, &d.Before, &d.Anomalies, &d.Leadup, &meta, &d.Log)
+ if len(meta) > 0 {
+ d.Meta = meta
+ }
+ return d, err
+ })
+ if err != nil {
+ return nil, err
+ }
+ for i := range out {
+ if out[i].Builds, err = s.buildsFor(ctx, out[i].KernelBuilt); err != nil {
+ return nil, err
+ }
+ }
+ return out, nil
+}
+
+// buildsFor is the firmware builds made within hours of the kernel: the
+// kernel is compiled in the build's job, so one of them is the firmware.
+func (s *Store) buildsFor(ctx context.Context, built *time.Time) ([]Build, error) {
+ if built == nil {
+ return nil, nil
+ }
+ rows, err := s.DB.Query(ctx, `
+ SELECT id, release, sha, built_at FROM builds
+ WHERE source = 'firmware' AND built_at BETWEEN $1::timestamptz - interval '6 hours' AND $1::timestamptz + interval '6 hours'
+ ORDER BY abs(extract(epoch FROM built_at - $1::timestamptz)) LIMIT 3`, *built)
+ if err != nil {
+ return nil, err
+ }
+ return pgx.CollectRows(rows, pgx.RowToStructByPos[Build])
+}
+
+// Mine is a member's crashes, newest first.
+type Mine struct {
+ ID string `json:"id"`
+ Signature string `json:"signature"`
+ Title string `json:"title"`
+ Kind string `json:"kind"`
+ Status string `json:"status"`
+ SoC string `json:"soc"`
+ Sensor string `json:"sensor"`
+ ReceivedAt time.Time `json:"received_at"`
+ // SelfInflicted crashes earn nothing; Camera says whether the crash came
+ // from a camera linked to the member.
+ SelfInflicted bool `json:"self_inflicted"`
+ Camera bool `json:"camera"`
+}
+
+// Mine is the crashes the member sent or their linked cameras did.
+func (s *Store) Mine(ctx context.Context, member string) ([]Mine, error) {
+ rows, err := s.DB.Query(ctx, `
+ SELECT e.id, coalesce(sig.merged_into, sig.id), e.title, e.kind, root.status, e.soc, e.sensor, e.received_at,
+ e.self_inflicted, l.member_id IS NOT NULL
+ FROM crash_events e
+ JOIN crash_signatures sig ON sig.id = e.signature_id
+ JOIN crash_signatures root ON root.id = coalesce(sig.merged_into, sig.id)
+ LEFT JOIN camera_links l ON e.mac_key <> '' AND l.mac_key = e.mac_key AND l.member_id = $1
+ WHERE e.member_id = $1 OR l.member_id IS NOT NULL
+ ORDER BY e.received_at DESC LIMIT 200`, member)
+ if err != nil {
+ return nil, err
+ }
+ return pgx.CollectRows(rows, pgx.RowToStructByPos[Mine])
+}
+
+// Triage is a maintainer's decision on a signature. A field left out (nil)
+// keeps what the signature has; "" clears it -- MergeInto "" unmerges.
+type Triage struct {
+ Status string `json:"status"`
+ FixedIn *string `json:"fixed_in"`
+ IssueURL *string `json:"issue_url"`
+ MergeInto *string `json:"merge_into"`
+ Note *string `json:"note"`
+}
+
+var statuses = map[string]bool{"open": true, "confirmed": true, "fixed": true, "wontfix": true, "bogus": true}
+
+// ErrInvalid is a triage that cannot be applied as sent.
+var ErrInvalid = errors.New("invalid triage")
+
+// Decide applies a maintainer's triage. Merging files the signature's
+// events (and whatever was merged into it) under the other's. A signature
+// marked bogus has every star it and the signatures merged into it paid
+// taken back; marking it anything else again lets the settlement pay them
+// anew. Bogus is a decision about a bug, so a signature merged into another
+// is not marked bogus on its own: the one it is merged into is.
+func (s *Store) Decide(ctx context.Context, id, by string, t Triage) (taken int, err error) {
+ if !statuses[t.Status] {
+ return 0, fmt.Errorf("%w: status is open, confirmed, fixed, wontfix or bogus", ErrInvalid)
+ }
+ if t.IssueURL != nil && *t.IssueURL != "" && !httpsURL(*t.IssueURL) {
+ return 0, fmt.Errorf("%w: issue_url is an https:// link", ErrInvalid)
+ }
+ if t.FixedIn != nil && len(*t.FixedIn) > 200 || t.Note != nil && len(*t.Note) > 4000 {
+ return 0, fmt.Errorf("%w: fixed_in is at most 200 characters, note 4000", ErrInvalid)
+ }
+ if t.MergeInto != nil && *t.MergeInto == id {
+ return 0, fmt.Errorf("%w: a signature is not merged into itself", ErrInvalid)
+ }
+ err = pgx.BeginFunc(ctx, s.DB, func(tx pgx.Tx) error {
+ // The settlement's lock: a decision and a payment never interleave.
+ if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended('crash-stars-settle', 0))`); err != nil {
+ return err
+ }
+ var cur Triage
+ var fixedIn, issue, note string
+ var merged *string
+ err := tx.QueryRow(ctx, `SELECT status, fixed_in, issue_url, note, merged_into FROM crash_signatures WHERE id = $1 FOR UPDATE`, id).
+ Scan(&cur.Status, &fixedIn, &issue, ¬e, &merged)
+ if errors.Is(err, pgx.ErrNoRows) {
+ return ErrNoSignature
+ }
+ if err != nil {
+ return err
+ }
+ keep := func(v *string, old string) string {
+ if v == nil {
+ return old
+ }
+ return *v
+ }
+ fixedIn, issue, note = keep(t.FixedIn, fixedIn), keep(t.IssueURL, issue), keep(t.Note, note)
+ if t.MergeInto != nil {
+ merged = nil
+ if *t.MergeInto != "" {
+ var target string
+ err := tx.QueryRow(ctx, `SELECT coalesce(merged_into, id) FROM crash_signatures WHERE id = $1`, *t.MergeInto).Scan(&target)
+ if errors.Is(err, pgx.ErrNoRows) {
+ return fmt.Errorf("merge_into: %w", ErrNoSignature)
+ }
+ if err != nil {
+ return err
+ }
+ if target == id {
+ return fmt.Errorf("%w: %s is already merged into this one", ErrInvalid, *t.MergeInto)
+ }
+ merged = &target
+ if _, err := tx.Exec(ctx, `UPDATE crash_signatures SET merged_into = $2, updated_at = now() WHERE merged_into = $1`, id, target); err != nil {
+ return err
+ }
+ }
+ }
+ if t.Status == "bogus" && merged != nil {
+ return fmt.Errorf("%w: this signature is merged into %s; mark that one bogus, or unmerge this one first", ErrInvalid, *merged)
+ }
+ if _, err := tx.Exec(ctx, `
+ UPDATE crash_signatures SET status = $2, fixed_in = $3, issue_url = $4, note = $5, merged_into = $6, updated_at = now()
+ WHERE id = $1`, id, t.Status, fixedIn, issue, note, merged); err != nil {
+ return err
+ }
+ if _, err := tx.Exec(ctx, `
+ INSERT INTO crash_triage (signature_id, by_member, status, fixed_in, issue_url, merged_into, note)
+ VALUES ($1, $2, $3, $4, $5, $6, $7)`, id, by, t.Status, fixedIn, issue, merged, note); err != nil {
+ return err
+ }
+ if t.Status != "bogus" {
+ return nil
+ }
+ // What is still held under this bug -- awards booked under it and
+ // under each signature merged into it -- is taken back, each row
+ // under the signature it was booked under.
+ return tx.QueryRow(ctx, `WITH held AS (
+ SELECT c.member_id, c.signature_id, c.mac_key, c.reason, sum(c.points) AS points
+ FROM crash_stars c JOIN crash_signatures sig ON sig.id = c.signature_id
+ WHERE coalesce(sig.merged_into, sig.id) = $1
+ GROUP BY 1, 2, 3, 4 HAVING sum(c.points) > 0
+ ), back AS (
+ INSERT INTO crash_stars (member_id, signature_id, mac_key, kind, reason, points)
+ SELECT member_id, signature_id, mac_key, 'revoke', reason, -points FROM held
+ RETURNING points
+ ) SELECT coalesce(-sum(points), 0)::int FROM back`, id).Scan(&taken)
+ })
+ return taken, err
+}
+
+func httpsURL(s string) bool {
+ return len(s) > len("https://") && len(s) <= 500 && s[:8] == "https://"
+}
+
+// StarsOf is the member's crash stars, net.
+func (s *Store) StarsOf(ctx context.Context, member string) (int, error) {
+ var n int
+ err := s.DB.QueryRow(ctx, `SELECT coalesce(sum(points), 0) FROM crash_stars WHERE member_id = $1`, member).Scan(&n)
+ return n, err
+}
+
+// Takedown removes a crash (sent by mistake, or with something in it that
+// must not be kept), and its bundle when no other crash came in it.
+func (s *Store) Takedown(ctx context.Context, id string) error {
+ return pgx.BeginFunc(ctx, s.DB, func(tx pgx.Tx) error {
+ if _, err := tx.Exec(ctx, `SET LOCAL openipc.crashes_guard = 'off'`); err != nil {
+ return err
+ }
+ var bundle string
+ err := tx.QueryRow(ctx, `DELETE FROM crash_events WHERE id = $1 RETURNING bundle_sha256`, id).Scan(&bundle)
+ if errors.Is(err, pgx.ErrNoRows) {
+ return fmt.Errorf("no crash %s", id)
+ }
+ if err != nil {
+ return err
+ }
+ _, err = tx.Exec(ctx, `DELETE FROM crash_bundles b WHERE sha256 = $1
+ AND NOT EXISTS (SELECT 1 FROM crash_events e WHERE e.bundle_sha256 = b.sha256)`, bundle)
+ return err
+ })
+}
diff --git a/service/internal/crashes/store_test.go b/service/internal/crashes/store_test.go
new file mode 100644
index 00000000..5ba933cf
--- /dev/null
+++ b/service/internal/crashes/store_test.go
@@ -0,0 +1,507 @@
+package crashes
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "mime/multipart"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/jackc/pgx/v5/pgxpool"
+
+ "github.com/OpenIPC/website/service/internal/db/dbtest"
+)
+
+type env struct {
+ t *testing.T
+ pool *pgxpool.Pool
+ api *API
+ mux *http.ServeMux
+ now time.Time
+}
+
+func newEnv(t *testing.T) *env {
+ t.Helper()
+ pool := dbtest.New(t)
+ e := &env{t: t, pool: pool, mux: http.NewServeMux(), now: time.Date(2026, 10, 8, 12, 0, 0, 0, time.UTC)}
+ e.api = &API{DB: pool, Log: slog.New(slog.NewTextHandler(io.Discard, nil)), SiteURL: "https://openipc.org",
+ Now: func() time.Time { return e.now }}
+ for k, h := range e.api.Handlers() {
+ e.mux.Handle(k, h)
+ }
+ return e
+}
+
+func (e *env) exec(sql string, args ...any) {
+ e.t.Helper()
+ if _, err := e.pool.Exec(context.Background(), sql, args...); err != nil {
+ e.t.Fatal(err)
+ }
+}
+
+// send posts a bundle as the WebUI does, from an address.
+func (e *env) send(bundle []byte, fields map[string]string, ip string) (int, map[string]any) {
+ e.t.Helper()
+ var body bytes.Buffer
+ mw := multipart.NewWriter(&body)
+ for k, v := range fields {
+ mw.WriteField(k, v)
+ }
+ fw, _ := mw.CreateFormFile("bundle", "crashlog.tar.gz")
+ fw.Write(bundle)
+ mw.Close()
+ req := httptest.NewRequest("POST", "/api/v1/crashes", &body)
+ req.Header.Set("Content-Type", mw.FormDataContentType())
+ req.RemoteAddr = ip + ":1234"
+ rec := httptest.NewRecorder()
+ e.mux.ServeHTTP(rec, req)
+ var out map[string]any
+ json.Unmarshal(rec.Body.Bytes(), &out)
+ return rec.Code, out
+}
+
+func (e *env) get(path string) map[string]any {
+ e.t.Helper()
+ rec := httptest.NewRecorder()
+ e.mux.ServeHTTP(rec, httptest.NewRequest("GET", path, nil))
+ if rec.Code != 200 {
+ e.t.Fatalf("GET %s: %d %s", path, rec.Code, rec.Body)
+ }
+ var out map[string]any
+ json.Unmarshal(rec.Body.Bytes(), &out)
+ return out
+}
+
+const labMAC = "f6:61:31:c5:bd:c1" // the random MAC in the lab bundle's log
+
+func TestUploadStoresOneCrashAndRedacts(t *testing.T) {
+ e := newEnv(t)
+ lab := bundle(t, "gk7205v300-imx335-rgn.tar.gz")
+ code, out := e.send(lab, map[string]string{"mac": labMAC, "firmware": "2.6.10.05-lite", "soc": "gk7205v300",
+ "meta": `{"hostname":"cam","ip":"10.216.128.74"}`}, "10.0.0.1")
+ if code != 201 || out["duplicate"] != false || out["kind"] != KindPanic {
+ t.Fatalf("%d %v", code, out)
+ }
+ sig := out["signature"].(string)
+ if !strings.HasSuffix(out["url"].(string), "/crashes/#"+sig) {
+ t.Fatalf("url %v", out["url"])
+ }
+ // The same crash again, repacked: one event.
+ files, _ := Unpack(lab)
+ code, again := e.send(tgz(t, files), map[string]string{"mac": labMAC}, "10.0.0.1")
+ if code != 200 || again["duplicate"] != true || again["id"] != out["id"] {
+ t.Fatalf("%d %v", code, again)
+ }
+
+ var log, meta string
+ if err := e.pool.QueryRow(context.Background(), `SELECT redacted, meta::text FROM crash_events`).Scan(&log, &meta); err != nil {
+ t.Fatal(err)
+ }
+ if strings.Contains(strings.ToLower(log), labMAC) || !strings.Contains(log, "Unable to handle kernel NULL pointer dereference at virtual address 00000000\n" +
+ "<0>Internal error: Oops: 5 [#1] ARM\n<0>CPU: 0 PID: 9 Comm: majestic\n<0>PC is at " + fn + "+0x1/0x2\n" +
+ "<0>[] (" + fn + " [open_vi]) from [] (caller+0x4/0x8 [open_vi])\n" +
+ "<4>---[ end trace 00000000000000aa ]---\n")
+}
+
+func TestSettle(t *testing.T) {
+ e := newEnv(t)
+ ctx := context.Background()
+ e.member("m-alice00000", "Alice")
+ e.member("m-bob0000000", "Bob")
+ e.link("02:00:00:00:00:0a", "m-alice00000")
+
+ // Alice's camera crashes; Bob's unlinked camera crashes the same way later.
+ lab := bundle(t, "gk7205v300-imx335-rgn.tar.gz")
+ _, out := e.send(lab, map[string]string{"mac": "02:00:00:00:00:0a"}, "10.0.0.1")
+ sig := out["signature"].(string)
+ files, _ := Unpack(lab)
+ files["dmesg-ramoops-1"] += "\n" // another crash, the same bug
+ e.now = e.now.Add(time.Hour)
+ e.send(tgz(t, files), map[string]string{"mac": "02:00:00:00:00:0b"}, "10.0.0.2")
+ // A self-inflicted crash pays nothing.
+ e.send(tgz(t, map[string]string{"dmesg-ramoops-0": sysrq}), map[string]string{"mac": "02:00:00:00:00:0a"}, "10.0.0.1")
+
+ if s := e.settle(); s.Awarded != ReportStars {
+ t.Fatalf("%+v", s)
+ }
+ if e.stars("m-alice00000") != 1 || e.stars("m-bob0000000") != 0 {
+ t.Fatal("wrong first payout")
+ }
+ if s := e.settle(); s.Awarded != 0 {
+ t.Fatalf("paid twice: %+v", s)
+ }
+ // Bob links his camera: his crash pays now; Alice reported first.
+ e.link("02:00:00:00:00:0b", "m-bob0000000")
+ e.settle()
+ if e.stars("m-bob0000000") != 1 {
+ t.Fatal("a crash from a camera linked later did not pay")
+ }
+ st := e.api.Store()
+ if _, err := st.Decide(ctx, sig, "m-maint", Triage{Status: "confirmed", IssueURL: ptr("https://github.com/OpenIPC/firmware/issues/1")}); err != nil {
+ t.Fatal(err)
+ }
+ e.settle()
+ if e.stars("m-alice00000") != 1+FirstStars {
+ t.Fatalf("alice %d after the confirmation", e.stars("m-alice00000"))
+ }
+ if _, err := st.Decide(ctx, sig, "m-maint", Triage{Status: "fixed", FixedIn: ptr("2.6.10.20")}); err != nil {
+ t.Fatal(err)
+ }
+ e.settle()
+ if e.stars("m-alice00000") != 1+FirstStars+FixedStars {
+ t.Fatalf("alice %d after the fix", e.stars("m-alice00000"))
+ }
+ // Found faked after all: everything it paid is taken back.
+ taken, err := st.Decide(ctx, sig, "m-maint", Triage{Status: "bogus"})
+ if err != nil || taken != 1+FirstStars+FixedStars+1 {
+ t.Fatalf("taken %d %v", taken, err)
+ }
+ if e.stars("m-alice00000") != 0 || e.stars("m-bob0000000") != 0 {
+ t.Fatal("a bogus signature's stars stayed")
+ }
+ e.settle()
+ if e.stars("m-alice00000") != 0 {
+ t.Fatal("a bogus signature paid again")
+ }
+ if _, err := e.pool.Exec(ctx, `UPDATE crash_stars SET points = 100`); err == nil {
+ t.Fatal("the ledger changed")
+ }
+ // The decision undone: the next settlement pays it all again.
+ if _, err := st.Decide(ctx, sig, "m-maint", Triage{Status: "fixed"}); err != nil {
+ t.Fatal(err)
+ }
+ e.settle()
+ if e.stars("m-alice00000") != 1+FirstStars+FixedStars || e.stars("m-bob0000000") != 1 {
+ t.Fatalf("restored: alice %d bob %d", e.stars("m-alice00000"), e.stars("m-bob0000000"))
+ }
+ // Re-triage keeps what it does not send: the issue link stays.
+ g, _ := st.Get(ctx, sig, true, e.now)
+ if g.IssueURL == "" || g.FixedIn != "2.6.10.20" {
+ t.Fatalf("triage fields lost: %+v", g)
+ }
+}
+
+func ptr(s string) *string { return &s }
+
+func TestMonthCap(t *testing.T) {
+ e := newEnv(t)
+ e.member("m-alice00000", "Alice")
+ e.link("02:00:00:00:00:0a", "m-alice00000")
+ for i := 0; i < MonthCap+3; i++ {
+ e.now = e.now.Add(5 * time.Hour) // under the camera's daily limit
+ if code, out := e.send(oops(i), map[string]string{"mac": "02:00:00:00:00:0a"}, "10.0.0.1"); code != 201 {
+ t.Fatalf("%d %v", code, out)
+ }
+ }
+ if s := e.settle(); s.Awarded != MonthCap || s.Held != 3 {
+ t.Fatalf("%+v", s)
+ }
+ e.now = e.now.Add(31 * 24 * time.Hour)
+ if s := e.settle(); s.Awarded != 3 {
+ t.Fatalf("held stars not paid a month later: %+v", s)
+ }
+}
+
+func TestRankingAndMerge(t *testing.T) {
+ e := newEnv(t)
+ ctx := context.Background()
+ // One warning-only crash on three cameras, one panic on one camera.
+ warn := []byte("<4>------------[ cut here ]------------\n<4>WARNING: CPU: 0 PID: 1 at drivers/x.c:10 x_fn+0x1/0x2\n" +
+ "<0>[] (x_fn) from [] (y_fn+0x1/0x2)\n<4>---[ end trace 0000000000000001 ]---\n")
+ for i, mac := range []string{"02:00:00:00:00:01", "02:00:00:00:00:02", "02:00:00:00:00:03"} {
+ e.send(append(warn, []byte(strings.Repeat(" ", i))...), map[string]string{"mac": mac}, "10.0.0.1")
+ }
+ _, lab := e.send(bundle(t, "gk7205v300-imx335-rgn.tar.gz"), map[string]string{"mac": "02:00:00:00:00:04"}, "10.0.0.2")
+ _, other := e.send(oops(1), map[string]string{"mac": "02:00:00:00:00:05"}, "10.0.0.2")
+ list, err := e.api.Store().Ranked(ctx, false, e.now)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(list) != 3 || list[0].ID != lab["signature"] || list[len(list)-1].Kind != KindWarning || list[len(list)-1].Cameras != 3 {
+ for _, g := range list {
+ t.Logf("%s %s %v cams %d", g.ID, g.Kind, g.Score, g.Cameras)
+ }
+ t.Fatal("ranking")
+ }
+ // The other oops is the same bug: merged, it counts under the lab's.
+ if _, err := e.api.Store().Decide(ctx, other["signature"].(string), "m-maint", Triage{Status: "open", MergeInto: ptr(lab["signature"].(string))}); err != nil {
+ t.Fatal(err)
+ }
+ g, err := e.api.Store().Get(ctx, lab["signature"].(string), false, e.now)
+ if err != nil || g.Cameras != 2 || g.Events != 2 {
+ t.Fatalf("%+v %v", g, err)
+ }
+ if _, err := e.api.Store().Get(ctx, other["signature"].(string), false, e.now); err != ErrNoSignature {
+ t.Fatalf("a merged signature is listed publicly: %v", err)
+ }
+ details, err := e.api.Store().Details(ctx, lab["signature"].(string))
+ if err != nil || len(details) != 2 || details[0].Log == "" {
+ t.Fatalf("%d %v", len(details), err)
+ }
+}
+
+// Crashes are redacted wherever they are stored, and two addresses one
+// separator apart are both found.
+func TestRedact(t *testing.T) {
+ in := "ip=10.0.0.2:10.0.0.1:10.0.0.254:255.255.255.0 then 1.2.3.4 5.6.7.8, v6 fe80::1ff:fe23:4567:890a and 2001:db8:0:0:0:0:2:1, " +
+ "cisco 0012.3456.789a, colon aa:bb:cc:dd:ee:ff, kept: Linux version 4.9.37.1.2, 17:57:19, 9dc0: bf12bef8, ::1, 127.0.0.1"
+ out := Redact(in, "", "k")
+ for _, leak := range []string{"10.0.0.2", "10.0.0.1", "10.0.0.254", "1.2.3.4", "5.6.7.8", "fe80::", "2001:db8", "0012.3456", "aa:bb:cc"} {
+ if strings.Contains(out, leak) {
+ t.Errorf("%q survived: %s", leak, out)
+ }
+ }
+ for _, kept := range []string{"255.255.255.0", "4.9.37.1.2", "17:57:19", "9dc0: bf12bef8", "::1", "127.0.0.1"} {
+ if !strings.Contains(out, kept) {
+ t.Errorf("%q was redacted: %s", kept, out)
+ }
+ }
+}
+
+func TestAddressesNeverReachThePublicListOrTheDetails(t *testing.T) {
+ e := newEnv(t)
+ panicky := []byte("<5>Kernel command line: mem=64M ip=192.168.1.40:192.168.1.1\n<6>eth0: up at 192.168.1.40\n" +
+ "<0>Kernel panic - not syncing: VFS: Unable to mount root fs via NFS from 192.168.1.4\n" +
+ "<0>[] (mount_root) from [] (prepare_namespace+0x1/0x2)\n")
+ code, out := e.send(panicky, map[string]string{"soc": "gk7205v300"}, "10.0.0.1")
+ if code != 201 {
+ t.Fatalf("%d %v", code, out)
+ }
+ raw, _ := json.Marshal(e.get("/api/v1/crashes"))
+ details, err := e.api.Store().Details(context.Background(), out["signature"].(string))
+ if err != nil {
+ t.Fatal(err)
+ }
+ all, _ := json.Marshal(details)
+ for _, leak := range []string{"192.168.1."} {
+ if bytes.Contains(raw, []byte(leak)) || bytes.Contains(all, []byte(leak)) || strings.Contains(out["title"].(string), leak) {
+ t.Fatalf("%q reached the list or the details:\n%s\n%s", leak, raw, all)
+ }
+ }
+ if code, _ := e.send(oops(1), map[string]string{"soc": strings.Repeat("a", 101)}, "10.0.0.1"); code != 400 {
+ t.Fatalf("a 101-character soc: %d", code)
+ }
+ if code, _ := e.send(oops(1), map[string]string{"sensor": "imx335 at 10.0.0.1"}, "10.0.0.1"); code != 400 {
+ t.Fatalf("an address as a sensor: %d", code)
+ }
+}
+
+// Two cameras' identical failsafe notes are two crashes; the same camera's,
+// one.
+func TestIdenticalRecordsFromTwoCameras(t *testing.T) {
+ e := newEnv(t)
+ b := tgz(t, map[string]string{"failsafe": "reason=bootlimit\nutc=0\n"})
+ _, one := e.send(b, map[string]string{"mac": "02:00:00:00:00:01"}, "10.0.0.1")
+ _, two := e.send(b, map[string]string{"mac": "02:00:00:00:00:02"}, "10.0.0.2")
+ _, again := e.send(b, map[string]string{"mac": "02:00:00:00:00:01"}, "10.0.0.1")
+ if one["id"] == two["id"] || again["id"] != one["id"] || two["duplicate"] != false {
+ t.Fatalf("%v %v %v", one, two, again)
+ }
+ g, _ := e.api.Store().Get(context.Background(), one["signature"].(string), false, e.now)
+ if g.Cameras != 2 || g.Kind != KindBootloop {
+ t.Fatalf("%+v", g)
+ }
+}
+
+// A crash with a camera's MAC pays the camera's owner, whoever sent it.
+func TestACameraCrashPaysItsOwner(t *testing.T) {
+ e := newEnv(t)
+ e.member("m-alice00000", "Alice")
+ e.member("m-bob0000000", "Bob")
+ e.link("02:00:00:00:00:0a", "m-alice00000")
+ // Bob's member id on an event naming Alice's camera (Submit refuses
+ // this from /club; the settlement must not trust it either).
+ e.exec(`INSERT INTO crash_bundles (sha256, bytes) VALUES (repeat('a', 64), '\x00')`)
+ e.exec(`INSERT INTO crash_signatures (id, class, kind, title, frames) VALUES ('aaaaaaaaaaaa', 'fatal', 'oops', 't', '[]')`)
+ e.exec(`INSERT INTO crash_events (id, content_sum, signature_id, channel, mac_key, member_id, client_key, kind, in_irq,
+ self_inflicted, title, records, fatal, redacted, bundle_sha256)
+ VALUES ('c-aaaaaaaa', repeat('b', 64), 'aaaaaaaaaaaa', 'club', '02000000000a', 'm-bob0000000', 'x', 'oops', false,
+ false, 't', 1, '{}', '', repeat('a', 64))`)
+ e.settle()
+ if e.stars("m-alice00000") != ReportStars || e.stars("m-bob0000000") != 0 {
+ t.Fatalf("alice %d bob %d", e.stars("m-alice00000"), e.stars("m-bob0000000"))
+ }
+}
+
+// Merging after payment pays nothing again; a bogus bug takes back what its
+// merged signatures paid; a merged signature is not marked bogus alone, and
+// saving it without a merge target keeps it merged.
+func TestMergesAndStars(t *testing.T) {
+ e := newEnv(t)
+ ctx := context.Background()
+ st := e.api.Store()
+ e.member("m-alice00000", "Alice")
+ e.link("02:00:00:00:00:0a", "m-alice00000")
+ _, a := e.send(oops(1), map[string]string{"mac": "02:00:00:00:00:0a"}, "10.0.0.1")
+ _, b := e.send(oops(2), map[string]string{"mac": "02:00:00:00:00:0a"}, "10.0.0.1")
+ sa, sb := a["signature"].(string), b["signature"].(string)
+ if _, err := st.Decide(ctx, sa, "m", Triage{Status: "confirmed"}); err != nil {
+ t.Fatal(err)
+ }
+ e.settle()
+ before := e.stars("m-alice00000") // two reports and A's first-reporter bonus
+ if before != 2*ReportStars+FirstStars {
+ t.Fatalf("before the merge: %d", before)
+ }
+ if _, err := st.Decide(ctx, sa, "m", Triage{Status: "confirmed", MergeInto: &sb}); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := st.Decide(ctx, sb, "m", Triage{Status: "confirmed"}); err != nil {
+ t.Fatal(err)
+ }
+ e.settle()
+ if got := e.stars("m-alice00000"); got != before {
+ t.Fatalf("the merge paid again: %d, was %d", got, before)
+ }
+ if _, err := st.Decide(ctx, sa, "m", Triage{Status: "bogus"}); !errors.Is(err, ErrInvalid) {
+ t.Fatalf("a merged signature marked bogus alone: %v", err)
+ }
+ if _, err := st.Decide(ctx, sa, "m", Triage{Status: "confirmed", Note: ptr("same bug")}); err != nil {
+ t.Fatal(err)
+ }
+ if g, _ := st.Get(ctx, sa, true, e.now); g.MergedInto != sb {
+ t.Fatalf("saving a note unmerged it: %+v", g)
+ }
+ taken, err := st.Decide(ctx, sb, "m", Triage{Status: "bogus"})
+ if err != nil || taken != before || e.stars("m-alice00000") != 0 {
+ t.Fatalf("taken %d (%v), left %d", taken, err, e.stars("m-alice00000"))
+ }
+}
+
+// Uploads racing each other do not pass the camera's daily limit together.
+func TestConcurrentUploadsKeepTheLimit(t *testing.T) {
+ e := newEnv(t)
+ var wg sync.WaitGroup
+ var mu sync.Mutex
+ created := 0
+ for i := 0; i < DailyPerCamera*3; i++ {
+ wg.Add(1)
+ go func(i int) {
+ defer wg.Done()
+ code, _ := e.send(oops(i), map[string]string{"mac": "02:00:00:00:00:0c"}, fmt.Sprintf("10.0.1.%d", i))
+ mu.Lock()
+ if code == 201 {
+ created++
+ }
+ mu.Unlock()
+ }(i)
+ }
+ wg.Wait()
+ if created != DailyPerCamera {
+ t.Fatalf("%d created, the limit is %d", created, DailyPerCamera)
+ }
+}
diff --git a/service/internal/crashes/testdata/gk7205v300-imx335-rgn.tar.gz b/service/internal/crashes/testdata/gk7205v300-imx335-rgn.tar.gz
new file mode 100644
index 00000000..ee3cc04a
Binary files /dev/null and b/service/internal/crashes/testdata/gk7205v300-imx335-rgn.tar.gz differ
diff --git a/service/internal/db/migrations/030_crashes.sql b/service/internal/db/migrations/030_crashes.sql
new file mode 100644
index 00000000..36b29bc1
--- /dev/null
+++ b/service/internal/db/migrations/030_crashes.sql
@@ -0,0 +1,169 @@
+-- Kernel crashes cameras recovered from (internal/crashes): the bundle the
+-- firmware's S98crashlog makes of pstore's records, sent by the WebUI
+-- (POST /api/v1/crashes) or by a member from /club.
+--
+-- A crash is filed under its signature, the top of its backtrace without
+-- offsets, so one bug sent by many cameras is one signature. The events and
+-- the bundles they came in are append-only, like owner reports; only a
+-- takedown (openipc.crashes_guard = 'off', its own transaction) removes one.
+-- Signatures are the maintainers' to triage, so they change.
+--
+-- Stars are paid by the nightly settlement (`openipc crashes settle`), never
+-- by an upload, into crash_stars -- the third ledger beside report_stars and
+-- wall_stars.
+
+CREATE TABLE crash_signatures (
+ id text PRIMARY KEY CHECK (id ~ '^[0-9a-f]{12}$'),
+ class text NOT NULL CHECK (class IN ('fatal', 'warning')),
+ -- the worst kind any event under it was
+ kind text NOT NULL CHECK (kind IN ('bootloop', 'panic', 'oops', 'bug', 'warning')),
+ title text NOT NULL,
+ frames jsonb NOT NULL,
+ first_seen timestamptz NOT NULL DEFAULT now(),
+ -- triage: open until a maintainer looks; bogus is a faked crash, which
+ -- pays nothing and is taken back
+ status text NOT NULL DEFAULT 'open' CHECK (status IN ('open', 'confirmed', 'fixed', 'wontfix', 'bogus')),
+ fixed_in text NOT NULL DEFAULT '' CHECK (length(fixed_in) <= 200),
+ issue_url text NOT NULL DEFAULT '' CHECK (issue_url = '' OR issue_url ~ '^https://'),
+ note text NOT NULL DEFAULT '' CHECK (length(note) <= 4000),
+ -- one bug found under two signatures: the events of this one count
+ -- under that one
+ merged_into text REFERENCES crash_signatures CHECK (merged_into <> id),
+ updated_at timestamptz NOT NULL DEFAULT now()
+);
+
+-- Who triaged what, as it happened.
+CREATE TABLE crash_triage (
+ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+ signature_id text NOT NULL REFERENCES crash_signatures,
+ by_member text NOT NULL,
+ status text NOT NULL,
+ fixed_in text NOT NULL,
+ issue_url text NOT NULL,
+ merged_into text,
+ note text NOT NULL,
+ at timestamptz NOT NULL DEFAULT now()
+);
+
+-- A bundle as it was sent, never served: the log maintainers read is the
+-- redacted copy in crash_events.
+CREATE TABLE crash_bundles (
+ sha256 text PRIMARY KEY CHECK (sha256 ~ '^[0-9a-f]{64}$'),
+ bytes bytea NOT NULL CHECK (length(bytes) <= 262144),
+ received_at timestamptz NOT NULL DEFAULT now()
+);
+
+CREATE TABLE crash_events (
+ id text PRIMARY KEY CHECK (id ~ '^c-[a-z0-9]{8}$'),
+ -- the records' content, whatever tar or gzip carried them: the same
+ -- crash sent twice from one camera is one event (UNIQUE below; two
+ -- cameras' identical failsafe notes are two)
+ content_sum text NOT NULL CHECK (content_sum ~ '^[0-9a-f]{64}$'),
+ signature_id text NOT NULL REFERENCES crash_signatures,
+ received_at timestamptz NOT NULL DEFAULT now(),
+ channel text NOT NULL CHECK (channel IN ('webui', 'club', 'api')),
+ -- the camera: its MAC as the wall keys it ('' when the sender did not
+ -- say), and the member who sent it from /club
+ mac_key text NOT NULL DEFAULT '',
+ member_id text REFERENCES club_members ON DELETE SET NULL,
+ client_key text NOT NULL,
+ kind text NOT NULL,
+ in_irq boolean NOT NULL,
+ self_inflicted boolean NOT NULL,
+ title text NOT NULL,
+ -- what the camera said it runs, and what the log says it ran
+ firmware text NOT NULL DEFAULT '' CHECK (length(firmware) <= 200),
+ majestic text NOT NULL DEFAULT '' CHECK (length(majestic) <= 200),
+ soc text NOT NULL DEFAULT '' CHECK (length(soc) <= 100),
+ sensor text NOT NULL DEFAULT '' CHECK (length(sensor) <= 100),
+ board text NOT NULL DEFAULT '',
+ machine text NOT NULL DEFAULT '',
+ kernel text NOT NULL DEFAULT '',
+ kernel_build text NOT NULL DEFAULT '',
+ kernel_built timestamptz,
+ cmdline text NOT NULL DEFAULT '',
+ uptime double precision,
+ records int NOT NULL,
+ modules text[] NOT NULL DEFAULT '{}',
+ fatal jsonb NOT NULL,
+ before jsonb NOT NULL DEFAULT '[]',
+ anomalies jsonb NOT NULL DEFAULT '{}',
+ leadup jsonb NOT NULL DEFAULT '[]',
+ -- the firmware's description of the camera (meta.json), redacted
+ meta jsonb,
+ -- every record, the identifiers replaced by keyed hashes
+ redacted text NOT NULL,
+ bundle_sha256 text NOT NULL REFERENCES crash_bundles ON DELETE RESTRICT,
+ UNIQUE (content_sum, mac_key)
+);
+CREATE INDEX crash_events_by_signature ON crash_events (signature_id, received_at);
+CREATE INDEX crash_events_by_member ON crash_events (member_id, received_at) WHERE member_id IS NOT NULL;
+CREATE INDEX crash_events_by_camera ON crash_events (mac_key, received_at) WHERE mac_key <> '';
+CREATE INDEX crash_events_by_client ON crash_events (client_key, received_at);
+
+CREATE FUNCTION crashes_guard() RETURNS trigger LANGUAGE plpgsql AS $$
+BEGIN
+ IF coalesce(current_setting('openipc.crashes_guard', true), '') = 'off' THEN
+ IF TG_OP = 'DELETE' THEN
+ RETURN OLD;
+ END IF;
+ RETURN NEW;
+ END IF;
+ -- a member who leaves: their events stay, no longer theirs
+ IF TG_OP = 'UPDATE' AND TG_TABLE_NAME = 'crash_events' THEN
+ IF NEW.member_id IS NULL AND NOT EXISTS (SELECT 1 FROM club_members WHERE id = OLD.member_id)
+ AND to_jsonb(NEW) - 'member_id' = to_jsonb(OLD) - 'member_id' THEN
+ RETURN NEW;
+ END IF;
+ END IF;
+ RAISE EXCEPTION 'crashes are never changed or deleted (% on %); see service/internal/crashes', TG_OP, TG_TABLE_NAME
+ USING ERRCODE = 'insufficient_privilege';
+END $$;
+CREATE TRIGGER crash_events_guard BEFORE UPDATE OR DELETE ON crash_events
+ FOR EACH ROW EXECUTE FUNCTION crashes_guard();
+CREATE TRIGGER crash_bundles_guard BEFORE UPDATE OR DELETE ON crash_bundles
+ FOR EACH ROW EXECUTE FUNCTION crashes_guard();
+CREATE TRIGGER crash_events_no_truncate BEFORE TRUNCATE ON crash_events
+ FOR EACH STATEMENT EXECUTE FUNCTION crashes_guard();
+CREATE TRIGGER crash_bundles_no_truncate BEFORE TRUNCATE ON crash_bundles
+ FOR EACH STATEMENT EXECUTE FUNCTION crashes_guard();
+
+-- The crashes' stars ledger, beside report_stars and wall_stars: inserted,
+-- never changed. What a member holds for a bug is the net of their rows
+-- under it -- its signature and every signature merged into it -- so a
+-- revoke (a signature found bogus) and a later award (the decision undone)
+-- are both just rows, and the settlement pays only what is not held.
+-- reason: report (a crash of the bug from the member's linked camera, or
+-- sent by them from /club without a camera: mac_key ''), first (they
+-- reported the bug first and a maintainer confirmed it), fixed (it was
+-- fixed); a revoke row repeats the reason of the award it takes back.
+CREATE TABLE crash_stars (
+ id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+ member_id text NOT NULL REFERENCES club_members ON DELETE CASCADE,
+ signature_id text NOT NULL REFERENCES crash_signatures,
+ mac_key text NOT NULL DEFAULT '',
+ kind text NOT NULL CHECK (kind IN ('award', 'revoke')),
+ reason text NOT NULL CHECK (reason IN ('report', 'first', 'fixed')),
+ points int NOT NULL,
+ at timestamptz NOT NULL DEFAULT now()
+);
+CREATE INDEX crash_stars_by_member ON crash_stars (member_id, at);
+CREATE INDEX crash_stars_by_signature ON crash_stars (signature_id);
+
+CREATE FUNCTION crash_stars_guard() RETURNS trigger LANGUAGE plpgsql AS $$
+BEGIN
+ IF TG_OP = 'DELETE' AND NOT EXISTS (SELECT 1 FROM club_members WHERE id = OLD.member_id) THEN
+ RETURN OLD;
+ END IF;
+ RAISE EXCEPTION 'the stars ledger is never changed (% on %); see service/internal/crashes', TG_OP, TG_TABLE_NAME
+ USING ERRCODE = 'insufficient_privilege';
+END $$;
+CREATE TRIGGER crash_stars_guard BEFORE UPDATE OR DELETE ON crash_stars
+ FOR EACH ROW EXECUTE FUNCTION crash_stars_guard();
+CREATE FUNCTION crash_stars_no_truncate() RETURNS trigger LANGUAGE plpgsql AS $$
+BEGIN
+ RAISE EXCEPTION 'the stars ledger is never truncated; see service/internal/crashes'
+ USING ERRCODE = 'insufficient_privilege';
+END $$;
+CREATE TRIGGER crash_stars_no_truncate BEFORE TRUNCATE ON crash_stars
+ FOR EACH STATEMENT EXECUTE FUNCTION crash_stars_no_truncate();
diff --git a/service/internal/wallstars/wallstars.go b/service/internal/wallstars/wallstars.go
index 9094c036..d923b1e4 100644
--- a/service/internal/wallstars/wallstars.go
+++ b/service/internal/wallstars/wallstars.go
@@ -543,6 +543,7 @@ type Leader struct {
Name string `json:"name"`
Reports int `json:"reports"`
Wall int `json:"wall"`
+ Crashes int `json:"crashes"`
Stars int `json:"stars"`
You bool `json:"you,omitempty"`
}
@@ -551,16 +552,19 @@ type Leader struct {
const LeaderboardSize = 100
// Leaderboard ranks the members who have not opted out by their stars from
-// both ledgers, since a moment (the zero time for all of them). Members with
-// no stars in the period are left out. you marks the asking member's row.
+// every ledger -- reports, the wall, crashes -- since a moment (the zero time
+// for all of them). Members with no stars in the period are left out. you marks the asking member's row.
func (s *Store) Leaderboard(ctx context.Context, since time.Time, you string) ([]Leader, error) {
rows, err := s.DB.Query(ctx, `
WITH ledger AS (
- SELECT member_id, points, 0 AS wall FROM report_stars WHERE at >= $1
+ SELECT member_id, points, 0 AS wall, 0 AS crash FROM report_stars WHERE at >= $1
UNION ALL
- SELECT member_id, points, points FROM wall_stars WHERE at >= $1
+ SELECT member_id, points, points, 0 FROM wall_stars WHERE at >= $1
+ UNION ALL
+ SELECT member_id, points, 0, points FROM crash_stars WHERE at >= $1
)
- SELECT m.id, m.name, (sum(l.points) - sum(l.wall))::int, sum(l.wall)::int, sum(l.points)::int
+ SELECT m.id, m.name, (sum(l.points) - sum(l.wall) - sum(l.crash))::int, sum(l.wall)::int, sum(l.crash)::int,
+ sum(l.points)::int
FROM ledger l JOIN club_members m ON m.id = l.member_id
WHERE m.listed
GROUP BY m.id, m.name
@@ -575,7 +579,7 @@ func (s *Store) Leaderboard(ctx context.Context, since time.Time, you string) ([
for rows.Next() {
var id string
var l Leader
- if err := rows.Scan(&id, &l.Name, &l.Reports, &l.Wall, &l.Stars); err != nil {
+ if err := rows.Scan(&id, &l.Name, &l.Reports, &l.Wall, &l.Crashes, &l.Stars); err != nil {
return nil, err
}
l.Rank, l.You = len(out)+1, id == you
@@ -599,6 +603,7 @@ func (s *Store) OwnerOf(ctx context.Context, macKey string) (*Owner, error) {
SELECT m.name, c.days,
(SELECT coalesce(sum(points), 0) FROM report_stars WHERE member_id = m.id)
+ (SELECT coalesce(sum(points), 0) FROM wall_stars WHERE member_id = m.id)
+ + (SELECT coalesce(sum(points), 0) FROM crash_stars WHERE member_id = m.id)
FROM camera_links l JOIN club_members m ON m.id = l.member_id JOIN cameras c ON c.mac_key = l.mac_key
WHERE l.mac_key = $1 AND l.show_owner
AND NOT EXISTS (SELECT 1 FROM wall_revoked r WHERE r.mac_key = l.mac_key)`, macKey).Scan(&o.Name, &o.Days, &o.Stars)
diff --git a/service/routes.json b/service/routes.json
index a136e803..1fad2fa3 100644
--- a/service/routes.json
+++ b/service/routes.json
@@ -269,6 +269,46 @@
"method": "GET",
"path": "/api/v1/club/leaderboard"
},
+ {
+ "role": "web",
+ "method": "POST",
+ "path": "/api/v1/club/crashes"
+ },
+ {
+ "role": "web",
+ "method": "GET",
+ "path": "/api/v1/club/crashes"
+ },
+ {
+ "role": "web",
+ "method": "GET",
+ "path": "/api/v1/club/crashes/triage"
+ },
+ {
+ "role": "web",
+ "method": "GET",
+ "path": "/api/v1/club/crashes/{id}"
+ },
+ {
+ "role": "web",
+ "method": "POST",
+ "path": "/api/v1/club/crashes/{id}"
+ },
+ {
+ "role": "web",
+ "method": "POST",
+ "path": "/api/v1/crashes"
+ },
+ {
+ "role": "web",
+ "method": "GET",
+ "path": "/api/v1/crashes"
+ },
+ {
+ "role": "web",
+ "method": "GET",
+ "path": "/api/v1/crashes/{id}"
+ },
{
"role": "web",
"method": "PUT",