Club: majestic's own crashes, symbolized against its CI's debuginfo

.github/workflows/build.yml

@@ -41,11 +41,13 @@ jobs:
       - uses: actions/checkout@v4
 
       # The keyframe and variants tests decode with ffmpeg, as the service does
-      # in its image. They fail rather than skip when CI is set and it is missing.
-      - name: Install ffmpeg
+      # in its image; the crash symbolizer's unwind with gdb-multiarch and
+      # unpack firmware with unsquashfs. They fail rather than skip when CI is
+      # set and one is missing.
+      - name: Install ffmpeg, gdb-multiarch and squashfs-tools
         run: |
           sudo apt-get update -qq
-          sudo apt-get install -y --no-install-recommends ffmpeg
+          sudo apt-get install -y --no-install-recommends ffmpeg gdb-multiarch squashfs-tools
 
       # --- the Go service (#287) ---------------------------------------------
       - uses: actions/setup-go@v5

CLAUDE.md

@@ -234,6 +234,21 @@ restores the image but never the schema, so keep migrations additive.
   merges: `bogus` takes back what a bug and the signatures merged into it
   paid, and undoing it lets the settlement pay again. A crash with a MAC is
   the camera's linked owner's, whoever sent it.
+  **majestic's own crashes** come the same way, as `/etc/crash/majestic.dump`
+  (class `user`, kind `signal`, migration 032), and are the maintainers'
+  only: never on `/crashes`, and a member is shown the signal, not where.
+  One arrives filed under a provisional signature (module and offset of the
+  faulting instruction) with its dump in `crash_dumps`; the firmware role's
+  `internal/crashes/symbolize` (woken by `NOTIFY crash_symbolize`) builds an
+  ELF core from it and unwinds it with `gdb-multiarch` against the
+  executable and debuginfo majestic's CI publishes by build-id
+  (`MAJESTIC_SYMBOLS_BASE`), naming library frames from the firmware build's
+  own rootfs (`unsquashfs` of its release tarball), and scanning the stack
+  for probable callers where gdb stops. Then the event is refiled under its
+  backtrace's signature (the crashes package's one change to an event) and
+  the dump -- majestic's memory -- is deleted. Retried for days when the
+  debuginfo is not there, then given up on. Cache under
+  `RELEASE_CACHE_ROOT/symbols`, swept after 30 days unused.
 - `internal/tools`, `internal/nfsro` — **ipctool for stock firmware**, which
   has no curl and no TLS. ipctool's release job pushes each build to
   `PUT /api/v1/tools/{name}` (OIDC, `internal/tools/PUSH.md`); nginx serves

data/locales/boards.en.yml

@@ -49,7 +49,7 @@ en:
     club_crashes:
       eyebrow: 'OpenIPC Club'
       title: 'Crash triage'
-      lede: 'Kernel crashes cameras sent, worst first, with what it takes to reproduce each one.'
+      lede: 'Crashes cameras sent, the kernel''s and majestic''s, worst first, with what it takes to reproduce each one.'
     crashes:
       eyebrow: 'OpenIPC firmware'
       title: 'Kernel crashes from the field'
@@ -632,6 +632,7 @@ en:
       kind_oops: 'Oops'
       kind_bug: 'BUG'
       kind_warning: 'Warning'
+      kind_signal: 'majestic crashed'
       in_irq: 'in an interrupt'
       status_open: 'Not yet looked at'
       status_confirmed: 'Confirmed'
@@ -691,3 +692,7 @@ en:
       d_cmdline: 'Command line'
       d_anomalies: 'Before it'
       d_before: 'Warnings'
+      d_backtrace: 'Backtrace'
+      sym_pending: 'Waiting for its backtrace (tried {n} times)'
+      sym_failed: 'No backtrace'
+      probable: 'probable'

data/locales/boards.ru.yml

@@ -49,7 +49,7 @@ ru:
     club_crashes:
       eyebrow: 'Клуб OpenIPC'
       title: 'Разбор падений'
-      lede: 'Падения ядра, присланные камерами, — сначала самые тяжёлые, с тем, что нужно, чтобы воспроизвести каждое.'
+      lede: 'Падения ядра и majestic, присланные камерами, — сначала самые тяжёлые, с тем, что нужно, чтобы воспроизвести каждое.'
     crashes:
       eyebrow: 'Прошивка OpenIPC'
       title: 'Падения ядра с реальных камер'
@@ -673,6 +673,7 @@ ru:
       kind_oops: 'Oops'
       kind_bug: 'BUG'
       kind_warning: 'Предупреждение'
+      kind_signal: 'Сбой majestic'
       in_irq: 'в прерывании'
       status_open: 'Ещё не разобрано'
       status_confirmed: 'Подтверждено'
@@ -736,3 +737,7 @@ ru:
       d_cmdline: 'Командная строка'
       d_anomalies: 'Перед этим'
       d_before: 'Предупреждения'
+      d_backtrace: 'Стек вызовов'
+      sym_pending: 'Ждёт стека вызовов (попыток: {n})'
+      sym_failed: 'Стека вызовов нет'
+      probable: 'вероятно'

frontend/apps/site/src/components/club/CrashTriage.tsx

@@ -1,8 +1,9 @@
 /**
- * /club/crashes: the maintainers' triage of kernel crashes. Every signature,
- * worst first; one opens to what it takes to reproduce it -- the chips,
- * sensors and builds it was seen on, the firmware builds whose time matches
- * its kernel, and each crash's redacted log, warnings and lead-up.
+ * /club/crashes: the maintainers' triage of kernel crashes and majestic's.
+ * Every signature, worst first; one opens to what it takes to reproduce it --
+ * the chips, sensors and builds it was seen on, the firmware builds whose time
+ * matches its kernel, and each crash's redacted log, warnings and lead-up, or
+ * for majestic's, its backtrace from the debuginfo of its build.
  *
  * Confirming a bug pays its first reporter, fixing it pays them again;
  * bogus takes back every star it paid. The same is `openipc crashes status`.
@@ -12,7 +13,8 @@ import { useBoardsTranslations, type BoardsT } from '../../lib/boards-i18n';
 import { pathFor, type Locale } from '../../lib/i18n';
 import { ClubError, fetchMe } from '../../lib/club';
 import {
-  decideCrash, fetchCrashDetail, fetchTriage, frame, KIND_TONE, STATUS_TONE, type Combo, type Detail, type Signature, type Status,
+  decideCrash, fetchCrashDetail, fetchTriage, frame, frameAt, KIND_TONE, STATUS_TONE, type Combo, type Detail, type Frame, type Signature,
+  type Status,
 } from '../../lib/crashes';
 
 type Load = { state: 'loading' } | { state: 'ok'; list: Signature[] } | { state: 'forbidden' } | { state: 'error' };
@@ -81,9 +83,7 @@ function Open({ g, t, onDone }: { g: Signature; t: BoardsT; onDone: () => void }
   const field = 'min-w-0 rounded-md border border-hairline px-2 py-1 text-sm';
   return (
     <div class="grid gap-3 border-t border-hairline pt-3 text-[13px]">
-      <pre class="m-0 overflow-x-auto rounded-md bg-surface-alt p-2.5 font-mono text-[12px] leading-relaxed">
-        {g.frames.slice(0, 10).map((f, i) => `${i === 0 ? '' : '← '}${frame(f)}`).join('\n')}
-      </pre>
+      <Backtrace frames={g.frames.slice(0, 10)} t={t} />
       <form class="grid gap-2 sm:grid-cols-2" onSubmit={save}>
         <label class="grid gap-1">{t('crashes.f_status')}
           <select class={field} value={status} onChange={(e) => setStatus((e.target as HTMLSelectElement).value as Status)}>
@@ -125,6 +125,15 @@ function Open({ g, t, onDone }: { g: Signature; t: BoardsT; onDone: () => void }
   );
 }
 
+/** A backtrace, innermost first; the scan's probable callers marked so. */
+function Backtrace({ frames, t }: { frames: Frame[]; t: BoardsT }) {
+  return (
+    <pre class="m-0 overflow-x-auto rounded-md bg-surface-alt p-2.5 font-mono text-[12px] leading-relaxed">
+      {frames.map((f, i) => `${i === 0 ? '' : '← '}${frameAt(f)}${f.probable ? `  (${t('crashes.probable')})` : ''}`).join('\n')}
+    </pre>
+  );
+}
+
 function Crash({ d, t }: { d: Detail; t: BoardsT }) {
   const anomalies = Object.entries(d.anomalies ?? {});
   return (
@@ -136,19 +145,35 @@ function Crash({ d, t }: { d: Detail; t: BoardsT }) {
       <dl class="m-0 mt-2 grid gap-x-3 gap-y-0.5 sm:grid-cols-[max-content_1fr]">
         <dt class="text-body-secondary">{t('crashes.d_hardware')}</dt><dd class="m-0 font-mono">{[d.soc, d.sensor, d.board, d.machine].filter(Boolean).join(' · ')}</dd>
         <dt class="text-body-secondary">{t('crashes.d_firmware')}</dt><dd class="m-0 font-mono">{[d.firmware, d.majestic && `majestic ${d.majestic}`].filter(Boolean).join(' · ') || '?'}</dd>
-        <dt class="text-body-secondary">{t('crashes.d_kernel')}</dt><dd class="m-0 font-mono">{d.kernel} {d.kernel_build}</dd>
+        {d.kernel && <><dt class="text-body-secondary">{t('crashes.d_kernel')}</dt><dd class="m-0 font-mono">{d.kernel} {d.kernel_build}</dd></>}
         {d.builds && d.builds.length > 0 && (
           <><dt class="text-body-secondary">{t('crashes.d_builds')}</dt><dd class="m-0 font-mono">{d.builds.map((b) => `${b.release} (${b.sha.slice(0, 7)})`).join(', ')}</dd></>
         )}
         <dt class="text-body-secondary">{t('crashes.d_task')}</dt><dd class="m-0 font-mono">{d.fatal.comm ?? '?'}{d.fatal.interrupted?.length ? ` · ${t('crashes.d_interrupted')} ${d.fatal.interrupted.map(frame).join(' ← ')}` : ''}</dd>
         {d.uptime != null && <><dt class="text-body-secondary">{t('crashes.d_uptime')}</dt><dd class="m-0 tabular-nums">{Math.round(d.uptime)} s</dd></>}
-        <dt class="text-body-secondary">{t('crashes.d_cmdline')}</dt><dd class="m-0 font-mono break-all">{d.cmdline}</dd>
+        {d.cmdline && <><dt class="text-body-secondary">{t('crashes.d_cmdline')}</dt><dd class="m-0 font-mono break-all">{d.cmdline}</dd></>}
         {anomalies.length > 0 && <><dt class="text-body-secondary">{t('crashes.d_anomalies')}</dt><dd class="m-0 font-mono">{anomalies.map(([k, v]) => `${k} ×${v}`).join(', ')}</dd></>}
         {d.before.length > 0 && <><dt class="text-body-secondary">{t('crashes.d_before')}</dt><dd class="m-0 font-mono">{d.before.map((b) => `${b.reason} (${b.comm ?? '?'}) ${b.frames.slice(0, 3).map(frame).join(' ← ')}`).join('; ')}</dd></>}
       </dl>
+      {d.symbolization && <Symbolized s={d.symbolization} t={t} />}
       {d.leadup.length > 0 && <pre class="mt-2 mb-0 overflow-x-auto rounded bg-surface-alt p-2 font-mono text-[11.5px]">{d.leadup.join('\n')}</pre>}
       {d.meta != null && <pre class="mt-2 mb-0 max-h-60 overflow-auto rounded bg-surface-alt p-2 font-mono text-[11.5px]">{JSON.stringify(d.meta, null, 2)}</pre>}
       {d.log && <pre class="mt-2 mb-0 max-h-96 overflow-auto rounded bg-ink p-2 font-mono text-[11.5px] text-white">{d.log}</pre>}
     </details>
   );
 }
+
+/** How a majestic crash's dump was unwound: its backtrace, or why not yet. */
+function Symbolized({ s, t }: { s: NonNullable<Detail['symbolization']>; t: BoardsT }) {
+  return (
+    <div class="mt-2 grid gap-1">
+      <span class="text-body-secondary">
+        {t('crashes.d_backtrace')}
+        {s.status === 'pending' && ` · ${t('crashes.sym_pending', { n: s.attempts })}`}
+        {s.status === 'failed' && ` · ${t('crashes.sym_failed')}`}
+        {s.error && <span class="font-mono"> · {s.error}</span>}
+      </span>
+      {s.frames.length > 0 && <Backtrace frames={s.frames} t={t} />}
+    </div>
+  );
+}

frontend/apps/site/src/components/club/Crashes.test.tsx

@@ -42,3 +42,18 @@ test('a crash log is sent and filed under its bug', async () => {
   // The list reloads: the crash is there too, under its bug.
   await waitFor(() => expect(getAllByText(/RGN_PutRegion/).length).toBe(2));
 });
+
+test("a crash of majestic's is named by its signal, and not linked to the public list", async () => {
+  vi.stubGlobal('fetch', vi.fn(async (url: string) => {
+    if (url === '/api/v1/club/crashes') {
+      return new Response(JSON.stringify({ crashes: [{ id: 'c-majestic2', signature: '0123456789ac', title: 'SIGSEGV (NULL pointer)',
+        kind: 'signal', status: 'open', soc: 'gk7205v300', sensor: 'imx335', received_at: '2026-10-09T01:00:00Z', self_inflicted: false,
+        camera: true }], stars: 0, rules: { report: 1, first: 3, fixed: 5, month_cap: 10 } }));
+    }
+    return new Response('{}', { status: 404 });
+  }));
+  const { findByText } = render(<Harness />);
+  const title = await findByText('SIGSEGV (NULL pointer)');
+  expect(title.closest('a')).toBeNull();
+  expect(await findByText('majestic crashed')).toBeTruthy();
+});

frontend/apps/site/src/components/club/Crashes.tsx

@@ -70,7 +70,9 @@ export default function Crashes({ locale, t, onChange }: { locale: Locale; t: Bo
         {state.s === 'sent' && (
           <p class="m-0 rounded-md bg-[#e7f5ee] px-3 py-2 text-sm text-[#1f7a4d]" role="status">
             {t(state.r.self_inflicted ? 'crashes.sent_self' : state.r.duplicate ? 'crashes.sent_again' : 'crashes.sent')}{' '}
-            <a href={`${pathFor(locale, '/crashes')}#${state.r.signature}`} class="font-mono">{state.r.title}</a>
+            {state.r.kind === 'signal'
+              ? <span class="font-mono">{state.r.title}</span>
+              : <a href={`${pathFor(locale, '/crashes')}#${state.r.signature}`} class="font-mono">{state.r.title}</a>}
           </p>
         )}
         {state.s === 'error' && <p class="m-0 text-sm text-[#a3262e]" role="alert">{state.error}</p>}
@@ -87,7 +89,10 @@ export default function Crashes({ locale, t, onChange }: { locale: Locale; t: Bo
                 {c.self_inflicted && <span class="text-[12px] text-body-secondary">{t('crashes.self_inflicted')}</span>}
                 <span class="text-[12px] text-body-secondary">{c.received_at.slice(0, 10)} · {[c.soc, c.sensor].filter(Boolean).join(' · ').toUpperCase()}</span>
               </span>
-              <a class="font-mono break-words" href={`${pathFor(locale, '/crashes')}#${c.signature}`}>{c.title}</a>
+              {/* majestic's crashes are not on the public list. */}
+              {c.kind === 'signal'
+                ? <span class="font-mono break-words">{c.title}</span>
+                : <a class="font-mono break-words" href={`${pathFor(locale, '/crashes')}#${c.signature}`}>{c.title}</a>}
             </li>
           ))}
         </ul>

frontend/apps/site/src/i18n/boards.en.json

@@ -251,6 +251,7 @@
       "col_soc": "SoC",
       "current": "In current firmware",
       "d_anomalies": "Before it",
+      "d_backtrace": "Backtrace",
       "d_before": "Warnings",
       "d_builds": "Matching builds",
       "d_cmdline": "Command line",
@@ -285,6 +286,7 @@
       "kind_bug": "BUG",
       "kind_oops": "Oops",
       "kind_panic": "Kernel panic",
+      "kind_signal": "majestic crashed",
       "kind_warning": "Warning",
       "last_seen": "last {date}",
       "list_label": "Kernel crashes, worst first",
@@ -293,6 +295,7 @@
       "mine_empty": "No crash sent yet, from you or your linked cameras.",
       "mine_lede": "Each bug a camera of yours hits earns {report} ★ once settled; if you were the first to send it, {first} ★ more when a maintainer confirms it and {fixed} ★ when it is fixed. At most {cap} ★ a month from crashes; a crash you caused on purpose earns nothing.",
       "mine_title": "My crash reports",
+      "probable": "probable",
       "saved": "Saved.",
       "self_inflicted": "caused on purpose",
       "send": "Send the crash log",
@@ -305,6 +308,8 @@
       "status_fixed": "Fixed",
       "status_open": "Not yet looked at",
       "status_wontfix": "Will not be fixed",
+      "sym_failed": "No backtrace",
+      "sym_pending": "Waiting for its backtrace (tried {n} times)",
       "taken_back": "Saved; {n} ★ taken back.",
       "triage_forbidden": "Only OpenIPC's maintainers triage crashes.",
       "triage_link": "Crash triage"

frontend/apps/site/src/i18n/boards.ru.json

@@ -255,6 +255,7 @@
       "col_soc": "SoC",
       "current": "В текущей прошивке",
       "d_anomalies": "Перед этим",
+      "d_backtrace": "Стек вызовов",
       "d_before": "Предупреждения",
       "d_builds": "Подходящие сборки",
       "d_cmdline": "Командная строка",
@@ -291,6 +292,7 @@
       "kind_bug": "BUG",
       "kind_oops": "Oops",
       "kind_panic": "Kernel panic",
+      "kind_signal": "Сбой majestic",
       "kind_warning": "Предупреждение",
       "last_seen": "последнее {date}",
       "list_label": "Падения ядра, сначала самые тяжёлые",
@@ -299,6 +301,7 @@
       "mine_empty": "Пока ни одного падения — ни от вас, ни от привязанных камер.",
       "mine_lede": "Каждая ошибка, на которую попала ваша камера, приносит {report} ★ после расчёта; если вы прислали её первым — ещё {first} ★, когда мейнтейнер подтвердит, и {fixed} ★, когда её исправят. Не больше {cap} ★ в месяц за падения; падение, вызванное нарочно, ничего не приносит.",
       "mine_title": "Мои отчёты о падениях",
+      "probable": "вероятно",
       "saved": "Сохранено.",
       "self_inflicted": "вызвано нарочно",
       "send": "Отправить лог падения",
@@ -311,6 +314,8 @@
       "status_fixed": "Исправлено",
       "status_open": "Ещё не разобрано",
       "status_wontfix": "Исправлено не будет",
+      "sym_failed": "Стека вызовов нет",
+      "sym_pending": "Ждёт стека вызовов (попыток: {n})",
       "taken_back": "Сохранено; списано {n} ★.",
       "triage_forbidden": "Разбирать падения могут только мейнтейнеры OpenIPC.",
       "triage_link": "Разбор падений"

frontend/apps/site/src/i18n/en.json

@@ -164,7 +164,7 @@
     },
     "club_crashes": {
       "eyebrow": "OpenIPC Club",
-      "lede": "Kernel crashes cameras sent, worst first, with what it takes to reproduce each one.",
+      "lede": "Crashes cameras sent, the kernel's and majestic's, worst first, with what it takes to reproduce each one.",
       "title": "Crash triage"
     },
     "club_leaderboard": {

frontend/apps/site/src/i18n/ru.json

@@ -164,7 +164,7 @@
     },
     "club_crashes": {
       "eyebrow": "Клуб OpenIPC",
-      "lede": "Падения ядра, присланные камерами, — сначала самые тяжёлые, с тем, что нужно, чтобы воспроизвести каждое.",
+      "lede": "Падения ядра и majestic, присланные камерами, — сначала самые тяжёлые, с тем, что нужно, чтобы воспроизвести каждое.",
       "title": "Разбор падений"
     },
     "club_leaderboard": {