From c20b40f353b6659a13b11d93ea93dd3a6450a2e8 Mon Sep 17 00:00:00 2001 From: AI Dev Date: Fri, 9 Oct 2026 01:31:22 +0000 Subject: [PATCH 1/3] Club: majestic's own crashes, symbolized against its CI's debuginfo majestic now leaves /etc/crash/majestic.dump when it dies of a fatal signal: the registers at the fault, a slice of the stack, the memory map and each module's build-id. A bundle carrying one is filed as class 'user', kind 'signal' (migration 032), maintainers' only: never on /crashes, and its sender is told the signal, not where. It arrives under a provisional signature (module and offset), with its dump in crash_dumps. The firmware role's new symbolize package, woken by NOTIFY crash_symbolize, writes an ELF core from the dump and unwinds it with gdb-multiarch against the executable and debuginfo majestic's CI publishes by build-id. Library frames are named from the firmware build's own rootfs, unsquashed from its release tarball. Where gdb stops (most camera libraries have no unwind tables) a scan of the stack for return addresses that follow a call finds the probable callers. The event is then refiled under its backtrace's signature and the dump, being majestic's memory, is deleted. Tested against dumps of a stand-in program that faulted on a gk7205v300 and on x86_64: in its own code, in a stripped library and in libc. The service image and CI gain gdb-multiarch and squashfs-tools. --- .github/workflows/build.yml | 8 +- CLAUDE.md | 15 + data/locales/boards.en.yml | 5 + data/locales/boards.ru.yml | 5 + .../site/src/components/club/CrashTriage.tsx | 45 +- .../site/src/components/club/Crashes.test.tsx | 15 + .../apps/site/src/components/club/Crashes.tsx | 9 +- frontend/apps/site/src/i18n/boards.en.json | 5 + frontend/apps/site/src/i18n/boards.ru.json | 5 + frontend/apps/site/src/lib/crashes.ts | 29 +- service/Dockerfile | 7 +- service/cmd/openipc/main.go | 10 + service/internal/config/config.go | 4 + service/internal/crashes/CRASH.md | 21 + service/internal/crashes/api.go | 29 +- service/internal/crashes/dump.go | 442 ++++++++++++++++++ service/internal/crashes/dump_test.go | 382 +++++++++++++++ service/internal/crashes/parse.go | 98 +++- service/internal/crashes/store.go | 55 ++- service/internal/crashes/symbolize/core.go | 239 ++++++++++ service/internal/crashes/symbolize/frames.py | 56 +++ service/internal/crashes/symbolize/gdb.go | 165 +++++++ .../internal/crashes/symbolize/rootfs_test.go | 114 +++++ service/internal/crashes/symbolize/scan.go | 133 ++++++ service/internal/crashes/symbolize/sources.go | 339 ++++++++++++++ .../internal/crashes/symbolize/symbolize.go | 312 +++++++++++++ .../crashes/symbolize/symbolize_test.go | 364 +++++++++++++++ service/internal/crashes/symbolize/worker.go | 161 +++++++ service/internal/crashes/symbolized.go | 238 ++++++++++ .../crashes/testdata/dump/arm-lib.dump | Bin 0 -> 2199 bytes .../crashes/testdata/dump/arm-libc.dump | Bin 0 -> 2199 bytes .../crashes/testdata/dump/arm-own.dump | Bin 0 -> 2191 bytes .../crashes/testdata/dump/arm-sent.dump | Bin 0 -> 2343 bytes .../crashes/testdata/dump/arm/libtoy.so | Bin 0 -> 5280 bytes .../internal/crashes/testdata/dump/arm/toy | Bin 0 -> 9848 bytes .../crashes/testdata/dump/arm/toy.debug | Bin 0 -> 16828 bytes service/internal/crashes/testdata/dump/gen.sh | 38 ++ .../internal/crashes/testdata/dump/libtoy.c | 9 + service/internal/crashes/testdata/dump/toy.c | 208 +++++++++ .../crashes/testdata/dump/x86_64-lib.dump | Bin 0 -> 12000 bytes .../crashes/testdata/dump/x86_64-libc.dump | Bin 0 -> 5816 bytes .../crashes/testdata/dump/x86_64-own.dump | Bin 0 -> 5752 bytes .../crashes/testdata/dump/x86_64-sent.dump | Bin 0 -> 6874 bytes .../crashes/testdata/dump/x86_64/libtoy.so | Bin 0 -> 14160 bytes .../internal/crashes/testdata/dump/x86_64/toy | Bin 0 -> 14768 bytes .../crashes/testdata/dump/x86_64/toy.debug | Bin 0 -> 27152 bytes .../migrations/032_crash_symbolizations.sql | 47 ++ 47 files changed, 3559 insertions(+), 53 deletions(-) create mode 100644 service/internal/crashes/dump.go create mode 100644 service/internal/crashes/dump_test.go create mode 100644 service/internal/crashes/symbolize/core.go create mode 100644 service/internal/crashes/symbolize/frames.py create mode 100644 service/internal/crashes/symbolize/gdb.go create mode 100644 service/internal/crashes/symbolize/rootfs_test.go create mode 100644 service/internal/crashes/symbolize/scan.go create mode 100644 service/internal/crashes/symbolize/sources.go create mode 100644 service/internal/crashes/symbolize/symbolize.go create mode 100644 service/internal/crashes/symbolize/symbolize_test.go create mode 100644 service/internal/crashes/symbolize/worker.go create mode 100644 service/internal/crashes/symbolized.go create mode 100644 service/internal/crashes/testdata/dump/arm-lib.dump create mode 100644 service/internal/crashes/testdata/dump/arm-libc.dump create mode 100644 service/internal/crashes/testdata/dump/arm-own.dump create mode 100644 service/internal/crashes/testdata/dump/arm-sent.dump create mode 100755 service/internal/crashes/testdata/dump/arm/libtoy.so create mode 100755 service/internal/crashes/testdata/dump/arm/toy create mode 100755 service/internal/crashes/testdata/dump/arm/toy.debug create mode 100755 service/internal/crashes/testdata/dump/gen.sh create mode 100644 service/internal/crashes/testdata/dump/libtoy.c create mode 100644 service/internal/crashes/testdata/dump/toy.c create mode 100644 service/internal/crashes/testdata/dump/x86_64-lib.dump create mode 100644 service/internal/crashes/testdata/dump/x86_64-libc.dump create mode 100644 service/internal/crashes/testdata/dump/x86_64-own.dump create mode 100644 service/internal/crashes/testdata/dump/x86_64-sent.dump create mode 100755 service/internal/crashes/testdata/dump/x86_64/libtoy.so create mode 100755 service/internal/crashes/testdata/dump/x86_64/toy create mode 100755 service/internal/crashes/testdata/dump/x86_64/toy.debug create mode 100644 service/internal/db/migrations/032_crash_symbolizations.sql diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b4088fe8..97fece42 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,11 +41,13 @@ jobs: - uses: actions/checkout@v4 # The keyframe and variants tests decode with ffmpeg, as the service does - # in its image. They fail rather than skip when CI is set and it is missing. - - name: Install ffmpeg + # in its image; the crash symbolizer's unwind with gdb-multiarch and + # unpack firmware with unsquashfs. They fail rather than skip when CI is + # set and one is missing. + - name: Install ffmpeg, gdb-multiarch and squashfs-tools run: | sudo apt-get update -qq - sudo apt-get install -y --no-install-recommends ffmpeg + sudo apt-get install -y --no-install-recommends ffmpeg gdb-multiarch squashfs-tools # --- the Go service (#287) --------------------------------------------- - uses: actions/setup-go@v5 diff --git a/CLAUDE.md b/CLAUDE.md index f328b0ec..b3f34e8f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -234,6 +234,21 @@ restores the image but never the schema, so keep migrations additive. merges: `bogus` takes back what a bug and the signatures merged into it paid, and undoing it lets the settlement pay again. A crash with a MAC is the camera's linked owner's, whoever sent it. + **majestic's own crashes** come the same way, as `/etc/crash/majestic.dump` + (class `user`, kind `signal`, migration 032), and are the maintainers' + only: never on `/crashes`, and a member is shown the signal, not where. + One arrives filed under a provisional signature (module and offset of the + faulting instruction) with its dump in `crash_dumps`; the firmware role's + `internal/crashes/symbolize` (woken by `NOTIFY crash_symbolize`) builds an + ELF core from it and unwinds it with `gdb-multiarch` against the + executable and debuginfo majestic's CI publishes by build-id + (`MAJESTIC_SYMBOLS_BASE`), naming library frames from the firmware build's + own rootfs (`unsquashfs` of its release tarball), and scanning the stack + for probable callers where gdb stops. Then the event is refiled under its + backtrace's signature (the crashes package's one change to an event) and + the dump -- majestic's memory -- is deleted. Retried for days when the + debuginfo is not there, then given up on. Cache under + `RELEASE_CACHE_ROOT/symbols`, swept after 30 days unused. - `internal/tools`, `internal/nfsro` — **ipctool for stock firmware**, which has no curl and no TLS. ipctool's release job pushes each build to `PUT /api/v1/tools/{name}` (OIDC, `internal/tools/PUSH.md`); nginx serves diff --git a/data/locales/boards.en.yml b/data/locales/boards.en.yml index 8ac9ea28..794ae2b6 100644 --- a/data/locales/boards.en.yml +++ b/data/locales/boards.en.yml @@ -632,6 +632,7 @@ en: kind_oops: 'Oops' kind_bug: 'BUG' kind_warning: 'Warning' + kind_signal: 'majestic crashed' in_irq: 'in an interrupt' status_open: 'Not yet looked at' status_confirmed: 'Confirmed' @@ -691,3 +692,7 @@ en: d_cmdline: 'Command line' d_anomalies: 'Before it' d_before: 'Warnings' + d_backtrace: 'Backtrace' + sym_pending: 'Waiting for its backtrace (tried {n} times)' + sym_failed: 'No backtrace' + probable: 'probable' diff --git a/data/locales/boards.ru.yml b/data/locales/boards.ru.yml index e9d6fc33..5da4d7c0 100644 --- a/data/locales/boards.ru.yml +++ b/data/locales/boards.ru.yml @@ -673,6 +673,7 @@ ru: kind_oops: 'Oops' kind_bug: 'BUG' kind_warning: 'Предупреждение' + kind_signal: 'Сбой majestic' in_irq: 'в прерывании' status_open: 'Ещё не разобрано' status_confirmed: 'Подтверждено' @@ -736,3 +737,7 @@ ru: d_cmdline: 'Командная строка' d_anomalies: 'Перед этим' d_before: 'Предупреждения' + d_backtrace: 'Стек вызовов' + sym_pending: 'Ждёт стека вызовов (попыток: {n})' + sym_failed: 'Стека вызовов нет' + probable: 'вероятно' diff --git a/frontend/apps/site/src/components/club/CrashTriage.tsx b/frontend/apps/site/src/components/club/CrashTriage.tsx index aa444a75..6eb9ed3e 100644 --- a/frontend/apps/site/src/components/club/CrashTriage.tsx +++ b/frontend/apps/site/src/components/club/CrashTriage.tsx @@ -1,8 +1,9 @@ /** - * /club/crashes: the maintainers' triage of kernel crashes. Every signature, - * worst first; one opens to what it takes to reproduce it -- the chips, - * sensors and builds it was seen on, the firmware builds whose time matches - * its kernel, and each crash's redacted log, warnings and lead-up. + * /club/crashes: the maintainers' triage of kernel crashes and majestic's. + * Every signature, worst first; one opens to what it takes to reproduce it -- + * the chips, sensors and builds it was seen on, the firmware builds whose time + * matches its kernel, and each crash's redacted log, warnings and lead-up, or + * for majestic's, its backtrace from the debuginfo of its build. * * Confirming a bug pays its first reporter, fixing it pays them again; * bogus takes back every star it paid. The same is `openipc crashes status`. @@ -12,7 +13,8 @@ import { useBoardsTranslations, type BoardsT } from '../../lib/boards-i18n'; import { pathFor, type Locale } from '../../lib/i18n'; import { ClubError, fetchMe } from '../../lib/club'; import { - decideCrash, fetchCrashDetail, fetchTriage, frame, KIND_TONE, STATUS_TONE, type Combo, type Detail, type Signature, type Status, + decideCrash, fetchCrashDetail, fetchTriage, frame, frameAt, KIND_TONE, STATUS_TONE, type Combo, type Detail, type Frame, type Signature, + type Status, } from '../../lib/crashes'; type Load = { state: 'loading' } | { state: 'ok'; list: Signature[] } | { state: 'forbidden' } | { state: 'error' }; @@ -81,9 +83,7 @@ function Open({ g, t, onDone }: { g: Signature; t: BoardsT; onDone: () => void } const field = 'min-w-0 rounded-md border border-hairline px-2 py-1 text-sm'; return (
-
-        {g.frames.slice(0, 10).map((f, i) => `${i === 0 ? '' : '← '}${frame(f)}`).join('\n')}
-      
+