From 0b0f5229582398def786779debe9077fa7ac9a89 Mon Sep 17 00:00:00 2001 From: AI Dev Date: Thu, 8 Oct 2026 17:54:53 +0000 Subject: [PATCH 1/3] Read the firmware's meta.json from a crash bundle, and its chip and sensor when the log has lost them OpenIPC/firmware now writes meta.json into each crash bundle: the build, the kernel, the command line, the chip, the sensor, the majestic version and the pipeline's settings. The WebUI sends it as the meta field, but a bundle downloaded and sent from /club carries it only inside, where it was ignored. A pstore record is the tail of the kernel's log, and on a camera whose log fills up the lines naming the chip and sensor are gone from it: a gk7205v300 + imx335 flooding its log with sensor i2c aborts sent a crash filed under no chip at all. meta.json still names them, and is now used for the SoC and sensor when neither the camera nor the log says. --- service/internal/crashes/CRASH.md | 2 +- service/internal/crashes/api.go | 17 +++++++++++++ service/internal/crashes/parse.go | 2 +- service/internal/crashes/store_test.go | 33 ++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/service/internal/crashes/CRASH.md b/service/internal/crashes/CRASH.md index 467f920b..d8d94271 100644 --- a/service/internal/crashes/CRASH.md +++ b/service/internal/crashes/CRASH.md @@ -23,7 +23,7 @@ instead; include it in the tar as `failsafe` and it is filed as a boot loop. | `firmware` | optional | `/etc/os-release`'s version, e.g. `2.6.10.05-lite` | | `majestic` | optional | `majestic -v` | | `soc`, `sensor` | optional | as `ipcinfo` says; the log's own `sensor=… chip=…` line is used when absent | -| `meta` | optional | JSON, at most 64 KB: anything else that helps reproduce it -- `/proc/cmdline`, `lsmod`, the flash size, uptime, and `majestic.yaml` **with passwords, keys, Wi-Fi and URLs removed** | +| `meta` | optional | JSON, at most 64 KB: what helps reproduce it. The firmware writes it as `meta.json` in the bundle (OpenIPC/firmware `S98crashlog`: the build, kernel, command line, chip, sensor, majestic version, and the pipeline's settings **without passwords, keys, accounts or servers**); a bundle carrying it needs no `meta` field, and its `soc` and `sensor` are used when the camera sends neither and the log no longer names them | ```sh curl -sS -L --post301 --max-time 30 \ diff --git a/service/internal/crashes/api.go b/service/internal/crashes/api.go index e99b0670..92c05926 100644 --- a/service/internal/crashes/api.go +++ b/service/internal/crashes/api.go @@ -131,6 +131,14 @@ func (a *API) Submit(w http.ResponseWriter, r *http.Request, member, channel str SignatureID: c.Fatal.Signature(), Redacted: Redact(c.Text, mac, key), } + // What the firmware wrote about the camera when it kept the crash: the + // meta field, or the meta.json in the bundle, which a bundle downloaded + // from the WebUI and sent from /club carries. The log is the tail of the + // kernel's; on a camera whose log filled up, the lines naming the chip and + // the sensor are gone from it, and meta.json still names them. + if in.meta == "" && len(files["meta.json"]) <= maxMeta { + in.meta = strings.TrimSpace(files["meta.json"]) + } if in.meta != "" { red := Redact(in.meta, mac, key) if !json.Valid([]byte(red)) { @@ -138,6 +146,15 @@ func (a *API) Submit(w http.ResponseWriter, r *http.Request, member, channel str return } e.Meta = json.RawMessage(red) + var named struct{ SoC, Sensor string } + if json.Unmarshal(e.Meta, &named) == nil { + if v := strings.ToLower(named.SoC); e.SoC == "" && hardwareRe.MatchString(v) { + e.SoC = v + } + if v := strings.ToLower(named.Sensor); e.Sensor == "" && hardwareRe.MatchString(v) { + e.Sensor = v + } + } } id, sig, dup, err := a.Store().Insert(ctx, e, a.now()) if errors.Is(err, ErrQuota) { diff --git a/service/internal/crashes/parse.go b/service/internal/crashes/parse.go index 571d98e2..38f73a13 100644 --- a/service/internal/crashes/parse.go +++ b/service/internal/crashes/parse.go @@ -194,7 +194,7 @@ func Unpack(data []byte) (map[string]string, error) { continue } name := h.Name[strings.LastIndex(h.Name, "/")+1:] - if !strings.HasPrefix(name, "dmesg-") && name != "failsafe" && name != "pending" { + if !strings.HasPrefix(name, "dmesg-") && name != "failsafe" && name != "pending" && name != "meta.json" { continue } if len(out) == maxRecords { diff --git a/service/internal/crashes/store_test.go b/service/internal/crashes/store_test.go index 5ba933cf..e0709fa8 100644 --- a/service/internal/crashes/store_test.go +++ b/service/internal/crashes/store_test.go @@ -505,3 +505,36 @@ func TestConcurrentUploadsKeepTheLimit(t *testing.T) { t.Fatalf("%d created, the limit is %d", created, DailyPerCamera) } } + +// A bundle the WebUI downloaded carries the firmware's meta.json: it is kept +// beside the log, and names the chip and sensor when the log no longer does. +func TestMetaInTheBundle(t *testing.T) { + e := newEnv(t) + meta := `{"soc": "GK7205V300", "sensor": "imx335", "cmdline": "ip=192.0.2.7", "config": ["video0:", " codec: h265"]}` + b := tgz(t, map[string]string{"dmesg-ramoops-0": string(oops(3)), "meta.json": meta}) + code, out := e.send(b, nil, "10.0.0.1") + if code != 201 { + t.Fatalf("%d %v", code, out) + } + var soc, sensor, stored string + if err := e.pool.QueryRow(context.Background(), `SELECT soc, sensor, meta::text FROM crash_events`).Scan(&soc, &sensor, &stored); err != nil { + t.Fatal(err) + } + if soc != "gk7205v300" || sensor != "imx335" { + t.Fatalf("soc %q sensor %q", soc, sensor) + } + if strings.Contains(stored, "192.0.2.7") || !strings.Contains(stored, "codec: h265") { + t.Fatalf("meta %s", stored) + } + // What the camera says outright wins over meta.json. + b = tgz(t, map[string]string{"dmesg-ramoops-0": string(oops(4)), "meta.json": `{"soc": "hi3516ev300"}`}) + if code, _ := e.send(b, map[string]string{"soc": "gk7205v200"}, "10.0.0.1"); code != 201 { + t.Fatal(code) + } + if err := e.pool.QueryRow(context.Background(), `SELECT soc FROM crash_events ORDER BY received_at DESC, id LIMIT 1`).Scan(&soc); err != nil { + t.Fatal(err) + } + if soc != "gk7205v200" { + t.Fatalf("form soc lost to meta: %q", soc) + } +} From ee372a519aafedf995c981d0122ec738bf2ae33a Mon Sep 17 00:00:00 2001 From: AI Dev Date: Thu, 8 Oct 2026 17:58:15 +0000 Subject: [PATCH 2/3] Address review: meta.json takes no record's place, and a broken one does not cost the crash The 16-record limit counts dmesg records only, so a bundle of sixteen and its meta.json is within it. A meta.json found in the bundle that is not JSON is left out; one sent as the meta field must still be JSON. --- service/internal/crashes/api.go | 6 +++++- service/internal/crashes/parse.go | 10 ++++++++-- service/internal/crashes/store_test.go | 13 +++++++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/service/internal/crashes/api.go b/service/internal/crashes/api.go index 92c05926..3eef1957 100644 --- a/service/internal/crashes/api.go +++ b/service/internal/crashes/api.go @@ -136,8 +136,12 @@ func (a *API) Submit(w http.ResponseWriter, r *http.Request, member, channel str // from the WebUI and sent from /club carries. The log is the tail of the // kernel's; on a camera whose log filled up, the lines naming the chip and // the sensor are gone from it, and meta.json still names them. + // Sent as the field, it must be JSON; found in the bundle, one that is not + // is left out rather than costing the crash it came with. if in.meta == "" && len(files["meta.json"]) <= maxMeta { - in.meta = strings.TrimSpace(files["meta.json"]) + if m := strings.TrimSpace(files["meta.json"]); json.Valid([]byte(m)) { + in.meta = m + } } if in.meta != "" { red := Redact(in.meta, mac, key) diff --git a/service/internal/crashes/parse.go b/service/internal/crashes/parse.go index 38f73a13..3cd1fbd2 100644 --- a/service/internal/crashes/parse.go +++ b/service/internal/crashes/parse.go @@ -180,6 +180,7 @@ func Unpack(data []byte) (map[string]string, error) { return nil, fmt.Errorf("the bundle unpacks to more than %d bytes", MaxUnpacked) } out := map[string]string{} + records := 0 if len(raw) >= 512 && bytes.Equal(raw[257:262], []byte("ustar")) { tr := tar.NewReader(bytes.NewReader(raw)) for { @@ -197,8 +198,13 @@ func Unpack(data []byte) (map[string]string, error) { if !strings.HasPrefix(name, "dmesg-") && name != "failsafe" && name != "pending" && name != "meta.json" { continue } - if len(out) == maxRecords { - return nil, fmt.Errorf("more than %d records", maxRecords) + // The limit is on records; failsafe, pending and meta.json are one + // each at most, and the tar's own entries say which. + if strings.HasPrefix(name, "dmesg-") { + if records == maxRecords { + return nil, fmt.Errorf("more than %d records", maxRecords) + } + records++ } b, err := io.ReadAll(tr) if err != nil { diff --git a/service/internal/crashes/store_test.go b/service/internal/crashes/store_test.go index e0709fa8..e9d45e0e 100644 --- a/service/internal/crashes/store_test.go +++ b/service/internal/crashes/store_test.go @@ -537,4 +537,17 @@ func TestMetaInTheBundle(t *testing.T) { if soc != "gk7205v200" { t.Fatalf("form soc lost to meta: %q", soc) } + // A broken meta.json in the bundle does not cost the crash. + b = tgz(t, map[string]string{"dmesg-ramoops-0": string(oops(5)), "meta.json": `{"soc": `}) + if code, out := e.send(b, nil, "10.0.0.1"); code != 201 { + t.Fatalf("a broken embedded meta.json refused the crash: %d %v", code, out) + } + // Sixteen records and a meta.json are within the limit. + files := map[string]string{"meta.json": meta} + for i := 0; i < maxRecords; i++ { + files[fmt.Sprintf("dmesg-ramoops-%d", i)] = string(oops(6)) + } + if _, err := Unpack(tgz(t, files)); err != nil { + t.Fatalf("16 records and meta.json: %v", err) + } } From 3172d2749764032318e04976afa876a3a5841c1b Mon Sep 17 00:00:00 2001 From: AI Dev Date: Thu, 8 Oct 2026 18:39:47 +0000 Subject: [PATCH 3/3] Find the event a test sent by its id The events in TestMetaInTheBundle share the test's one fixed time, so the newest by received_at was any of them. --- service/internal/crashes/store_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/service/internal/crashes/store_test.go b/service/internal/crashes/store_test.go index e9d45e0e..81aebbf8 100644 --- a/service/internal/crashes/store_test.go +++ b/service/internal/crashes/store_test.go @@ -528,10 +528,12 @@ func TestMetaInTheBundle(t *testing.T) { } // What the camera says outright wins over meta.json. b = tgz(t, map[string]string{"dmesg-ramoops-0": string(oops(4)), "meta.json": `{"soc": "hi3516ev300"}`}) - if code, _ := e.send(b, map[string]string{"soc": "gk7205v200"}, "10.0.0.1"); code != 201 { + code, out = e.send(b, map[string]string{"soc": "gk7205v200"}, "10.0.0.1") + if code != 201 { t.Fatal(code) } - if err := e.pool.QueryRow(context.Background(), `SELECT soc FROM crash_events ORDER BY received_at DESC, id LIMIT 1`).Scan(&soc); err != nil { + // Every event here has the test's one fixed time: find this one by its id. + if err := e.pool.QueryRow(context.Background(), `SELECT soc FROM crash_events WHERE id = $1`, out["id"]).Scan(&soc); err != nil { t.Fatal(err) } if soc != "gk7205v200" {