Read the firmware's meta.json from a crash bundle, and its chip and sensor when the log has lost them

service/internal/crashes/CRASH.md

@@ -23,7 +23,7 @@ instead; include it in the tar as `failsafe` and it is filed as a boot loop.
 | `firmware` | optional | `/etc/os-release`'s version, e.g. `2.6.10.05-lite` |
 | `majestic` | optional | `majestic -v` |
 | `soc`, `sensor` | optional | as `ipcinfo` says; the log's own `sensor=… chip=…` line is used when absent |
-| `meta` | optional | JSON, at most 64 KB: anything else that helps reproduce it -- `/proc/cmdline`, `lsmod`, the flash size, uptime, and `majestic.yaml` **with passwords, keys, Wi-Fi and URLs removed** |
+| `meta` | optional | JSON, at most 64 KB: what helps reproduce it. The firmware writes it as `meta.json` in the bundle (OpenIPC/firmware `S98crashlog`: the build, kernel, command line, chip, sensor, majestic version, and the pipeline's settings **without passwords, keys, accounts or servers**); a bundle carrying it needs no `meta` field, and its `soc` and `sensor` are used when the camera sends neither and the log no longer names them |
 
 ```sh
 curl -sS -L --post301 --max-time 30 \

service/internal/crashes/api.go

@@ -131,13 +131,34 @@ func (a *API) Submit(w http.ResponseWriter, r *http.Request, member, channel str
 		SignatureID: c.Fatal.Signature(),
 		Redacted:    Redact(c.Text, mac, key),
 	}
+	// What the firmware wrote about the camera when it kept the crash: the
+	// meta field, or the meta.json in the bundle, which a bundle downloaded
+	// from the WebUI and sent from /club carries. The log is the tail of the
+	// kernel's; on a camera whose log filled up, the lines naming the chip and
+	// the sensor are gone from it, and meta.json still names them.
+	// Sent as the field, it must be JSON; found in the bundle, one that is not
+	// is left out rather than costing the crash it came with.
+	if in.meta == "" && len(files["meta.json"]) <= maxMeta {
+		if m := strings.TrimSpace(files["meta.json"]); json.Valid([]byte(m)) {
+			in.meta = m
+		}
+	}
 	if in.meta != "" {
 		red := Redact(in.meta, mac, key)
 		if !json.Valid([]byte(red)) {
 			a.refuse(w, http.StatusBadRequest, "meta is not JSON")
 			return
 		}
 		e.Meta = json.RawMessage(red)
+		var named struct{ SoC, Sensor string }
+		if json.Unmarshal(e.Meta, &named) == nil {
+			if v := strings.ToLower(named.SoC); e.SoC == "" && hardwareRe.MatchString(v) {
+				e.SoC = v
+			}
+			if v := strings.ToLower(named.Sensor); e.Sensor == "" && hardwareRe.MatchString(v) {
+				e.Sensor = v
+			}
+		}
 	}
 	id, sig, dup, err := a.Store().Insert(ctx, e, a.now())
 	if errors.Is(err, ErrQuota) {

service/internal/crashes/parse.go

@@ -180,6 +180,7 @@ func Unpack(data []byte) (map[string]string, error) {
 		return nil, fmt.Errorf("the bundle unpacks to more than %d bytes", MaxUnpacked)
 	}
 	out := map[string]string{}
+	records := 0
 	if len(raw) >= 512 && bytes.Equal(raw[257:262], []byte("ustar")) {
 		tr := tar.NewReader(bytes.NewReader(raw))
 		for {
@@ -194,11 +195,16 @@ func Unpack(data []byte) (map[string]string, error) {
 				continue
 			}
 			name := h.Name[strings.LastIndex(h.Name, "/")+1:]
-			if !strings.HasPrefix(name, "dmesg-") && name != "failsafe" && name != "pending" {
+			if !strings.HasPrefix(name, "dmesg-") && name != "failsafe" && name != "pending" && name != "meta.json" {
 				continue
 			}
-			if len(out) == maxRecords {
-				return nil, fmt.Errorf("more than %d records", maxRecords)
+			// The limit is on records; failsafe, pending and meta.json are one
+			// each at most, and the tar's own entries say which.
+			if strings.HasPrefix(name, "dmesg-") {
+				if records == maxRecords {
+					return nil, fmt.Errorf("more than %d records", maxRecords)
+				}
+				records++
 			}
 			b, err := io.ReadAll(tr)
 			if err != nil {

service/internal/crashes/store_test.go

@@ -505,3 +505,51 @@ func TestConcurrentUploadsKeepTheLimit(t *testing.T) {
 		t.Fatalf("%d created, the limit is %d", created, DailyPerCamera)
 	}
 }
+
+// A bundle the WebUI downloaded carries the firmware's meta.json: it is kept
+// beside the log, and names the chip and sensor when the log no longer does.
+func TestMetaInTheBundle(t *testing.T) {
+	e := newEnv(t)
+	meta := `{"soc": "GK7205V300", "sensor": "imx335", "cmdline": "ip=192.0.2.7", "config": ["video0:", "  codec: h265"]}`
+	b := tgz(t, map[string]string{"dmesg-ramoops-0": string(oops(3)), "meta.json": meta})
+	code, out := e.send(b, nil, "10.0.0.1")
+	if code != 201 {
+		t.Fatalf("%d %v", code, out)
+	}
+	var soc, sensor, stored string
+	if err := e.pool.QueryRow(context.Background(), `SELECT soc, sensor, meta::text FROM crash_events`).Scan(&soc, &sensor, &stored); err != nil {
+		t.Fatal(err)
+	}
+	if soc != "gk7205v300" || sensor != "imx335" {
+		t.Fatalf("soc %q sensor %q", soc, sensor)
+	}
+	if strings.Contains(stored, "192.0.2.7") || !strings.Contains(stored, "codec: h265") {
+		t.Fatalf("meta %s", stored)
+	}
+	// What the camera says outright wins over meta.json.
+	b = tgz(t, map[string]string{"dmesg-ramoops-0": string(oops(4)), "meta.json": `{"soc": "hi3516ev300"}`})
+	code, out = e.send(b, map[string]string{"soc": "gk7205v200"}, "10.0.0.1")
+	if code != 201 {
+		t.Fatal(code)
+	}
+	// Every event here has the test's one fixed time: find this one by its id.
+	if err := e.pool.QueryRow(context.Background(), `SELECT soc FROM crash_events WHERE id = $1`, out["id"]).Scan(&soc); err != nil {
+		t.Fatal(err)
+	}
+	if soc != "gk7205v200" {
+		t.Fatalf("form soc lost to meta: %q", soc)
+	}
+	// A broken meta.json in the bundle does not cost the crash.
+	b = tgz(t, map[string]string{"dmesg-ramoops-0": string(oops(5)), "meta.json": `{"soc": `})
+	if code, out := e.send(b, nil, "10.0.0.1"); code != 201 {
+		t.Fatalf("a broken embedded meta.json refused the crash: %d %v", code, out)
+	}
+	// Sixteen records and a meta.json are within the limit.
+	files := map[string]string{"meta.json": meta}
+	for i := 0; i < maxRecords; i++ {
+		files[fmt.Sprintf("dmesg-ramoops-%d", i)] = string(oops(6))
+	}
+	if _, err := Unpack(tgz(t, files)); err != nil {
+		t.Fatalf("16 records and meta.json: %v", err)
+	}
+}