diff --git a/CLAUDE.md b/CLAUDE.md index 962bb887..79775892 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -112,7 +112,11 @@ restores the image but never the schema, so keep migrations additive. publishes each upload as-is -- a camera's HEIF keyframe (and its substream's, sent as the optional `thumb` part), checked by `internal/keyframe`, or a legacy JPEG with its metadata dropped. **Nothing a camera sends is - re-encoded.** + re-encoded.** Each frame's brightness is measured from the decode it is + checked with, and the home page's mosaic (`Store.Showcase`) leaves out + flat, black or blown-out frames and any camera first seen less than 30 + days ago (`cameras`, migration 022), so a new camera cannot deface the + front page. - `internal/wall`, `internal/wallsocket` — the wall's JSON and the frame socket (a small JSON protocol at `/api/v1/wall/socket`), with signed grants keyed by `WALL_GRANT_KEY`. diff --git a/service/conformance/conformance_test.go b/service/conformance/conformance_test.go index f445977d..2ce3d628 100644 --- a/service/conformance/conformance_test.go +++ b/service/conformance/conformance_test.go @@ -223,6 +223,7 @@ func start(t *testing.T, surface string) *suite { } for _, mac := range s.macs { s.db.Exec(context.Background(), "DELETE FROM snapshots WHERE mac_address = $1", mac) + s.db.Exec(context.Background(), "DELETE FROM cameras WHERE mac_key = lower(translate($1, ':-', ''))", mac) } s.db.Close(context.Background()) }) diff --git a/service/conformance/wall_api_test.go b/service/conformance/wall_api_test.go index 4f82dca5..a5752950 100644 --- a/service/conformance/wall_api_test.go +++ b/service/conformance/wall_api_test.go @@ -1,8 +1,14 @@ package conformance import ( + "bytes" + "context" "encoding/json" "fmt" + "image" + "image/color" + imagejpeg "image/jpeg" + mathrand "math/rand/v2" "reflect" "regexp" "strings" @@ -33,10 +39,16 @@ func (s *suite) wallSetup() { // frames uploads count frames for one camera, oldest first, and returns their ids. func (s *suite) frames(mac string, count int, fields map[string]string) []string { + return s.framesOf(mac, count, fields, nil) +} + +// framesOf is frames with a picture of the caller's: nil sends the default +// JPEG, which is a header and padding and which the wall refuses to publish. +func (s *suite) framesOf(mac string, count int, fields map[string]string, f *file) []string { s.t.Helper() var ids []string for range count { - r := s.upload(upload{mac: &mac, headers: map[string]string{"X-Forwarded-For": whitelisted}, fields: fields}) + r := s.upload(upload{mac: &mac, file: f, headers: map[string]string{"X-Forwarded-For": whitelisted}, fields: fields}) if r.StatusCode != 201 { s.t.Fatalf("upload: %d %q", r.StatusCode, r.Header.Get("X-Error")) } @@ -80,16 +92,68 @@ func assertKeys(t *testing.T, got, want []string, what string) { } } -func TestTheMosaicIsTheNewestFrameOfEachCameraWithAThumbGrant(t *testing.T) { +// picture is a JPEG with something in it -- a gradient under noise -- over +// the upload's minimum size, so the wall publishes it and measures it as a +// scene rather than a blank frame. +func picture() *file { + img := image.NewRGBA(image.Rect(0, 0, 320, 180)) + rng := mathrand.New(mathrand.NewPCG(1, 2)) + for y := range 180 { + for x := range 320 { + v := uint8(x*200/320 + rng.IntN(48)) + img.SetRGBA(x, y, color.RGBA{v, uint8(y), 255 - v, 255}) + } + } + var buf bytes.Buffer + imagejpeg.Encode(&buf, img, &imagejpeg.Options{Quality: 95}) + return &file{name: "snapshot.jpg", declared: str("image/jpeg"), data: buf.Bytes()} +} + +// measured waits for the wall to publish and measure a frame. +func (s *suite) measured(id string) { + s.t.Helper() + for range 100 { + var done bool + if err := s.db.QueryRow(context.Background(), + "SELECT luma_p50 IS NOT NULL FROM snapshots WHERE public_id = $1", id).Scan(&done); err != nil { + s.t.Fatal(err) + } + if done { + return + } + time.Sleep(100 * time.Millisecond) + } + s.t.Fatalf("%s was not published and measured in 10 s", id) +} + +// The mosaic is the home page's, not the wall's: a camera reaches it once it +// has been uploading for a month on many days (snapshots.Showcase), so a +// camera that has just uploaded is not on it until the database says it is +// established. +func TestTheMosaicIsTheNewestFrameOfEachEstablishedCameraWithAThumbGrant(t *testing.T) { s := start(t, "wall") s.wallSetup() - ids := s.frames(s.freshMAC(), 2, map[string]string{"soc": "hi3516ev300", "sensor": "imx335"}) + mac := s.freshMAC() + ids := s.framesOf(mac, 2, map[string]string{"soc": "hi3516ev300", "sensor": "imx335"}, picture()) + s.measured(ids[1]) var body struct { Variant string Grant *string Tiles []map[string]any } + s.wallJSON("/api/v1/wall/mosaic.json", &body) + for _, tile := range body.Tiles { + if tile["id"] == ids[0] || tile["id"] == ids[1] { + t.Error("a camera that started uploading a moment ago is on the front page") + } + } + if _, err := s.db.Exec(context.Background(), `UPDATE cameras + SET first_seen = now() - interval '31 days', days = 31 WHERE mac_key = lower(translate($1, ':-', ''))`, mac); err != nil { + t.Fatal(err) + } + + body.Tiles = nil r := s.wallJSON("/api/v1/wall/mosaic.json", &body) s.assertWallHeaders(r, "mosaic") assertKeys(t, keys(t, r.body), []string{"variant", "grant", "tiles"}, "mosaic") @@ -107,7 +171,7 @@ func TestTheMosaicIsTheNewestFrameOfEachCameraWithAThumbGrant(t *testing.T) { } want := map[string]any{"id": ids[1], "soc": "hi3516ev300", "sensor": "imx335"} if !reflect.DeepEqual(ours, want) { - t.Errorf("the newest frame of a camera that just uploaded: %v, want %v", ours, want) + t.Errorf("the newest frame of an established camera: %v, want %v", ours, want) } assertKeys(t, keys(t, firstObject(t, r.body, "tiles")), []string{"id", "soc", "sensor"}, "tile") } diff --git a/service/internal/db/migrations/022_wall_showcase.sql b/service/internal/db/migrations/022_wall_showcase.sql new file mode 100644 index 00000000..d78b2263 --- /dev/null +++ b/service/internal/db/migrations/022_wall_showcase.sql @@ -0,0 +1,39 @@ +-- What the home page's mosaic may show (internal/snapshots, Showcase). +-- +-- The mosaic is the first thing a visitor to openipc.org sees, and the wall +-- publishes whatever a camera sends. Two kinds of frame do not belong there: +-- +-- A frame with nothing in it. A lens cap, a sensor stuck white, an IR scene +-- with no light: 7.5% of the frames on the wall on 2026-10-03 were flat grey, +-- white, black or blown out. Each frame's brightness percentiles over a 64x36 +-- greyscale copy are measured once, from the decode the frame is checked +-- with, and the mosaic leaves out a camera whose newest frame is flat. The +-- rule is applied when the mosaic is read, so moving a threshold needs no +-- recomputation. +-- +-- A camera nobody knows yet. Anyone can make a camera upload anything, and a +-- new one could put a picture of its choosing on the front page within +-- fifteen minutes. A camera appears there only once it has been uploading +-- for a month, on many separate days -- one upload and a month of silence is +-- not a camera anyone has watched. That needs a camera's history, and +-- snapshots are purged after two days, so cameras keeps it: the first frame +-- the wall accepted, and on how many UTC days it has accepted one. Written +-- by snapshots.Store.MarkGenerated, so an upload the wall refused counts for +-- nothing; never purged, one row per camera. +ALTER TABLE snapshots ADD COLUMN luma_p5 smallint; +ALTER TABLE snapshots ADD COLUMN luma_p50 smallint; +ALTER TABLE snapshots ADD COLUMN luma_p95 smallint; + +CREATE TABLE cameras ( + mac_key text PRIMARY KEY, + first_seen timestamptz NOT NULL, + last_day date NOT NULL, + days integer NOT NULL +); + +-- The frames on the wall at the moment this runs: those with a picture (a +-- refused upload has no dimensions). +INSERT INTO cameras (mac_key, first_seen, last_day, days) + SELECT mac_key, min(created_at), max((created_at AT TIME ZONE 'UTC')::date), + count(DISTINCT (created_at AT TIME ZONE 'UTC')::date) + FROM snapshots WHERE width IS NOT NULL GROUP BY mac_key; diff --git a/service/internal/keyframe/check.go b/service/internal/keyframe/check.go index ba91f9da..ae21134f 100644 --- a/service/internal/keyframe/check.go +++ b/service/internal/keyframe/check.go @@ -20,16 +20,18 @@ var ErrCheckTimeout = errors.New("the decode check timed out") // Check has ffmpeg decode the frame exactly as a browser will receive it -- // the parameter sets from the configuration record followed by the access -// unit -- and requires one picture out with no decode error. +// unit -- and requires one picture out with no decode error. That picture is +// scaled to a LumaW x LumaH full-range greyscale copy on the way out, which is +// what the frame's brightness (Luma) is measured on. // // Parse proves the file is shaped like a keyframe; only a decoder proves the // bitstream inside is one. The wall passes these bytes on untouched to // visitors' hardware decoders, so one that will not decode here is not -// published. Nothing Check produces is kept. -func Check(ctx context.Context, ffmpeg string, f *Frame) error { +// published. Nothing Check produces is kept but the measurement. +func Check(ctx context.Context, ffmpeg string, f *Frame) (Luma, error) { stream, err := AnnexB(f) if err != nil { - return err + return Luma{}, err } format := "h264" if f.HEVC { @@ -39,23 +41,22 @@ func Check(ctx context.Context, ffmpeg string, f *Frame) error { ctx, cancel := context.WithTimeout(ctx, CheckTimeout) defer cancel() cmd := exec.CommandContext(ctx, ffmpeg, "-nostdin", "-hide_banner", "-v", "error", "-xerror", "-err_detect", "explode", - "-f", format, "-i", "pipe:0", "-frames:v", "1", "-f", "framemd5", "pipe:1") + "-f", format, "-i", "pipe:0", "-frames:v", "1", + "-vf", fmt.Sprintf("scale=%d:%d:out_range=full,format=gray", LumaW, LumaH), "-f", "rawvideo", "pipe:1") cmd.Stdin = bytes.NewReader(stream) var stdout, stderr bytes.Buffer cmd.Stdout, cmd.Stderr = &stdout, &stderr if err := cmd.Run(); err != nil { if parent.Err() == nil && errors.Is(ctx.Err(), context.DeadlineExceeded) { - return ErrCheckTimeout + return Luma{}, ErrCheckTimeout } - return fmt.Errorf("%s: %w: %s", ffmpeg, err, strings.TrimSpace(stderr.String())) + return Luma{}, fmt.Errorf("%s: %w: %s", ffmpeg, err, strings.TrimSpace(stderr.String())) } if msg := strings.TrimSpace(stderr.String()); msg != "" { - return fmt.Errorf("decoder complained: %s", msg) + return Luma{}, fmt.Errorf("decoder complained: %s", msg) } - for _, line := range strings.Split(stdout.String(), "\n") { - if line != "" && !strings.HasPrefix(line, "#") { - return nil - } + if stdout.Len() != LumaW*LumaH { + return Luma{}, errors.New("no picture decoded") } - return errors.New("no picture decoded") + return lumaOf(stdout.Bytes()), nil } diff --git a/service/internal/keyframe/jpeg.go b/service/internal/keyframe/jpeg.go index 5189b2a3..41ff5cec 100644 --- a/service/internal/keyframe/jpeg.go +++ b/service/internal/keyframe/jpeg.go @@ -7,8 +7,8 @@ import ( "image/jpeg" ) -// StripJPEG returns a JPEG without its application and comment segments, and -// the picture's size. No pixel changes: every entropy-coded scan is copied +// StripJPEG returns a JPEG without its application and comment segments, the +// picture's size, and its brightness (Luma). No pixel changes: every entropy-coded scan is copied // verbatim. // // Only cameras that cannot be updated still send JPEG, and the wall keeps @@ -18,22 +18,23 @@ import ( // wherever they sit, between scans of a progressive file as well as before // the first, and the file must run to its end-of-image marker. // -// The result is then decoded once and thrown away. The browser is handed -// these bytes as they are, so a file cut short or corrupted inside a scan -// would otherwise be published and paint nothing. +// The result is then decoded once, measured and thrown away. The browser is +// handed these bytes as they are, so a file cut short or corrupted inside a +// scan would otherwise be published and paint nothing. // // REMOVE AFTER 2027-06, with the legacy frame path in wallsocket and in // frontend/apps/site/src/lib/wall-decode.ts: by then a camera still uploading // JPEG has had a year of firmware that sends HEIF. -func StripJPEG(b []byte) ([]byte, int, int, error) { +func StripJPEG(b []byte) ([]byte, int, int, Luma, error) { out, w, h, err := strip(b) if err != nil { - return nil, 0, 0, err + return nil, 0, 0, Luma{}, err } - if _, err := jpeg.Decode(bytes.NewReader(out)); err != nil { - return nil, 0, 0, errors.New("JPEG: does not decode: " + err.Error()) + img, err := jpeg.Decode(bytes.NewReader(out)) + if err != nil { + return nil, 0, 0, Luma{}, errors.New("JPEG: does not decode: " + err.Error()) } - return out, w, h, nil + return out, w, h, lumaOfImage(img), nil } func strip(b []byte) ([]byte, int, int, error) { diff --git a/service/internal/keyframe/keyframe_test.go b/service/internal/keyframe/keyframe_test.go index 1d83889a..b6d9106a 100644 --- a/service/internal/keyframe/keyframe_test.go +++ b/service/internal/keyframe/keyframe_test.go @@ -297,9 +297,14 @@ func TestCheckDecodes(t *testing.T) { if err != nil { t.Fatal(err) } - if err := Check(context.Background(), bin, f); err != nil { + l, err := Check(context.Background(), bin, f) + if err != nil { t.Errorf("hevc=%v: %v", s.hevc, err) } + // testsrc is bars, a gradient and a counter: anything but flat. + if l.P95-l.P5 < 100 { + t.Errorf("hevc=%v: testsrc measured as %+v", s.hevc, l) + } } } @@ -325,7 +330,7 @@ func TestCheckRefusesGarbage(t *testing.T) { if _, err := NALs(bad, 4); err != nil { t.Fatal(err) } - if err := Check(context.Background(), bin, f); err == nil { + if _, err := Check(context.Background(), bin, f); err == nil { t.Errorf("hevc=%v: noise decoded cleanly", s.hevc) } } @@ -344,12 +349,12 @@ func TestSamples(t *testing.T) { } for _, p := range jpegs { b, _ := os.ReadFile(p) - out, w, h, err := StripJPEG(b) + out, w, h, l, err := StripJPEG(b) if err != nil { t.Errorf("%s: %v", p, err) continue } - t.Logf("%s: JPEG %dx%d, %d of %d bytes kept", filepath.Base(p), w, h, len(out), len(b)) + t.Logf("%s: JPEG %dx%d, %d of %d bytes kept, luma %d %d %d", filepath.Base(p), w, h, len(out), len(b), l.P5, l.P50, l.P95) } for _, p := range files { b, _ := os.ReadFile(p) @@ -358,9 +363,11 @@ func TestSamples(t *testing.T) { t.Errorf("%s: %v", p, err) continue } - if err := Check(context.Background(), bin, f); err != nil { + l, err := Check(context.Background(), bin, f) + if err != nil { t.Errorf("%s: %v", p, err) } + t.Logf("%s: luma %d %d %d", filepath.Base(p), l.P5, l.P50, l.P95) full, rerr := false, error(nil) if f.HEVC { full, rerr = FullRange(f) @@ -386,7 +393,7 @@ func TestStripJPEG(t *testing.T) { com = append(com, "hello"...) tagged := append(append(append([]byte{0xFF, 0xD8}, exif...), com...), plain[2:]...) - out, w, h, err := StripJPEG(tagged) + out, w, h, _, err := StripJPEG(tagged) if err != nil { t.Fatal(err) } @@ -405,7 +412,7 @@ func TestStripJPEG(t *testing.T) { t.Fatal(err) } for n := range len(tagged) / 2 { - if _, _, _, err := StripJPEG(tagged[:n]); err == nil { + if _, _, _, _, err := StripJPEG(tagged[:n]); err == nil { t.Fatalf("accepted a JPEG cut at %d", n) } } @@ -465,7 +472,7 @@ func TestCheckTimeoutIsItsOwnError(t *testing.T) { if err != nil { t.Fatal(err) } - if err := Check(context.Background(), slow, f); !errors.Is(err, ErrCheckTimeout) { + if _, err := Check(context.Background(), slow, f); !errors.Is(err, ErrCheckTimeout) { t.Fatalf("got %v", err) } } @@ -490,7 +497,7 @@ func TestStripJPEGWalksTheWholeFile(t *testing.T) { com := append([]byte{0xFF, 0xFE}, u16(2+11)...) com = append(com, "after scan!"...) late := append(append(append([]byte{}, plain[:eoi]...), com...), plain[eoi:]...) - out, _, _, err := StripJPEG(late) + out, _, _, _, err := StripJPEG(late) if err != nil { t.Fatal(err) } @@ -501,7 +508,7 @@ func TestStripJPEGWalksTheWholeFile(t *testing.T) { "cut inside the scan": plain[:eoi-40], "no end of image": plain[:eoi], } { - if _, _, _, err := StripJPEG(b); err == nil { + if _, _, _, _, err := StripJPEG(b); err == nil { t.Errorf("%s: accepted", name) } } diff --git a/service/internal/keyframe/luma.go b/service/internal/keyframe/luma.go new file mode 100644 index 00000000..e9f275b9 --- /dev/null +++ b/service/internal/keyframe/luma.go @@ -0,0 +1,65 @@ +package keyframe + +import ( + "image" + "image/color" + "slices" +) + +// LumaW and LumaH are the size of the greyscale copy a frame's brightness is +// measured on: small enough that a camera's timestamp overlay is a few +// percent of it, and the same for both decoders. +const LumaW, LumaH = 64, 36 + +// Luma is a frame's brightness percentiles, 0..255, measured on a LumaW x +// LumaH greyscale copy. The 5th and 95th rather than the darkest and the +// brightest pixel, so an on-screen clock on a flat grey frame does not make +// it look like a picture. The wall's mosaic uses them to leave out frames +// with nothing in them (internal/snapshots, Showcase). +type Luma struct{ P5, P50, P95 uint8 } + +// lumaOf takes the percentiles of a greyscale copy, one byte per pixel. +func lumaOf(grey []byte) Luma { + if len(grey) == 0 { + return Luma{} + } + px := slices.Clone(grey) + slices.Sort(px) + n := len(px) + return Luma{P5: px[n*5/100], P50: px[n/2], P95: px[n*95/100]} +} + +// lumaOfImage box-averages a decoded picture down to LumaW x LumaH and takes +// its percentiles. A picture smaller than that in either direction is sampled +// instead, each cell reading at least one source pixel. A JPEG decodes to +// YCbCr, whose Y plane is read directly; anything else goes through +// color.GrayModel. +func lumaOfImage(img image.Image) Luma { + b := img.Bounds() + if b.Empty() { + return Luma{} + } + yc, isYCbCr := img.(*image.YCbCr) + grey := make([]byte, 0, LumaW*LumaH) + for cy := range LumaH { + y0, y1 := b.Min.Y+cy*b.Dy()/LumaH, b.Min.Y+(cy+1)*b.Dy()/LumaH + y1 = max(y1, y0+1) + for cx := range LumaW { + x0, x1 := b.Min.X+cx*b.Dx()/LumaW, b.Min.X+(cx+1)*b.Dx()/LumaW + x1 = max(x1, x0+1) + var sum, count int + for y := y0; y < y1; y++ { + for x := x0; x < x1; x++ { + if isYCbCr { + sum += int(yc.Y[yc.YOffset(x, y)]) + } else { + sum += int(color.GrayModel.Convert(img.At(x, y)).(color.Gray).Y) + } + count++ + } + } + grey = append(grey, byte(sum/count)) + } + } + return lumaOf(grey) +} diff --git a/service/internal/keyframe/luma_test.go b/service/internal/keyframe/luma_test.go new file mode 100644 index 00000000..eed1b546 --- /dev/null +++ b/service/internal/keyframe/luma_test.go @@ -0,0 +1,86 @@ +package keyframe + +import ( + "bytes" + "image" + "image/color" + "image/jpeg" + "testing" +) + +// encodeJPEG makes a colour JPEG of the size cameras send, painted grey by +// paint. Colour, so it decodes to YCbCr as a camera's does. +func encodeJPEG(t *testing.T, w, h int, paint func(x, y int) uint8) []byte { + t.Helper() + img := image.NewRGBA(image.Rect(0, 0, w, h)) + for y := range h { + for x := range w { + v := paint(x, y) + img.SetRGBA(x, y, color.RGBA{v, v, v, 255}) + } + } + var buf bytes.Buffer + if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 90}); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// The two frames reported on 2026-10-03: a flat grey frame carrying the +// camera's clock in the top-left corner (280228abf9b1e84c5ee8), and a white +// one (878a7b98ed37538b9b91). The clock must not make the grey one look like +// a picture; a real scene must not look flat. +func TestLumaOfJPEG(t *testing.T) { + const w, h = 1920, 1080 + clock := func(x, y int) bool { return x < 480 && y < 60 } + for _, c := range []struct { + name string + paint func(x, y int) uint8 + flat bool + median uint8 + }{ + {"grey with a clock", func(x, y int) uint8 { + if clock(x, y) && (x/8+y/8)%2 == 0 { + return 250 + } + return 54 + }, true, 54}, + {"white", func(int, int) uint8 { return 254 }, true, 254}, + {"a gradient", func(x, _ int) uint8 { return uint8(x * 255 / w) }, false, 127}, + } { + _, _, _, l, err := StripJPEG(encodeJPEG(t, w, h, c.paint)) + if err != nil { + t.Fatalf("%s: %v", c.name, err) + } + if flat := l.P95-l.P5 < 16; flat != c.flat { + t.Errorf("%s: measured %+v, flat %v", c.name, l, flat) + } + if d := int(l.P50) - int(c.median); d < -3 || d > 3 { + t.Errorf("%s: median %d, want about %d", c.name, l.P50, c.median) + } + } +} + +// A picture smaller than the 64x36 copy is sampled, not measured as black. +func TestLumaOfASmallJPEG(t *testing.T) { + _, _, _, l, err := StripJPEG(encodeJPEG(t, 32, 20, func(x, _ int) uint8 { return uint8(x * 8) })) + if err != nil { + t.Fatal(err) + } + if l.P95-l.P5 < 100 || l.P50 < 100 || l.P50 > 160 { + t.Errorf("a 32x20 gradient measured as %+v", l) + } +} + +func TestLumaOfPercentiles(t *testing.T) { + grey := make([]byte, 100) + for i := range grey { + grey[i] = byte(i) + } + if l := lumaOf(grey); l != (Luma{P5: 5, P50: 50, P95: 95}) { + t.Errorf("%+v", l) + } + if l := lumaOf(nil); l != (Luma{}) { + t.Errorf("empty: %+v", l) + } +} diff --git a/service/internal/snapshots/snapshots_test.go b/service/internal/snapshots/snapshots_test.go index d1a2c22b..7e6ef661 100644 --- a/service/internal/snapshots/snapshots_test.go +++ b/service/internal/snapshots/snapshots_test.go @@ -14,14 +14,17 @@ import ( "net/textproto" "os" "regexp" + "slices" "strconv" "strings" "sync" "testing" + "time" "github.com/jackc/pgx/v5/pgxpool" "github.com/OpenIPC/website/service/internal/db/dbtest" + "github.com/OpenIPC/website/service/internal/keyframe" "github.com/OpenIPC/website/service/internal/snapshots" "github.com/OpenIPC/website/service/internal/variants" ) @@ -304,6 +307,120 @@ func TestConcurrentFramesFromOneCamera(t *testing.T) { } } +// The home page's mosaic: a camera that has been uploading for a month on many +// days, whose newest measured frame has something in it. The two frames +// reported on 2026-10-03 are here by their measurements -- a flat grey frame +// with a clock on it (54/54/54) and a white one (254/254/254). +func TestShowcase(t *testing.T) { + r := newRig(t) + ctx := context.Background() + frame := func(mac string, minutesAgo int, luma *[3]int) { + t.Helper() + var p5, p50, p95 any + if luma != nil { + p5, p50, p95 = luma[0], luma[1], luma[2] + } + if _, err := r.pool.Exec(ctx, `INSERT INTO snapshots + (public_id, mac_address, camera_token, content_type, byte_size, luma_p5, luma_p50, luma_p95, created_at) + VALUES ($1, $2, 'x', 'image/jpeg', 1, $3, $4, $5, now() - make_interval(mins => $6))`, + snapshots.NewPublicID(), mac, p5, p50, p95, minutesAgo); err != nil { + t.Fatal(err) + } + } + camera := func(mac string, ageDays, days int) { + t.Helper() + if _, err := r.pool.Exec(ctx, `INSERT INTO cameras (mac_key, first_seen, last_day, days) + VALUES (lower(translate($1, ':-', '')), now() - make_interval(days => $2), current_date, $3) + ON CONFLICT (mac_key) DO UPDATE SET first_seen = EXCLUDED.first_seen, days = EXCLUDED.days`, + mac, ageDays, days); err != nil { + t.Fatal(err) + } + } + scene, grey, white, black := &[3]int{18, 96, 210}, &[3]int{54, 54, 54}, &[3]int{254, 254, 254}, &[3]int{3, 5, 9} + frame("02:00:00:00:00:01", 5, scene) // shown + camera("02:00:00:00:00:01", 31, 30) + frame("02:00:00:00:00:02", 30, scene) // went flat after a good frame: not shown + frame("02:00:00:00:00:02", 5, grey) + camera("02:00:00:00:00:02", 31, 30) + frame("02:00:00:00:00:03", 5, white) + camera("02:00:00:00:00:03", 31, 30) + frame("02:00:00:00:00:04", 5, black) + camera("02:00:00:00:00:04", 31, 30) + frame("02:00:00:00:00:05", 20, scene) // newest frame not measured yet: the one before it stands + frame("02:00:00:00:00:05", 1, nil) + camera("02:00:00:00:00:05", 31, 30) + frame("02:00:00:00:00:06", 5, scene) // new to the wall + camera("02:00:00:00:00:06", 2, 2) + frame("02:00:00:00:00:07", 5, scene) // one upload a month ago, then silence + camera("02:00:00:00:00:07", 31, 1) + frame("02:00:00:00:00:08", 5, scene) // never counted: no history at all + rows, err := r.store.Showcase(ctx, 0) + if err != nil { + t.Fatal(err) + } + var got []string + for _, s := range rows { + got = append(got, s.MACKey) + } + if want := []string{"020000000001", "020000000005"}; !slices.Equal(got, want) { + t.Errorf("showcase %v, want %v", got, want) + } + + // Ageing in: the new camera, a month and twenty days of uploads later. + camera("02:00:00:00:00:06", 31, snapshots.ShowcaseMinDays) + if rows, _ := r.store.Showcase(ctx, 0); len(rows) != 3 { + t.Errorf("%d cameras once the new one is established, want 3", len(rows)) + } +} + +// A camera's history is written by the frames the wall accepted: the first +// one, and one day for each UTC day with any. A refused upload counts for +// nothing, and the history outlives the purge. +func TestCameraHistory(t *testing.T) { + r := newRig(t) + ctx := context.Background() + luma := keyframe.Luma{P5: 20, P50: 100, P95: 200} + at := func(mac, when string) string { + t.Helper() + id := snapshots.NewPublicID() + if _, err := r.pool.Exec(ctx, `INSERT INTO snapshots (public_id, mac_address, camera_token, content_type, byte_size, created_at) + VALUES ($1, $2, 'x', 'image/jpeg', 1, $3::timestamptz)`, id, mac, when); err != nil { + t.Fatal(err) + } + return id + } + history := func() (first time.Time, days int, ok bool) { + err := r.pool.QueryRow(ctx, `SELECT first_seen, days FROM cameras WHERE mac_key = 'aabbccddee10'`).Scan(&first, &days) + return first, days, err == nil + } + + refused := at("aa:bb:cc:dd:ee:10", "2026-09-01 10:00:00+00") + if err := r.store.MarkRefused(ctx, refused); err != nil { + t.Fatal(err) + } + if _, _, ok := history(); ok { + t.Fatal("a refused upload started a camera's history") + } + for _, f := range []struct{ mac, when string }{ + {"aa:bb:cc:dd:ee:10", "2026-09-02 10:00:00+00"}, + {"AA-BB-CC-DD-EE-10", "2026-09-02 23:45:00+00"}, // same day, another spelling + {"aa:bb:cc:dd:ee:10", "2026-09-04 00:15:00+00"}, + {"aa:bb:cc:dd:ee:10", "2026-09-03 12:00:00+00"}, // processed late: an earlier day, not a new one + } { + if still, err := r.store.MarkGenerated(ctx, at(f.mac, f.when), 640, 360, luma); err != nil || !still { + t.Fatalf("%v %v", still, err) + } + } + first, days, _ := history() + if want := time.Date(2026, 9, 2, 10, 0, 0, 0, time.UTC); !first.Equal(want) || days != 2 { + t.Errorf("first seen %v on %d days, want %v on 2 (the 2nd and the 4th; the 3rd arrived after the 4th)", first, days, want) + } + r.pool.Exec(ctx, `DELETE FROM snapshots`) + if _, _, ok := history(); !ok { + t.Error("purging the snapshots took the camera's history with it") + } +} + // One camera, three spellings, one tile; and the tie-break on id that the // homepage once lost two of its five tiles without. func TestLatestPerCamera(t *testing.T) { diff --git a/service/internal/snapshots/store.go b/service/internal/snapshots/store.go index a3881c20..8b155896 100644 --- a/service/internal/snapshots/store.go +++ b/service/internal/snapshots/store.go @@ -14,6 +14,8 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" + + "github.com/OpenIPC/website/service/internal/keyframe" ) // PublicIDFormat is what a snapshot's address looks like. @@ -199,6 +201,52 @@ func (st *Store) LatestPerCamera(ctx context.Context, limit int) ([]*Snapshot, e return scanAll(rows) } +// What the home page's mosaic leaves out (migration 022). Measured on the +// wall's 4,123 frames of 2026-10-03, where 309 fell under one of these and +// every one of them was a frame with nothing to see: flat grey or white with +// at most the camera's clock on it, a night scene with no light, or a garden +// camera blown out to white. +const ( + // ShowcaseMinSpread: the 5th to 95th percentile of brightness. Flat + // frames measure 0-7; the dimmest picture still worth showing, about 24. + ShowcaseMinSpread = 16 + // ShowcaseMaxMedian: half the frame at least this bright is blown out. + ShowcaseMaxMedian = 245 + // ShowcaseMinBright: the brightest 5% darker than this is a black frame. + ShowcaseMinBright = 20 + // ShowcaseMinAge: how long a camera has to have been uploading before its + // frames reach the front page, so a new one cannot deface it. + ShowcaseMinAge = 30 * 24 * time.Hour + // ShowcaseMinDays: and on how many separate days. A camera that uploaded + // once a month ago, or a MAC invented then, has one. + ShowcaseMinDays = 20 +) + +// Showcase is LatestPerCamera for the home page: the newest measured frame of +// every camera seen in the last day, leaving out a camera that has been +// uploading for less than ShowcaseMinAge, or on fewer than ShowcaseMinDays +// days, or whose newest frame has nothing to see in it. Its newest frame, not its best one: a camera that has gone dark +// is not shown by a picture from before it did. +func (st *Store) Showcase(ctx context.Context, limit int) ([]*Snapshot, error) { + q := `SELECT ` + columns + ` FROM ( + SELECT DISTINCT ON (s.mac_key) s.* FROM snapshots s + JOIN cameras c ON c.mac_key = s.mac_key + WHERE s.created_at > now() - interval '1 day' AND s.luma_p50 IS NOT NULL + AND c.first_seen <= now() - make_interval(secs => $1) AND c.days >= $5 + ORDER BY s.mac_key, s.created_at DESC, s.id DESC + ) latest + WHERE luma_p95 - luma_p5 >= $2 AND luma_p50 < $3 AND luma_p95 >= $4 + ORDER BY created_at DESC, id DESC` + if limit > 0 { + q += fmt.Sprintf(" LIMIT %d", limit) + } + rows, err := st.DB.Query(ctx, q, ShowcaseMinAge.Seconds(), ShowcaseMinSpread, ShowcaseMaxMedian, ShowcaseMinBright, ShowcaseMinDays) + if err != nil { + return nil, err + } + return scanAll(rows) +} + // ByPublicID finds one frame; nil when there is none. func (st *Store) ByPublicID(ctx context.Context, id string) (*Snapshot, error) { s, err := scan(st.DB.QueryRow(ctx, `SELECT `+columns+` FROM snapshots WHERE public_id = $1`, id)) @@ -272,16 +320,31 @@ func (st *Store) Generated(ctx context.Context) ([]string, error) { return ids, rows.Err() } -// MarkGenerated records the variants and the image's dimensions. It reports -// whether the row still exists: a frame purged while its variants were being -// made must have its files removed by the caller. -func (st *Store) MarkGenerated(ctx context.Context, publicID string, width, height int) (bool, error) { - tag, err := st.DB.Exec(ctx, `UPDATE snapshots SET variants_generated_at = now(), - width = $2, height = $3 WHERE public_id = $1`, publicID, width, height) +// MarkGenerated records the variants, the image's dimensions and its +// brightness, and counts the frame toward its camera's history (cameras, +// migration 022) -- here and not on insert, so only a frame the wall accepted +// counts. It reports whether the row still exists: a frame purged while its +// variants were being made must have its files removed by the caller. +func (st *Store) MarkGenerated(ctx context.Context, publicID string, width, height int, luma keyframe.Luma) (bool, error) { + var n int + err := st.DB.QueryRow(ctx, `WITH frame AS ( + UPDATE snapshots SET variants_generated_at = now(), + width = $2, height = $3, luma_p5 = $4, luma_p50 = $5, luma_p95 = $6 + WHERE public_id = $1 RETURNING mac_key, created_at + ), seen AS ( + INSERT INTO cameras (mac_key, first_seen, last_day, days) + SELECT mac_key, created_at, (created_at AT TIME ZONE 'UTC')::date, 1 FROM frame + ON CONFLICT (mac_key) DO UPDATE SET + first_seen = LEAST(cameras.first_seen, EXCLUDED.first_seen), + days = cameras.days + (EXCLUDED.last_day > cameras.last_day)::int, + last_day = GREATEST(cameras.last_day, EXCLUDED.last_day) + ) + SELECT count(*) FROM frame`, + publicID, width, height, int16(luma.P5), int16(luma.P50), int16(luma.P95)).Scan(&n) if err != nil { return false, err } - return tag.RowsAffected() == 1, nil + return n == 1, nil } // Exists is a cheap check for the worker. diff --git a/service/internal/variants/variants.go b/service/internal/variants/variants.go index 12458caa..0e43cbfb 100644 --- a/service/internal/variants/variants.go +++ b/service/internal/variants/variants.go @@ -117,7 +117,7 @@ func writeAtomically(dir, name string, fill func(*os.File) error) error { type Store interface { Exists(ctx context.Context, publicID string) (bool, error) MarkRefused(ctx context.Context, publicID string) error - MarkGenerated(ctx context.Context, publicID string, width, height int) (bool, error) + MarkGenerated(ctx context.Context, publicID string, width, height int, luma keyframe.Luma) (bool, error) Pending(ctx context.Context) ([]string, error) Generated(ctx context.Context) ([]string, error) } @@ -226,7 +226,7 @@ func (p *Processor) process(ctx context.Context, id string) { _ = p.Wall.Purge(id) return } - width, height, err := p.Generate(ctx, id) + width, height, luma, err := p.Generate(ctx, id) var refused ErrRefused if errors.As(err, &refused) { // Not a frame the wall can show. Files first: if closing the row then @@ -243,7 +243,7 @@ func (p *Processor) process(ctx context.Context, id string) { p.Log.Error("variants: failed", "public_id", id, "err", err) return } - still, err := p.Store.MarkGenerated(ctx, id, width, height) + still, err := p.Store.MarkGenerated(ctx, id, width, height, luma) if err != nil { p.Log.Error("variants: could not mark", "public_id", id, "err", err) return @@ -262,50 +262,52 @@ type ErrRefused struct{ Reason string } func (e ErrRefused) Error() string { return "refused: " + e.Reason } -// Generate publishes id's original as-is and returns the picture's size. -func (p *Processor) Generate(ctx context.Context, id string) (int, int, error) { +// Generate publishes id's original as-is and returns the picture's size and +// brightness, measured from the decode it was checked with. +func (p *Processor) Generate(ctx context.Context, id string) (int, int, keyframe.Luma, error) { data, err := os.ReadFile(p.Wall.Original(id)) if err != nil { - return 0, 0, err + return 0, 0, keyframe.Luma{}, err } dir := p.Wall.Dir(id) switch { case len(data) >= 12 && string(data[4:8]) == "ftyp": - f, err := p.keyframe(ctx, data) + f, luma, err := p.keyframe(ctx, data) if err != nil { - return 0, 0, err + return 0, 0, keyframe.Luma{}, err } if err := publish(dir, MainHEIF, data); err != nil { - return 0, 0, err + return 0, 0, keyframe.Luma{}, err } p.thumb(ctx, id) - return f.Width, f.Height, nil + return f.Width, f.Height, luma, nil case bytes.HasPrefix(data, []byte{0xFF, 0xD8, 0xFF}): // REMOVE AFTER 2027-06, with keyframe.StripJPEG. - out, w, h, err := keyframe.StripJPEG(data) + out, w, h, luma, err := keyframe.StripJPEG(data) if err != nil { - return 0, 0, ErrRefused{err.Error()} + return 0, 0, keyframe.Luma{}, ErrRefused{err.Error()} } - return w, h, publish(dir, MainJPEG, out) + return w, h, luma, publish(dir, MainJPEG, out) } - return 0, 0, ErrRefused{"neither a HEIF keyframe nor a JPEG"} + return 0, 0, keyframe.Luma{}, ErrRefused{"neither a HEIF keyframe nor a JPEG"} } // keyframe parses and decodes a HEIF. A file that is not one keyframe, or // that the decoder rejects, is refused; a decoder that could not be run at all // is a failure to retry, not a verdict on the frame. -func (p *Processor) keyframe(ctx context.Context, data []byte) (*keyframe.Frame, error) { +func (p *Processor) keyframe(ctx context.Context, data []byte) (*keyframe.Frame, keyframe.Luma, error) { f, err := keyframe.Parse(data) if err != nil { - return nil, ErrRefused{err.Error()} + return nil, keyframe.Luma{}, ErrRefused{err.Error()} } - if err := keyframe.Check(ctx, p.FFmpeg, f); err != nil { + luma, err := keyframe.Check(ctx, p.FFmpeg, f) + if err != nil { if errors.Is(err, exec.ErrNotFound) || errors.Is(err, keyframe.ErrCheckTimeout) || ctx.Err() != nil { - return nil, err + return nil, keyframe.Luma{}, err } - return nil, ErrRefused{err.Error()} + return nil, keyframe.Luma{}, ErrRefused{err.Error()} } - return f, nil + return f, luma, nil } // thumb publishes the substream keyframe if the upload carried a good one. @@ -315,7 +317,7 @@ func (p *Processor) thumb(ctx context.Context, id string) { if err != nil { return } - if _, err := p.keyframe(ctx, data); err != nil { + if _, _, err := p.keyframe(ctx, data); err != nil { p.Log.Warn("variants: substream keyframe not published", "public_id", id, "err", err) return } diff --git a/service/internal/variants/variants_test.go b/service/internal/variants/variants_test.go index 4da030af..7b3903a2 100644 --- a/service/internal/variants/variants_test.go +++ b/service/internal/variants/variants_test.go @@ -41,7 +41,7 @@ func (s *fakeStore) MarkRefused(_ context.Context, id string) error { return nil } -func (s *fakeStore) MarkGenerated(_ context.Context, id string, w, h int) (bool, error) { +func (s *fakeStore) MarkGenerated(_ context.Context, id string, w, h int, _ keyframe.Luma) (bool, error) { s.mu.Lock() defer s.mu.Unlock() s.marked[id] = [2]int{w, h} diff --git a/service/internal/wall/api.go b/service/internal/wall/api.go index 7bdca120..80705f83 100644 --- a/service/internal/wall/api.go +++ b/service/internal/wall/api.go @@ -31,6 +31,7 @@ var ( // Snapshots is what the API reads. type Snapshots interface { LatestPerCamera(ctx context.Context, limit int) ([]*snapshots.Snapshot, error) + Showcase(ctx context.Context, limit int) ([]*snapshots.Snapshot, error) ByPublicID(ctx context.Context, id string) (*snapshots.Snapshot, error) ByCameraToken(ctx context.Context, token string) (*snapshots.Snapshot, error) DayOf(ctx context.Context, subject *snapshots.Snapshot, limit int) ([]*snapshots.Snapshot, error) @@ -186,7 +187,9 @@ func (a *API) fail(w http.ResponseWriter, err error) { } func (a *API) mosaic(w http.ResponseWriter, r *http.Request) { - rows, err := a.Store.LatestPerCamera(r.Context(), MosaicTiles) + // The home page's mosaic, which is not the wall: only cameras that have + // been uploading for a month, and only frames with something in them. + rows, err := a.Store.Showcase(r.Context(), MosaicTiles) if err != nil { a.fail(w, err) return diff --git a/service/internal/wall/wall_test.go b/service/internal/wall/wall_test.go index 8d347d0d..337a632b 100644 --- a/service/internal/wall/wall_test.go +++ b/service/internal/wall/wall_test.go @@ -146,9 +146,9 @@ func seed(t *testing.T, store *snapshots.Store, mac string, secondsAgo int, soc id := snapshots.NewPublicID() _, err := store.DB.Exec(context.Background(), `INSERT INTO snapshots (public_id, mac_address, camera_token, content_type, byte_size, width, height, soc, sensor, - firmware, streamer, uptime, soc_temperature, caption, created_at) + firmware, streamer, uptime, soc_temperature, caption, luma_p5, luma_p50, luma_p95, created_at) VALUES ($1, $2, $3, 'image/jpeg', 534513, 2592, 1520, $4, 'imx335', '2.6.09.15-lite', 'majestic', - '8 days', '55.73', '', now() - make_interval(secs => $5))`, + '8 days', '55.73', '', 30, 120, 220, now() - make_interval(secs => $5))`, id, mac, store.CameraToken(mac), soc, secondsAgo) if err != nil { t.Fatal(err) @@ -176,6 +176,11 @@ func TestWallAddresses(t *testing.T) { ids = append(ids, seed(t, store, "aa:bb:cc:00:00:01", i*900, &gk)) } seed(t, store, "AA-BB-CC-00-00-02", 60, nil) // no soc: null, not an empty string + // Cameras the front page knows (snapshots.ShowcaseMinAge). + if _, err := pool.Exec(context.Background(), `INSERT INTO cameras (mac_key, first_seen, last_day, days) + SELECT DISTINCT mac_key, now() - interval '60 days', current_date, 60 FROM snapshots`); err != nil { + t.Fatal(err) + } // mosaic: the key order the pages read, byte for byte. rec := get(t, mux, "/api/v1/wall/mosaic.json")