Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 62 additions & 7 deletions src/core/vision/NetImage.res
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// Reads a picture of a cube net (a screenshot or an export): the cross with U
// above F and L F R B in a row, D below F, for 2x2 to 7x7. Stickers are the
// above F and L F R B in a row, D below F, for 2x2 to 15x15. Stickers are the
// bright shapes the dark gaps, lines and panels leave; the cross' columns and
// rows tell the size, and the six colors found are named by the faces'
// centers (odd cubes) or by the nearest standard color (even cubes).
Expand All @@ -20,7 +20,16 @@ type ints

// Stickers of different colors differ by more than this (RGB distance).
let sameColor = 45.0
let maxSize = 7
// The largest cube a net can show.
let maxSize = 15
// A net of the largest cube has 6 * 15 * 15 = 1350 stickers; a picture with
// more sticker-like shapes than about three times that is no net.
let maxCandidates = 4000
// A sticker is read from a box at least this many pixels on a side.
let minStickerSide = 6
let minStickerArea = minStickerSide * minStickerSide
// Fewer sticker-like shapes than the smallest net (2x2: 6 * 2 * 2).
let minCandidates = 24

type component = {
mutable area: int,
Expand Down Expand Up @@ -92,7 +101,20 @@ let boxHeight = c => c.maxY - c.minY + 1
let looksLikeSticker = c => {
let w = Int.toFloat(boxWidth(c))
let h = Int.toFloat(boxHeight(c))
c.area >= 36 && w /. h > 0.7 && w /. h < 1.4 && Int.toFloat(c.area) /. (w *. h) > 0.6
c.area >= minStickerArea && w /. h > 0.7 && w /. h < 1.4 && Int.toFloat(c.area) /. (w *. h) > 0.6
}

// Square-ish, filled shapes from 4 x 4 pixels up to the sticker minimum: the
// stickers of a picture whose resolution is too low to read.
let minTinyArea = 16
let looksLikeTinySticker = c => {
let w = Int.toFloat(boxWidth(c))
let h = Int.toFloat(boxHeight(c))
c.area >= minTinyArea &&
c.area < minStickerArea &&
w /. h > 0.7 &&
w /. h < 1.4 &&
Int.toFloat(c.area) /. (w *. h) > 0.6
}

// The sticker size: the area most shapes share, within a quarter.
Expand Down Expand Up @@ -301,10 +323,43 @@ let faceBoxes = (stickers, slots: array<option<(int, int)>>, width, height, area

let unreadable = message => Unreadable({message: message})

// How many too small shapes of one common size the picture holds, when they
// are what the picture mostly shows: at least a net's worth of them and ten
// times the readable candidates. Specks of dust, noise and a net with a
// sticker missing do not qualify; 0 then.
let tooSmallStickers = (all, candidateCount) => {
let tiny = all->Array.filter(looksLikeTinySticker)
if Array.length(tiny) < minCandidates || Array.length(tiny) > maxCandidates {
0
} else {
let area = Int.toFloat(referenceArea(tiny))
let same =
tiny
->Array.filter(c => {
let a = Int.toFloat(c.area)
a >= area *. 0.8 && a <= area *. 1.25
})
->Array.length
same >= minCandidates && candidateCount * 10 < same ? same : 0
}
}

let readNetImage = (data: Uint8ClampedArray.t, width: int, height: int): netRead => {
let all = components(data, width, height)
let candidates = all->Array.filter(looksLikeSticker)
if Array.length(candidates) < 24 || Array.length(candidates) > 4000 {
let tooSmall =
Array.length(candidates) < minCandidates ? tooSmallStickers(all, Array.length(candidates)) : 0
if tooSmall > 0 {
unreadable(
`The stickers are too small to read: the picture shows ${Int.toString(
tooSmall,
)} shapes smaller than ${Int.toString(minStickerSide)} x ${Int.toString(
minStickerSide,
)} pixels. Use a bigger picture, at least ${Int.toString(
minStickerSide,
)} pixels per sticker.`,
)
} else if Array.length(candidates) < minCandidates || Array.length(candidates) > maxCandidates {
unreadable("No cube net found in the picture.")
} else {
let area = Int.toFloat(referenceArea(candidates))
Expand All @@ -326,9 +381,9 @@ let readNetImage = (data: Uint8ClampedArray.t, width: int, height: int): netRead
unreadable(
`Found ${Int.toString(Array.length(stickers))} stickers in ${Int.toString(
Array.length(columns),
)} columns and ${Int.toString(
Array.length(rows),
)} rows; a net of a 2x2 to 7x7 cube has 6 faces of n x n in a cross.`,
)} columns and ${Int.toString(Array.length(rows))} rows; a net of a 2x2 to ${Int.toString(
maxSize,
)}x${Int.toString(maxSize)} cube has 6 faces of n x n in a cross.`,
)
} else {
// Each sticker's face and place in it.
Expand Down
3 changes: 2 additions & 1 deletion src/cube/GuidedCaptureSetup.res
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ let cross = ((a0, a1, a2): vector, (b0, b1, b2): vector): vector => (
a0 * b1 - a1 * b0,
)

let isOddSize = n => n == 3 || n == 5 || n == 7
// Odd cubes have a fixed center sticker on every face, at any size.
let isOddSize = n => n >= 3 && mod(n, 2) == 1

// The sticker in the middle of a photo, if it has one there.
let centerOf = (photo: grid, mid) => photo[mid]->Option.flatMap(row => row[mid])
Expand Down
29 changes: 28 additions & 1 deletion test/cube/guidedCapture.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ describe('predictGuidedCenters', () => {
const face = (n: number, color: string) =>
Array.from({ length: n }, () => Array(n).fill(color))

for (const n of [3, 5, 7]) {
for (const n of [3, 5, 7, 9, 11, 13, 15]) {
it(`shows the opposite after one ${n}x${n} face, then all remaining centers after adjacent faces`, () => {
const first = face(n, 'G'),
second = face(n, 'R')
Expand Down Expand Up @@ -284,6 +284,33 @@ describe('predictGuidedCenters', () => {
})
}

for (const n of [3, 5, 7, 9, 11, 13, 15]) {
it(`reserves center slots and routes faces by center on ${n}x${n}`, () => {
const first = face(n, 'G'),
second = face(n, 'R')
expect(captureCenterSlots([first, second])).toEqual([
'G',
'R',
'B',
'O',
'W',
'Y',
])
// The opposite of the first face goes to slot 3, a duplicate nowhere.
expect(captureSlotForCenter([first], 1, face(n, 'B'))).toBe(2)
expect(captureSlotForCenter([first], 1, face(n, 'G'))).toBeNull()
expect(captureSlotForCenter([first, second], 3, face(n, 'W'))).toBe(4)
})
}

for (const n of [2, 4, 6, 8, 10, 12, 14]) {
it(`predicts nothing on the even ${n}x${n} size`, () => {
const blank = [null, null, null, null, null, null]
expect(predictGuidedCenters([face(n, 'G'), face(n, 'R')])).toEqual(blank)
expect(captureCenterSlots([face(n, 'G'), face(n, 'R')])).toEqual(blank)
})
}

it('works when the first two captures are on other slots', () => {
expect(
predictGuidedCenters([
Expand Down
115 changes: 113 additions & 2 deletions test/netImage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ import { DEFAULT_SAMPLING } from '../src/client/vision/stickerColorGeometry'
import { randomCube } from '../src/cube/randomState'
import { readNetImage } from '../src/core/vision/NetImage.gen'

// Every cube size the app handles.
const SIZES = Array.from({ length: 14 }, (_, i) => i + 2)

type RGB = [number, number, number]
const STANDARD: Record<string, RGB> = {
W: [250, 250, 245],
Expand Down Expand Up @@ -169,7 +172,7 @@ describe('reading a net image', () => {
['light panels with titles', LIGHT_PANELS],
]
for (const [name, style] of styles)
for (const size of [2, 3, 4, 5, 7])
for (const size of SIZES)
it(`reads a ${size}x${size} net, ${name}`, () => {
const cube = randomCube(size, 11 + size)
const { data, width, height } = renderNet(cube, style)
Expand Down Expand Up @@ -231,6 +234,90 @@ describe('reading a net image', () => {
expect(read.TAG).toBe('unreadable')
})

it('says the stickers are too small when the picture has too few pixels', () => {
// 15x15 at 5 pixels a sticker: 1350 shapes, each below the 6 x 6 minimum.
const cube = randomCube(15, 2)
const tiny: NetStyle = { ...DARK_TIGHT, cell: 5, gap: 2, margin: 4 }
const { data, width, height } = renderNet(cube, tiny)
const read = readNetImage(data, width, height)
if (read.TAG !== 'unreadable')
throw new Error('read a net of tiny stickers')
expect(read.message).toMatch(/too small/i)
expect(read.message).toMatch(/6 x 6 pixels/)
})

it('does not call a noise picture too small', () => {
let seed = 7
const data = new Uint8ClampedArray(200 * 200 * 4)
for (let i = 0; i < data.length; i += 4) {
seed = (seed * 1103515245 + 12345) % 2147483648
const v = seed % 2 ? 255 : 0
data.set([v, v, v, 255], i)
}
const read = readNetImage(data, 200, 200)
if (read.TAG !== 'unreadable') throw new Error('read noise')
expect(read.message).toBe('No cube net found in the picture.')
})

it('reads a net of full-size stickers among many specks', () => {
const cube = randomCube(2, 12)
const { data, width, height } = renderNet(cube, {
...DARK_GAPPED,
margin: 40,
specks: 200,
})
expect(cubeFromRead(readNetImage(data, width, height))).toEqual(cube)
})

it('reports a missing sticker, not small stickers, when specks are around', () => {
const cube = randomCube(2, 3)
const { data, width, height } = renderNet(cube, {
...DARK_GAPPED,
margin: 40,
specks: 24,
})
// Paint the front face's first sticker over with the background.
for (let y = 136; y < 164; y++)
for (let x = 108; x < 136; x++) {
const i = (y * width + x) * 4
data.set([13, 15, 25, 255], i)
}
const read = readNetImage(data, width, height)
if (read.TAG !== 'unreadable') throw new Error('read a net with a gap')
expect(read.message).toBe('No cube net found in the picture.')
})

it('does not look for a net among more than 4000 sticker-sized shapes', () => {
const side = 70
const data = new Uint8ClampedArray(side * 10 * side * 10 * 4)
for (let y = 0; y < side * 10; y++)
for (let x = 0; x < side * 10; x++) {
const lit = x % 10 < 8 && y % 10 < 8
data.set(
lit ? [200, 60, 60, 255] : [0, 0, 0, 255],
(y * side * 10 + x) * 4,
)
}
const read = readNetImage(data, side * 10, side * 10)
if (read.TAG !== 'unreadable') throw new Error('read a grid')
expect(read.message).toBe('No cube net found in the picture.')
})

it('reads a 15x15 net at the smallest sticker size', () => {
const cube = randomCube(15, 9)
const small: NetStyle = { ...DARK_TIGHT, cell: 6, gap: 2, margin: 4 }
const { data, width, height } = renderNet(cube, small)
expect(cubeFromRead(readNetImage(data, width, height))).toEqual(cube)
})

it('refuses a 16x16 net and names the supported sizes', () => {
const cube = randomCube(16, 5)
const { data, width, height } = renderNet(cube, DARK_TIGHT)
const read = readNetImage(data, width, height)
if (read.TAG !== 'unreadable') throw new Error('read a 16x16 net')
expect(read.message).toContain('2x2 to 15x15')
})

it('says so when two face centers share a color', () => {
const cube = randomCube(3, 5)
cube.d[4] = 'W'
Expand Down Expand Up @@ -292,8 +379,32 @@ describe('splitting a net into six face pictures', () => {
['dark and tight', DARK_TIGHT],
['light panels with titles', LIGHT_PANELS],
]
// The color pipeline's own limits above 7x7 belong to face detection.
for (const [name, style] of styles)
for (const size of SIZES)
it(`gives six face rectangles that hold their stickers, ${size}x${size}, ${name}`, () => {
const cube = randomCube(size, 40 + size)
const { data, width, height } = renderNet(cube, style)
const read = readNetImage(data, width, height)
if (read.TAG !== 'read') throw new Error(read.message)
expect(read.faces).toHaveLength(6)
const pitch = style.cell + style.gap
for (const box of read.faces) {
expect(box.x).toBeGreaterThanOrEqual(0)
expect(box.y).toBeGreaterThanOrEqual(0)
expect(box.x + box.width).toBeLessThanOrEqual(width)
expect(box.y + box.height).toBeLessThanOrEqual(height)
expect(box.width).toBeGreaterThanOrEqual(size * pitch - style.gap)
expect(box.height).toBeGreaterThanOrEqual(size * pitch - style.gap)
}
const [, right, front, , left, back] = read.faces
expect(left.x + left.width).toBeLessThanOrEqual(front.x)
expect(front.x + front.width).toBeLessThanOrEqual(right.x)
expect(right.x + right.width).toBeLessThanOrEqual(back.x)
})

for (const [name, style] of styles)
for (const size of [3, 4, 5])
for (const size of [3, 4, 5, 6, 7])
it(`gives six faces the pipeline reads, ${size}x${size}, ${name}`, () => {
const cube = randomCube(size, 30 + size)
const { data, width, height } = renderNet(cube, style)
Expand Down
Loading