diff --git a/src/client/vision/colorMath.ts b/src/client/vision/colorMath.ts index e662cf68..cb9cb882 100644 --- a/src/client/vision/colorMath.ts +++ b/src/client/vision/colorMath.ts @@ -8,6 +8,7 @@ import type { RGB } from './stickerColorGeometry' export { applyGains, applyGainsToPixels, + clusterDistance, hueCircularRange, hueRangesOverlap, linearChannelToSrgb, diff --git a/src/client/vision/stickerLearning.ts b/src/client/vision/stickerLearning.ts index 8d26972c..449fa86d 100644 --- a/src/client/vision/stickerLearning.ts +++ b/src/client/vision/stickerLearning.ts @@ -10,6 +10,7 @@ import { import { STICKER_COLORS, type RGB } from './stickerColorGeometry' export { + balancedAssign, glareWarningStickers as GLARE_WARNING_STICKERS, hungarianAssignment, mixedUpClusters, diff --git a/src/core/color/StickerLearning.res b/src/core/color/StickerLearning.res index 6db95db4..32013013 100644 --- a/src/core/color/StickerLearning.res +++ b/src/core/color/StickerLearning.res @@ -77,12 +77,22 @@ let hungarianAssignment = (cost: array>) => { // Assigns each point to one of `centroids`, each centroid taking as close // as possible to points / centroids of them - a valid NxN capture always // has exactly N^2 stickers of each color. Solved as a genuine minimum-cost -// assignment: each centroid becomes `capacity` equal-cost slots, padded -// with zero-cost dummy points when they don't divide evenly, so the -// globally cheapest assignment wins. A greedy version once stranded a -// point 0.49 from its assigned color while its own color, 0.04 away, had -// filled up. `pinned[i]`, when set, is the cluster point i must take (see -// clearStickerColors); the rest share the slots left over. +// assignment under per-centroid capacities, so the globally cheapest +// assignment wins. A greedy version once stranded a point 0.49 from its +// assigned color while its own color, 0.04 away, had filled up. `pinned[i]`, +// when set, is the cluster point i must take (see clearStickerColors); the +// rest share the capacity left over. +// +// The assignment is a transportation problem with k sinks. Points enter +// one by one in index order; each entry follows the cheapest augmenting +// path over the k clusters, where moving from cluster a to b costs the +// least increase from displacing one member of a to b. Every step keeps +// the assignment optimal for the points entered so far, so the final one +// is optimal: the same minimum as expanding each centroid into `capacity` +// slots and solving the square matching with hungarianAssignment (O(n^6) +// for an NxN cube), in O(m^2 k) with small constants. Equal costs resolve +// by strict comparisons: the lowest cluster index and the lowest member +// index win, so the result is deterministic. let balancedAssign = ( points: array, centroids: array, @@ -93,31 +103,120 @@ let balancedAssign = ( let capacity = Math.ceil(Int.toFloat(n) /. Int.toFloat(k))->Float.toInt let result = Array.make(~length=n, 0) let free = [] - let slots = Array.make(~length=k, capacity) + let room = Array.make(~length=k, capacity) points->Array.forEachWithIndex((_, pi) => switch pinned[pi]->Option.flatMap(Nullable.toOption) { - | Some(pin) if pin >= 0 && pin < k && slots->Array.getUnsafe(pin) > 0 => + | Some(pin) if pin >= 0 && pin < k && room->Array.getUnsafe(pin) > 0 => result->Array.setUnsafe(pi, pin) - slots->Array.setUnsafe(pin, slots->Array.getUnsafe(pin) - 1) + room->Array.setUnsafe(pin, room->Array.getUnsafe(pin) - 1) | _ => free->Array.push(pi) } ) - let slotCluster = slots->Array.flatMapWithIndex((count, ci) => Array.make(~length=count, ci)) - let totalSlots = Array.length(slotCluster) - let cost = free->Array.map(pi => { - let point = points->Array.getUnsafe(pi) - let distances = centroids->Array.map(centroid => clusterDistance(point, centroid)) - slotCluster->Array.map(ci => distances->Array.getUnsafe(ci)) + let m = Array.length(free) + let infinity = Float.Constants.positiveInfinity + // A path must beat the current one by more than rounding noise, so a + // cycle of equal costs can never loop. + let epsilon = 1e-12 + // clusterDistance(a, b) is clusterOklabDistance of both conversions; + // converting each color once gives the same values. + let centroidLabs = centroids->Array.map(rgbToOklab) + let dist = free->Array.map(pi => { + let lab = rgbToOklab(points->Array.getUnsafe(pi)) + centroidLabs->Array.map(centroidLab => clusterOklabDistance(lab, centroidLab)) }) - // Dummy rows cost nothing anywhere, so they take whichever leftover slots - // are cheapest to leave empty. - for _ in Array.length(free) to totalSlots - 1 { - cost->Array.push(Array.make(~length=totalSlots, 0.0)) + let owner = Array.make(~length=m, -1) + let load = Array.make(~length=k, 0) + // move[a * k + b]: the least extra cost of sending one member of a to b, + // and which member; recomputed for clusters whose membership changed. + let move = Array.make(~length=k * k, infinity) + let mover = Array.make(~length=k * k, -1) + let dirty = Array.make(~length=k, false) + let refresh = a => { + for b in 0 to k - 1 { + move->Array.setUnsafe(a * k + b, infinity) + mover->Array.setUnsafe(a * k + b, -1) + } + for q in 0 to m - 1 { + if owner->Array.getUnsafe(q) == a { + let row = dist->Array.getUnsafe(q) + let own = row->Array.getUnsafe(a) + for b in 0 to k - 1 { + if b != a { + let extra = row->Array.getUnsafe(b) -. own + if extra < move->Array.getUnsafe(a * k + b) { + move->Array.setUnsafe(a * k + b, extra) + mover->Array.setUnsafe(a * k + b, q) + } + } + } + } + } + dirty->Array.setUnsafe(a, false) } - let slotAssignment = hungarianAssignment(cost) - free->Array.forEachWithIndex((pi, j) => - result->Array.setUnsafe(pi, slotCluster->Array.getUnsafe(slotAssignment->Array.getUnsafe(j))) - ) + let best = Array.make(~length=k, 0.0) + let via = Array.make(~length=k, -1) + for i in 0 to m - 1 { + for a in 0 to k - 1 { + if dirty->Array.getUnsafe(a) { + refresh(a) + } + } + let row = dist->Array.getUnsafe(i) + for a in 0 to k - 1 { + best->Array.setUnsafe(a, row->Array.getUnsafe(a)) + via->Array.setUnsafe(a, -1) + } + // Bellman-Ford over the k clusters (no negative cycles: the + // assignment so far is optimal). + let changed = ref(true) + let rounds = ref(0) + while changed.contents && rounds.contents < k { + changed := false + rounds := rounds.contents + 1 + for a in 0 to k - 1 { + for b in 0 to k - 1 { + let cost = move->Array.getUnsafe(a * k + b) + if b != a && cost < infinity { + let through = best->Array.getUnsafe(a) +. cost + if through < best->Array.getUnsafe(b) -. epsilon { + best->Array.setUnsafe(b, through) + via->Array.setUnsafe(b, a) + changed := true + } + } + } + } + } + // The cheapest cluster that still has room ends the path. + let target = ref(-1) + for a in 0 to k - 1 { + if load->Array.getUnsafe(a) < room->Array.getUnsafe(a) { + if ( + target.contents < 0 || best->Array.getUnsafe(a) < best->Array.getUnsafe(target.contents) + ) { + target := a + } + } + } + // With capacity for every point a target always exists; the fallback + // only keeps a malformed call from looping. + let stop = target.contents < 0 ? 0 : target.contents + load->Array.setUnsafe(stop, load->Array.getUnsafe(stop) + 1) + let cluster = ref(stop) + let hops = ref(0) + while via->Array.getUnsafe(cluster.contents) >= 0 && hops.contents < k { + let from = via->Array.getUnsafe(cluster.contents) + let q = mover->Array.getUnsafe(from * k + cluster.contents) + owner->Array.setUnsafe(q, cluster.contents) + dirty->Array.setUnsafe(from, true) + dirty->Array.setUnsafe(cluster.contents, true) + cluster := from + hops := hops.contents + 1 + } + owner->Array.setUnsafe(i, cluster.contents) + dirty->Array.setUnsafe(cluster.contents, true) + } + free->Array.forEachWithIndex((pi, i) => result->Array.setUnsafe(pi, owner->Array.getUnsafe(i))) result } @@ -182,16 +281,22 @@ let splitMixedPairs = (points: array, centroids: array, iterations) => : sum ) let seeds = ref([members->Array.getUnsafe(0), members->Array.getUnsafe(1)]) + // The distance is symmetric, so the first farthest ordered pair + // has p at or before q; Oklab is converted once per member. + let memberLabs = members->Array.map(rgbToOklab) let farthest = ref(-1.0) - members->Array.forEach(p => - members->Array.forEach(q => { - let d = clusterDistance(p, q) + for pi in 0 to Array.length(members) - 1 { + for qi in pi to Array.length(members) - 1 { + let d = clusterOklabDistance( + memberLabs->Array.getUnsafe(pi), + memberLabs->Array.getUnsafe(qi), + ) if d > farthest.contents { farthest := d - seeds := [p, q] + seeds := [members->Array.getUnsafe(pi), members->Array.getUnsafe(qi)] } - }) - ) + } + } let size = assignment.contents->Array.filter(ci => ci == x)->Array.length let split = ref(splitInTwo(members, seeds.contents, size)) for _ in 1 to iterations { diff --git a/test/balancedAssign.test.ts b/test/balancedAssign.test.ts new file mode 100644 index 00000000..9f11f3b8 --- /dev/null +++ b/test/balancedAssign.test.ts @@ -0,0 +1,375 @@ +// balancedAssign: equivalence with the reference (slot expansion solved by +// the O(M^3) Hungarian algorithm) and a speed budget at the largest sizes. +import { describe, it, expect } from 'vitest' +import { clusterDistance } from '../src/client/vision/colorMath' +import { + balancedAssign, + hungarianAssignment, + learnStickerColors, +} from '../src/client/vision/stickerLearning' +import { + STICKER_COLORS, + type RGB, +} from '../src/client/vision/stickerColorGeometry' + +type Pins = Array + +// The implementation this project used before the transportation solver: +// each centroid becomes `capacity` equal-cost slots, dummy zero-cost rows +// pad the matrix to a square, and the Hungarian algorithm solves it. +function referenceBalancedAssign( + points: RGB[], + centroids: RGB[], + pinned: Pins, +): number[] { + const k = centroids.length + const n = points.length + const capacity = Math.ceil(n / k) + const result = new Array(n).fill(0) + const free: number[] = [] + const slots = new Array(k).fill(capacity) + points.forEach((_, pi) => { + const pin = pinned[pi] + if (pin != null && pin >= 0 && pin < k && slots[pin] > 0) { + result[pi] = pin + slots[pin]-- + } else free.push(pi) + }) + const slotCluster = slots.flatMap((count, ci) => new Array(count).fill(ci)) + const total = slotCluster.length + const cost = free.map((pi) => { + const d = centroids.map((c) => clusterDistance(points[pi], c)) + return slotCluster.map((ci) => d[ci]) + }) + for (let i = free.length; i < total; i++) cost.push(new Array(total).fill(0)) + const slotAssignment = hungarianAssignment(cost) + free.forEach((pi, j) => { + result[pi] = slotCluster[slotAssignment[j]] + }) + return result +} + +function lcg(seed: number) { + let s = seed >>> 0 + return () => { + s = (Math.imul(s, 1664525) + 1013904223) >>> 0 + return s / 4294967296 + } +} + +const NAMES = ['W', 'Y', 'R', 'O', 'G', 'B'] +const PALETTE: RGB[] = NAMES.map((name) => STICKER_COLORS[name]) + +function jitter(rand: () => number, c: RGB, spread: number): RGB { + const j = () => (rand() - 0.5) * 2 * spread + const q = (v: number) => Math.max(0, Math.min(255, Math.round(v))) + return { r: q(c.r + j()), g: q(c.g + j()), b: q(c.b + j()) } +} + +// n^2 stickers per color, jittered; `spread` large enough makes the colors +// overlap so capacities bind. +function balancedPoints(n: number, seed: number, spread: number): RGB[] { + const rand = lcg(seed) + const points: RGB[] = [] + for (const color of PALETTE) + for (let i = 0; i < n * n; i++) points.push(jitter(rand, color, spread)) + // Fisher-Yates with the same generator. + for (let i = points.length - 1; i > 0; i--) { + const j = Math.floor(rand() * (i + 1)) + ;[points[i], points[j]] = [points[j], points[i]] + } + return points +} + +function totalCost(points: RGB[], centroids: RGB[], assignment: number[]) { + return assignment.reduce( + (sum, ci, pi) => sum + clusterDistance(points[pi], centroids[ci]), + 0, + ) +} + +function counts(assignment: number[], k: number) { + const c = new Array(k).fill(0) + for (const ci of assignment) c[ci]++ + return c +} + +// Same minimum, valid capacities and honored pins; returns whether the +// assignment is identical to the reference. +function checkAgainstReference( + points: RGB[], + centroids: RGB[], + pinned: Pins, +): boolean { + const k = centroids.length + const capacity = Math.ceil(points.length / k) + const expected = referenceBalancedAssign(points, centroids, pinned) + const actual = balancedAssign( + points, + centroids, + pinned as Array, + ) + expect(actual).toHaveLength(points.length) + actual.forEach((ci) => { + expect(Number.isInteger(ci)).toBe(true) + expect(ci).toBeGreaterThanOrEqual(0) + expect(ci).toBeLessThan(k) + }) + counts(actual, k).forEach((c) => expect(c).toBeLessThanOrEqual(capacity)) + // Pins are honored in point order while their cluster has room. + const room = new Array(k).fill(capacity) + pinned.forEach((pin, pi) => { + if (pin != null && pin >= 0 && pin < k && room[pin] > 0) { + room[pin]-- + expect(actual[pi]).toBe(pin) + } + }) + const want = totalCost(points, centroids, expected) + const got = totalCost(points, centroids, actual) + expect(Math.abs(got - want)).toBeLessThanOrEqual(1e-9 * Math.max(1, want)) + return actual.every((ci, i) => ci === expected[i]) +} + +const NO_PINS: Pins = [] + +describe('balancedAssign against the Hungarian reference', () => { + // Continuous random jitter: the optimum is unique, so the assignment + // itself must match, not just its cost. + for (const n of [2, 3, 4, 5, 6, 7]) { + for (const spread of [6, 40, 120]) { + for (const seed of [1, 2, 3]) { + it(`n=${n} spread=${spread} seed=${seed}: same assignment`, () => { + const points = balancedPoints(n, seed * 1000 + n, spread) + expect(checkAgainstReference(points, PALETTE, NO_PINS)).toBe(true) + }) + } + } + } + + for (const n of [8, 9, 10]) { + it(`n=${n} (sampled): same assignment, overlapping colors`, () => { + const points = balancedPoints(n, 77 + n, 90) + expect(checkAgainstReference(points, PALETTE, NO_PINS)).toBe(true) + }) + } + + it('random centroids (not the palette) match', () => { + for (let seed = 1; seed <= 20; seed++) { + const rand = lcg(seed) + const centroids = Array.from({ length: 6 }, () => ({ + r: Math.floor(rand() * 256), + g: Math.floor(rand() * 256), + b: Math.floor(rand() * 256), + })) + const points = Array.from({ length: 30 + seed * 7 }, () => ({ + r: Math.floor(rand() * 256), + g: Math.floor(rand() * 256), + b: Math.floor(rand() * 256), + })) + expect(checkAgainstReference(points, centroids, NO_PINS)).toBe(true) + } + }) + + it('point counts not divisible by k (dummy rows) match', () => { + for (const total of [1, 2, 5, 6, 7, 11, 13, 29, 50, 97]) { + const rand = lcg(total) + const points = Array.from({ length: total }, () => + jitter(rand, PALETTE[Math.floor(rand() * 6)], 70), + ) + expect(checkAgainstReference(points, PALETTE, NO_PINS)).toBe(true) + } + }) + + it('handles zero points and other cluster counts', () => { + expect(balancedAssign([], PALETTE, [])).toEqual([]) + const rand = lcg(5) + for (const k of [1, 2, 3, 6]) { + const centroids = PALETTE.slice(0, k) + const points = Array.from({ length: 17 }, () => + jitter(rand, PALETTE[Math.floor(rand() * 6)], 60), + ) + expect(checkAgainstReference(points, centroids, NO_PINS)).toBe(true) + } + }) + + it('an empty class (all points near five colors) still fills capacities', () => { + const rand = lcg(9) + const points: RGB[] = [] + for (const c of PALETTE.slice(0, 5)) + for (let i = 0; i < 36; i++) points.push(jitter(rand, c, 10)) + for (let i = 0; i < 6; i++) points.push(jitter(rand, PALETTE[0], 10)) + expect(checkAgainstReference(points, PALETTE, NO_PINS)).toBe(true) + }) + + it('a full class (every point at one centroid) spreads by capacity', () => { + const rand = lcg(11) + const points = Array.from({ length: 54 }, () => + jitter(rand, PALETTE[2], 25), + ) + expect(checkAgainstReference(points, PALETTE, NO_PINS)).toBe(true) + }) + + describe('pins', () => { + it('honors pins, including over-pinned and out-of-range values', () => { + for (let seed = 1; seed <= 12; seed++) { + const rand = lcg(seed * 31) + const points = balancedPoints(3, seed, 60) + const pinned: Pins = points.map(() => { + const r = rand() + if (r < 0.5) return null + if (r < 0.9) return Math.floor(rand() * 6) + return r < 0.95 ? -1 : 6 // out of range: treated as free + }) + checkAgainstReference(points, PALETTE, pinned) + } + }) + + it('pins the first points of an over-subscribed cluster only', () => { + const points = balancedPoints(2, 4, 30) + const pinned: Pins = points.map(() => 0) // 24 points pin 4 slots + const actual = balancedAssign( + points, + PALETTE, + pinned as Array, + ) + expect(actual.slice(0, 4)).toEqual([0, 0, 0, 0]) + expect(counts(actual, 6)[0]).toBeLessThanOrEqual(4) + checkAgainstReference(points, PALETTE, pinned) + }) + + it('a pin array shorter than the points leaves the rest free', () => { + const points = balancedPoints(3, 8, 50) + checkAgainstReference(points, PALETTE, [1, 1, 1]) + }) + + it('unique-optimum pinned cases match the reference assignment', () => { + for (let seed = 1; seed <= 12; seed++) { + const rand = lcg(seed * 7) + const points = balancedPoints(4, seed, 60) + const pinned: Pins = points.map(() => + rand() < 0.4 ? Math.floor(rand() * 6) : null, + ) + expect(checkAgainstReference(points, PALETTE, pinned)).toBe(true) + } + }) + }) + + // Exact ties: identical points are interchangeable, so only the cost, the + // capacities and the number of points of each color per cluster can be + // asserted. Which of two identical points is displaced is not specified. + describe('exact ties', () => { + function tieCheck(points: RGB[], centroids: RGB[], pinned: Pins) { + checkAgainstReference(points, centroids, pinned) + const expected = referenceBalancedAssign(points, centroids, pinned) + const actual = balancedAssign( + points, + centroids, + pinned as Array, + ) + const histogram = (assignment: number[]) => { + const h = new Map() + assignment.forEach((ci, pi) => { + const key = `${points[pi].r},${points[pi].g},${points[pi].b}->${ci}` + h.set(key, (h.get(key) ?? 0) + 1) + }) + return [...h.entries()].sort() + } + return ( + JSON.stringify(histogram(actual)) === + JSON.stringify(histogram(expected)) + ) + } + + it('all points identical', () => { + const points = new Array(24).fill(0).map(() => ({ ...PALETTE[3] })) + checkAgainstReference(points, PALETTE, NO_PINS) + }) + + it('exact palette colors, n^2 each (capacities exactly fit)', () => { + for (const n of [2, 3, 5, 7]) { + const points: RGB[] = [] + for (const c of PALETTE) + for (let i = 0; i < n * n; i++) points.push({ ...c }) + expect(tieCheck(points, PALETTE, NO_PINS)).toBe(true) + } + }) + + it('one color oversubscribed by identical points', () => { + const points: RGB[] = [] + for (const [ci, c] of PALETTE.entries()) + for (let i = 0; i < (ci === 0 ? 11 : 8); i++) points.push({ ...c }) + expect(tieCheck(points, PALETTE, NO_PINS)).toBe(true) + }) + + it('points equidistant from two centroids', () => { + const a = { r: 0, g: 0, b: 0 } + const b = { r: 100, g: 0, b: 0 } + const mid = { r: 50, g: 0, b: 0 } + const points = new Array(8).fill(0).map(() => ({ ...mid })) + points.push({ ...a }, { ...b }) + const centroids = [a, b, PALETTE[2], PALETTE[3], PALETTE[4], PALETTE[5]] + checkAgainstReference(points, centroids, NO_PINS) + }) + }) +}) + +describe('balancedAssign ties', () => { + // Coarse color grids create many equal-cost optima. The minimum, the + // capacities and the result for a repeated call must hold; which of + // several equal-cost assignments is returned is not specified (the + // reference picks another one in about two thirds of these cases). + it('keeps the minimum on coarse grids with many equal costs', () => { + const rand = lcg(99) + for (let t = 0; t < 120; t++) { + const levels = [2, 3, 4, 8][t % 4] + const q = () => Math.floor(rand() * levels) * 60 + const points = Array.from( + { length: 20 + Math.floor(rand() * 100) }, + () => ({ r: q(), g: q(), b: q() }), + ) + checkAgainstReference(points, PALETTE, NO_PINS) + } + }) + + it('returns the same assignment for the same input', () => { + const points = balancedPoints(4, 3, 40) + expect(balancedAssign(points, PALETTE, [])).toEqual( + balancedAssign(points, PALETTE, []), + ) + }) +}) + +describe('balancedAssign speed', () => { + // The reference needs about O(M^3) for M = 6 n^2 slots: seconds at n=12. + for (const n of [12, 15]) { + it(`n=${n} (${6 * n * n} points) takes under one second`, () => { + const points = balancedPoints(n, 5, 70) + const start = performance.now() + const assignment = balancedAssign( + points, + PALETTE, + NO_PINS as Array, + ) + const elapsed = performance.now() - start + expect(counts(assignment, 6).every((c) => c === n * n)).toBe(true) + expect(elapsed).toBeLessThan(1000) + }) + } + + // learnStickerColors runs balancedAssign about 40 times and searches + // farthest pairs: the whole learning pass at 15x15 took over a minute + // before the transportation solver. + it('learns the six colors of a 15x15 capture in under ten seconds', () => { + const n = 15 + const samples = balancedPoints(n, 6, 25).map((rgb) => ({ + rgb, + colorGuess: 'W', + })) + const start = performance.now() + const learned = learnStickerColors(samples, STICKER_COLORS, []) + const elapsed = performance.now() - start + expect(learned).not.toBeNull() + expect(Object.keys(learned!.colors).sort()).toEqual([...NAMES].sort()) + expect(elapsed).toBeLessThan(10_000) + }) +})