Migrate from .NET MAUI to Expo + Convex

Complete rewrite of Scry using TypeScript stack:

- Expo/React Native with Expo Router (file-based routing)
- Convex backend (serverless functions + real-time database)
- Adaptive camera system (expo-camera in Expo Go, Vision Camera in production)
- React Native Skia + fast-opencv for image processing
- GDPR-compliant auth setup with Zitadel OIDC (pending configuration)

Key features:
- Card recognition pipeline ported to TypeScript
- Perceptual hashing (192-bit color pHash)
- CLAHE preprocessing for lighting normalization
- Local SQLite cache with Convex sync
- Collection management with offline support

Removes all .NET/MAUI code (src/, test/, tools/).

💘 Generated with Crush

Assisted-by: Claude Opus 4.5 via Crush <crush@charm.land>
This commit is contained in:
Chris Kruining 2026-02-09 16:16:34 +01:00
parent 56499d5af9
commit 83ab4df537
No known key found for this signature in database
GPG key ID: EB894A3560CCCAD2
138 changed files with 19136 additions and 7681 deletions

View file

@ -0,0 +1,492 @@
/**
* Card detection using edge detection and contour analysis.
* Detects card boundaries in images and returns the four corner points.
*
* This is a pure TypeScript implementation for when OpenCV is not available.
*/
import {
Point,
toGrayscale,
gaussianBlur,
distance,
crossProduct,
pointToLineDistance,
} from "./imageUtils";
/** Standard MTG card aspect ratio (height / width). Cards are 63mm x 88mm. */
const CARD_ASPECT_RATIO = 88 / 63; // ~1.397
const ASPECT_RATIO_TOLERANCE = 0.25;
const MIN_CARD_AREA_RATIO = 0.05;
const MAX_CARD_AREA_RATIO = 0.98;
export interface CardDetectionResult {
found: boolean;
corners: Point[];
confidence: number;
debugMessage?: string;
}
/**
* Detect a card in the image and return its corner points.
*/
export function detectCard(
pixels: Uint8Array | Uint8ClampedArray,
width: number,
height: number
): CardDetectionResult {
// Step 1: Convert to grayscale
const grayscale = toGrayscale(pixels);
// Step 2: Apply Gaussian blur to reduce noise
const blurred = gaussianBlur(grayscale, width, height, 5);
// Step 3: Apply Canny edge detection
const edges = applyCannyEdgeDetection(blurred, width, height, 50, 150);
// Step 4: Find contours
const contours = findContours(edges, width, height);
if (contours.length === 0) {
return { found: false, corners: [], confidence: 0, debugMessage: "No contours found" };
}
// Step 5: Find the best card-like quadrilateral
const imageArea = width * height;
const bestQuad = findBestCardQuadrilateral(contours, imageArea);
if (!bestQuad) {
return { found: false, corners: [], confidence: 0, debugMessage: "No card-like quadrilateral found" };
}
// Step 6: Order corners consistently
const orderedCorners = orderCorners(bestQuad);
// Calculate confidence
const confidence = calculateConfidence(orderedCorners, imageArea);
return { found: true, corners: orderedCorners, confidence };
}
/**
* Apply Canny edge detection.
*/
function applyCannyEdgeDetection(
gray: Uint8Array,
width: number,
height: number,
lowThreshold: number,
highThreshold: number
): Uint8Array {
// Step 1: Compute gradients using Sobel operators
const gradientX = new Float32Array(width * height);
const gradientY = new Float32Array(width * height);
const magnitude = new Float32Array(width * height);
const direction = new Float32Array(width * height);
// Sobel kernels
const sobelX = [[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]];
const sobelY = [[-1, -2, -1], [0, 0, 0], [1, 2, 1]];
for (let y = 1; y < height - 1; y++) {
for (let x = 1; x < width - 1; x++) {
let gx = 0, gy = 0;
for (let ky = -1; ky <= 1; ky++) {
for (let kx = -1; kx <= 1; kx++) {
const pixel = gray[(y + ky) * width + (x + kx)];
gx += pixel * sobelX[ky + 1][kx + 1];
gy += pixel * sobelY[ky + 1][kx + 1];
}
}
const idx = y * width + x;
gradientX[idx] = gx;
gradientY[idx] = gy;
magnitude[idx] = Math.sqrt(gx * gx + gy * gy);
direction[idx] = Math.atan2(gy, gx);
}
}
// Step 2: Non-maximum suppression
const suppressed = new Float32Array(width * height);
for (let y = 1; y < height - 1; y++) {
for (let x = 1; x < width - 1; x++) {
const idx = y * width + x;
let angle = direction[idx] * 180 / Math.PI;
if (angle < 0) angle += 180;
let neighbor1: number, neighbor2: number;
if (angle < 22.5 || angle >= 157.5) {
neighbor1 = magnitude[y * width + (x - 1)];
neighbor2 = magnitude[y * width + (x + 1)];
} else if (angle >= 22.5 && angle < 67.5) {
neighbor1 = magnitude[(y - 1) * width + (x + 1)];
neighbor2 = magnitude[(y + 1) * width + (x - 1)];
} else if (angle >= 67.5 && angle < 112.5) {
neighbor1 = magnitude[(y - 1) * width + x];
neighbor2 = magnitude[(y + 1) * width + x];
} else {
neighbor1 = magnitude[(y - 1) * width + (x - 1)];
neighbor2 = magnitude[(y + 1) * width + (x + 1)];
}
if (magnitude[idx] >= neighbor1 && magnitude[idx] >= neighbor2) {
suppressed[idx] = magnitude[idx];
}
}
}
// Step 3: Double thresholding and edge tracking
const result = new Uint8Array(width * height);
const strong = new Uint8Array(width * height);
const weak = new Uint8Array(width * height);
for (let i = 0; i < width * height; i++) {
if (suppressed[i] >= highThreshold) {
strong[i] = 1;
} else if (suppressed[i] >= lowThreshold) {
weak[i] = 1;
}
}
// Edge tracking by hysteresis
for (let y = 1; y < height - 1; y++) {
for (let x = 1; x < width - 1; x++) {
const idx = y * width + x;
if (strong[idx]) {
result[idx] = 255;
} else if (weak[idx]) {
// Check if connected to strong edge
outer: for (let dy = -1; dy <= 1; dy++) {
for (let dx = -1; dx <= 1; dx++) {
if (strong[(y + dy) * width + (x + dx)]) {
result[idx] = 255;
break outer;
}
}
}
}
}
}
return result;
}
/**
* Find contours in a binary edge image using flood fill.
*/
function findContours(
edges: Uint8Array,
width: number,
height: number
): Point[][] {
const visited = new Uint8Array(width * height);
const contours: Point[][] = [];
const dx = [-1, 0, 1, 1, 1, 0, -1, -1];
const dy = [-1, -1, -1, 0, 1, 1, 1, 0];
for (let y = 1; y < height - 1; y++) {
for (let x = 1; x < width - 1; x++) {
const idx = y * width + x;
if (visited[idx]) continue;
if (edges[idx] < 128) continue;
// Trace contour using BFS
const contour: Point[] = [];
const queue: Array<{ x: number; y: number }> = [{ x, y }];
while (queue.length > 0 && contour.length < 10000) {
const point = queue.shift()!;
const pidx = point.y * width + point.x;
if (point.x < 0 || point.x >= width || point.y < 0 || point.y >= height) continue;
if (visited[pidx]) continue;
if (edges[pidx] < 128) continue;
visited[pidx] = 1;
contour.push(point);
for (let i = 0; i < 8; i++) {
queue.push({ x: point.x + dx[i], y: point.y + dy[i] });
}
}
if (contour.length >= 4) {
contours.push(contour);
}
}
}
return contours;
}
/**
* Find the best quadrilateral that matches a card shape.
*/
function findBestCardQuadrilateral(
contours: Point[][],
imageArea: number
): Point[] | null {
let bestQuad: Point[] | null = null;
let bestScore = -Infinity;
for (const contour of contours) {
// Simplify contour using Douglas-Peucker
const simplified = simplifyContour(contour, contour.length * 0.02);
// Try to approximate as quadrilateral
const quad = approximateQuadrilateral(simplified);
if (!quad) continue;
// Check if valid card shape
const area = calculateQuadArea(quad);
const areaRatio = area / imageArea;
if (areaRatio < MIN_CARD_AREA_RATIO || areaRatio > MAX_CARD_AREA_RATIO) {
continue;
}
// Check aspect ratio
const aspectScore = calculateAspectRatioScore(quad);
if (aspectScore < 0.5) continue;
// Check convexity
if (!isConvex(quad)) continue;
// Score based on area and aspect ratio
const score = areaRatio * aspectScore;
if (score > bestScore) {
bestScore = score;
bestQuad = quad;
}
}
return bestQuad;
}
/**
* Simplify a contour using Douglas-Peucker algorithm.
*/
function simplifyContour(contour: Point[], epsilon: number): Point[] {
if (contour.length < 3) return contour;
const first = contour[0];
const last = contour[contour.length - 1];
let maxDist = 0;
let maxIndex = 0;
for (let i = 1; i < contour.length - 1; i++) {
const dist = pointToLineDistance(contour[i], first, last);
if (dist > maxDist) {
maxDist = dist;
maxIndex = i;
}
}
if (maxDist > epsilon) {
const left = simplifyContour(contour.slice(0, maxIndex + 1), epsilon);
const right = simplifyContour(contour.slice(maxIndex), epsilon);
return [...left.slice(0, -1), ...right];
}
return [first, last];
}
/**
* Try to approximate a contour as a quadrilateral.
*/
function approximateQuadrilateral(contour: Point[]): Point[] | null {
if (contour.length < 4) return null;
if (contour.length === 4) return contour;
// Find convex hull
const hull = convexHull(contour);
if (hull.length < 4) return null;
if (hull.length === 4) return hull;
// Find 4 extreme points
return findExtremePoints(hull);
}
/**
* Calculate convex hull using Graham scan.
*/
function convexHull(points: Point[]): Point[] {
if (points.length < 3) return points;
// Find bottom-most point
const start = points.reduce((min, p) =>
p.y < min.y || (p.y === min.y && p.x < min.x) ? p : min
);
// Sort by polar angle
const sorted = points
.filter(p => p !== start)
.sort((a, b) => {
const angleA = Math.atan2(a.y - start.y, a.x - start.x);
const angleB = Math.atan2(b.y - start.y, b.x - start.x);
if (angleA !== angleB) return angleA - angleB;
return distance(a, start) - distance(b, start);
});
const hull: Point[] = [start];
for (const point of sorted) {
while (hull.length > 1 && crossProduct(hull[hull.length - 2], hull[hull.length - 1], point) <= 0) {
hull.pop();
}
hull.push(point);
}
return hull;
}
/**
* Find 4 extreme points of a convex hull.
*/
function findExtremePoints(hull: Point[]): Point[] {
if (hull.length <= 4) return hull;
const minX = hull.reduce((min, p) => p.x < min.x ? p : min, hull[0]);
const maxX = hull.reduce((max, p) => p.x > max.x ? p : max, hull[0]);
const minY = hull.reduce((min, p) => p.y < min.y ? p : min, hull[0]);
const maxY = hull.reduce((max, p) => p.y > max.y ? p : max, hull[0]);
const extremes = new Set([minX, maxX, minY, maxY]);
if (extremes.size === 4) {
return Array.from(extremes);
}
// Fallback: take points at regular angular intervals
const centerX = hull.reduce((s, p) => s + p.x, 0) / hull.length;
const centerY = hull.reduce((s, p) => s + p.y, 0) / hull.length;
const sorted = [...hull].sort((a, b) =>
Math.atan2(a.y - centerY, a.x - centerX) -
Math.atan2(b.y - centerY, b.x - centerX)
);
const step = Math.floor(sorted.length / 4);
return [sorted[0], sorted[step], sorted[step * 2], sorted[step * 3]];
}
/**
* Calculate area of a quadrilateral using shoelace formula.
*/
function calculateQuadArea(quad: Point[]): number {
let area = 0;
for (let i = 0; i < 4; i++) {
const j = (i + 1) % 4;
area += quad[i].x * quad[j].y;
area -= quad[j].x * quad[i].y;
}
return Math.abs(area) / 2;
}
/**
* Calculate how well the aspect ratio matches a card.
*/
function calculateAspectRatioScore(quad: Point[]): number {
const width1 = distance(quad[0], quad[1]);
const width2 = distance(quad[2], quad[3]);
const height1 = distance(quad[1], quad[2]);
const height2 = distance(quad[3], quad[0]);
const avgWidth = (width1 + width2) / 2;
const avgHeight = (height1 + height2) / 2;
// Ensure we get the right ratio regardless of orientation
const aspectRatio = avgWidth > avgHeight
? avgWidth / avgHeight
: avgHeight / avgWidth;
const deviation = Math.abs(aspectRatio - CARD_ASPECT_RATIO) / CARD_ASPECT_RATIO;
return Math.max(0, 1 - deviation / ASPECT_RATIO_TOLERANCE);
}
/**
* Check if a quadrilateral is convex.
*/
function isConvex(quad: Point[]): boolean {
let sign = 0;
for (let i = 0; i < 4; i++) {
const cross = crossProduct(
quad[i],
quad[(i + 1) % 4],
quad[(i + 2) % 4]
);
if (Math.abs(cross) < 0.0001) continue;
const currentSign = cross > 0 ? 1 : -1;
if (sign === 0) {
sign = currentSign;
} else if (sign !== currentSign) {
return false;
}
}
return true;
}
/**
* Order corners consistently: top-left, top-right, bottom-right, bottom-left.
*/
function orderCorners(corners: Point[]): Point[] {
const centerX = corners.reduce((s, c) => s + c.x, 0) / 4;
const centerY = corners.reduce((s, c) => s + c.y, 0) / 4;
const topLeft = corners.filter(c => c.x < centerX && c.y < centerY)
.sort((a, b) => (a.x + a.y) - (b.x + b.y))[0];
const topRight = corners.filter(c => c.x >= centerX && c.y < centerY)
.sort((a, b) => (a.y - a.x) - (b.y - b.x))[0];
const bottomRight = corners.filter(c => c.x >= centerX && c.y >= centerY)
.sort((a, b) => (b.x + b.y) - (a.x + a.y))[0];
const bottomLeft = corners.filter(c => c.x < centerX && c.y >= centerY)
.sort((a, b) => (b.y - b.x) - (a.y - a.x))[0];
// Handle edge cases by sorting by angle
if (!topLeft || !topRight || !bottomRight || !bottomLeft) {
const sorted = [...corners].sort((a, b) =>
Math.atan2(a.y - centerY, a.x - centerX) -
Math.atan2(b.y - centerY, b.x - centerX)
);
// Find index of point with minimum sum (top-left)
let minSumIdx = 0;
let minSum = Infinity;
sorted.forEach((c, i) => {
const sum = c.x + c.y;
if (sum < minSum) {
minSum = sum;
minSumIdx = i;
}
});
return [...sorted.slice(minSumIdx), ...sorted.slice(0, minSumIdx)];
}
return [topLeft, topRight, bottomRight, bottomLeft];
}
/**
* Calculate confidence of the detection.
*/
function calculateConfidence(corners: Point[], imageArea: number): number {
const area = calculateQuadArea(corners);
const areaScore = Math.min(area / imageArea / 0.5, 1);
const aspectScore = calculateAspectRatioScore(corners);
return areaScore * 0.4 + aspectScore * 0.6;
}

278
lib/recognition/clahe.ts Normal file
View file

@ -0,0 +1,278 @@
/**
* CLAHE (Contrast Limited Adaptive Histogram Equalization) implementation.
*
* This is a pure TypeScript implementation for when OpenCV is not available.
* For better performance, use OpenCV's createCLAHE when available.
*/
interface CLAHEOptions {
clipLimit?: number;
tileGridSize?: { width: number; height: number };
}
const DEFAULT_OPTIONS: Required<CLAHEOptions> = {
clipLimit: 2.0,
tileGridSize: { width: 8, height: 8 },
};
/**
* Convert RGB to LAB color space.
* Returns L channel (0-100 range, scaled to 0-255 for processing).
*/
function rgbToLab(r: number, g: number, b: number): { l: number; a: number; b: number } {
// Normalize RGB to 0-1
let rNorm = r / 255;
let gNorm = g / 255;
let bNorm = b / 255;
// Apply gamma correction
rNorm = rNorm > 0.04045 ? Math.pow((rNorm + 0.055) / 1.055, 2.4) : rNorm / 12.92;
gNorm = gNorm > 0.04045 ? Math.pow((gNorm + 0.055) / 1.055, 2.4) : gNorm / 12.92;
bNorm = bNorm > 0.04045 ? Math.pow((bNorm + 0.055) / 1.055, 2.4) : bNorm / 12.92;
// Convert to XYZ
const x = (rNorm * 0.4124564 + gNorm * 0.3575761 + bNorm * 0.1804375) / 0.95047;
const y = rNorm * 0.2126729 + gNorm * 0.7151522 + bNorm * 0.0721750;
const z = (rNorm * 0.0193339 + gNorm * 0.1191920 + bNorm * 0.9503041) / 1.08883;
// Convert to LAB
const xNorm = x > 0.008856 ? Math.pow(x, 1 / 3) : 7.787 * x + 16 / 116;
const yNorm = y > 0.008856 ? Math.pow(y, 1 / 3) : 7.787 * y + 16 / 116;
const zNorm = z > 0.008856 ? Math.pow(z, 1 / 3) : 7.787 * z + 16 / 116;
return {
l: Math.max(0, 116 * yNorm - 16), // 0-100
a: 500 * (xNorm - yNorm),
b: 200 * (yNorm - zNorm),
};
}
/**
* Convert LAB to RGB color space.
*/
function labToRgb(l: number, a: number, b: number): { r: number; g: number; b: number } {
// Convert LAB to XYZ
const yNorm = (l + 16) / 116;
const xNorm = a / 500 + yNorm;
const zNorm = yNorm - b / 200;
const x3 = Math.pow(xNorm, 3);
const y3 = Math.pow(yNorm, 3);
const z3 = Math.pow(zNorm, 3);
const x = (x3 > 0.008856 ? x3 : (xNorm - 16 / 116) / 7.787) * 0.95047;
const y = y3 > 0.008856 ? y3 : (yNorm - 16 / 116) / 7.787;
const z = (z3 > 0.008856 ? z3 : (zNorm - 16 / 116) / 7.787) * 1.08883;
// Convert XYZ to RGB
let rNorm = x * 3.2404542 + y * -1.5371385 + z * -0.4985314;
let gNorm = x * -0.9692660 + y * 1.8760108 + z * 0.0415560;
let bNorm = x * 0.0556434 + y * -0.2040259 + z * 1.0572252;
// Apply gamma correction
rNorm = rNorm > 0.0031308 ? 1.055 * Math.pow(rNorm, 1 / 2.4) - 0.055 : 12.92 * rNorm;
gNorm = gNorm > 0.0031308 ? 1.055 * Math.pow(gNorm, 1 / 2.4) - 0.055 : 12.92 * gNorm;
bNorm = bNorm > 0.0031308 ? 1.055 * Math.pow(bNorm, 1 / 2.4) - 0.055 : 12.92 * bNorm;
return {
r: Math.round(Math.max(0, Math.min(255, rNorm * 255))),
g: Math.round(Math.max(0, Math.min(255, gNorm * 255))),
b: Math.round(Math.max(0, Math.min(255, bNorm * 255))),
};
}
/**
* Compute histogram for a region.
*/
function computeHistogram(
data: Uint8Array,
startX: number,
startY: number,
width: number,
height: number,
stride: number
): number[] {
const histogram = new Array(256).fill(0);
for (let y = startY; y < startY + height; y++) {
for (let x = startX; x < startX + width; x++) {
const value = data[y * stride + x];
histogram[value]++;
}
}
return histogram;
}
/**
* Clip histogram and redistribute excess.
*/
function clipHistogram(histogram: number[], clipLimit: number, numPixels: number): number[] {
const limit = Math.floor(clipLimit * numPixels / 256);
const clipped = [...histogram];
let excess = 0;
for (let i = 0; i < 256; i++) {
if (clipped[i] > limit) {
excess += clipped[i] - limit;
clipped[i] = limit;
}
}
// Redistribute excess
const increment = Math.floor(excess / 256);
const remainder = excess % 256;
for (let i = 0; i < 256; i++) {
clipped[i] += increment;
if (i < remainder) {
clipped[i]++;
}
}
return clipped;
}
/**
* Create lookup table from clipped histogram.
*/
function createLUT(histogram: number[], numPixels: number): Uint8Array {
const lut = new Uint8Array(256);
let cumSum = 0;
for (let i = 0; i < 256; i++) {
cumSum += histogram[i];
lut[i] = Math.round((cumSum / numPixels) * 255);
}
return lut;
}
/**
* Bilinear interpolation between four values.
*/
function bilinearInterpolate(
topLeft: number,
topRight: number,
bottomLeft: number,
bottomRight: number,
xFrac: number,
yFrac: number
): number {
const top = topLeft + (topRight - topLeft) * xFrac;
const bottom = bottomLeft + (bottomRight - bottomLeft) * xFrac;
return Math.round(top + (bottom - top) * yFrac);
}
/**
* Apply CLAHE to RGBA pixel data.
* Processes only the L channel in LAB color space.
*
* @param pixels RGBA pixel data
* @param width Image width
* @param height Image height
* @param options CLAHE options
* @returns Processed RGBA pixel data
*/
export function applyCLAHE(
pixels: Uint8Array | Uint8ClampedArray,
width: number,
height: number,
options: CLAHEOptions = {}
): Uint8Array {
const opts = { ...DEFAULT_OPTIONS, ...options };
const { clipLimit, tileGridSize } = opts;
const tileWidth = Math.floor(width / tileGridSize.width);
const tileHeight = Math.floor(height / tileGridSize.height);
// Extract L channel from LAB
const lChannel = new Uint8Array(width * height);
const aChannel = new Float32Array(width * height);
const bChannel = new Float32Array(width * height);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
const idx = (y * width + x) * 4;
const lab = rgbToLab(pixels[idx], pixels[idx + 1], pixels[idx + 2]);
const pixelIdx = y * width + x;
lChannel[pixelIdx] = Math.round((lab.l / 100) * 255);
aChannel[pixelIdx] = lab.a;
bChannel[pixelIdx] = lab.b;
}
}
// Compute LUTs for each tile
const luts: Uint8Array[][] = [];
const numPixelsPerTile = tileWidth * tileHeight;
for (let ty = 0; ty < tileGridSize.height; ty++) {
luts[ty] = [];
for (let tx = 0; tx < tileGridSize.width; tx++) {
const startX = tx * tileWidth;
const startY = ty * tileHeight;
const histogram = computeHistogram(
lChannel,
startX,
startY,
tileWidth,
tileHeight,
width
);
const clipped = clipHistogram(histogram, clipLimit, numPixelsPerTile);
luts[ty][tx] = createLUT(clipped, numPixelsPerTile);
}
}
// Apply CLAHE with bilinear interpolation
const result = new Uint8Array(pixels.length);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
const pixelIdx = y * width + x;
const rgbaIdx = pixelIdx * 4;
// Determine tile position
const tileX = Math.min(x / tileWidth, tileGridSize.width - 1);
const tileY = Math.min(y / tileHeight, tileGridSize.height - 1);
const tx = Math.floor(tileX);
const ty = Math.floor(tileY);
const xFrac = tileX - tx;
const yFrac = tileY - ty;
// Get surrounding tiles (handle edges)
const tx1 = Math.min(tx + 1, tileGridSize.width - 1);
const ty1 = Math.min(ty + 1, tileGridSize.height - 1);
const lValue = lChannel[pixelIdx];
// Interpolate LUT values
const newL = bilinearInterpolate(
luts[ty][tx][lValue],
luts[ty][tx1][lValue],
luts[ty1][tx][lValue],
luts[ty1][tx1][lValue],
xFrac,
yFrac
);
// Convert back to RGB
const rgb = labToRgb(
(newL / 255) * 100,
aChannel[pixelIdx],
bChannel[pixelIdx]
);
result[rgbaIdx] = rgb.r;
result[rgbaIdx + 1] = rgb.g;
result[rgbaIdx + 2] = rgb.b;
result[rgbaIdx + 3] = pixels[rgbaIdx + 3]; // Preserve alpha
}
}
return result;
}

View file

@ -0,0 +1,95 @@
/**
* Image loader utilities for React Native.
* Loads images from file paths and returns pixel data for recognition.
*/
import * as ImageManipulator from "expo-image-manipulator";
import { Platform } from "react-native";
export interface LoadedImage {
pixels: Uint8Array;
width: number;
height: number;
}
/**
* Load image from a file path and return RGBA pixel data.
* Uses expo-image-manipulator to resize the image to a manageable size first.
*
* Note: expo-image-manipulator doesn't directly provide pixel data.
* We need to use a canvas (web) or Skia (native) to decode pixels.
* For now, this provides the resized image URI for the Skia-based decoder.
*/
export async function loadImageForRecognition(
uri: string,
targetWidth: number = 480, // Reasonable size for processing
targetHeight: number = 640
): Promise<{ uri: string; width: number; height: number }> {
// Normalize the URI for the platform
const normalizedUri =
Platform.OS === "android" && !uri.startsWith("file://")
? `file://${uri}`
: uri;
// Resize the image for faster processing
const result = await ImageManipulator.manipulateAsync(
normalizedUri,
[
{
resize: {
width: targetWidth,
height: targetHeight,
},
},
],
{
compress: 1,
format: ImageManipulator.SaveFormat.PNG,
}
);
return {
uri: result.uri,
width: result.width,
height: result.height,
};
}
/**
* Load image and get base64 data.
* Useful for passing to native modules or Skia.
*/
export async function loadImageAsBase64(
uri: string,
targetWidth: number = 480,
targetHeight: number = 640
): Promise<{ base64: string; width: number; height: number }> {
// Normalize the URI
const normalizedUri =
Platform.OS === "android" && !uri.startsWith("file://")
? `file://${uri}`
: uri;
const result = await ImageManipulator.manipulateAsync(
normalizedUri,
[
{
resize: {
width: targetWidth,
height: targetHeight,
},
},
],
{
compress: 1,
format: ImageManipulator.SaveFormat.PNG,
base64: true,
}
);
return {
base64: result.base64 || "",
width: result.width,
height: result.height,
};
}

View file

@ -0,0 +1,292 @@
/**
* Image utility functions for the recognition pipeline.
* Provides resize, rotation, and pixel manipulation helpers.
*/
export interface Point {
x: number;
y: number;
}
export interface Size {
width: number;
height: number;
}
/**
* Resize RGBA pixel data to target dimensions using bilinear interpolation.
*/
export function resizeImage(
pixels: Uint8Array | Uint8ClampedArray,
srcWidth: number,
srcHeight: number,
dstWidth: number,
dstHeight: number
): Uint8Array {
const result = new Uint8Array(dstWidth * dstHeight * 4);
const xRatio = srcWidth / dstWidth;
const yRatio = srcHeight / dstHeight;
for (let y = 0; y < dstHeight; y++) {
for (let x = 0; x < dstWidth; x++) {
const srcX = x * xRatio;
const srcY = y * yRatio;
const x0 = Math.floor(srcX);
const y0 = Math.floor(srcY);
const x1 = Math.min(x0 + 1, srcWidth - 1);
const y1 = Math.min(y0 + 1, srcHeight - 1);
const xFrac = srcX - x0;
const yFrac = srcY - y0;
const dstIdx = (y * dstWidth + x) * 4;
for (let c = 0; c < 4; c++) {
const idx00 = (y0 * srcWidth + x0) * 4 + c;
const idx10 = (y0 * srcWidth + x1) * 4 + c;
const idx01 = (y1 * srcWidth + x0) * 4 + c;
const idx11 = (y1 * srcWidth + x1) * 4 + c;
const top = pixels[idx00] + (pixels[idx10] - pixels[idx00]) * xFrac;
const bottom = pixels[idx01] + (pixels[idx11] - pixels[idx01]) * xFrac;
result[dstIdx + c] = Math.round(top + (bottom - top) * yFrac);
}
}
}
return result;
}
/**
* Rotate RGBA pixel data by 90, 180, or 270 degrees.
*/
export function rotateImage(
pixels: Uint8Array | Uint8ClampedArray,
width: number,
height: number,
degrees: 0 | 90 | 180 | 270
): { pixels: Uint8Array; width: number; height: number } {
if (degrees === 0) {
return { pixels: new Uint8Array(pixels), width, height };
}
const [newWidth, newHeight] = degrees === 90 || degrees === 270
? [height, width]
: [width, height];
const result = new Uint8Array(newWidth * newHeight * 4);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
const srcIdx = (y * width + x) * 4;
let dstX: number, dstY: number;
switch (degrees) {
case 90:
dstX = height - 1 - y;
dstY = x;
break;
case 180:
dstX = width - 1 - x;
dstY = height - 1 - y;
break;
case 270:
dstX = y;
dstY = width - 1 - x;
break;
default:
dstX = x;
dstY = y;
}
const dstIdx = (dstY * newWidth + dstX) * 4;
result[dstIdx] = pixels[srcIdx];
result[dstIdx + 1] = pixels[srcIdx + 1];
result[dstIdx + 2] = pixels[srcIdx + 2];
result[dstIdx + 3] = pixels[srcIdx + 3];
}
}
return { pixels: result, width: newWidth, height: newHeight };
}
/**
* Convert RGBA to grayscale using luminance formula.
*/
export function toGrayscale(pixels: Uint8Array | Uint8ClampedArray): Uint8Array {
const numPixels = pixels.length / 4;
const gray = new Uint8Array(numPixels);
for (let i = 0; i < numPixels; i++) {
const idx = i * 4;
gray[i] = Math.round(
0.299 * pixels[idx] +
0.587 * pixels[idx + 1] +
0.114 * pixels[idx + 2]
);
}
return gray;
}
/**
* Convert grayscale back to RGBA.
*/
export function grayscaleToRgba(
gray: Uint8Array,
width: number,
height: number
): Uint8Array {
const rgba = new Uint8Array(width * height * 4);
for (let i = 0; i < gray.length; i++) {
const idx = i * 4;
rgba[idx] = gray[i];
rgba[idx + 1] = gray[i];
rgba[idx + 2] = gray[i];
rgba[idx + 3] = 255;
}
return rgba;
}
/**
* Apply Gaussian blur to grayscale image.
* Uses separable 1D convolution for efficiency.
*/
export function gaussianBlur(
gray: Uint8Array,
width: number,
height: number,
radius: number = 5
): Uint8Array {
// Generate Gaussian kernel
const sigma = radius / 2;
const size = radius * 2 + 1;
const kernel = new Float32Array(size);
let sum = 0;
for (let i = 0; i < size; i++) {
const x = i - radius;
kernel[i] = Math.exp(-(x * x) / (2 * sigma * sigma));
sum += kernel[i];
}
// Normalize
for (let i = 0; i < size; i++) {
kernel[i] /= sum;
}
// Horizontal pass
const temp = new Uint8Array(width * height);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
let value = 0;
for (let k = -radius; k <= radius; k++) {
const sx = Math.max(0, Math.min(width - 1, x + k));
value += gray[y * width + sx] * kernel[k + radius];
}
temp[y * width + x] = Math.round(value);
}
}
// Vertical pass
const result = new Uint8Array(width * height);
for (let y = 0; y < height; y++) {
for (let x = 0; x < width; x++) {
let value = 0;
for (let k = -radius; k <= radius; k++) {
const sy = Math.max(0, Math.min(height - 1, y + k));
value += temp[sy * width + x] * kernel[k + radius];
}
result[y * width + x] = Math.round(value);
}
}
return result;
}
/**
* Calculate distance between two points.
*/
export function distance(a: Point, b: Point): number {
const dx = b.x - a.x;
const dy = b.y - a.y;
return Math.sqrt(dx * dx + dy * dy);
}
/**
* Calculate cross product of vectors (b-a) and (c-b).
*/
export function crossProduct(a: Point, b: Point, c: Point): number {
return (b.x - a.x) * (c.y - a.y) - (b.y - a.y) * (c.x - a.x);
}
/**
* Calculate point-to-line distance.
*/
export function pointToLineDistance(point: Point, lineStart: Point, lineEnd: Point): number {
const dx = lineEnd.x - lineStart.x;
const dy = lineEnd.y - lineStart.y;
const lengthSquared = dx * dx + dy * dy;
if (lengthSquared === 0) {
return distance(point, lineStart);
}
let t = ((point.x - lineStart.x) * dx + (point.y - lineStart.y) * dy) / lengthSquared;
t = Math.max(0, Math.min(1, t));
const projection = {
x: lineStart.x + t * dx,
y: lineStart.y + t * dy,
};
return distance(point, projection);
}
/**
* Sample a pixel from RGBA data using bilinear interpolation.
*/
export function sampleBilinear(
pixels: Uint8Array | Uint8ClampedArray,
width: number,
height: number,
x: number,
y: number
): [number, number, number, number] {
// Clamp to valid range
x = Math.max(0, Math.min(width - 1, x));
y = Math.max(0, Math.min(height - 1, y));
const x0 = Math.floor(x);
const y0 = Math.floor(y);
const x1 = Math.min(x0 + 1, width - 1);
const y1 = Math.min(y0 + 1, height - 1);
const xFrac = x - x0;
const yFrac = y - y0;
const idx00 = (y0 * width + x0) * 4;
const idx10 = (y0 * width + x1) * 4;
const idx01 = (y1 * width + x0) * 4;
const idx11 = (y1 * width + x1) * 4;
const result: [number, number, number, number] = [0, 0, 0, 0];
for (let c = 0; c < 4; c++) {
const c00 = pixels[idx00 + c];
const c10 = pixels[idx10 + c];
const c01 = pixels[idx01 + c];
const c11 = pixels[idx11 + c];
const top = c00 + (c10 - c00) * xFrac;
const bottom = c01 + (c11 - c01) * xFrac;
result[c] = Math.round(top + (bottom - top) * yFrac);
}
return result;
}

67
lib/recognition/index.ts Normal file
View file

@ -0,0 +1,67 @@
/**
* Card recognition module exports.
*/
// Recognition service
export {
recognizeCard,
computeImageHash,
calculateConfidence,
createRecognitionService,
type RecognitionOptions,
type RecognitionResult,
type CardMatch,
type CardHashEntry,
} from "./recognitionService";
// Card detection
export {
detectCard,
type CardDetectionResult,
} from "./cardDetector";
// Perspective correction
export {
warpPerspective,
OUTPUT_WIDTH,
OUTPUT_HEIGHT,
} from "./perspectiveCorrection";
// CLAHE preprocessing
export { applyCLAHE } from "./clahe";
// Perceptual hashing
export {
computeColorHash,
hammingDistance,
hashToHex,
hexToHash,
HASH_VERSION,
MATCH_THRESHOLD,
HASH_BITS,
} from "./perceptualHash";
// Image utilities
export {
resizeImage,
rotateImage,
toGrayscale,
grayscaleToRgba,
gaussianBlur,
distance,
type Point,
type Size,
} from "./imageUtils";
// Image loading
export {
loadImageForRecognition,
loadImageAsBase64,
} from "./imageLoader";
// Skia image decoding
export {
decodeImageBase64,
decodeImageFromUri,
useDecodedImage,
} from "./skiaDecoder";

View file

@ -0,0 +1,211 @@
/**
* Perceptual hashing implementation using DCT (Discrete Cosine Transform).
* Computes a 192-bit (24 byte) color hash from an image.
*
* The hash is computed by:
* 1. Resizing to 32x32
* 2. For each RGB channel:
* - Apply 2D DCT
* - Extract 8x8 low-frequency coefficients (skip DC)
* - Compare each to median -> 63 bits per channel
* 3. Concatenate R, G, B hashes -> 24 bytes (192 bits)
*/
const DCT_SIZE = 32;
const HASH_SIZE = 8;
const BITS_PER_CHANNEL = 63; // 8x8 - 1 (skip DC)
/**
* Precomputed cosine values for DCT.
*/
const cosineCache: number[][] = [];
function initCosineCache(): void {
if (cosineCache.length > 0) return;
for (let i = 0; i < DCT_SIZE; i++) {
cosineCache[i] = [];
for (let j = 0; j < DCT_SIZE; j++) {
cosineCache[i][j] = Math.cos((Math.PI / DCT_SIZE) * (j + 0.5) * i);
}
}
}
/**
* Apply 2D DCT to a matrix.
*/
function applyDCT2D(matrix: number[][]): number[][] {
initCosineCache();
const result: number[][] = [];
for (let u = 0; u < DCT_SIZE; u++) {
result[u] = [];
for (let v = 0; v < DCT_SIZE; v++) {
let sum = 0;
for (let i = 0; i < DCT_SIZE; i++) {
for (let j = 0; j < DCT_SIZE; j++) {
sum += matrix[i][j] * cosineCache[u][i] * cosineCache[v][j];
}
}
const cu = u === 0 ? 1 / Math.sqrt(2) : 1;
const cv = v === 0 ? 1 / Math.sqrt(2) : 1;
result[u][v] = (cu * cv * sum) / 4;
}
}
return result;
}
/**
* Get the median of an array of numbers.
*/
function getMedian(values: number[]): number {
const sorted = [...values].sort((a, b) => a - b);
const mid = Math.floor(sorted.length / 2);
return sorted.length % 2 !== 0
? sorted[mid]
: (sorted[mid - 1] + sorted[mid]) / 2;
}
/**
* Convert a BigInt to a Uint8Array of specified length.
*/
function bigintToBytes(value: bigint, length: number): Uint8Array {
const bytes = new Uint8Array(length);
for (let i = 0; i < length; i++) {
bytes[i] = Number((value >> BigInt(i * 8)) & 0xFFn);
}
return bytes;
}
/**
* Compute hash for a single color channel.
*/
function computeChannelHash(channel: number[][]): Uint8Array {
const dct = applyDCT2D(channel);
// Extract 8x8 low-frequency coefficients, skip DC (0,0)
const lowFreq: number[] = [];
for (let i = 0; i < HASH_SIZE; i++) {
for (let j = 0; j < HASH_SIZE; j++) {
if (i === 0 && j === 0) continue; // Skip DC component
lowFreq.push(dct[i][j]);
}
}
const median = getMedian(lowFreq);
// Generate 63-bit hash
let bits = 0n;
for (let i = 0; i < lowFreq.length; i++) {
if (lowFreq[i] > median) {
bits |= 1n << BigInt(i);
}
}
return bigintToBytes(bits, 8);
}
/**
* Extract a color channel from RGBA pixel data.
* @param pixels RGBA pixel data (width * height * 4)
* @param width Image width
* @param height Image height
* @param channel 0=R, 1=G, 2=B
*/
function extractChannel(
pixels: Uint8Array | Uint8ClampedArray,
width: number,
height: number,
channel: 0 | 1 | 2
): number[][] {
const matrix: number[][] = [];
for (let y = 0; y < height; y++) {
matrix[y] = [];
for (let x = 0; x < width; x++) {
const idx = (y * width + x) * 4;
matrix[y][x] = pixels[idx + channel];
}
}
return matrix;
}
/**
* Compute a 192-bit perceptual color hash from RGBA pixel data.
* The image should already be resized to 32x32.
*
* @param pixels RGBA pixel data (32 * 32 * 4 = 4096 bytes)
* @returns 24-byte hash (8 bytes per RGB channel)
*/
export function computeColorHash(pixels: Uint8Array | Uint8ClampedArray): Uint8Array {
if (pixels.length !== DCT_SIZE * DCT_SIZE * 4) {
throw new Error(`Expected ${DCT_SIZE * DCT_SIZE * 4} bytes, got ${pixels.length}`);
}
const rChannel = extractChannel(pixels, DCT_SIZE, DCT_SIZE, 0);
const gChannel = extractChannel(pixels, DCT_SIZE, DCT_SIZE, 1);
const bChannel = extractChannel(pixels, DCT_SIZE, DCT_SIZE, 2);
const rHash = computeChannelHash(rChannel);
const gHash = computeChannelHash(gChannel);
const bHash = computeChannelHash(bChannel);
// Combine all channels
const combined = new Uint8Array(24);
combined.set(rHash, 0);
combined.set(gHash, 8);
combined.set(bHash, 16);
return combined;
}
/**
* Compute Hamming distance between two hashes.
* Lower distance = more similar.
*/
export function hammingDistance(a: Uint8Array, b: Uint8Array): number {
if (a.length !== b.length) {
throw new Error(`Hash length mismatch: ${a.length} vs ${b.length}`);
}
let distance = 0;
for (let i = 0; i < a.length; i++) {
let xor = a[i] ^ b[i];
while (xor) {
distance += xor & 1;
xor >>>= 1;
}
}
return distance;
}
/**
* Convert hash to hex string for display/storage.
*/
export function hashToHex(hash: Uint8Array): string {
return Array.from(hash)
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
}
/**
* Convert hex string back to hash.
*/
export function hexToHash(hex: string): Uint8Array {
const bytes = new Uint8Array(hex.length / 2);
for (let i = 0; i < bytes.length; i++) {
bytes[i] = parseInt(hex.substr(i * 2, 2), 16);
}
return bytes;
}
// Hash algorithm version for migrations
export const HASH_VERSION = 1;
// Matching thresholds
export const MATCH_THRESHOLD = 25; // Max Hamming distance for a match
export const HASH_BITS = 192; // Total bits in hash

View file

@ -0,0 +1,207 @@
/**
* Perspective correction to transform a quadrilateral region into a rectangle.
* Uses homography transformation with bilinear sampling.
*/
import { Point, sampleBilinear } from "./imageUtils";
/** Standard output size for corrected card images (63:88 aspect ratio). */
export const OUTPUT_WIDTH = 480;
export const OUTPUT_HEIGHT = 670;
/**
* Apply perspective correction to extract and normalize a card from an image.
*
* @param pixels - Source RGBA pixel data
* @param width - Source image width
* @param height - Source image height
* @param corners - Four corners in order: top-left, top-right, bottom-right, bottom-left
* @param outputWidth - Width of output image
* @param outputHeight - Height of output image
* @returns Perspective-corrected RGBA pixel data
*/
export function warpPerspective(
pixels: Uint8Array | Uint8ClampedArray,
width: number,
height: number,
corners: Point[],
outputWidth: number = OUTPUT_WIDTH,
outputHeight: number = OUTPUT_HEIGHT
): { pixels: Uint8Array; width: number; height: number } {
if (corners.length !== 4) {
throw new Error("Exactly 4 corners required");
}
// Determine if card is landscape (rotated 90°)
const width1 = distance(corners[0], corners[1]);
const height1 = distance(corners[1], corners[2]);
let orderedCorners: Point[];
if (width1 > height1) {
// Card is landscape - rotate corners to portrait
orderedCorners = [corners[1], corners[2], corners[3], corners[0]];
} else {
orderedCorners = corners;
}
// Compute the perspective transform matrix
const matrix = computePerspectiveTransform(orderedCorners, outputWidth, outputHeight);
const inverse = invertMatrix3x3(matrix);
// Apply the transform
const result = new Uint8Array(outputWidth * outputHeight * 4);
for (let y = 0; y < outputHeight; y++) {
for (let x = 0; x < outputWidth; x++) {
// Apply inverse transform to find source coordinates
const srcPoint = applyTransform(inverse, x, y);
// Bilinear interpolation for smooth sampling
const [r, g, b, a] = sampleBilinear(pixels, width, height, srcPoint.x, srcPoint.y);
const idx = (y * outputWidth + x) * 4;
result[idx] = r;
result[idx + 1] = g;
result[idx + 2] = b;
result[idx + 3] = a;
}
}
return { pixels: result, width: outputWidth, height: outputHeight };
}
/**
* Compute a perspective transform matrix from quad corners to rectangle.
* Uses the Direct Linear Transform (DLT) algorithm.
*/
function computePerspectiveTransform(
src: Point[],
dstWidth: number,
dstHeight: number
): number[] {
// Destination corners (rectangle)
const dst: Point[] = [
{ x: 0, y: 0 },
{ x: dstWidth - 1, y: 0 },
{ x: dstWidth - 1, y: dstHeight - 1 },
{ x: 0, y: dstHeight - 1 },
];
// Build the 8x8 matrix for solving the homography
const A: number[][] = [];
const b: number[] = [];
for (let i = 0; i < 4; i++) {
const sx = src[i].x;
const sy = src[i].y;
const dx = dst[i].x;
const dy = dst[i].y;
A.push([sx, sy, 1, 0, 0, 0, -dx * sx, -dx * sy]);
b.push(dx);
A.push([0, 0, 0, sx, sy, 1, -dy * sx, -dy * sy]);
b.push(dy);
}
// Solve using Gaussian elimination
const h = solveLinearSystem(A, b);
// Return 3x3 matrix as flat array [h11, h12, h13, h21, h22, h23, h31, h32, h33]
return [h[0], h[1], h[2], h[3], h[4], h[5], h[6], h[7], 1];
}
/**
* Solve a linear system Ax = b using Gaussian elimination with partial pivoting.
*/
function solveLinearSystem(A: number[][], b: number[]): number[] {
const n = b.length;
// Create augmented matrix
const augmented: number[][] = A.map((row, i) => [...row, b[i]]);
// Forward elimination with partial pivoting
for (let col = 0; col < n; col++) {
// Find pivot
let maxRow = col;
for (let row = col + 1; row < n; row++) {
if (Math.abs(augmented[row][col]) > Math.abs(augmented[maxRow][col])) {
maxRow = row;
}
}
// Swap rows
[augmented[col], augmented[maxRow]] = [augmented[maxRow], augmented[col]];
// Eliminate
for (let row = col + 1; row < n; row++) {
if (Math.abs(augmented[col][col]) < 1e-10) continue;
const factor = augmented[row][col] / augmented[col][col];
for (let j = col; j <= n; j++) {
augmented[row][j] -= factor * augmented[col][j];
}
}
}
// Back substitution
const x = new Array(n).fill(0);
for (let i = n - 1; i >= 0; i--) {
x[i] = augmented[i][n];
for (let j = i + 1; j < n; j++) {
x[i] -= augmented[i][j] * x[j];
}
if (Math.abs(augmented[i][i]) > 1e-10) {
x[i] /= augmented[i][i];
}
}
return x;
}
/**
* Apply a 3x3 transform matrix to a point.
*/
function applyTransform(H: number[], x: number, y: number): Point {
let w = H[6] * x + H[7] * y + H[8];
if (Math.abs(w) < 1e-10) w = 1e-10;
return {
x: (H[0] * x + H[1] * y + H[2]) / w,
y: (H[3] * x + H[4] * y + H[5]) / w,
};
}
/**
* Invert a 3x3 matrix.
*/
function invertMatrix3x3(m: number[]): number[] {
const det =
m[0] * (m[4] * m[8] - m[5] * m[7]) -
m[1] * (m[3] * m[8] - m[5] * m[6]) +
m[2] * (m[3] * m[7] - m[4] * m[6]);
const invDet = Math.abs(det) < 1e-10 ? 1e10 : 1 / det;
return [
(m[4] * m[8] - m[5] * m[7]) * invDet,
(m[2] * m[7] - m[1] * m[8]) * invDet,
(m[1] * m[5] - m[2] * m[4]) * invDet,
(m[5] * m[6] - m[3] * m[8]) * invDet,
(m[0] * m[8] - m[2] * m[6]) * invDet,
(m[2] * m[3] - m[0] * m[5]) * invDet,
(m[3] * m[7] - m[4] * m[6]) * invDet,
(m[1] * m[6] - m[0] * m[7]) * invDet,
(m[0] * m[4] - m[1] * m[3]) * invDet,
];
}
/**
* Calculate distance between two points.
*/
function distance(a: Point, b: Point): number {
const dx = b.x - a.x;
const dy = b.y - a.y;
return Math.sqrt(dx * dx + dy * dy);
}

View file

@ -0,0 +1,313 @@
/**
* Card recognition service that orchestrates the full pipeline.
*
* Pipeline:
* 1. Card detection (optional) - find card boundaries
* 2. Perspective correction - warp to rectangle
* 3. CLAHE preprocessing - normalize lighting
* 4. Resize to 32x32
* 5. Compute perceptual hash
* 6. Match against database via Hamming distance
*/
import { detectCard, CardDetectionResult } from "./cardDetector";
import { warpPerspective } from "./perspectiveCorrection";
import { applyCLAHE } from "./clahe";
import {
computeColorHash,
hammingDistance,
MATCH_THRESHOLD,
HASH_BITS,
} from "./perceptualHash";
import { resizeImage, rotateImage } from "./imageUtils";
export interface RecognitionOptions {
/** Enable card detection and perspective correction. */
enableCardDetection?: boolean;
/** Enable rotation matching (try 0°, 90°, 180°, 270°). */
enableRotationMatching?: boolean;
/** Minimum confidence to accept a match (0-1). */
minConfidence?: number;
/** Maximum Hamming distance to accept a match. */
matchThreshold?: number;
}
export interface CardMatch {
/** Card ID from database. */
cardId: string;
/** Match confidence (0-1). */
confidence: number;
/** Hamming distance between hashes. */
distance: number;
/** Rotation used for match (0, 90, 180, or 270). */
rotation: number;
}
export interface RecognitionResult {
success: boolean;
match?: CardMatch;
cardDetection?: CardDetectionResult;
error?: string;
processingTimeMs: number;
}
export interface CardHashEntry {
id: string;
name: string;
setCode: string;
collectorNumber?: string;
imageUri?: string;
hash: Uint8Array;
}
const DEFAULT_OPTIONS: Required<RecognitionOptions> = {
enableCardDetection: true,
enableRotationMatching: true,
minConfidence: 0.85,
matchThreshold: MATCH_THRESHOLD,
};
/**
* Calculate confidence from Hamming distance.
*/
export function calculateConfidence(distance: number, totalBits: number = HASH_BITS): number {
return 1 - distance / totalBits;
}
/**
* Recognize a card from RGBA pixel data.
*
* @param pixels - RGBA pixel data
* @param width - Image width
* @param height - Image height
* @param cardHashes - Array of card hashes to match against
* @param options - Recognition options
* @returns Recognition result with best match
*/
export function recognizeCard(
pixels: Uint8Array | Uint8ClampedArray,
width: number,
height: number,
cardHashes: CardHashEntry[],
options: RecognitionOptions = {}
): RecognitionResult {
const startTime = performance.now();
const opts = { ...DEFAULT_OPTIONS, ...options };
try {
if (cardHashes.length === 0) {
return {
success: false,
error: "No cards in database",
processingTimeMs: performance.now() - startTime,
};
}
let cardPixels = pixels;
let cardWidth = width;
let cardHeight = height;
let detection: CardDetectionResult | undefined;
// Step 1: Detect and extract card (if enabled)
if (opts.enableCardDetection) {
detection = detectCard(pixels, width, height);
if (detection.found) {
const warped = warpPerspective(
pixels,
width,
height,
detection.corners
);
cardPixels = warped.pixels;
cardWidth = warped.width;
cardHeight = warped.height;
}
}
// Step 2: Find best match (with or without rotation)
const match = opts.enableRotationMatching
? findBestMatchWithRotations(cardPixels, cardWidth, cardHeight, cardHashes, opts)
: findBestMatchSingle(cardPixels, cardWidth, cardHeight, cardHashes, opts);
const processingTimeMs = performance.now() - startTime;
if (!match) {
return {
success: false,
cardDetection: detection,
error: "No match found",
processingTimeMs,
};
}
return {
success: true,
match,
cardDetection: detection,
processingTimeMs,
};
} catch (error) {
return {
success: false,
error: error instanceof Error ? error.message : "Unknown error",
processingTimeMs: performance.now() - startTime,
};
}
}
/**
* Compute hash for an image with full preprocessing pipeline.
*/
export function computeImageHash(
pixels: Uint8Array | Uint8ClampedArray,
width: number,
height: number
): Uint8Array {
// Apply CLAHE
const clahePixels = applyCLAHE(pixels, width, height);
// Resize to 32x32
const resized = resizeImage(clahePixels, width, height, 32, 32);
// Compute hash
return computeColorHash(resized);
}
/**
* Find best match trying all 4 rotations.
*/
function findBestMatchWithRotations(
pixels: Uint8Array | Uint8ClampedArray,
width: number,
height: number,
cardHashes: CardHashEntry[],
opts: Required<RecognitionOptions>
): CardMatch | null {
let bestMatch: CardMatch | null = null;
const rotations: Array<0 | 90 | 180 | 270> = [0, 90, 180, 270];
for (const rotation of rotations) {
const { pixels: rotatedPixels, width: rotatedWidth, height: rotatedHeight } =
rotation === 0
? { pixels, width, height }
: rotateImage(pixels, width, height, rotation);
// Apply CLAHE
const clahePixels = applyCLAHE(rotatedPixels, rotatedWidth, rotatedHeight);
// Resize to 32x32
const resized = resizeImage(clahePixels, rotatedWidth, rotatedHeight, 32, 32);
// Compute hash
const queryHash = computeColorHash(resized);
// Find best match for this rotation
for (const card of cardHashes) {
if (!card.hash || card.hash.length !== queryHash.length) continue;
const distance = hammingDistance(queryHash, card.hash);
const confidence = calculateConfidence(distance);
if (distance <= opts.matchThreshold && confidence >= opts.minConfidence) {
if (!bestMatch || distance < bestMatch.distance) {
bestMatch = {
cardId: card.id,
confidence,
distance,
rotation,
};
}
// Early exit on perfect match
if (distance === 0) {
return bestMatch;
}
}
}
}
return bestMatch;
}
/**
* Find best match without rotation.
*/
function findBestMatchSingle(
pixels: Uint8Array | Uint8ClampedArray,
width: number,
height: number,
cardHashes: CardHashEntry[],
opts: Required<RecognitionOptions>
): CardMatch | null {
// Apply CLAHE
const clahePixels = applyCLAHE(pixels, width, height);
// Resize to 32x32
const resized = resizeImage(clahePixels, width, height, 32, 32);
// Compute hash
const queryHash = computeColorHash(resized);
let bestMatch: CardMatch | null = null;
for (const card of cardHashes) {
if (!card.hash || card.hash.length !== queryHash.length) continue;
const distance = hammingDistance(queryHash, card.hash);
const confidence = calculateConfidence(distance);
if (distance <= opts.matchThreshold && confidence >= opts.minConfidence) {
if (!bestMatch || distance < bestMatch.distance) {
bestMatch = {
cardId: card.id,
confidence,
distance,
rotation: 0,
};
}
if (distance === 0) {
return bestMatch;
}
}
}
return bestMatch;
}
/**
* Create a recognition service instance with cached card hashes.
*/
export function createRecognitionService(cardHashes: CardHashEntry[]) {
let cachedHashes = cardHashes;
return {
/**
* Recognize a card from RGBA pixel data.
*/
recognize(
pixels: Uint8Array | Uint8ClampedArray,
width: number,
height: number,
options?: RecognitionOptions
): RecognitionResult {
return recognizeCard(pixels, width, height, cachedHashes, options);
},
/**
* Update the cached card hashes.
*/
updateHashes(hashes: CardHashEntry[]) {
cachedHashes = hashes;
},
/**
* Get the number of cached hashes.
*/
getHashCount(): number {
return cachedHashes.length;
},
};
}

View file

@ -0,0 +1,121 @@
/**
* Skia-based image decoder for getting RGBA pixel data.
* Uses react-native-skia to decode images and extract pixel buffers.
*/
import {
Skia,
useImage,
SkImage,
} from "@shopify/react-native-skia";
/**
* Decode a base64 PNG/JPEG image and return RGBA pixel data.
*
* @param base64 - Base64 encoded image data (without data URI prefix)
* @returns RGBA pixel data with dimensions
*/
export function decodeImageBase64(base64: string): {
pixels: Uint8Array;
width: number;
height: number;
} | null {
try {
// Decode base64 to data
const data = Skia.Data.fromBase64(base64);
if (!data) return null;
// Create image from data
const image = Skia.Image.MakeImageFromEncoded(data);
if (!image) return null;
const width = image.width();
const height = image.height();
// Read pixels from the image
// Note: Skia images are in RGBA format
const pixels = image.readPixels(0, 0, {
width,
height,
colorType: 4, // RGBA_8888
alphaType: 1, // Unpremultiplied
});
if (!pixels) return null;
return {
pixels: new Uint8Array(pixels),
width,
height,
};
} catch (error) {
console.error("[SkiaDecoder] Failed to decode image:", error);
return null;
}
}
/**
* Decode an image from a file URI and return RGBA pixel data.
*
* @param uri - File URI (e.g., file:///path/to/image.png)
* @returns Promise with RGBA pixel data
*/
export async function decodeImageFromUri(uri: string): Promise<{
pixels: Uint8Array;
width: number;
height: number;
} | null> {
try {
// Fetch the image data
const response = await fetch(uri);
const arrayBuffer = await response.arrayBuffer();
const base64 = arrayBufferToBase64(arrayBuffer);
return decodeImageBase64(base64);
} catch (error) {
console.error("[SkiaDecoder] Failed to load image from URI:", error);
return null;
}
}
/**
* Convert ArrayBuffer to base64 string.
*/
function arrayBufferToBase64(buffer: ArrayBuffer): string {
let binary = "";
const bytes = new Uint8Array(buffer);
const len = bytes.byteLength;
for (let i = 0; i < len; i++) {
binary += String.fromCharCode(bytes[i]);
}
return btoa(binary);
}
/**
* React hook to decode an image from URI.
* Uses Skia's useImage hook for caching.
*/
export function useDecodedImage(uri: string | null) {
const skiaImage = useImage(uri);
if (!skiaImage) {
return { loading: true, pixels: null, width: 0, height: 0 };
}
const width = skiaImage.width();
const height = skiaImage.height();
const pixels = skiaImage.readPixels(0, 0, {
width,
height,
colorType: 4, // RGBA_8888
alphaType: 1, // Unpremultiplied
});
return {
loading: false,
pixels: pixels ? new Uint8Array(pixels) : null,
width,
height,
};
}