Migrate from .NET MAUI to Expo + Convex
Complete rewrite of Scry using TypeScript stack:
- Expo/React Native with Expo Router (file-based routing)
- Convex backend (serverless functions + real-time database)
- Adaptive camera system (expo-camera in Expo Go, Vision Camera in production)
- React Native Skia + fast-opencv for image processing
- GDPR-compliant auth setup with Zitadel OIDC (pending configuration)
Key features:
- Card recognition pipeline ported to TypeScript
- Perceptual hashing (192-bit color pHash)
- CLAHE preprocessing for lighting normalization
- Local SQLite cache with Convex sync
- Collection management with offline support
Removes all .NET/MAUI code (src/, test/, tools/).
💘 Generated with Crush
Assisted-by: Claude Opus 4.5 via Crush <crush@charm.land>
This commit is contained in:
Chris Kruining
parent
56499d5af9
commit
83ab4df537
138 changed files with 19136 additions and 7681 deletions
492
lib/recognition/cardDetector.ts
Normal file
492
lib/recognition/cardDetector.ts
Normal file
|
|
@ -0,0 +1,492 @@
|
|||
/**
|
||||
* Card detection using edge detection and contour analysis.
|
||||
* Detects card boundaries in images and returns the four corner points.
|
||||
*
|
||||
* This is a pure TypeScript implementation for when OpenCV is not available.
|
||||
*/
|
||||
|
||||
import {
|
||||
Point,
|
||||
toGrayscale,
|
||||
gaussianBlur,
|
||||
distance,
|
||||
crossProduct,
|
||||
pointToLineDistance,
|
||||
} from "./imageUtils";
|
||||
|
||||
/** Standard MTG card aspect ratio (height / width). Cards are 63mm x 88mm. */
|
||||
const CARD_ASPECT_RATIO = 88 / 63; // ~1.397
|
||||
const ASPECT_RATIO_TOLERANCE = 0.25;
|
||||
const MIN_CARD_AREA_RATIO = 0.05;
|
||||
const MAX_CARD_AREA_RATIO = 0.98;
|
||||
|
||||
export interface CardDetectionResult {
|
||||
found: boolean;
|
||||
corners: Point[];
|
||||
confidence: number;
|
||||
debugMessage?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect a card in the image and return its corner points.
|
||||
*/
|
||||
export function detectCard(
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
width: number,
|
||||
height: number
|
||||
): CardDetectionResult {
|
||||
// Step 1: Convert to grayscale
|
||||
const grayscale = toGrayscale(pixels);
|
||||
|
||||
// Step 2: Apply Gaussian blur to reduce noise
|
||||
const blurred = gaussianBlur(grayscale, width, height, 5);
|
||||
|
||||
// Step 3: Apply Canny edge detection
|
||||
const edges = applyCannyEdgeDetection(blurred, width, height, 50, 150);
|
||||
|
||||
// Step 4: Find contours
|
||||
const contours = findContours(edges, width, height);
|
||||
|
||||
if (contours.length === 0) {
|
||||
return { found: false, corners: [], confidence: 0, debugMessage: "No contours found" };
|
||||
}
|
||||
|
||||
// Step 5: Find the best card-like quadrilateral
|
||||
const imageArea = width * height;
|
||||
const bestQuad = findBestCardQuadrilateral(contours, imageArea);
|
||||
|
||||
if (!bestQuad) {
|
||||
return { found: false, corners: [], confidence: 0, debugMessage: "No card-like quadrilateral found" };
|
||||
}
|
||||
|
||||
// Step 6: Order corners consistently
|
||||
const orderedCorners = orderCorners(bestQuad);
|
||||
|
||||
// Calculate confidence
|
||||
const confidence = calculateConfidence(orderedCorners, imageArea);
|
||||
|
||||
return { found: true, corners: orderedCorners, confidence };
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply Canny edge detection.
|
||||
*/
|
||||
function applyCannyEdgeDetection(
|
||||
gray: Uint8Array,
|
||||
width: number,
|
||||
height: number,
|
||||
lowThreshold: number,
|
||||
highThreshold: number
|
||||
): Uint8Array {
|
||||
// Step 1: Compute gradients using Sobel operators
|
||||
const gradientX = new Float32Array(width * height);
|
||||
const gradientY = new Float32Array(width * height);
|
||||
const magnitude = new Float32Array(width * height);
|
||||
const direction = new Float32Array(width * height);
|
||||
|
||||
// Sobel kernels
|
||||
const sobelX = [[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]];
|
||||
const sobelY = [[-1, -2, -1], [0, 0, 0], [1, 2, 1]];
|
||||
|
||||
for (let y = 1; y < height - 1; y++) {
|
||||
for (let x = 1; x < width - 1; x++) {
|
||||
let gx = 0, gy = 0;
|
||||
|
||||
for (let ky = -1; ky <= 1; ky++) {
|
||||
for (let kx = -1; kx <= 1; kx++) {
|
||||
const pixel = gray[(y + ky) * width + (x + kx)];
|
||||
gx += pixel * sobelX[ky + 1][kx + 1];
|
||||
gy += pixel * sobelY[ky + 1][kx + 1];
|
||||
}
|
||||
}
|
||||
|
||||
const idx = y * width + x;
|
||||
gradientX[idx] = gx;
|
||||
gradientY[idx] = gy;
|
||||
magnitude[idx] = Math.sqrt(gx * gx + gy * gy);
|
||||
direction[idx] = Math.atan2(gy, gx);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: Non-maximum suppression
|
||||
const suppressed = new Float32Array(width * height);
|
||||
|
||||
for (let y = 1; y < height - 1; y++) {
|
||||
for (let x = 1; x < width - 1; x++) {
|
||||
const idx = y * width + x;
|
||||
let angle = direction[idx] * 180 / Math.PI;
|
||||
if (angle < 0) angle += 180;
|
||||
|
||||
let neighbor1: number, neighbor2: number;
|
||||
|
||||
if (angle < 22.5 || angle >= 157.5) {
|
||||
neighbor1 = magnitude[y * width + (x - 1)];
|
||||
neighbor2 = magnitude[y * width + (x + 1)];
|
||||
} else if (angle >= 22.5 && angle < 67.5) {
|
||||
neighbor1 = magnitude[(y - 1) * width + (x + 1)];
|
||||
neighbor2 = magnitude[(y + 1) * width + (x - 1)];
|
||||
} else if (angle >= 67.5 && angle < 112.5) {
|
||||
neighbor1 = magnitude[(y - 1) * width + x];
|
||||
neighbor2 = magnitude[(y + 1) * width + x];
|
||||
} else {
|
||||
neighbor1 = magnitude[(y - 1) * width + (x - 1)];
|
||||
neighbor2 = magnitude[(y + 1) * width + (x + 1)];
|
||||
}
|
||||
|
||||
if (magnitude[idx] >= neighbor1 && magnitude[idx] >= neighbor2) {
|
||||
suppressed[idx] = magnitude[idx];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3: Double thresholding and edge tracking
|
||||
const result = new Uint8Array(width * height);
|
||||
const strong = new Uint8Array(width * height);
|
||||
const weak = new Uint8Array(width * height);
|
||||
|
||||
for (let i = 0; i < width * height; i++) {
|
||||
if (suppressed[i] >= highThreshold) {
|
||||
strong[i] = 1;
|
||||
} else if (suppressed[i] >= lowThreshold) {
|
||||
weak[i] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Edge tracking by hysteresis
|
||||
for (let y = 1; y < height - 1; y++) {
|
||||
for (let x = 1; x < width - 1; x++) {
|
||||
const idx = y * width + x;
|
||||
|
||||
if (strong[idx]) {
|
||||
result[idx] = 255;
|
||||
} else if (weak[idx]) {
|
||||
// Check if connected to strong edge
|
||||
outer: for (let dy = -1; dy <= 1; dy++) {
|
||||
for (let dx = -1; dx <= 1; dx++) {
|
||||
if (strong[(y + dy) * width + (x + dx)]) {
|
||||
result[idx] = 255;
|
||||
break outer;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find contours in a binary edge image using flood fill.
|
||||
*/
|
||||
function findContours(
|
||||
edges: Uint8Array,
|
||||
width: number,
|
||||
height: number
|
||||
): Point[][] {
|
||||
const visited = new Uint8Array(width * height);
|
||||
const contours: Point[][] = [];
|
||||
|
||||
const dx = [-1, 0, 1, 1, 1, 0, -1, -1];
|
||||
const dy = [-1, -1, -1, 0, 1, 1, 1, 0];
|
||||
|
||||
for (let y = 1; y < height - 1; y++) {
|
||||
for (let x = 1; x < width - 1; x++) {
|
||||
const idx = y * width + x;
|
||||
if (visited[idx]) continue;
|
||||
if (edges[idx] < 128) continue;
|
||||
|
||||
// Trace contour using BFS
|
||||
const contour: Point[] = [];
|
||||
const queue: Array<{ x: number; y: number }> = [{ x, y }];
|
||||
|
||||
while (queue.length > 0 && contour.length < 10000) {
|
||||
const point = queue.shift()!;
|
||||
const pidx = point.y * width + point.x;
|
||||
|
||||
if (point.x < 0 || point.x >= width || point.y < 0 || point.y >= height) continue;
|
||||
if (visited[pidx]) continue;
|
||||
if (edges[pidx] < 128) continue;
|
||||
|
||||
visited[pidx] = 1;
|
||||
contour.push(point);
|
||||
|
||||
for (let i = 0; i < 8; i++) {
|
||||
queue.push({ x: point.x + dx[i], y: point.y + dy[i] });
|
||||
}
|
||||
}
|
||||
|
||||
if (contour.length >= 4) {
|
||||
contours.push(contour);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return contours;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the best quadrilateral that matches a card shape.
|
||||
*/
|
||||
function findBestCardQuadrilateral(
|
||||
contours: Point[][],
|
||||
imageArea: number
|
||||
): Point[] | null {
|
||||
let bestQuad: Point[] | null = null;
|
||||
let bestScore = -Infinity;
|
||||
|
||||
for (const contour of contours) {
|
||||
// Simplify contour using Douglas-Peucker
|
||||
const simplified = simplifyContour(contour, contour.length * 0.02);
|
||||
|
||||
// Try to approximate as quadrilateral
|
||||
const quad = approximateQuadrilateral(simplified);
|
||||
if (!quad) continue;
|
||||
|
||||
// Check if valid card shape
|
||||
const area = calculateQuadArea(quad);
|
||||
const areaRatio = area / imageArea;
|
||||
|
||||
if (areaRatio < MIN_CARD_AREA_RATIO || areaRatio > MAX_CARD_AREA_RATIO) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check aspect ratio
|
||||
const aspectScore = calculateAspectRatioScore(quad);
|
||||
if (aspectScore < 0.5) continue;
|
||||
|
||||
// Check convexity
|
||||
if (!isConvex(quad)) continue;
|
||||
|
||||
// Score based on area and aspect ratio
|
||||
const score = areaRatio * aspectScore;
|
||||
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
bestQuad = quad;
|
||||
}
|
||||
}
|
||||
|
||||
return bestQuad;
|
||||
}
|
||||
|
||||
/**
|
||||
* Simplify a contour using Douglas-Peucker algorithm.
|
||||
*/
|
||||
function simplifyContour(contour: Point[], epsilon: number): Point[] {
|
||||
if (contour.length < 3) return contour;
|
||||
|
||||
const first = contour[0];
|
||||
const last = contour[contour.length - 1];
|
||||
|
||||
let maxDist = 0;
|
||||
let maxIndex = 0;
|
||||
|
||||
for (let i = 1; i < contour.length - 1; i++) {
|
||||
const dist = pointToLineDistance(contour[i], first, last);
|
||||
if (dist > maxDist) {
|
||||
maxDist = dist;
|
||||
maxIndex = i;
|
||||
}
|
||||
}
|
||||
|
||||
if (maxDist > epsilon) {
|
||||
const left = simplifyContour(contour.slice(0, maxIndex + 1), epsilon);
|
||||
const right = simplifyContour(contour.slice(maxIndex), epsilon);
|
||||
return [...left.slice(0, -1), ...right];
|
||||
}
|
||||
|
||||
return [first, last];
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to approximate a contour as a quadrilateral.
|
||||
*/
|
||||
function approximateQuadrilateral(contour: Point[]): Point[] | null {
|
||||
if (contour.length < 4) return null;
|
||||
if (contour.length === 4) return contour;
|
||||
|
||||
// Find convex hull
|
||||
const hull = convexHull(contour);
|
||||
if (hull.length < 4) return null;
|
||||
if (hull.length === 4) return hull;
|
||||
|
||||
// Find 4 extreme points
|
||||
return findExtremePoints(hull);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate convex hull using Graham scan.
|
||||
*/
|
||||
function convexHull(points: Point[]): Point[] {
|
||||
if (points.length < 3) return points;
|
||||
|
||||
// Find bottom-most point
|
||||
const start = points.reduce((min, p) =>
|
||||
p.y < min.y || (p.y === min.y && p.x < min.x) ? p : min
|
||||
);
|
||||
|
||||
// Sort by polar angle
|
||||
const sorted = points
|
||||
.filter(p => p !== start)
|
||||
.sort((a, b) => {
|
||||
const angleA = Math.atan2(a.y - start.y, a.x - start.x);
|
||||
const angleB = Math.atan2(b.y - start.y, b.x - start.x);
|
||||
if (angleA !== angleB) return angleA - angleB;
|
||||
return distance(a, start) - distance(b, start);
|
||||
});
|
||||
|
||||
const hull: Point[] = [start];
|
||||
|
||||
for (const point of sorted) {
|
||||
while (hull.length > 1 && crossProduct(hull[hull.length - 2], hull[hull.length - 1], point) <= 0) {
|
||||
hull.pop();
|
||||
}
|
||||
hull.push(point);
|
||||
}
|
||||
|
||||
return hull;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find 4 extreme points of a convex hull.
|
||||
*/
|
||||
function findExtremePoints(hull: Point[]): Point[] {
|
||||
if (hull.length <= 4) return hull;
|
||||
|
||||
const minX = hull.reduce((min, p) => p.x < min.x ? p : min, hull[0]);
|
||||
const maxX = hull.reduce((max, p) => p.x > max.x ? p : max, hull[0]);
|
||||
const minY = hull.reduce((min, p) => p.y < min.y ? p : min, hull[0]);
|
||||
const maxY = hull.reduce((max, p) => p.y > max.y ? p : max, hull[0]);
|
||||
|
||||
const extremes = new Set([minX, maxX, minY, maxY]);
|
||||
|
||||
if (extremes.size === 4) {
|
||||
return Array.from(extremes);
|
||||
}
|
||||
|
||||
// Fallback: take points at regular angular intervals
|
||||
const centerX = hull.reduce((s, p) => s + p.x, 0) / hull.length;
|
||||
const centerY = hull.reduce((s, p) => s + p.y, 0) / hull.length;
|
||||
|
||||
const sorted = [...hull].sort((a, b) =>
|
||||
Math.atan2(a.y - centerY, a.x - centerX) -
|
||||
Math.atan2(b.y - centerY, b.x - centerX)
|
||||
);
|
||||
|
||||
const step = Math.floor(sorted.length / 4);
|
||||
return [sorted[0], sorted[step], sorted[step * 2], sorted[step * 3]];
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate area of a quadrilateral using shoelace formula.
|
||||
*/
|
||||
function calculateQuadArea(quad: Point[]): number {
|
||||
let area = 0;
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const j = (i + 1) % 4;
|
||||
area += quad[i].x * quad[j].y;
|
||||
area -= quad[j].x * quad[i].y;
|
||||
}
|
||||
return Math.abs(area) / 2;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate how well the aspect ratio matches a card.
|
||||
*/
|
||||
function calculateAspectRatioScore(quad: Point[]): number {
|
||||
const width1 = distance(quad[0], quad[1]);
|
||||
const width2 = distance(quad[2], quad[3]);
|
||||
const height1 = distance(quad[1], quad[2]);
|
||||
const height2 = distance(quad[3], quad[0]);
|
||||
|
||||
const avgWidth = (width1 + width2) / 2;
|
||||
const avgHeight = (height1 + height2) / 2;
|
||||
|
||||
// Ensure we get the right ratio regardless of orientation
|
||||
const aspectRatio = avgWidth > avgHeight
|
||||
? avgWidth / avgHeight
|
||||
: avgHeight / avgWidth;
|
||||
|
||||
const deviation = Math.abs(aspectRatio - CARD_ASPECT_RATIO) / CARD_ASPECT_RATIO;
|
||||
|
||||
return Math.max(0, 1 - deviation / ASPECT_RATIO_TOLERANCE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a quadrilateral is convex.
|
||||
*/
|
||||
function isConvex(quad: Point[]): boolean {
|
||||
let sign = 0;
|
||||
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const cross = crossProduct(
|
||||
quad[i],
|
||||
quad[(i + 1) % 4],
|
||||
quad[(i + 2) % 4]
|
||||
);
|
||||
|
||||
if (Math.abs(cross) < 0.0001) continue;
|
||||
|
||||
const currentSign = cross > 0 ? 1 : -1;
|
||||
|
||||
if (sign === 0) {
|
||||
sign = currentSign;
|
||||
} else if (sign !== currentSign) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Order corners consistently: top-left, top-right, bottom-right, bottom-left.
|
||||
*/
|
||||
function orderCorners(corners: Point[]): Point[] {
|
||||
const centerX = corners.reduce((s, c) => s + c.x, 0) / 4;
|
||||
const centerY = corners.reduce((s, c) => s + c.y, 0) / 4;
|
||||
|
||||
const topLeft = corners.filter(c => c.x < centerX && c.y < centerY)
|
||||
.sort((a, b) => (a.x + a.y) - (b.x + b.y))[0];
|
||||
const topRight = corners.filter(c => c.x >= centerX && c.y < centerY)
|
||||
.sort((a, b) => (a.y - a.x) - (b.y - b.x))[0];
|
||||
const bottomRight = corners.filter(c => c.x >= centerX && c.y >= centerY)
|
||||
.sort((a, b) => (b.x + b.y) - (a.x + a.y))[0];
|
||||
const bottomLeft = corners.filter(c => c.x < centerX && c.y >= centerY)
|
||||
.sort((a, b) => (b.y - b.x) - (a.y - a.x))[0];
|
||||
|
||||
// Handle edge cases by sorting by angle
|
||||
if (!topLeft || !topRight || !bottomRight || !bottomLeft) {
|
||||
const sorted = [...corners].sort((a, b) =>
|
||||
Math.atan2(a.y - centerY, a.x - centerX) -
|
||||
Math.atan2(b.y - centerY, b.x - centerX)
|
||||
);
|
||||
|
||||
// Find index of point with minimum sum (top-left)
|
||||
let minSumIdx = 0;
|
||||
let minSum = Infinity;
|
||||
sorted.forEach((c, i) => {
|
||||
const sum = c.x + c.y;
|
||||
if (sum < minSum) {
|
||||
minSum = sum;
|
||||
minSumIdx = i;
|
||||
}
|
||||
});
|
||||
|
||||
return [...sorted.slice(minSumIdx), ...sorted.slice(0, minSumIdx)];
|
||||
}
|
||||
|
||||
return [topLeft, topRight, bottomRight, bottomLeft];
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate confidence of the detection.
|
||||
*/
|
||||
function calculateConfidence(corners: Point[], imageArea: number): number {
|
||||
const area = calculateQuadArea(corners);
|
||||
const areaScore = Math.min(area / imageArea / 0.5, 1);
|
||||
const aspectScore = calculateAspectRatioScore(corners);
|
||||
|
||||
return areaScore * 0.4 + aspectScore * 0.6;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue