.
This commit is contained in:
parent
83ab4df537
commit
b4e4ff73ec
8 changed files with 427 additions and 10586 deletions
278
lib/recognition/debugSaver.ts
Normal file
278
lib/recognition/debugSaver.ts
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
/**
|
||||
* Debug image saver for the recognition pipeline.
|
||||
* Saves RGBA pixel data as images to the device's pictures folder.
|
||||
*/
|
||||
|
||||
import { File, Paths } from "expo-file-system";
|
||||
import * as MediaLibrary from "expo-media-library";
|
||||
import { Skia, AlphaType, ColorType } from "@shopify/react-native-skia";
|
||||
|
||||
/** Debug output configuration */
|
||||
export interface DebugConfig {
|
||||
/** Enable debug output. When true, saves images at each pipeline step. */
|
||||
enabled: boolean;
|
||||
/** Album name in device pictures. Default: "Scry Debug" */
|
||||
albumName?: string;
|
||||
/** Prefix for debug images. Default: timestamp */
|
||||
prefix?: string;
|
||||
}
|
||||
|
||||
/** Steps in the recognition pipeline */
|
||||
export type PipelineStep =
|
||||
| "01_input"
|
||||
| "02_resized"
|
||||
| "03_detection"
|
||||
| "04_perspective"
|
||||
| "05_clahe"
|
||||
| "06_final_32x32";
|
||||
|
||||
let debugConfig: DebugConfig = { enabled: false };
|
||||
let currentPrefix: string = "";
|
||||
let hasMediaPermission: boolean | null = null;
|
||||
|
||||
/**
|
||||
* Configure debug output.
|
||||
*/
|
||||
export function configureDebug(config: DebugConfig): void {
|
||||
debugConfig = { ...config };
|
||||
if (config.enabled) {
|
||||
currentPrefix = config.prefix || Date.now().toString();
|
||||
console.log("[DebugSaver] Enabled with prefix:", currentPrefix);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if debug mode is enabled.
|
||||
*/
|
||||
export function isDebugEnabled(): boolean {
|
||||
return debugConfig.enabled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Request media library permission.
|
||||
*/
|
||||
async function ensureMediaPermission(): Promise<boolean> {
|
||||
if (hasMediaPermission !== null) return hasMediaPermission;
|
||||
|
||||
try {
|
||||
const { status } = await MediaLibrary.requestPermissionsAsync();
|
||||
hasMediaPermission = status === "granted";
|
||||
if (!hasMediaPermission) {
|
||||
console.warn("[DebugSaver] Media library permission denied");
|
||||
}
|
||||
return hasMediaPermission;
|
||||
} catch (error) {
|
||||
console.error("[DebugSaver] Failed to request permission:", error);
|
||||
hasMediaPermission = false;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert RGBA pixels to PNG base64 using Skia.
|
||||
*/
|
||||
function pixelsToPngBase64(
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
width: number,
|
||||
height: number
|
||||
): string | null {
|
||||
try {
|
||||
// Create Skia image from raw pixels
|
||||
const data = Skia.Data.fromBytes(new Uint8Array(pixels));
|
||||
const image = Skia.Image.MakeImage(
|
||||
{
|
||||
width,
|
||||
height,
|
||||
colorType: ColorType.RGBA_8888,
|
||||
alphaType: AlphaType.Unpremul,
|
||||
},
|
||||
data,
|
||||
width * 4 // bytes per row
|
||||
);
|
||||
|
||||
if (!image) {
|
||||
console.error("[DebugSaver] Failed to create Skia image");
|
||||
return null;
|
||||
}
|
||||
|
||||
// Encode to PNG
|
||||
const encoded = image.encodeToBase64();
|
||||
return encoded;
|
||||
} catch (error) {
|
||||
console.error("[DebugSaver] Failed to encode image:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save debug image at a pipeline step.
|
||||
*
|
||||
* @param step - Pipeline step identifier
|
||||
* @param pixels - RGBA pixel data
|
||||
* @param width - Image width
|
||||
* @param height - Image height
|
||||
* @param extraInfo - Optional extra info to append to filename
|
||||
*/
|
||||
export async function saveDebugImage(
|
||||
step: PipelineStep,
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
width: number,
|
||||
height: number,
|
||||
extraInfo?: string
|
||||
): Promise<void> {
|
||||
if (!debugConfig.enabled) return;
|
||||
|
||||
try {
|
||||
// Check permission
|
||||
const hasPermission = await ensureMediaPermission();
|
||||
if (!hasPermission) {
|
||||
console.warn("[DebugSaver] No permission, skipping:", step);
|
||||
return;
|
||||
}
|
||||
|
||||
// Convert to PNG
|
||||
const base64 = pixelsToPngBase64(pixels, width, height);
|
||||
if (!base64) {
|
||||
console.error("[DebugSaver] Failed to encode:", step);
|
||||
return;
|
||||
}
|
||||
|
||||
// Build filename
|
||||
const suffix = extraInfo ? `_${extraInfo}` : "";
|
||||
const filename = `${currentPrefix}_${step}${suffix}.png`;
|
||||
|
||||
// Write to temporary file using new File API
|
||||
const tempFile = new File(Paths.cache, filename);
|
||||
await tempFile.write(base64, { encoding: "base64" });
|
||||
|
||||
// Save to media library
|
||||
const asset = await MediaLibrary.createAssetAsync(tempFile.uri);
|
||||
|
||||
// Create or get album
|
||||
const albumName = debugConfig.albumName || "Scry Debug";
|
||||
let album = await MediaLibrary.getAlbumAsync(albumName);
|
||||
if (!album) {
|
||||
album = await MediaLibrary.createAlbumAsync(albumName, asset, false);
|
||||
} else {
|
||||
await MediaLibrary.addAssetsToAlbumAsync([asset], album, false);
|
||||
}
|
||||
|
||||
// Clean up temp file
|
||||
await tempFile.delete();
|
||||
|
||||
console.log(`[DebugSaver] Saved: ${filename} (${width}x${height})`);
|
||||
} catch (error) {
|
||||
console.error(`[DebugSaver] Failed to save ${step}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Draw corners on an image for detection visualization.
|
||||
* Returns a new pixel buffer with the corners drawn.
|
||||
*
|
||||
* @param pixels - RGBA pixel data
|
||||
* @param width - Image width
|
||||
* @param height - Image height
|
||||
* @param corners - Array of 4 corner points [topLeft, topRight, bottomRight, bottomLeft]
|
||||
* @param color - RGB color for corners [R, G, B]
|
||||
*/
|
||||
export function drawCorners(
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
width: number,
|
||||
height: number,
|
||||
corners: Array<{ x: number; y: number }>,
|
||||
color: [number, number, number] = [255, 0, 0]
|
||||
): Uint8Array {
|
||||
const result = new Uint8Array(pixels);
|
||||
|
||||
const drawCircle = (cx: number, cy: number, radius: number) => {
|
||||
for (let dy = -radius; dy <= radius; dy++) {
|
||||
for (let dx = -radius; dx <= radius; dx++) {
|
||||
if (dx * dx + dy * dy <= radius * radius) {
|
||||
const x = Math.round(cx + dx);
|
||||
const y = Math.round(cy + dy);
|
||||
if (x >= 0 && x < width && y >= 0 && y < height) {
|
||||
const idx = (y * width + x) * 4;
|
||||
result[idx] = color[0];
|
||||
result[idx + 1] = color[1];
|
||||
result[idx + 2] = color[2];
|
||||
result[idx + 3] = 255;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const drawLine = (
|
||||
x0: number,
|
||||
y0: number,
|
||||
x1: number,
|
||||
y1: number,
|
||||
thickness: number = 2
|
||||
) => {
|
||||
const dx = Math.abs(x1 - x0);
|
||||
const dy = Math.abs(y1 - y0);
|
||||
const sx = x0 < x1 ? 1 : -1;
|
||||
const sy = y0 < y1 ? 1 : -1;
|
||||
let err = dx - dy;
|
||||
|
||||
let x = x0;
|
||||
let y = y0;
|
||||
|
||||
while (true) {
|
||||
// Draw thick point
|
||||
for (let t = -thickness; t <= thickness; t++) {
|
||||
for (let s = -thickness; s <= thickness; s++) {
|
||||
const px = Math.round(x + s);
|
||||
const py = Math.round(y + t);
|
||||
if (px >= 0 && px < width && py >= 0 && py < height) {
|
||||
const idx = (py * width + px) * 4;
|
||||
result[idx] = color[0];
|
||||
result[idx + 1] = color[1];
|
||||
result[idx + 2] = color[2];
|
||||
result[idx + 3] = 255;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Math.abs(x - x1) < 1 && Math.abs(y - y1) < 1) break;
|
||||
|
||||
const e2 = 2 * err;
|
||||
if (e2 > -dy) {
|
||||
err -= dy;
|
||||
x += sx;
|
||||
}
|
||||
if (e2 < dx) {
|
||||
err += dx;
|
||||
y += sy;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Draw lines between corners
|
||||
if (corners.length >= 4) {
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const next = (i + 1) % 4;
|
||||
drawLine(corners[i].x, corners[i].y, corners[next].x, corners[next].y);
|
||||
}
|
||||
}
|
||||
|
||||
// Draw corner circles
|
||||
const radius = Math.min(10, Math.max(3, Math.min(width, height) / 50));
|
||||
for (const corner of corners) {
|
||||
drawCircle(corner.x, corner.y, radius);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset debug state for a new recognition run.
|
||||
* Call this before starting a new recognition to get fresh timestamps.
|
||||
*/
|
||||
export function resetDebugPrefix(prefix?: string): void {
|
||||
if (debugConfig.enabled) {
|
||||
currentPrefix = prefix || Date.now().toString();
|
||||
console.log("[DebugSaver] New prefix:", currentPrefix);
|
||||
}
|
||||
}
|
||||
|
|
@ -65,3 +65,12 @@ export {
|
|||
decodeImageFromUri,
|
||||
useDecodedImage,
|
||||
} from "./skiaDecoder";
|
||||
|
||||
// Debug utilities
|
||||
export {
|
||||
configureDebug,
|
||||
isDebugEnabled,
|
||||
resetDebugPrefix,
|
||||
type DebugConfig,
|
||||
type PipelineStep,
|
||||
} from "./debugSaver";
|
||||
|
|
|
|||
|
|
@ -20,6 +20,12 @@ import {
|
|||
HASH_BITS,
|
||||
} from "./perceptualHash";
|
||||
import { resizeImage, rotateImage } from "./imageUtils";
|
||||
import {
|
||||
isDebugEnabled,
|
||||
saveDebugImage,
|
||||
drawCorners,
|
||||
resetDebugPrefix,
|
||||
} from "./debugSaver";
|
||||
|
||||
export interface RecognitionOptions {
|
||||
/** Enable card detection and perspective correction. */
|
||||
|
|
@ -30,6 +36,8 @@ export interface RecognitionOptions {
|
|||
minConfidence?: number;
|
||||
/** Maximum Hamming distance to accept a match. */
|
||||
matchThreshold?: number;
|
||||
/** Enable debug image saving at each pipeline step. */
|
||||
debug?: boolean;
|
||||
}
|
||||
|
||||
export interface CardMatch {
|
||||
|
|
@ -65,6 +73,7 @@ const DEFAULT_OPTIONS: Required<RecognitionOptions> = {
|
|||
enableRotationMatching: true,
|
||||
minConfidence: 0.85,
|
||||
matchThreshold: MATCH_THRESHOLD,
|
||||
debug: false,
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -93,8 +102,18 @@ export function recognizeCard(
|
|||
): RecognitionResult {
|
||||
const startTime = performance.now();
|
||||
const opts = { ...DEFAULT_OPTIONS, ...options };
|
||||
|
||||
|
||||
// Reset debug prefix for this recognition run
|
||||
if (opts.debug && isDebugEnabled()) {
|
||||
resetDebugPrefix();
|
||||
}
|
||||
|
||||
try {
|
||||
// Save input image
|
||||
if (opts.debug && isDebugEnabled()) {
|
||||
saveDebugImage("01_input", pixels, width, height);
|
||||
}
|
||||
|
||||
if (cardHashes.length === 0) {
|
||||
return {
|
||||
success: false,
|
||||
|
|
@ -111,7 +130,19 @@ export function recognizeCard(
|
|||
// Step 1: Detect and extract card (if enabled)
|
||||
if (opts.enableCardDetection) {
|
||||
detection = detectCard(pixels, width, height);
|
||||
|
||||
|
||||
// Save detection visualization
|
||||
if (opts.debug && isDebugEnabled() && detection.found) {
|
||||
const detectionViz = drawCorners(
|
||||
pixels,
|
||||
width,
|
||||
height,
|
||||
detection.corners,
|
||||
[0, 255, 0]
|
||||
);
|
||||
saveDebugImage("03_detection", detectionViz, width, height);
|
||||
}
|
||||
|
||||
if (detection.found) {
|
||||
const warped = warpPerspective(
|
||||
pixels,
|
||||
|
|
@ -122,6 +153,11 @@ export function recognizeCard(
|
|||
cardPixels = warped.pixels;
|
||||
cardWidth = warped.width;
|
||||
cardHeight = warped.height;
|
||||
|
||||
// Save perspective corrected image
|
||||
if (opts.debug && isDebugEnabled()) {
|
||||
saveDebugImage("04_perspective", cardPixels, cardWidth, cardHeight);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -196,9 +232,19 @@ function findBestMatchWithRotations(
|
|||
|
||||
// Apply CLAHE
|
||||
const clahePixels = applyCLAHE(rotatedPixels, rotatedWidth, rotatedHeight);
|
||||
|
||||
// Save CLAHE output (only for rotation 0 to avoid spam)
|
||||
if (opts.debug && isDebugEnabled() && rotation === 0) {
|
||||
saveDebugImage("05_clahe", clahePixels, rotatedWidth, rotatedHeight);
|
||||
}
|
||||
|
||||
// Resize to 32x32
|
||||
const resized = resizeImage(clahePixels, rotatedWidth, rotatedHeight, 32, 32);
|
||||
|
||||
// Save final 32x32 (only for rotation 0)
|
||||
if (opts.debug && isDebugEnabled() && rotation === 0) {
|
||||
saveDebugImage("06_final_32x32", resized, 32, 32);
|
||||
}
|
||||
|
||||
// Compute hash
|
||||
const queryHash = computeColorHash(resized);
|
||||
|
|
@ -243,9 +289,19 @@ function findBestMatchSingle(
|
|||
): CardMatch | null {
|
||||
// Apply CLAHE
|
||||
const clahePixels = applyCLAHE(pixels, width, height);
|
||||
|
||||
// Save CLAHE output
|
||||
if (opts.debug && isDebugEnabled()) {
|
||||
saveDebugImage("05_clahe", clahePixels, width, height);
|
||||
}
|
||||
|
||||
// Resize to 32x32
|
||||
const resized = resizeImage(clahePixels, width, height, 32, 32);
|
||||
|
||||
// Save final 32x32
|
||||
if (opts.debug && isDebugEnabled()) {
|
||||
saveDebugImage("06_final_32x32", resized, 32, 32);
|
||||
}
|
||||
|
||||
// Compute hash
|
||||
const queryHash = computeColorHash(resized);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue