Migrate from .NET MAUI to Expo + Convex
Complete rewrite of Scry using TypeScript stack:
- Expo/React Native with Expo Router (file-based routing)
- Convex backend (serverless functions + real-time database)
- Adaptive camera system (expo-camera in Expo Go, Vision Camera in production)
- React Native Skia + fast-opencv for image processing
- GDPR-compliant auth setup with Zitadel OIDC (pending configuration)
Key features:
- Card recognition pipeline ported to TypeScript
- Perceptual hashing (192-bit color pHash)
- CLAHE preprocessing for lighting normalization
- Local SQLite cache with Convex sync
- Collection management with offline support
Removes all .NET/MAUI code (src/, test/, tools/).
💘 Generated with Crush
Assisted-by: Claude Opus 4.5 via Crush <crush@charm.land>
This commit is contained in:
Chris Kruining
parent
56499d5af9
commit
83ab4df537
138 changed files with 19136 additions and 7681 deletions
64
lib/context/HashCacheContext.tsx
Normal file
64
lib/context/HashCacheContext.tsx
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
/**
|
||||
* React Context for card hash cache.
|
||||
* Provides offline-first hash data for card recognition.
|
||||
*/
|
||||
|
||||
import React, { createContext, useContext, useState, useCallback, type ReactNode } from "react";
|
||||
import type { CardHashEntry } from "../recognition";
|
||||
|
||||
interface HashCacheState {
|
||||
cardHashes: CardHashEntry[];
|
||||
hashesLoaded: boolean;
|
||||
lastHashSync: number;
|
||||
}
|
||||
|
||||
interface HashCacheContextValue extends HashCacheState {
|
||||
setCardHashes: (hashes: CardHashEntry[]) => void;
|
||||
clearHashes: () => void;
|
||||
}
|
||||
|
||||
const HashCacheContext = createContext<HashCacheContextValue | null>(null);
|
||||
|
||||
export function HashCacheProvider({ children }: { children: ReactNode }) {
|
||||
const [state, setState] = useState<HashCacheState>({
|
||||
cardHashes: [],
|
||||
hashesLoaded: false,
|
||||
lastHashSync: 0,
|
||||
});
|
||||
|
||||
const setCardHashes = useCallback((hashes: CardHashEntry[]) => {
|
||||
setState({
|
||||
cardHashes: hashes,
|
||||
hashesLoaded: true,
|
||||
lastHashSync: Date.now(),
|
||||
});
|
||||
}, []);
|
||||
|
||||
const clearHashes = useCallback(() => {
|
||||
setState({
|
||||
cardHashes: [],
|
||||
hashesLoaded: false,
|
||||
lastHashSync: 0,
|
||||
});
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<HashCacheContext.Provider
|
||||
value={{
|
||||
...state,
|
||||
setCardHashes,
|
||||
clearHashes,
|
||||
}}
|
||||
>
|
||||
{children}
|
||||
</HashCacheContext.Provider>
|
||||
);
|
||||
}
|
||||
|
||||
export function useHashCache() {
|
||||
const context = useContext(HashCacheContext);
|
||||
if (!context) {
|
||||
throw new Error("useHashCache must be used within HashCacheProvider");
|
||||
}
|
||||
return context;
|
||||
}
|
||||
5
lib/context/index.ts
Normal file
5
lib/context/index.ts
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
/**
|
||||
* Context exports.
|
||||
*/
|
||||
|
||||
export { HashCacheProvider, useHashCache } from "./HashCacheContext";
|
||||
20
lib/db/index.ts
Normal file
20
lib/db/index.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
/**
|
||||
* Local database module exports.
|
||||
*/
|
||||
|
||||
export {
|
||||
initDatabase,
|
||||
getLastSyncTimestamp,
|
||||
setLastSyncTimestamp,
|
||||
getCachedHashes,
|
||||
getCachedCardCount,
|
||||
upsertCards,
|
||||
clearCache,
|
||||
closeDatabase,
|
||||
} from "./localDatabase";
|
||||
|
||||
export {
|
||||
createSyncService,
|
||||
type SyncService,
|
||||
type SyncStatus,
|
||||
} from "./syncService";
|
||||
180
lib/db/localDatabase.ts
Normal file
180
lib/db/localDatabase.ts
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
/**
|
||||
* Local SQLite database for offline card hash caching.
|
||||
* Syncs with Convex and provides offline recognition support.
|
||||
*/
|
||||
|
||||
import * as SQLite from "expo-sqlite";
|
||||
import type { CardHashEntry } from "../recognition";
|
||||
|
||||
const DB_NAME = "scry_cache.db";
|
||||
|
||||
let db: SQLite.SQLiteDatabase | null = null;
|
||||
|
||||
/**
|
||||
* Initialize the local database.
|
||||
*/
|
||||
export async function initDatabase(): Promise<void> {
|
||||
if (db) return;
|
||||
|
||||
db = await SQLite.openDatabaseAsync(DB_NAME);
|
||||
|
||||
// Create tables
|
||||
await db.execAsync(`
|
||||
CREATE TABLE IF NOT EXISTS cards (
|
||||
scryfall_id TEXT PRIMARY KEY,
|
||||
oracle_id TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
set_code TEXT NOT NULL,
|
||||
collector_number TEXT,
|
||||
rarity TEXT,
|
||||
artist TEXT,
|
||||
image_uri TEXT,
|
||||
hash BLOB NOT NULL,
|
||||
hash_version INTEGER DEFAULT 1,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sync_metadata (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_cards_name ON cards(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_cards_updated ON cards(updated_at);
|
||||
`);
|
||||
|
||||
console.log("[LocalDB] Database initialized");
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the last sync timestamp.
|
||||
*/
|
||||
export async function getLastSyncTimestamp(): Promise<number> {
|
||||
if (!db) await initDatabase();
|
||||
|
||||
const result = await db!.getFirstAsync<{ value: string }>(
|
||||
"SELECT value FROM sync_metadata WHERE key = ?",
|
||||
["last_sync"]
|
||||
);
|
||||
|
||||
return result ? parseInt(result.value, 10) : 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the last sync timestamp.
|
||||
*/
|
||||
export async function setLastSyncTimestamp(timestamp: number): Promise<void> {
|
||||
if (!db) await initDatabase();
|
||||
|
||||
await db!.runAsync(
|
||||
"INSERT OR REPLACE INTO sync_metadata (key, value) VALUES (?, ?)",
|
||||
["last_sync", timestamp.toString()]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all cached card hashes for recognition.
|
||||
*/
|
||||
export async function getCachedHashes(): Promise<CardHashEntry[]> {
|
||||
if (!db) await initDatabase();
|
||||
|
||||
const rows = await db!.getAllAsync<{
|
||||
scryfall_id: string;
|
||||
name: string;
|
||||
set_code: string;
|
||||
collector_number: string | null;
|
||||
image_uri: string | null;
|
||||
hash: Uint8Array;
|
||||
}>("SELECT scryfall_id, name, set_code, collector_number, image_uri, hash FROM cards");
|
||||
|
||||
return rows.map((row) => ({
|
||||
id: row.scryfall_id,
|
||||
name: row.name,
|
||||
setCode: row.set_code,
|
||||
collectorNumber: row.collector_number || undefined,
|
||||
imageUri: row.image_uri || undefined,
|
||||
hash: new Uint8Array(row.hash),
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the count of cached cards.
|
||||
*/
|
||||
export async function getCachedCardCount(): Promise<number> {
|
||||
if (!db) await initDatabase();
|
||||
|
||||
const result = await db!.getFirstAsync<{ count: number }>(
|
||||
"SELECT COUNT(*) as count FROM cards"
|
||||
);
|
||||
|
||||
return result?.count || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert or update cards from Convex.
|
||||
*/
|
||||
export async function upsertCards(cards: Array<{
|
||||
scryfallId: string;
|
||||
oracleId: string;
|
||||
name: string;
|
||||
setCode: string;
|
||||
collectorNumber: string;
|
||||
rarity: string;
|
||||
artist?: string;
|
||||
imageUri?: string;
|
||||
hash: Uint8Array;
|
||||
hashVersion: number;
|
||||
updatedAt: number;
|
||||
}>): Promise<void> {
|
||||
if (!db) await initDatabase();
|
||||
|
||||
// Use a transaction for batch insert
|
||||
await db!.withTransactionAsync(async () => {
|
||||
for (const card of cards) {
|
||||
await db!.runAsync(
|
||||
`INSERT OR REPLACE INTO cards
|
||||
(scryfall_id, oracle_id, name, set_code, collector_number, rarity, artist, image_uri, hash, hash_version, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
card.scryfallId,
|
||||
card.oracleId,
|
||||
card.name,
|
||||
card.setCode,
|
||||
card.collectorNumber,
|
||||
card.rarity,
|
||||
card.artist || null,
|
||||
card.imageUri || null,
|
||||
card.hash,
|
||||
card.hashVersion,
|
||||
card.updatedAt,
|
||||
]
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`[LocalDB] Upserted ${cards.length} cards`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all cached data.
|
||||
*/
|
||||
export async function clearCache(): Promise<void> {
|
||||
if (!db) await initDatabase();
|
||||
|
||||
await db!.execAsync(`
|
||||
DELETE FROM cards;
|
||||
DELETE FROM sync_metadata;
|
||||
`);
|
||||
|
||||
console.log("[LocalDB] Cache cleared");
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the database connection.
|
||||
*/
|
||||
export async function closeDatabase(): Promise<void> {
|
||||
if (db) {
|
||||
await db.closeAsync();
|
||||
db = null;
|
||||
}
|
||||
}
|
||||
183
lib/db/syncService.ts
Normal file
183
lib/db/syncService.ts
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
/**
|
||||
* Sync service for keeping local SQLite cache in sync with Convex.
|
||||
* Provides offline-first functionality with background sync.
|
||||
*/
|
||||
|
||||
import { ConvexReactClient } from "convex/react";
|
||||
import { api } from "../../convex/_generated/api";
|
||||
import {
|
||||
initDatabase,
|
||||
getLastSyncTimestamp,
|
||||
setLastSyncTimestamp,
|
||||
getCachedHashes,
|
||||
getCachedCardCount,
|
||||
upsertCards,
|
||||
clearCache,
|
||||
} from "./localDatabase";
|
||||
import type { CardHashEntry } from "../recognition";
|
||||
|
||||
export interface SyncStatus {
|
||||
isInitialized: boolean;
|
||||
isSyncing: boolean;
|
||||
lastSync: number;
|
||||
localCardCount: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface SyncService {
|
||||
initialize: () => Promise<void>;
|
||||
sync: () => Promise<void>;
|
||||
getHashes: () => Promise<CardHashEntry[]>;
|
||||
getStatus: () => SyncStatus;
|
||||
clearLocalCache: () => Promise<void>;
|
||||
onStatusChange: (callback: (status: SyncStatus) => void) => () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a sync service instance.
|
||||
*/
|
||||
export function createSyncService(convexClient: ConvexReactClient): SyncService {
|
||||
let status: SyncStatus = {
|
||||
isInitialized: false,
|
||||
isSyncing: false,
|
||||
lastSync: 0,
|
||||
localCardCount: 0,
|
||||
};
|
||||
|
||||
const listeners = new Set<(status: SyncStatus) => void>();
|
||||
|
||||
function notifyListeners() {
|
||||
listeners.forEach((cb) => cb(status));
|
||||
}
|
||||
|
||||
function updateStatus(partial: Partial<SyncStatus>) {
|
||||
status = { ...status, ...partial };
|
||||
notifyListeners();
|
||||
}
|
||||
|
||||
return {
|
||||
/**
|
||||
* Initialize the local database and load cached data.
|
||||
*/
|
||||
async initialize() {
|
||||
try {
|
||||
await initDatabase();
|
||||
|
||||
const [lastSync, cardCount] = await Promise.all([
|
||||
getLastSyncTimestamp(),
|
||||
getCachedCardCount(),
|
||||
]);
|
||||
|
||||
updateStatus({
|
||||
isInitialized: true,
|
||||
lastSync,
|
||||
localCardCount: cardCount,
|
||||
error: undefined,
|
||||
});
|
||||
|
||||
console.log(`[SyncService] Initialized with ${cardCount} cached cards`);
|
||||
} catch (error) {
|
||||
console.error("[SyncService] Initialization failed:", error);
|
||||
updateStatus({
|
||||
error: error instanceof Error ? error.message : "Initialization failed",
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Sync with Convex - fetch new/updated cards.
|
||||
*/
|
||||
async sync() {
|
||||
if (status.isSyncing) {
|
||||
console.log("[SyncService] Sync already in progress");
|
||||
return;
|
||||
}
|
||||
|
||||
updateStatus({ isSyncing: true, error: undefined });
|
||||
|
||||
try {
|
||||
// Get cards updated since last sync
|
||||
const lastSync = status.lastSync;
|
||||
|
||||
// Query Convex for updated cards
|
||||
// Note: This uses the HTTP client for one-off queries
|
||||
const cards = lastSync > 0
|
||||
? await convexClient.query(api.cards.listUpdatedAfter, { since: lastSync })
|
||||
: await convexClient.query(api.cards.list, {});
|
||||
|
||||
if (cards && cards.length > 0) {
|
||||
// Convert to local format and save
|
||||
const localCards = cards.map((card) => ({
|
||||
scryfallId: card.scryfallId,
|
||||
oracleId: card.oracleId,
|
||||
name: card.name,
|
||||
setCode: card.setCode,
|
||||
collectorNumber: card.collectorNumber,
|
||||
rarity: card.rarity,
|
||||
artist: card.artist,
|
||||
imageUri: card.imageUri,
|
||||
hash: new Uint8Array(card.hash),
|
||||
hashVersion: card.hashVersion,
|
||||
updatedAt: card.updatedAt,
|
||||
}));
|
||||
|
||||
await upsertCards(localCards);
|
||||
console.log(`[SyncService] Synced ${localCards.length} cards`);
|
||||
}
|
||||
|
||||
// Update sync timestamp
|
||||
const now = Date.now();
|
||||
await setLastSyncTimestamp(now);
|
||||
|
||||
// Update status
|
||||
const cardCount = await getCachedCardCount();
|
||||
updateStatus({
|
||||
isSyncing: false,
|
||||
lastSync: now,
|
||||
localCardCount: cardCount,
|
||||
});
|
||||
|
||||
console.log(`[SyncService] Sync complete. ${cardCount} cards cached.`);
|
||||
} catch (error) {
|
||||
console.error("[SyncService] Sync failed:", error);
|
||||
updateStatus({
|
||||
isSyncing: false,
|
||||
error: error instanceof Error ? error.message : "Sync failed",
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Get cached hashes for recognition.
|
||||
*/
|
||||
async getHashes() {
|
||||
return getCachedHashes();
|
||||
},
|
||||
|
||||
/**
|
||||
* Get current sync status.
|
||||
*/
|
||||
getStatus() {
|
||||
return status;
|
||||
},
|
||||
|
||||
/**
|
||||
* Clear local cache and re-sync.
|
||||
*/
|
||||
async clearLocalCache() {
|
||||
await clearCache();
|
||||
updateStatus({
|
||||
lastSync: 0,
|
||||
localCardCount: 0,
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* Subscribe to status changes.
|
||||
*/
|
||||
onStatusChange(callback: (status: SyncStatus) => void) {
|
||||
listeners.add(callback);
|
||||
return () => listeners.delete(callback);
|
||||
},
|
||||
};
|
||||
}
|
||||
9
lib/hooks/index.ts
Normal file
9
lib/hooks/index.ts
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
/**
|
||||
* Hooks module exports.
|
||||
*/
|
||||
|
||||
export { useCameraPermission } from "./useCamera";
|
||||
export { useCardHashes, useCardCount, useSearchCards, useCollection, useCurrentUser } from "./useConvex";
|
||||
export { useSync, SyncInitializer } from "./useSync";
|
||||
export { useAuth } from "./useAuth";
|
||||
export { useUserProfile, storeAccessToken, clearAccessToken } from "./useUserProfile";
|
||||
112
lib/hooks/useAuth.ts
Normal file
112
lib/hooks/useAuth.ts
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
/**
|
||||
* Authentication hook and utilities.
|
||||
*
|
||||
* NOTE: Auth is currently disabled (using ConvexProvider instead of ConvexAuthProvider).
|
||||
* This hook provides a stub implementation until Zitadel is configured.
|
||||
*/
|
||||
|
||||
import { useState, useCallback } from "react";
|
||||
|
||||
/**
|
||||
* Hook for authentication state and actions.
|
||||
* Currently returns unauthenticated state - enable ConvexAuthProvider when Zitadel is ready.
|
||||
*/
|
||||
export function useAuth() {
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const handleSignIn = useCallback(async () => {
|
||||
setError("Authentication not configured. Set up Zitadel environment variables.");
|
||||
}, []);
|
||||
|
||||
const handleSignOut = useCallback(async () => {
|
||||
// No-op when not authenticated
|
||||
}, []);
|
||||
|
||||
return {
|
||||
isAuthenticated: false,
|
||||
isLoading: false,
|
||||
error,
|
||||
signIn: handleSignIn,
|
||||
signOut: handleSignOut,
|
||||
clearError: () => setError(null),
|
||||
};
|
||||
}
|
||||
|
||||
// Full implementation for when ConvexAuthProvider is enabled:
|
||||
/*
|
||||
import { useConvexAuth } from "convex/react";
|
||||
import { useAuthActions } from "@convex-dev/auth/react";
|
||||
import { makeRedirectUri } from "expo-auth-session";
|
||||
import { openAuthSessionAsync } from "expo-web-browser";
|
||||
import { Platform } from "react-native";
|
||||
|
||||
const redirectUri = makeRedirectUri({
|
||||
scheme: "scry",
|
||||
path: "auth",
|
||||
});
|
||||
|
||||
export function useAuth() {
|
||||
const { isAuthenticated, isLoading } = useConvexAuth();
|
||||
const { signIn, signOut } = useAuthActions();
|
||||
const [isSigningIn, setIsSigningIn] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const handleSignIn = useCallback(async () => {
|
||||
setIsSigningIn(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
const result = await signIn("zitadel", { redirectTo: redirectUri });
|
||||
|
||||
if (Platform.OS === "web") {
|
||||
return;
|
||||
}
|
||||
|
||||
if (result?.redirect) {
|
||||
const authResult = await openAuthSessionAsync(
|
||||
result.redirect.toString(),
|
||||
redirectUri,
|
||||
{
|
||||
showInRecents: true,
|
||||
preferEphemeralSession: false,
|
||||
}
|
||||
);
|
||||
|
||||
if (authResult.type === "success") {
|
||||
const url = new URL(authResult.url);
|
||||
const code = url.searchParams.get("code");
|
||||
|
||||
if (code) {
|
||||
await signIn("zitadel", { code });
|
||||
}
|
||||
} else if (authResult.type === "cancel") {
|
||||
setError("Sign-in was cancelled");
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Sign-in error:", err);
|
||||
setError(err instanceof Error ? err.message : "Sign-in failed");
|
||||
} finally {
|
||||
setIsSigningIn(false);
|
||||
}
|
||||
}, [signIn]);
|
||||
|
||||
const handleSignOut = useCallback(async () => {
|
||||
try {
|
||||
await signOut();
|
||||
} catch (err) {
|
||||
console.error("Sign-out error:", err);
|
||||
setError(err instanceof Error ? err.message : "Sign-out failed");
|
||||
}
|
||||
}, [signOut]);
|
||||
|
||||
return {
|
||||
isAuthenticated,
|
||||
isLoading: isLoading || isSigningIn,
|
||||
error,
|
||||
signIn: handleSignIn,
|
||||
signOut: handleSignOut,
|
||||
clearError: () => setError(null),
|
||||
};
|
||||
}
|
||||
*/
|
||||
116
lib/hooks/useCamera.ts
Normal file
116
lib/hooks/useCamera.ts
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
/**
|
||||
* Hook for camera permissions.
|
||||
* Supports both expo-camera (Expo Go) and react-native-vision-camera (production).
|
||||
*/
|
||||
|
||||
import { useState, useEffect, useRef } from "react";
|
||||
import { Platform, Linking, Alert } from "react-native";
|
||||
import Constants from "expo-constants";
|
||||
|
||||
// Detect if running in Expo Go
|
||||
const isExpoGo = Constants.appOwnership === "expo";
|
||||
|
||||
export interface CameraPermissionState {
|
||||
hasPermission: boolean;
|
||||
isLoading: boolean;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook for camera permissions that works in both Expo Go and production builds.
|
||||
*/
|
||||
export function useCameraPermission(): CameraPermissionState & { requestPermission: () => Promise<void> } {
|
||||
const [hasPermission, setHasPermission] = useState(false);
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
checkPermission();
|
||||
}, []);
|
||||
|
||||
const checkPermission = async () => {
|
||||
try {
|
||||
if (isExpoGo) {
|
||||
// Use expo-camera
|
||||
const { Camera } = await import("expo-camera");
|
||||
const { status } = await Camera.getCameraPermissionsAsync();
|
||||
setHasPermission(status === "granted");
|
||||
} else {
|
||||
// Use react-native-vision-camera
|
||||
const { Camera } = await import("react-native-vision-camera");
|
||||
const status = await Camera.getCameraPermissionStatus();
|
||||
setHasPermission(status === "granted");
|
||||
}
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : "Failed to check camera permission");
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const requestPermission = async () => {
|
||||
try {
|
||||
setIsLoading(true);
|
||||
setError(null);
|
||||
|
||||
if (isExpoGo) {
|
||||
// Use expo-camera
|
||||
const { Camera } = await import("expo-camera");
|
||||
const { status, canAskAgain } = await Camera.requestCameraPermissionsAsync();
|
||||
|
||||
if (status === "granted") {
|
||||
setHasPermission(true);
|
||||
} else {
|
||||
if (!canAskAgain) {
|
||||
showSettingsAlert();
|
||||
}
|
||||
setError("Camera permission denied");
|
||||
}
|
||||
} else {
|
||||
// Use react-native-vision-camera
|
||||
const { Camera } = await import("react-native-vision-camera");
|
||||
const status = await Camera.requestCameraPermission();
|
||||
|
||||
if (status === "granted") {
|
||||
setHasPermission(true);
|
||||
} else if (status === "denied") {
|
||||
showSettingsAlert();
|
||||
setError("Camera permission denied");
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : "Failed to request camera permission");
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
return { hasPermission, isLoading, error, requestPermission };
|
||||
}
|
||||
|
||||
function showSettingsAlert() {
|
||||
Alert.alert(
|
||||
"Camera Permission Required",
|
||||
"Scry needs camera access to scan cards. Please enable it in Settings.",
|
||||
[
|
||||
{ text: "Cancel", style: "cancel" },
|
||||
{
|
||||
text: "Open Settings",
|
||||
onPress: () => {
|
||||
if (Platform.OS === "ios") {
|
||||
Linking.openURL("app-settings:");
|
||||
} else {
|
||||
Linking.openSettings();
|
||||
}
|
||||
},
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if using expo-camera (Expo Go), false for Vision Camera (production).
|
||||
*/
|
||||
export function useIsExpoGo(): boolean {
|
||||
return isExpoGo;
|
||||
}
|
||||
101
lib/hooks/useConvex.ts
Normal file
101
lib/hooks/useConvex.ts
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
/**
|
||||
* Hooks for Convex data access.
|
||||
*/
|
||||
|
||||
import { useQuery, useMutation } from "convex/react";
|
||||
import { api } from "../../convex/_generated/api";
|
||||
import { useHashCache } from "../context";
|
||||
import { useEffect } from "react";
|
||||
import type { CardHashEntry } from "../recognition";
|
||||
import type { Id } from "../../convex/_generated/dataModel";
|
||||
|
||||
type User = { _id: Id<"users"> } | null | undefined;
|
||||
|
||||
/**
|
||||
* Hook to get the current authenticated user.
|
||||
* Returns unauthenticated state when auth is not configured.
|
||||
*/
|
||||
export function useCurrentUser(): { user: User; isLoading: boolean; isAuthenticated: boolean } {
|
||||
// When using ConvexProvider (no auth), just return unauthenticated state
|
||||
// Switch to ConvexAuthProvider + useConvexAuth when Zitadel is configured
|
||||
return {
|
||||
user: null,
|
||||
isLoading: false,
|
||||
isAuthenticated: false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook to fetch and cache card hashes from Convex.
|
||||
*/
|
||||
export function useCardHashes() {
|
||||
const { setCardHashes, hashesLoaded, lastHashSync, cardHashes } = useHashCache();
|
||||
|
||||
// Fetch all cards from Convex
|
||||
const cards = useQuery(api.cards.list);
|
||||
|
||||
useEffect(() => {
|
||||
if (cards) {
|
||||
// Convert Convex cards to CardHashEntry format
|
||||
const entries: CardHashEntry[] = cards.map((card) => ({
|
||||
id: card.scryfallId,
|
||||
name: card.name,
|
||||
setCode: card.setCode,
|
||||
collectorNumber: card.collectorNumber,
|
||||
imageUri: card.imageUri,
|
||||
hash: new Uint8Array(card.hash),
|
||||
}));
|
||||
|
||||
setCardHashes(entries);
|
||||
}
|
||||
}, [cards, setCardHashes]);
|
||||
|
||||
return {
|
||||
loading: cards === undefined && !hashesLoaded,
|
||||
loaded: hashesLoaded,
|
||||
count: cardHashes.length,
|
||||
lastSync: lastHashSync,
|
||||
hashes: cardHashes,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook to get card count.
|
||||
*/
|
||||
export function useCardCount() {
|
||||
const count = useQuery(api.cards.count);
|
||||
return count ?? 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook to search cards by name.
|
||||
*/
|
||||
export function useSearchCards(query: string) {
|
||||
const results = useQuery(
|
||||
api.cards.searchByName,
|
||||
query.length >= 2 ? { name: query } : "skip"
|
||||
);
|
||||
return results ?? [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Hook to manage collection.
|
||||
*/
|
||||
export function useCollection(userId?: string) {
|
||||
const collection = useQuery(
|
||||
api.collections.getByUser,
|
||||
userId ? { userId: userId as any } : "skip"
|
||||
);
|
||||
|
||||
const addCard = useMutation(api.collections.add);
|
||||
const removeCard = useMutation(api.collections.remove);
|
||||
const updateQuantity = useMutation(api.collections.updateQuantity);
|
||||
|
||||
return {
|
||||
collection: collection ?? [],
|
||||
addCard,
|
||||
removeCard,
|
||||
updateQuantity,
|
||||
loading: collection === undefined,
|
||||
};
|
||||
}
|
||||
102
lib/hooks/useSync.ts
Normal file
102
lib/hooks/useSync.ts
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
/**
|
||||
* Hook for managing offline sync with local SQLite cache.
|
||||
*/
|
||||
|
||||
import { useEffect, useState, useCallback, useRef } from "react";
|
||||
import { useConvex } from "convex/react";
|
||||
import { createSyncService, type SyncService, type SyncStatus } from "../db/syncService";
|
||||
import { useHashCache } from "../context";
|
||||
|
||||
/**
|
||||
* Hook to manage sync service lifecycle and status.
|
||||
* Initializes local database, syncs with Convex, and provides hashes for recognition.
|
||||
*/
|
||||
export function useSync() {
|
||||
const convex = useConvex();
|
||||
const syncServiceRef = useRef<SyncService | null>(null);
|
||||
const [status, setStatus] = useState<SyncStatus>({
|
||||
isInitialized: false,
|
||||
isSyncing: false,
|
||||
lastSync: 0,
|
||||
localCardCount: 0,
|
||||
});
|
||||
|
||||
const { setCardHashes } = useHashCache();
|
||||
|
||||
// Initialize sync service
|
||||
useEffect(() => {
|
||||
if (!convex || syncServiceRef.current) return;
|
||||
|
||||
const service = createSyncService(convex);
|
||||
syncServiceRef.current = service;
|
||||
|
||||
// Subscribe to status changes
|
||||
const unsubscribe = service.onStatusChange(setStatus);
|
||||
|
||||
// Initialize and load cached hashes
|
||||
const init = async () => {
|
||||
await service.initialize();
|
||||
|
||||
// Load cached hashes into app store
|
||||
const hashes = await service.getHashes();
|
||||
if (hashes.length > 0) {
|
||||
setCardHashes(hashes);
|
||||
console.log(`[useSync] Loaded ${hashes.length} cached hashes`);
|
||||
}
|
||||
|
||||
// Start background sync
|
||||
service.sync().then(async () => {
|
||||
// Reload hashes after sync
|
||||
const updatedHashes = await service.getHashes();
|
||||
setCardHashes(updatedHashes);
|
||||
});
|
||||
};
|
||||
|
||||
init();
|
||||
|
||||
return () => {
|
||||
unsubscribe();
|
||||
};
|
||||
}, [convex, setCardHashes]);
|
||||
|
||||
// Manual sync trigger
|
||||
const sync = useCallback(async () => {
|
||||
if (!syncServiceRef.current) return;
|
||||
|
||||
await syncServiceRef.current.sync();
|
||||
|
||||
// Reload hashes after sync
|
||||
const hashes = await syncServiceRef.current.getHashes();
|
||||
setCardHashes(hashes);
|
||||
}, [setCardHashes]);
|
||||
|
||||
// Clear local cache
|
||||
const clearCache = useCallback(async () => {
|
||||
if (!syncServiceRef.current) return;
|
||||
|
||||
await syncServiceRef.current.clearLocalCache();
|
||||
setCardHashes([]);
|
||||
}, [setCardHashes]);
|
||||
|
||||
// Get current hashes from cache
|
||||
const getHashes = useCallback(async () => {
|
||||
if (!syncServiceRef.current) return [];
|
||||
return syncServiceRef.current.getHashes();
|
||||
}, []);
|
||||
|
||||
return {
|
||||
...status,
|
||||
sync,
|
||||
clearCache,
|
||||
getHashes,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Context-free sync initialization for use in _layout.tsx
|
||||
* Returns a component that initializes sync when mounted.
|
||||
*/
|
||||
export function SyncInitializer() {
|
||||
useSync();
|
||||
return null;
|
||||
}
|
||||
115
lib/hooks/useUserProfile.ts
Normal file
115
lib/hooks/useUserProfile.ts
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
/**
|
||||
* Hook for fetching user profile from Zitadel OIDC userinfo endpoint.
|
||||
*
|
||||
* GDPR Compliance: User profile data is never stored in our database.
|
||||
* This hook fetches it directly from Zitadel when needed for display.
|
||||
*/
|
||||
|
||||
import { useState, useEffect, useCallback } from "react";
|
||||
import * as SecureStore from "expo-secure-store";
|
||||
|
||||
interface UserProfile {
|
||||
sub: string;
|
||||
name?: string;
|
||||
preferredUsername?: string;
|
||||
email?: string;
|
||||
emailVerified?: boolean;
|
||||
picture?: string;
|
||||
}
|
||||
|
||||
const ACCESS_TOKEN_KEY = "zitadel_access_token";
|
||||
|
||||
/**
|
||||
* Hook to fetch and cache user profile from Zitadel userinfo endpoint.
|
||||
* Profile is held in memory only - never persisted to our database.
|
||||
*/
|
||||
export function useUserProfile() {
|
||||
const [profile, setProfile] = useState<UserProfile | null>(null);
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
const fetchProfile = useCallback(async () => {
|
||||
setIsLoading(true);
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
// Get stored access token
|
||||
const accessToken = await SecureStore.getItemAsync(ACCESS_TOKEN_KEY);
|
||||
|
||||
if (!accessToken) {
|
||||
setProfile(null);
|
||||
return null;
|
||||
}
|
||||
|
||||
const issuer = process.env.EXPO_PUBLIC_ZITADEL_ISSUER;
|
||||
if (!issuer) {
|
||||
throw new Error("ZITADEL_ISSUER not configured");
|
||||
}
|
||||
|
||||
// Fetch from OIDC userinfo endpoint
|
||||
const response = await fetch(`${issuer}/oidc/v1/userinfo`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 401) {
|
||||
// Token expired - clear it
|
||||
await SecureStore.deleteItemAsync(ACCESS_TOKEN_KEY);
|
||||
setProfile(null);
|
||||
return null;
|
||||
}
|
||||
throw new Error(`Failed to fetch user profile: ${response.status}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
const userProfile: UserProfile = {
|
||||
sub: data.sub,
|
||||
name: data.name,
|
||||
preferredUsername: data.preferred_username,
|
||||
email: data.email,
|
||||
emailVerified: data.email_verified,
|
||||
picture: data.picture,
|
||||
};
|
||||
|
||||
setProfile(userProfile);
|
||||
return userProfile;
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : "Failed to fetch profile";
|
||||
setError(message);
|
||||
console.error("[useUserProfile]", message);
|
||||
return null;
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
const clearProfile = useCallback(() => {
|
||||
setProfile(null);
|
||||
setError(null);
|
||||
}, []);
|
||||
|
||||
return {
|
||||
profile,
|
||||
isLoading,
|
||||
error,
|
||||
fetchProfile,
|
||||
clearProfile,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Store access token after successful auth (called from useAuth).
|
||||
*/
|
||||
export async function storeAccessToken(token: string): Promise<void> {
|
||||
await SecureStore.setItemAsync(ACCESS_TOKEN_KEY, token);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear access token on sign out.
|
||||
*/
|
||||
export async function clearAccessToken(): Promise<void> {
|
||||
await SecureStore.deleteItemAsync(ACCESS_TOKEN_KEY);
|
||||
}
|
||||
492
lib/recognition/cardDetector.ts
Normal file
492
lib/recognition/cardDetector.ts
Normal file
|
|
@ -0,0 +1,492 @@
|
|||
/**
|
||||
* Card detection using edge detection and contour analysis.
|
||||
* Detects card boundaries in images and returns the four corner points.
|
||||
*
|
||||
* This is a pure TypeScript implementation for when OpenCV is not available.
|
||||
*/
|
||||
|
||||
import {
|
||||
Point,
|
||||
toGrayscale,
|
||||
gaussianBlur,
|
||||
distance,
|
||||
crossProduct,
|
||||
pointToLineDistance,
|
||||
} from "./imageUtils";
|
||||
|
||||
/** Standard MTG card aspect ratio (height / width). Cards are 63mm x 88mm. */
|
||||
const CARD_ASPECT_RATIO = 88 / 63; // ~1.397
|
||||
const ASPECT_RATIO_TOLERANCE = 0.25;
|
||||
const MIN_CARD_AREA_RATIO = 0.05;
|
||||
const MAX_CARD_AREA_RATIO = 0.98;
|
||||
|
||||
export interface CardDetectionResult {
|
||||
found: boolean;
|
||||
corners: Point[];
|
||||
confidence: number;
|
||||
debugMessage?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect a card in the image and return its corner points.
|
||||
*/
|
||||
export function detectCard(
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
width: number,
|
||||
height: number
|
||||
): CardDetectionResult {
|
||||
// Step 1: Convert to grayscale
|
||||
const grayscale = toGrayscale(pixels);
|
||||
|
||||
// Step 2: Apply Gaussian blur to reduce noise
|
||||
const blurred = gaussianBlur(grayscale, width, height, 5);
|
||||
|
||||
// Step 3: Apply Canny edge detection
|
||||
const edges = applyCannyEdgeDetection(blurred, width, height, 50, 150);
|
||||
|
||||
// Step 4: Find contours
|
||||
const contours = findContours(edges, width, height);
|
||||
|
||||
if (contours.length === 0) {
|
||||
return { found: false, corners: [], confidence: 0, debugMessage: "No contours found" };
|
||||
}
|
||||
|
||||
// Step 5: Find the best card-like quadrilateral
|
||||
const imageArea = width * height;
|
||||
const bestQuad = findBestCardQuadrilateral(contours, imageArea);
|
||||
|
||||
if (!bestQuad) {
|
||||
return { found: false, corners: [], confidence: 0, debugMessage: "No card-like quadrilateral found" };
|
||||
}
|
||||
|
||||
// Step 6: Order corners consistently
|
||||
const orderedCorners = orderCorners(bestQuad);
|
||||
|
||||
// Calculate confidence
|
||||
const confidence = calculateConfidence(orderedCorners, imageArea);
|
||||
|
||||
return { found: true, corners: orderedCorners, confidence };
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply Canny edge detection.
|
||||
*/
|
||||
function applyCannyEdgeDetection(
|
||||
gray: Uint8Array,
|
||||
width: number,
|
||||
height: number,
|
||||
lowThreshold: number,
|
||||
highThreshold: number
|
||||
): Uint8Array {
|
||||
// Step 1: Compute gradients using Sobel operators
|
||||
const gradientX = new Float32Array(width * height);
|
||||
const gradientY = new Float32Array(width * height);
|
||||
const magnitude = new Float32Array(width * height);
|
||||
const direction = new Float32Array(width * height);
|
||||
|
||||
// Sobel kernels
|
||||
const sobelX = [[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]];
|
||||
const sobelY = [[-1, -2, -1], [0, 0, 0], [1, 2, 1]];
|
||||
|
||||
for (let y = 1; y < height - 1; y++) {
|
||||
for (let x = 1; x < width - 1; x++) {
|
||||
let gx = 0, gy = 0;
|
||||
|
||||
for (let ky = -1; ky <= 1; ky++) {
|
||||
for (let kx = -1; kx <= 1; kx++) {
|
||||
const pixel = gray[(y + ky) * width + (x + kx)];
|
||||
gx += pixel * sobelX[ky + 1][kx + 1];
|
||||
gy += pixel * sobelY[ky + 1][kx + 1];
|
||||
}
|
||||
}
|
||||
|
||||
const idx = y * width + x;
|
||||
gradientX[idx] = gx;
|
||||
gradientY[idx] = gy;
|
||||
magnitude[idx] = Math.sqrt(gx * gx + gy * gy);
|
||||
direction[idx] = Math.atan2(gy, gx);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: Non-maximum suppression
|
||||
const suppressed = new Float32Array(width * height);
|
||||
|
||||
for (let y = 1; y < height - 1; y++) {
|
||||
for (let x = 1; x < width - 1; x++) {
|
||||
const idx = y * width + x;
|
||||
let angle = direction[idx] * 180 / Math.PI;
|
||||
if (angle < 0) angle += 180;
|
||||
|
||||
let neighbor1: number, neighbor2: number;
|
||||
|
||||
if (angle < 22.5 || angle >= 157.5) {
|
||||
neighbor1 = magnitude[y * width + (x - 1)];
|
||||
neighbor2 = magnitude[y * width + (x + 1)];
|
||||
} else if (angle >= 22.5 && angle < 67.5) {
|
||||
neighbor1 = magnitude[(y - 1) * width + (x + 1)];
|
||||
neighbor2 = magnitude[(y + 1) * width + (x - 1)];
|
||||
} else if (angle >= 67.5 && angle < 112.5) {
|
||||
neighbor1 = magnitude[(y - 1) * width + x];
|
||||
neighbor2 = magnitude[(y + 1) * width + x];
|
||||
} else {
|
||||
neighbor1 = magnitude[(y - 1) * width + (x - 1)];
|
||||
neighbor2 = magnitude[(y + 1) * width + (x + 1)];
|
||||
}
|
||||
|
||||
if (magnitude[idx] >= neighbor1 && magnitude[idx] >= neighbor2) {
|
||||
suppressed[idx] = magnitude[idx];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3: Double thresholding and edge tracking
|
||||
const result = new Uint8Array(width * height);
|
||||
const strong = new Uint8Array(width * height);
|
||||
const weak = new Uint8Array(width * height);
|
||||
|
||||
for (let i = 0; i < width * height; i++) {
|
||||
if (suppressed[i] >= highThreshold) {
|
||||
strong[i] = 1;
|
||||
} else if (suppressed[i] >= lowThreshold) {
|
||||
weak[i] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Edge tracking by hysteresis
|
||||
for (let y = 1; y < height - 1; y++) {
|
||||
for (let x = 1; x < width - 1; x++) {
|
||||
const idx = y * width + x;
|
||||
|
||||
if (strong[idx]) {
|
||||
result[idx] = 255;
|
||||
} else if (weak[idx]) {
|
||||
// Check if connected to strong edge
|
||||
outer: for (let dy = -1; dy <= 1; dy++) {
|
||||
for (let dx = -1; dx <= 1; dx++) {
|
||||
if (strong[(y + dy) * width + (x + dx)]) {
|
||||
result[idx] = 255;
|
||||
break outer;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find contours in a binary edge image using flood fill.
|
||||
*/
|
||||
function findContours(
|
||||
edges: Uint8Array,
|
||||
width: number,
|
||||
height: number
|
||||
): Point[][] {
|
||||
const visited = new Uint8Array(width * height);
|
||||
const contours: Point[][] = [];
|
||||
|
||||
const dx = [-1, 0, 1, 1, 1, 0, -1, -1];
|
||||
const dy = [-1, -1, -1, 0, 1, 1, 1, 0];
|
||||
|
||||
for (let y = 1; y < height - 1; y++) {
|
||||
for (let x = 1; x < width - 1; x++) {
|
||||
const idx = y * width + x;
|
||||
if (visited[idx]) continue;
|
||||
if (edges[idx] < 128) continue;
|
||||
|
||||
// Trace contour using BFS
|
||||
const contour: Point[] = [];
|
||||
const queue: Array<{ x: number; y: number }> = [{ x, y }];
|
||||
|
||||
while (queue.length > 0 && contour.length < 10000) {
|
||||
const point = queue.shift()!;
|
||||
const pidx = point.y * width + point.x;
|
||||
|
||||
if (point.x < 0 || point.x >= width || point.y < 0 || point.y >= height) continue;
|
||||
if (visited[pidx]) continue;
|
||||
if (edges[pidx] < 128) continue;
|
||||
|
||||
visited[pidx] = 1;
|
||||
contour.push(point);
|
||||
|
||||
for (let i = 0; i < 8; i++) {
|
||||
queue.push({ x: point.x + dx[i], y: point.y + dy[i] });
|
||||
}
|
||||
}
|
||||
|
||||
if (contour.length >= 4) {
|
||||
contours.push(contour);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return contours;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the best quadrilateral that matches a card shape.
|
||||
*/
|
||||
function findBestCardQuadrilateral(
|
||||
contours: Point[][],
|
||||
imageArea: number
|
||||
): Point[] | null {
|
||||
let bestQuad: Point[] | null = null;
|
||||
let bestScore = -Infinity;
|
||||
|
||||
for (const contour of contours) {
|
||||
// Simplify contour using Douglas-Peucker
|
||||
const simplified = simplifyContour(contour, contour.length * 0.02);
|
||||
|
||||
// Try to approximate as quadrilateral
|
||||
const quad = approximateQuadrilateral(simplified);
|
||||
if (!quad) continue;
|
||||
|
||||
// Check if valid card shape
|
||||
const area = calculateQuadArea(quad);
|
||||
const areaRatio = area / imageArea;
|
||||
|
||||
if (areaRatio < MIN_CARD_AREA_RATIO || areaRatio > MAX_CARD_AREA_RATIO) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check aspect ratio
|
||||
const aspectScore = calculateAspectRatioScore(quad);
|
||||
if (aspectScore < 0.5) continue;
|
||||
|
||||
// Check convexity
|
||||
if (!isConvex(quad)) continue;
|
||||
|
||||
// Score based on area and aspect ratio
|
||||
const score = areaRatio * aspectScore;
|
||||
|
||||
if (score > bestScore) {
|
||||
bestScore = score;
|
||||
bestQuad = quad;
|
||||
}
|
||||
}
|
||||
|
||||
return bestQuad;
|
||||
}
|
||||
|
||||
/**
|
||||
* Simplify a contour using Douglas-Peucker algorithm.
|
||||
*/
|
||||
function simplifyContour(contour: Point[], epsilon: number): Point[] {
|
||||
if (contour.length < 3) return contour;
|
||||
|
||||
const first = contour[0];
|
||||
const last = contour[contour.length - 1];
|
||||
|
||||
let maxDist = 0;
|
||||
let maxIndex = 0;
|
||||
|
||||
for (let i = 1; i < contour.length - 1; i++) {
|
||||
const dist = pointToLineDistance(contour[i], first, last);
|
||||
if (dist > maxDist) {
|
||||
maxDist = dist;
|
||||
maxIndex = i;
|
||||
}
|
||||
}
|
||||
|
||||
if (maxDist > epsilon) {
|
||||
const left = simplifyContour(contour.slice(0, maxIndex + 1), epsilon);
|
||||
const right = simplifyContour(contour.slice(maxIndex), epsilon);
|
||||
return [...left.slice(0, -1), ...right];
|
||||
}
|
||||
|
||||
return [first, last];
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to approximate a contour as a quadrilateral.
|
||||
*/
|
||||
function approximateQuadrilateral(contour: Point[]): Point[] | null {
|
||||
if (contour.length < 4) return null;
|
||||
if (contour.length === 4) return contour;
|
||||
|
||||
// Find convex hull
|
||||
const hull = convexHull(contour);
|
||||
if (hull.length < 4) return null;
|
||||
if (hull.length === 4) return hull;
|
||||
|
||||
// Find 4 extreme points
|
||||
return findExtremePoints(hull);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate convex hull using Graham scan.
|
||||
*/
|
||||
function convexHull(points: Point[]): Point[] {
|
||||
if (points.length < 3) return points;
|
||||
|
||||
// Find bottom-most point
|
||||
const start = points.reduce((min, p) =>
|
||||
p.y < min.y || (p.y === min.y && p.x < min.x) ? p : min
|
||||
);
|
||||
|
||||
// Sort by polar angle
|
||||
const sorted = points
|
||||
.filter(p => p !== start)
|
||||
.sort((a, b) => {
|
||||
const angleA = Math.atan2(a.y - start.y, a.x - start.x);
|
||||
const angleB = Math.atan2(b.y - start.y, b.x - start.x);
|
||||
if (angleA !== angleB) return angleA - angleB;
|
||||
return distance(a, start) - distance(b, start);
|
||||
});
|
||||
|
||||
const hull: Point[] = [start];
|
||||
|
||||
for (const point of sorted) {
|
||||
while (hull.length > 1 && crossProduct(hull[hull.length - 2], hull[hull.length - 1], point) <= 0) {
|
||||
hull.pop();
|
||||
}
|
||||
hull.push(point);
|
||||
}
|
||||
|
||||
return hull;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find 4 extreme points of a convex hull.
|
||||
*/
|
||||
function findExtremePoints(hull: Point[]): Point[] {
|
||||
if (hull.length <= 4) return hull;
|
||||
|
||||
const minX = hull.reduce((min, p) => p.x < min.x ? p : min, hull[0]);
|
||||
const maxX = hull.reduce((max, p) => p.x > max.x ? p : max, hull[0]);
|
||||
const minY = hull.reduce((min, p) => p.y < min.y ? p : min, hull[0]);
|
||||
const maxY = hull.reduce((max, p) => p.y > max.y ? p : max, hull[0]);
|
||||
|
||||
const extremes = new Set([minX, maxX, minY, maxY]);
|
||||
|
||||
if (extremes.size === 4) {
|
||||
return Array.from(extremes);
|
||||
}
|
||||
|
||||
// Fallback: take points at regular angular intervals
|
||||
const centerX = hull.reduce((s, p) => s + p.x, 0) / hull.length;
|
||||
const centerY = hull.reduce((s, p) => s + p.y, 0) / hull.length;
|
||||
|
||||
const sorted = [...hull].sort((a, b) =>
|
||||
Math.atan2(a.y - centerY, a.x - centerX) -
|
||||
Math.atan2(b.y - centerY, b.x - centerX)
|
||||
);
|
||||
|
||||
const step = Math.floor(sorted.length / 4);
|
||||
return [sorted[0], sorted[step], sorted[step * 2], sorted[step * 3]];
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate area of a quadrilateral using shoelace formula.
|
||||
*/
|
||||
function calculateQuadArea(quad: Point[]): number {
|
||||
let area = 0;
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const j = (i + 1) % 4;
|
||||
area += quad[i].x * quad[j].y;
|
||||
area -= quad[j].x * quad[i].y;
|
||||
}
|
||||
return Math.abs(area) / 2;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate how well the aspect ratio matches a card.
|
||||
*/
|
||||
function calculateAspectRatioScore(quad: Point[]): number {
|
||||
const width1 = distance(quad[0], quad[1]);
|
||||
const width2 = distance(quad[2], quad[3]);
|
||||
const height1 = distance(quad[1], quad[2]);
|
||||
const height2 = distance(quad[3], quad[0]);
|
||||
|
||||
const avgWidth = (width1 + width2) / 2;
|
||||
const avgHeight = (height1 + height2) / 2;
|
||||
|
||||
// Ensure we get the right ratio regardless of orientation
|
||||
const aspectRatio = avgWidth > avgHeight
|
||||
? avgWidth / avgHeight
|
||||
: avgHeight / avgWidth;
|
||||
|
||||
const deviation = Math.abs(aspectRatio - CARD_ASPECT_RATIO) / CARD_ASPECT_RATIO;
|
||||
|
||||
return Math.max(0, 1 - deviation / ASPECT_RATIO_TOLERANCE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a quadrilateral is convex.
|
||||
*/
|
||||
function isConvex(quad: Point[]): boolean {
|
||||
let sign = 0;
|
||||
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const cross = crossProduct(
|
||||
quad[i],
|
||||
quad[(i + 1) % 4],
|
||||
quad[(i + 2) % 4]
|
||||
);
|
||||
|
||||
if (Math.abs(cross) < 0.0001) continue;
|
||||
|
||||
const currentSign = cross > 0 ? 1 : -1;
|
||||
|
||||
if (sign === 0) {
|
||||
sign = currentSign;
|
||||
} else if (sign !== currentSign) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Order corners consistently: top-left, top-right, bottom-right, bottom-left.
|
||||
*/
|
||||
function orderCorners(corners: Point[]): Point[] {
|
||||
const centerX = corners.reduce((s, c) => s + c.x, 0) / 4;
|
||||
const centerY = corners.reduce((s, c) => s + c.y, 0) / 4;
|
||||
|
||||
const topLeft = corners.filter(c => c.x < centerX && c.y < centerY)
|
||||
.sort((a, b) => (a.x + a.y) - (b.x + b.y))[0];
|
||||
const topRight = corners.filter(c => c.x >= centerX && c.y < centerY)
|
||||
.sort((a, b) => (a.y - a.x) - (b.y - b.x))[0];
|
||||
const bottomRight = corners.filter(c => c.x >= centerX && c.y >= centerY)
|
||||
.sort((a, b) => (b.x + b.y) - (a.x + a.y))[0];
|
||||
const bottomLeft = corners.filter(c => c.x < centerX && c.y >= centerY)
|
||||
.sort((a, b) => (b.y - b.x) - (a.y - a.x))[0];
|
||||
|
||||
// Handle edge cases by sorting by angle
|
||||
if (!topLeft || !topRight || !bottomRight || !bottomLeft) {
|
||||
const sorted = [...corners].sort((a, b) =>
|
||||
Math.atan2(a.y - centerY, a.x - centerX) -
|
||||
Math.atan2(b.y - centerY, b.x - centerX)
|
||||
);
|
||||
|
||||
// Find index of point with minimum sum (top-left)
|
||||
let minSumIdx = 0;
|
||||
let minSum = Infinity;
|
||||
sorted.forEach((c, i) => {
|
||||
const sum = c.x + c.y;
|
||||
if (sum < minSum) {
|
||||
minSum = sum;
|
||||
minSumIdx = i;
|
||||
}
|
||||
});
|
||||
|
||||
return [...sorted.slice(minSumIdx), ...sorted.slice(0, minSumIdx)];
|
||||
}
|
||||
|
||||
return [topLeft, topRight, bottomRight, bottomLeft];
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate confidence of the detection.
|
||||
*/
|
||||
function calculateConfidence(corners: Point[], imageArea: number): number {
|
||||
const area = calculateQuadArea(corners);
|
||||
const areaScore = Math.min(area / imageArea / 0.5, 1);
|
||||
const aspectScore = calculateAspectRatioScore(corners);
|
||||
|
||||
return areaScore * 0.4 + aspectScore * 0.6;
|
||||
}
|
||||
278
lib/recognition/clahe.ts
Normal file
278
lib/recognition/clahe.ts
Normal file
|
|
@ -0,0 +1,278 @@
|
|||
/**
|
||||
* CLAHE (Contrast Limited Adaptive Histogram Equalization) implementation.
|
||||
*
|
||||
* This is a pure TypeScript implementation for when OpenCV is not available.
|
||||
* For better performance, use OpenCV's createCLAHE when available.
|
||||
*/
|
||||
|
||||
interface CLAHEOptions {
|
||||
clipLimit?: number;
|
||||
tileGridSize?: { width: number; height: number };
|
||||
}
|
||||
|
||||
const DEFAULT_OPTIONS: Required<CLAHEOptions> = {
|
||||
clipLimit: 2.0,
|
||||
tileGridSize: { width: 8, height: 8 },
|
||||
};
|
||||
|
||||
/**
|
||||
* Convert RGB to LAB color space.
|
||||
* Returns L channel (0-100 range, scaled to 0-255 for processing).
|
||||
*/
|
||||
function rgbToLab(r: number, g: number, b: number): { l: number; a: number; b: number } {
|
||||
// Normalize RGB to 0-1
|
||||
let rNorm = r / 255;
|
||||
let gNorm = g / 255;
|
||||
let bNorm = b / 255;
|
||||
|
||||
// Apply gamma correction
|
||||
rNorm = rNorm > 0.04045 ? Math.pow((rNorm + 0.055) / 1.055, 2.4) : rNorm / 12.92;
|
||||
gNorm = gNorm > 0.04045 ? Math.pow((gNorm + 0.055) / 1.055, 2.4) : gNorm / 12.92;
|
||||
bNorm = bNorm > 0.04045 ? Math.pow((bNorm + 0.055) / 1.055, 2.4) : bNorm / 12.92;
|
||||
|
||||
// Convert to XYZ
|
||||
const x = (rNorm * 0.4124564 + gNorm * 0.3575761 + bNorm * 0.1804375) / 0.95047;
|
||||
const y = rNorm * 0.2126729 + gNorm * 0.7151522 + bNorm * 0.0721750;
|
||||
const z = (rNorm * 0.0193339 + gNorm * 0.1191920 + bNorm * 0.9503041) / 1.08883;
|
||||
|
||||
// Convert to LAB
|
||||
const xNorm = x > 0.008856 ? Math.pow(x, 1 / 3) : 7.787 * x + 16 / 116;
|
||||
const yNorm = y > 0.008856 ? Math.pow(y, 1 / 3) : 7.787 * y + 16 / 116;
|
||||
const zNorm = z > 0.008856 ? Math.pow(z, 1 / 3) : 7.787 * z + 16 / 116;
|
||||
|
||||
return {
|
||||
l: Math.max(0, 116 * yNorm - 16), // 0-100
|
||||
a: 500 * (xNorm - yNorm),
|
||||
b: 200 * (yNorm - zNorm),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert LAB to RGB color space.
|
||||
*/
|
||||
function labToRgb(l: number, a: number, b: number): { r: number; g: number; b: number } {
|
||||
// Convert LAB to XYZ
|
||||
const yNorm = (l + 16) / 116;
|
||||
const xNorm = a / 500 + yNorm;
|
||||
const zNorm = yNorm - b / 200;
|
||||
|
||||
const x3 = Math.pow(xNorm, 3);
|
||||
const y3 = Math.pow(yNorm, 3);
|
||||
const z3 = Math.pow(zNorm, 3);
|
||||
|
||||
const x = (x3 > 0.008856 ? x3 : (xNorm - 16 / 116) / 7.787) * 0.95047;
|
||||
const y = y3 > 0.008856 ? y3 : (yNorm - 16 / 116) / 7.787;
|
||||
const z = (z3 > 0.008856 ? z3 : (zNorm - 16 / 116) / 7.787) * 1.08883;
|
||||
|
||||
// Convert XYZ to RGB
|
||||
let rNorm = x * 3.2404542 + y * -1.5371385 + z * -0.4985314;
|
||||
let gNorm = x * -0.9692660 + y * 1.8760108 + z * 0.0415560;
|
||||
let bNorm = x * 0.0556434 + y * -0.2040259 + z * 1.0572252;
|
||||
|
||||
// Apply gamma correction
|
||||
rNorm = rNorm > 0.0031308 ? 1.055 * Math.pow(rNorm, 1 / 2.4) - 0.055 : 12.92 * rNorm;
|
||||
gNorm = gNorm > 0.0031308 ? 1.055 * Math.pow(gNorm, 1 / 2.4) - 0.055 : 12.92 * gNorm;
|
||||
bNorm = bNorm > 0.0031308 ? 1.055 * Math.pow(bNorm, 1 / 2.4) - 0.055 : 12.92 * bNorm;
|
||||
|
||||
return {
|
||||
r: Math.round(Math.max(0, Math.min(255, rNorm * 255))),
|
||||
g: Math.round(Math.max(0, Math.min(255, gNorm * 255))),
|
||||
b: Math.round(Math.max(0, Math.min(255, bNorm * 255))),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute histogram for a region.
|
||||
*/
|
||||
function computeHistogram(
|
||||
data: Uint8Array,
|
||||
startX: number,
|
||||
startY: number,
|
||||
width: number,
|
||||
height: number,
|
||||
stride: number
|
||||
): number[] {
|
||||
const histogram = new Array(256).fill(0);
|
||||
|
||||
for (let y = startY; y < startY + height; y++) {
|
||||
for (let x = startX; x < startX + width; x++) {
|
||||
const value = data[y * stride + x];
|
||||
histogram[value]++;
|
||||
}
|
||||
}
|
||||
|
||||
return histogram;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clip histogram and redistribute excess.
|
||||
*/
|
||||
function clipHistogram(histogram: number[], clipLimit: number, numPixels: number): number[] {
|
||||
const limit = Math.floor(clipLimit * numPixels / 256);
|
||||
const clipped = [...histogram];
|
||||
|
||||
let excess = 0;
|
||||
for (let i = 0; i < 256; i++) {
|
||||
if (clipped[i] > limit) {
|
||||
excess += clipped[i] - limit;
|
||||
clipped[i] = limit;
|
||||
}
|
||||
}
|
||||
|
||||
// Redistribute excess
|
||||
const increment = Math.floor(excess / 256);
|
||||
const remainder = excess % 256;
|
||||
|
||||
for (let i = 0; i < 256; i++) {
|
||||
clipped[i] += increment;
|
||||
if (i < remainder) {
|
||||
clipped[i]++;
|
||||
}
|
||||
}
|
||||
|
||||
return clipped;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create lookup table from clipped histogram.
|
||||
*/
|
||||
function createLUT(histogram: number[], numPixels: number): Uint8Array {
|
||||
const lut = new Uint8Array(256);
|
||||
let cumSum = 0;
|
||||
|
||||
for (let i = 0; i < 256; i++) {
|
||||
cumSum += histogram[i];
|
||||
lut[i] = Math.round((cumSum / numPixels) * 255);
|
||||
}
|
||||
|
||||
return lut;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bilinear interpolation between four values.
|
||||
*/
|
||||
function bilinearInterpolate(
|
||||
topLeft: number,
|
||||
topRight: number,
|
||||
bottomLeft: number,
|
||||
bottomRight: number,
|
||||
xFrac: number,
|
||||
yFrac: number
|
||||
): number {
|
||||
const top = topLeft + (topRight - topLeft) * xFrac;
|
||||
const bottom = bottomLeft + (bottomRight - bottomLeft) * xFrac;
|
||||
return Math.round(top + (bottom - top) * yFrac);
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply CLAHE to RGBA pixel data.
|
||||
* Processes only the L channel in LAB color space.
|
||||
*
|
||||
* @param pixels RGBA pixel data
|
||||
* @param width Image width
|
||||
* @param height Image height
|
||||
* @param options CLAHE options
|
||||
* @returns Processed RGBA pixel data
|
||||
*/
|
||||
export function applyCLAHE(
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
width: number,
|
||||
height: number,
|
||||
options: CLAHEOptions = {}
|
||||
): Uint8Array {
|
||||
const opts = { ...DEFAULT_OPTIONS, ...options };
|
||||
const { clipLimit, tileGridSize } = opts;
|
||||
|
||||
const tileWidth = Math.floor(width / tileGridSize.width);
|
||||
const tileHeight = Math.floor(height / tileGridSize.height);
|
||||
|
||||
// Extract L channel from LAB
|
||||
const lChannel = new Uint8Array(width * height);
|
||||
const aChannel = new Float32Array(width * height);
|
||||
const bChannel = new Float32Array(width * height);
|
||||
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
const idx = (y * width + x) * 4;
|
||||
const lab = rgbToLab(pixels[idx], pixels[idx + 1], pixels[idx + 2]);
|
||||
const pixelIdx = y * width + x;
|
||||
lChannel[pixelIdx] = Math.round((lab.l / 100) * 255);
|
||||
aChannel[pixelIdx] = lab.a;
|
||||
bChannel[pixelIdx] = lab.b;
|
||||
}
|
||||
}
|
||||
|
||||
// Compute LUTs for each tile
|
||||
const luts: Uint8Array[][] = [];
|
||||
const numPixelsPerTile = tileWidth * tileHeight;
|
||||
|
||||
for (let ty = 0; ty < tileGridSize.height; ty++) {
|
||||
luts[ty] = [];
|
||||
for (let tx = 0; tx < tileGridSize.width; tx++) {
|
||||
const startX = tx * tileWidth;
|
||||
const startY = ty * tileHeight;
|
||||
|
||||
const histogram = computeHistogram(
|
||||
lChannel,
|
||||
startX,
|
||||
startY,
|
||||
tileWidth,
|
||||
tileHeight,
|
||||
width
|
||||
);
|
||||
|
||||
const clipped = clipHistogram(histogram, clipLimit, numPixelsPerTile);
|
||||
luts[ty][tx] = createLUT(clipped, numPixelsPerTile);
|
||||
}
|
||||
}
|
||||
|
||||
// Apply CLAHE with bilinear interpolation
|
||||
const result = new Uint8Array(pixels.length);
|
||||
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
const pixelIdx = y * width + x;
|
||||
const rgbaIdx = pixelIdx * 4;
|
||||
|
||||
// Determine tile position
|
||||
const tileX = Math.min(x / tileWidth, tileGridSize.width - 1);
|
||||
const tileY = Math.min(y / tileHeight, tileGridSize.height - 1);
|
||||
|
||||
const tx = Math.floor(tileX);
|
||||
const ty = Math.floor(tileY);
|
||||
|
||||
const xFrac = tileX - tx;
|
||||
const yFrac = tileY - ty;
|
||||
|
||||
// Get surrounding tiles (handle edges)
|
||||
const tx1 = Math.min(tx + 1, tileGridSize.width - 1);
|
||||
const ty1 = Math.min(ty + 1, tileGridSize.height - 1);
|
||||
|
||||
const lValue = lChannel[pixelIdx];
|
||||
|
||||
// Interpolate LUT values
|
||||
const newL = bilinearInterpolate(
|
||||
luts[ty][tx][lValue],
|
||||
luts[ty][tx1][lValue],
|
||||
luts[ty1][tx][lValue],
|
||||
luts[ty1][tx1][lValue],
|
||||
xFrac,
|
||||
yFrac
|
||||
);
|
||||
|
||||
// Convert back to RGB
|
||||
const rgb = labToRgb(
|
||||
(newL / 255) * 100,
|
||||
aChannel[pixelIdx],
|
||||
bChannel[pixelIdx]
|
||||
);
|
||||
|
||||
result[rgbaIdx] = rgb.r;
|
||||
result[rgbaIdx + 1] = rgb.g;
|
||||
result[rgbaIdx + 2] = rgb.b;
|
||||
result[rgbaIdx + 3] = pixels[rgbaIdx + 3]; // Preserve alpha
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
95
lib/recognition/imageLoader.ts
Normal file
95
lib/recognition/imageLoader.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
/**
|
||||
* Image loader utilities for React Native.
|
||||
* Loads images from file paths and returns pixel data for recognition.
|
||||
*/
|
||||
|
||||
import * as ImageManipulator from "expo-image-manipulator";
|
||||
import { Platform } from "react-native";
|
||||
|
||||
export interface LoadedImage {
|
||||
pixels: Uint8Array;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load image from a file path and return RGBA pixel data.
|
||||
* Uses expo-image-manipulator to resize the image to a manageable size first.
|
||||
*
|
||||
* Note: expo-image-manipulator doesn't directly provide pixel data.
|
||||
* We need to use a canvas (web) or Skia (native) to decode pixels.
|
||||
* For now, this provides the resized image URI for the Skia-based decoder.
|
||||
*/
|
||||
export async function loadImageForRecognition(
|
||||
uri: string,
|
||||
targetWidth: number = 480, // Reasonable size for processing
|
||||
targetHeight: number = 640
|
||||
): Promise<{ uri: string; width: number; height: number }> {
|
||||
// Normalize the URI for the platform
|
||||
const normalizedUri =
|
||||
Platform.OS === "android" && !uri.startsWith("file://")
|
||||
? `file://${uri}`
|
||||
: uri;
|
||||
|
||||
// Resize the image for faster processing
|
||||
const result = await ImageManipulator.manipulateAsync(
|
||||
normalizedUri,
|
||||
[
|
||||
{
|
||||
resize: {
|
||||
width: targetWidth,
|
||||
height: targetHeight,
|
||||
},
|
||||
},
|
||||
],
|
||||
{
|
||||
compress: 1,
|
||||
format: ImageManipulator.SaveFormat.PNG,
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
uri: result.uri,
|
||||
width: result.width,
|
||||
height: result.height,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Load image and get base64 data.
|
||||
* Useful for passing to native modules or Skia.
|
||||
*/
|
||||
export async function loadImageAsBase64(
|
||||
uri: string,
|
||||
targetWidth: number = 480,
|
||||
targetHeight: number = 640
|
||||
): Promise<{ base64: string; width: number; height: number }> {
|
||||
// Normalize the URI
|
||||
const normalizedUri =
|
||||
Platform.OS === "android" && !uri.startsWith("file://")
|
||||
? `file://${uri}`
|
||||
: uri;
|
||||
|
||||
const result = await ImageManipulator.manipulateAsync(
|
||||
normalizedUri,
|
||||
[
|
||||
{
|
||||
resize: {
|
||||
width: targetWidth,
|
||||
height: targetHeight,
|
||||
},
|
||||
},
|
||||
],
|
||||
{
|
||||
compress: 1,
|
||||
format: ImageManipulator.SaveFormat.PNG,
|
||||
base64: true,
|
||||
}
|
||||
);
|
||||
|
||||
return {
|
||||
base64: result.base64 || "",
|
||||
width: result.width,
|
||||
height: result.height,
|
||||
};
|
||||
}
|
||||
292
lib/recognition/imageUtils.ts
Normal file
292
lib/recognition/imageUtils.ts
Normal file
|
|
@ -0,0 +1,292 @@
|
|||
/**
|
||||
* Image utility functions for the recognition pipeline.
|
||||
* Provides resize, rotation, and pixel manipulation helpers.
|
||||
*/
|
||||
|
||||
export interface Point {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
export interface Size {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resize RGBA pixel data to target dimensions using bilinear interpolation.
|
||||
*/
|
||||
export function resizeImage(
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
srcWidth: number,
|
||||
srcHeight: number,
|
||||
dstWidth: number,
|
||||
dstHeight: number
|
||||
): Uint8Array {
|
||||
const result = new Uint8Array(dstWidth * dstHeight * 4);
|
||||
|
||||
const xRatio = srcWidth / dstWidth;
|
||||
const yRatio = srcHeight / dstHeight;
|
||||
|
||||
for (let y = 0; y < dstHeight; y++) {
|
||||
for (let x = 0; x < dstWidth; x++) {
|
||||
const srcX = x * xRatio;
|
||||
const srcY = y * yRatio;
|
||||
|
||||
const x0 = Math.floor(srcX);
|
||||
const y0 = Math.floor(srcY);
|
||||
const x1 = Math.min(x0 + 1, srcWidth - 1);
|
||||
const y1 = Math.min(y0 + 1, srcHeight - 1);
|
||||
|
||||
const xFrac = srcX - x0;
|
||||
const yFrac = srcY - y0;
|
||||
|
||||
const dstIdx = (y * dstWidth + x) * 4;
|
||||
|
||||
for (let c = 0; c < 4; c++) {
|
||||
const idx00 = (y0 * srcWidth + x0) * 4 + c;
|
||||
const idx10 = (y0 * srcWidth + x1) * 4 + c;
|
||||
const idx01 = (y1 * srcWidth + x0) * 4 + c;
|
||||
const idx11 = (y1 * srcWidth + x1) * 4 + c;
|
||||
|
||||
const top = pixels[idx00] + (pixels[idx10] - pixels[idx00]) * xFrac;
|
||||
const bottom = pixels[idx01] + (pixels[idx11] - pixels[idx01]) * xFrac;
|
||||
result[dstIdx + c] = Math.round(top + (bottom - top) * yFrac);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rotate RGBA pixel data by 90, 180, or 270 degrees.
|
||||
*/
|
||||
export function rotateImage(
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
width: number,
|
||||
height: number,
|
||||
degrees: 0 | 90 | 180 | 270
|
||||
): { pixels: Uint8Array; width: number; height: number } {
|
||||
if (degrees === 0) {
|
||||
return { pixels: new Uint8Array(pixels), width, height };
|
||||
}
|
||||
|
||||
const [newWidth, newHeight] = degrees === 90 || degrees === 270
|
||||
? [height, width]
|
||||
: [width, height];
|
||||
|
||||
const result = new Uint8Array(newWidth * newHeight * 4);
|
||||
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
const srcIdx = (y * width + x) * 4;
|
||||
|
||||
let dstX: number, dstY: number;
|
||||
|
||||
switch (degrees) {
|
||||
case 90:
|
||||
dstX = height - 1 - y;
|
||||
dstY = x;
|
||||
break;
|
||||
case 180:
|
||||
dstX = width - 1 - x;
|
||||
dstY = height - 1 - y;
|
||||
break;
|
||||
case 270:
|
||||
dstX = y;
|
||||
dstY = width - 1 - x;
|
||||
break;
|
||||
default:
|
||||
dstX = x;
|
||||
dstY = y;
|
||||
}
|
||||
|
||||
const dstIdx = (dstY * newWidth + dstX) * 4;
|
||||
result[dstIdx] = pixels[srcIdx];
|
||||
result[dstIdx + 1] = pixels[srcIdx + 1];
|
||||
result[dstIdx + 2] = pixels[srcIdx + 2];
|
||||
result[dstIdx + 3] = pixels[srcIdx + 3];
|
||||
}
|
||||
}
|
||||
|
||||
return { pixels: result, width: newWidth, height: newHeight };
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert RGBA to grayscale using luminance formula.
|
||||
*/
|
||||
export function toGrayscale(pixels: Uint8Array | Uint8ClampedArray): Uint8Array {
|
||||
const numPixels = pixels.length / 4;
|
||||
const gray = new Uint8Array(numPixels);
|
||||
|
||||
for (let i = 0; i < numPixels; i++) {
|
||||
const idx = i * 4;
|
||||
gray[i] = Math.round(
|
||||
0.299 * pixels[idx] +
|
||||
0.587 * pixels[idx + 1] +
|
||||
0.114 * pixels[idx + 2]
|
||||
);
|
||||
}
|
||||
|
||||
return gray;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert grayscale back to RGBA.
|
||||
*/
|
||||
export function grayscaleToRgba(
|
||||
gray: Uint8Array,
|
||||
width: number,
|
||||
height: number
|
||||
): Uint8Array {
|
||||
const rgba = new Uint8Array(width * height * 4);
|
||||
|
||||
for (let i = 0; i < gray.length; i++) {
|
||||
const idx = i * 4;
|
||||
rgba[idx] = gray[i];
|
||||
rgba[idx + 1] = gray[i];
|
||||
rgba[idx + 2] = gray[i];
|
||||
rgba[idx + 3] = 255;
|
||||
}
|
||||
|
||||
return rgba;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply Gaussian blur to grayscale image.
|
||||
* Uses separable 1D convolution for efficiency.
|
||||
*/
|
||||
export function gaussianBlur(
|
||||
gray: Uint8Array,
|
||||
width: number,
|
||||
height: number,
|
||||
radius: number = 5
|
||||
): Uint8Array {
|
||||
// Generate Gaussian kernel
|
||||
const sigma = radius / 2;
|
||||
const size = radius * 2 + 1;
|
||||
const kernel = new Float32Array(size);
|
||||
let sum = 0;
|
||||
|
||||
for (let i = 0; i < size; i++) {
|
||||
const x = i - radius;
|
||||
kernel[i] = Math.exp(-(x * x) / (2 * sigma * sigma));
|
||||
sum += kernel[i];
|
||||
}
|
||||
|
||||
// Normalize
|
||||
for (let i = 0; i < size; i++) {
|
||||
kernel[i] /= sum;
|
||||
}
|
||||
|
||||
// Horizontal pass
|
||||
const temp = new Uint8Array(width * height);
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
let value = 0;
|
||||
for (let k = -radius; k <= radius; k++) {
|
||||
const sx = Math.max(0, Math.min(width - 1, x + k));
|
||||
value += gray[y * width + sx] * kernel[k + radius];
|
||||
}
|
||||
temp[y * width + x] = Math.round(value);
|
||||
}
|
||||
}
|
||||
|
||||
// Vertical pass
|
||||
const result = new Uint8Array(width * height);
|
||||
for (let y = 0; y < height; y++) {
|
||||
for (let x = 0; x < width; x++) {
|
||||
let value = 0;
|
||||
for (let k = -radius; k <= radius; k++) {
|
||||
const sy = Math.max(0, Math.min(height - 1, y + k));
|
||||
value += temp[sy * width + x] * kernel[k + radius];
|
||||
}
|
||||
result[y * width + x] = Math.round(value);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate distance between two points.
|
||||
*/
|
||||
export function distance(a: Point, b: Point): number {
|
||||
const dx = b.x - a.x;
|
||||
const dy = b.y - a.y;
|
||||
return Math.sqrt(dx * dx + dy * dy);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate cross product of vectors (b-a) and (c-b).
|
||||
*/
|
||||
export function crossProduct(a: Point, b: Point, c: Point): number {
|
||||
return (b.x - a.x) * (c.y - a.y) - (b.y - a.y) * (c.x - a.x);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate point-to-line distance.
|
||||
*/
|
||||
export function pointToLineDistance(point: Point, lineStart: Point, lineEnd: Point): number {
|
||||
const dx = lineEnd.x - lineStart.x;
|
||||
const dy = lineEnd.y - lineStart.y;
|
||||
const lengthSquared = dx * dx + dy * dy;
|
||||
|
||||
if (lengthSquared === 0) {
|
||||
return distance(point, lineStart);
|
||||
}
|
||||
|
||||
let t = ((point.x - lineStart.x) * dx + (point.y - lineStart.y) * dy) / lengthSquared;
|
||||
t = Math.max(0, Math.min(1, t));
|
||||
|
||||
const projection = {
|
||||
x: lineStart.x + t * dx,
|
||||
y: lineStart.y + t * dy,
|
||||
};
|
||||
|
||||
return distance(point, projection);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sample a pixel from RGBA data using bilinear interpolation.
|
||||
*/
|
||||
export function sampleBilinear(
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
width: number,
|
||||
height: number,
|
||||
x: number,
|
||||
y: number
|
||||
): [number, number, number, number] {
|
||||
// Clamp to valid range
|
||||
x = Math.max(0, Math.min(width - 1, x));
|
||||
y = Math.max(0, Math.min(height - 1, y));
|
||||
|
||||
const x0 = Math.floor(x);
|
||||
const y0 = Math.floor(y);
|
||||
const x1 = Math.min(x0 + 1, width - 1);
|
||||
const y1 = Math.min(y0 + 1, height - 1);
|
||||
|
||||
const xFrac = x - x0;
|
||||
const yFrac = y - y0;
|
||||
|
||||
const idx00 = (y0 * width + x0) * 4;
|
||||
const idx10 = (y0 * width + x1) * 4;
|
||||
const idx01 = (y1 * width + x0) * 4;
|
||||
const idx11 = (y1 * width + x1) * 4;
|
||||
|
||||
const result: [number, number, number, number] = [0, 0, 0, 0];
|
||||
|
||||
for (let c = 0; c < 4; c++) {
|
||||
const c00 = pixels[idx00 + c];
|
||||
const c10 = pixels[idx10 + c];
|
||||
const c01 = pixels[idx01 + c];
|
||||
const c11 = pixels[idx11 + c];
|
||||
|
||||
const top = c00 + (c10 - c00) * xFrac;
|
||||
const bottom = c01 + (c11 - c01) * xFrac;
|
||||
result[c] = Math.round(top + (bottom - top) * yFrac);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
67
lib/recognition/index.ts
Normal file
67
lib/recognition/index.ts
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
/**
|
||||
* Card recognition module exports.
|
||||
*/
|
||||
|
||||
// Recognition service
|
||||
export {
|
||||
recognizeCard,
|
||||
computeImageHash,
|
||||
calculateConfidence,
|
||||
createRecognitionService,
|
||||
type RecognitionOptions,
|
||||
type RecognitionResult,
|
||||
type CardMatch,
|
||||
type CardHashEntry,
|
||||
} from "./recognitionService";
|
||||
|
||||
// Card detection
|
||||
export {
|
||||
detectCard,
|
||||
type CardDetectionResult,
|
||||
} from "./cardDetector";
|
||||
|
||||
// Perspective correction
|
||||
export {
|
||||
warpPerspective,
|
||||
OUTPUT_WIDTH,
|
||||
OUTPUT_HEIGHT,
|
||||
} from "./perspectiveCorrection";
|
||||
|
||||
// CLAHE preprocessing
|
||||
export { applyCLAHE } from "./clahe";
|
||||
|
||||
// Perceptual hashing
|
||||
export {
|
||||
computeColorHash,
|
||||
hammingDistance,
|
||||
hashToHex,
|
||||
hexToHash,
|
||||
HASH_VERSION,
|
||||
MATCH_THRESHOLD,
|
||||
HASH_BITS,
|
||||
} from "./perceptualHash";
|
||||
|
||||
// Image utilities
|
||||
export {
|
||||
resizeImage,
|
||||
rotateImage,
|
||||
toGrayscale,
|
||||
grayscaleToRgba,
|
||||
gaussianBlur,
|
||||
distance,
|
||||
type Point,
|
||||
type Size,
|
||||
} from "./imageUtils";
|
||||
|
||||
// Image loading
|
||||
export {
|
||||
loadImageForRecognition,
|
||||
loadImageAsBase64,
|
||||
} from "./imageLoader";
|
||||
|
||||
// Skia image decoding
|
||||
export {
|
||||
decodeImageBase64,
|
||||
decodeImageFromUri,
|
||||
useDecodedImage,
|
||||
} from "./skiaDecoder";
|
||||
211
lib/recognition/perceptualHash.ts
Normal file
211
lib/recognition/perceptualHash.ts
Normal file
|
|
@ -0,0 +1,211 @@
|
|||
/**
|
||||
* Perceptual hashing implementation using DCT (Discrete Cosine Transform).
|
||||
* Computes a 192-bit (24 byte) color hash from an image.
|
||||
*
|
||||
* The hash is computed by:
|
||||
* 1. Resizing to 32x32
|
||||
* 2. For each RGB channel:
|
||||
* - Apply 2D DCT
|
||||
* - Extract 8x8 low-frequency coefficients (skip DC)
|
||||
* - Compare each to median -> 63 bits per channel
|
||||
* 3. Concatenate R, G, B hashes -> 24 bytes (192 bits)
|
||||
*/
|
||||
|
||||
const DCT_SIZE = 32;
|
||||
const HASH_SIZE = 8;
|
||||
const BITS_PER_CHANNEL = 63; // 8x8 - 1 (skip DC)
|
||||
|
||||
/**
|
||||
* Precomputed cosine values for DCT.
|
||||
*/
|
||||
const cosineCache: number[][] = [];
|
||||
|
||||
function initCosineCache(): void {
|
||||
if (cosineCache.length > 0) return;
|
||||
|
||||
for (let i = 0; i < DCT_SIZE; i++) {
|
||||
cosineCache[i] = [];
|
||||
for (let j = 0; j < DCT_SIZE; j++) {
|
||||
cosineCache[i][j] = Math.cos((Math.PI / DCT_SIZE) * (j + 0.5) * i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply 2D DCT to a matrix.
|
||||
*/
|
||||
function applyDCT2D(matrix: number[][]): number[][] {
|
||||
initCosineCache();
|
||||
|
||||
const result: number[][] = [];
|
||||
|
||||
for (let u = 0; u < DCT_SIZE; u++) {
|
||||
result[u] = [];
|
||||
for (let v = 0; v < DCT_SIZE; v++) {
|
||||
let sum = 0;
|
||||
for (let i = 0; i < DCT_SIZE; i++) {
|
||||
for (let j = 0; j < DCT_SIZE; j++) {
|
||||
sum += matrix[i][j] * cosineCache[u][i] * cosineCache[v][j];
|
||||
}
|
||||
}
|
||||
const cu = u === 0 ? 1 / Math.sqrt(2) : 1;
|
||||
const cv = v === 0 ? 1 / Math.sqrt(2) : 1;
|
||||
result[u][v] = (cu * cv * sum) / 4;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the median of an array of numbers.
|
||||
*/
|
||||
function getMedian(values: number[]): number {
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
const mid = Math.floor(sorted.length / 2);
|
||||
return sorted.length % 2 !== 0
|
||||
? sorted[mid]
|
||||
: (sorted[mid - 1] + sorted[mid]) / 2;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a BigInt to a Uint8Array of specified length.
|
||||
*/
|
||||
function bigintToBytes(value: bigint, length: number): Uint8Array {
|
||||
const bytes = new Uint8Array(length);
|
||||
for (let i = 0; i < length; i++) {
|
||||
bytes[i] = Number((value >> BigInt(i * 8)) & 0xFFn);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute hash for a single color channel.
|
||||
*/
|
||||
function computeChannelHash(channel: number[][]): Uint8Array {
|
||||
const dct = applyDCT2D(channel);
|
||||
|
||||
// Extract 8x8 low-frequency coefficients, skip DC (0,0)
|
||||
const lowFreq: number[] = [];
|
||||
for (let i = 0; i < HASH_SIZE; i++) {
|
||||
for (let j = 0; j < HASH_SIZE; j++) {
|
||||
if (i === 0 && j === 0) continue; // Skip DC component
|
||||
lowFreq.push(dct[i][j]);
|
||||
}
|
||||
}
|
||||
|
||||
const median = getMedian(lowFreq);
|
||||
|
||||
// Generate 63-bit hash
|
||||
let bits = 0n;
|
||||
for (let i = 0; i < lowFreq.length; i++) {
|
||||
if (lowFreq[i] > median) {
|
||||
bits |= 1n << BigInt(i);
|
||||
}
|
||||
}
|
||||
|
||||
return bigintToBytes(bits, 8);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a color channel from RGBA pixel data.
|
||||
* @param pixels RGBA pixel data (width * height * 4)
|
||||
* @param width Image width
|
||||
* @param height Image height
|
||||
* @param channel 0=R, 1=G, 2=B
|
||||
*/
|
||||
function extractChannel(
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
width: number,
|
||||
height: number,
|
||||
channel: 0 | 1 | 2
|
||||
): number[][] {
|
||||
const matrix: number[][] = [];
|
||||
|
||||
for (let y = 0; y < height; y++) {
|
||||
matrix[y] = [];
|
||||
for (let x = 0; x < width; x++) {
|
||||
const idx = (y * width + x) * 4;
|
||||
matrix[y][x] = pixels[idx + channel];
|
||||
}
|
||||
}
|
||||
|
||||
return matrix;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute a 192-bit perceptual color hash from RGBA pixel data.
|
||||
* The image should already be resized to 32x32.
|
||||
*
|
||||
* @param pixels RGBA pixel data (32 * 32 * 4 = 4096 bytes)
|
||||
* @returns 24-byte hash (8 bytes per RGB channel)
|
||||
*/
|
||||
export function computeColorHash(pixels: Uint8Array | Uint8ClampedArray): Uint8Array {
|
||||
if (pixels.length !== DCT_SIZE * DCT_SIZE * 4) {
|
||||
throw new Error(`Expected ${DCT_SIZE * DCT_SIZE * 4} bytes, got ${pixels.length}`);
|
||||
}
|
||||
|
||||
const rChannel = extractChannel(pixels, DCT_SIZE, DCT_SIZE, 0);
|
||||
const gChannel = extractChannel(pixels, DCT_SIZE, DCT_SIZE, 1);
|
||||
const bChannel = extractChannel(pixels, DCT_SIZE, DCT_SIZE, 2);
|
||||
|
||||
const rHash = computeChannelHash(rChannel);
|
||||
const gHash = computeChannelHash(gChannel);
|
||||
const bHash = computeChannelHash(bChannel);
|
||||
|
||||
// Combine all channels
|
||||
const combined = new Uint8Array(24);
|
||||
combined.set(rHash, 0);
|
||||
combined.set(gHash, 8);
|
||||
combined.set(bHash, 16);
|
||||
|
||||
return combined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute Hamming distance between two hashes.
|
||||
* Lower distance = more similar.
|
||||
*/
|
||||
export function hammingDistance(a: Uint8Array, b: Uint8Array): number {
|
||||
if (a.length !== b.length) {
|
||||
throw new Error(`Hash length mismatch: ${a.length} vs ${b.length}`);
|
||||
}
|
||||
|
||||
let distance = 0;
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
let xor = a[i] ^ b[i];
|
||||
while (xor) {
|
||||
distance += xor & 1;
|
||||
xor >>>= 1;
|
||||
}
|
||||
}
|
||||
|
||||
return distance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert hash to hex string for display/storage.
|
||||
*/
|
||||
export function hashToHex(hash: Uint8Array): string {
|
||||
return Array.from(hash)
|
||||
.map((b) => b.toString(16).padStart(2, "0"))
|
||||
.join("");
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert hex string back to hash.
|
||||
*/
|
||||
export function hexToHash(hex: string): Uint8Array {
|
||||
const bytes = new Uint8Array(hex.length / 2);
|
||||
for (let i = 0; i < bytes.length; i++) {
|
||||
bytes[i] = parseInt(hex.substr(i * 2, 2), 16);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
// Hash algorithm version for migrations
|
||||
export const HASH_VERSION = 1;
|
||||
|
||||
// Matching thresholds
|
||||
export const MATCH_THRESHOLD = 25; // Max Hamming distance for a match
|
||||
export const HASH_BITS = 192; // Total bits in hash
|
||||
207
lib/recognition/perspectiveCorrection.ts
Normal file
207
lib/recognition/perspectiveCorrection.ts
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
/**
|
||||
* Perspective correction to transform a quadrilateral region into a rectangle.
|
||||
* Uses homography transformation with bilinear sampling.
|
||||
*/
|
||||
|
||||
import { Point, sampleBilinear } from "./imageUtils";
|
||||
|
||||
/** Standard output size for corrected card images (63:88 aspect ratio). */
|
||||
export const OUTPUT_WIDTH = 480;
|
||||
export const OUTPUT_HEIGHT = 670;
|
||||
|
||||
/**
|
||||
* Apply perspective correction to extract and normalize a card from an image.
|
||||
*
|
||||
* @param pixels - Source RGBA pixel data
|
||||
* @param width - Source image width
|
||||
* @param height - Source image height
|
||||
* @param corners - Four corners in order: top-left, top-right, bottom-right, bottom-left
|
||||
* @param outputWidth - Width of output image
|
||||
* @param outputHeight - Height of output image
|
||||
* @returns Perspective-corrected RGBA pixel data
|
||||
*/
|
||||
export function warpPerspective(
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
width: number,
|
||||
height: number,
|
||||
corners: Point[],
|
||||
outputWidth: number = OUTPUT_WIDTH,
|
||||
outputHeight: number = OUTPUT_HEIGHT
|
||||
): { pixels: Uint8Array; width: number; height: number } {
|
||||
if (corners.length !== 4) {
|
||||
throw new Error("Exactly 4 corners required");
|
||||
}
|
||||
|
||||
// Determine if card is landscape (rotated 90°)
|
||||
const width1 = distance(corners[0], corners[1]);
|
||||
const height1 = distance(corners[1], corners[2]);
|
||||
|
||||
let orderedCorners: Point[];
|
||||
|
||||
if (width1 > height1) {
|
||||
// Card is landscape - rotate corners to portrait
|
||||
orderedCorners = [corners[1], corners[2], corners[3], corners[0]];
|
||||
} else {
|
||||
orderedCorners = corners;
|
||||
}
|
||||
|
||||
// Compute the perspective transform matrix
|
||||
const matrix = computePerspectiveTransform(orderedCorners, outputWidth, outputHeight);
|
||||
const inverse = invertMatrix3x3(matrix);
|
||||
|
||||
// Apply the transform
|
||||
const result = new Uint8Array(outputWidth * outputHeight * 4);
|
||||
|
||||
for (let y = 0; y < outputHeight; y++) {
|
||||
for (let x = 0; x < outputWidth; x++) {
|
||||
// Apply inverse transform to find source coordinates
|
||||
const srcPoint = applyTransform(inverse, x, y);
|
||||
|
||||
// Bilinear interpolation for smooth sampling
|
||||
const [r, g, b, a] = sampleBilinear(pixels, width, height, srcPoint.x, srcPoint.y);
|
||||
|
||||
const idx = (y * outputWidth + x) * 4;
|
||||
result[idx] = r;
|
||||
result[idx + 1] = g;
|
||||
result[idx + 2] = b;
|
||||
result[idx + 3] = a;
|
||||
}
|
||||
}
|
||||
|
||||
return { pixels: result, width: outputWidth, height: outputHeight };
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute a perspective transform matrix from quad corners to rectangle.
|
||||
* Uses the Direct Linear Transform (DLT) algorithm.
|
||||
*/
|
||||
function computePerspectiveTransform(
|
||||
src: Point[],
|
||||
dstWidth: number,
|
||||
dstHeight: number
|
||||
): number[] {
|
||||
// Destination corners (rectangle)
|
||||
const dst: Point[] = [
|
||||
{ x: 0, y: 0 },
|
||||
{ x: dstWidth - 1, y: 0 },
|
||||
{ x: dstWidth - 1, y: dstHeight - 1 },
|
||||
{ x: 0, y: dstHeight - 1 },
|
||||
];
|
||||
|
||||
// Build the 8x8 matrix for solving the homography
|
||||
const A: number[][] = [];
|
||||
const b: number[] = [];
|
||||
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const sx = src[i].x;
|
||||
const sy = src[i].y;
|
||||
const dx = dst[i].x;
|
||||
const dy = dst[i].y;
|
||||
|
||||
A.push([sx, sy, 1, 0, 0, 0, -dx * sx, -dx * sy]);
|
||||
b.push(dx);
|
||||
|
||||
A.push([0, 0, 0, sx, sy, 1, -dy * sx, -dy * sy]);
|
||||
b.push(dy);
|
||||
}
|
||||
|
||||
// Solve using Gaussian elimination
|
||||
const h = solveLinearSystem(A, b);
|
||||
|
||||
// Return 3x3 matrix as flat array [h11, h12, h13, h21, h22, h23, h31, h32, h33]
|
||||
return [h[0], h[1], h[2], h[3], h[4], h[5], h[6], h[7], 1];
|
||||
}
|
||||
|
||||
/**
|
||||
* Solve a linear system Ax = b using Gaussian elimination with partial pivoting.
|
||||
*/
|
||||
function solveLinearSystem(A: number[][], b: number[]): number[] {
|
||||
const n = b.length;
|
||||
|
||||
// Create augmented matrix
|
||||
const augmented: number[][] = A.map((row, i) => [...row, b[i]]);
|
||||
|
||||
// Forward elimination with partial pivoting
|
||||
for (let col = 0; col < n; col++) {
|
||||
// Find pivot
|
||||
let maxRow = col;
|
||||
for (let row = col + 1; row < n; row++) {
|
||||
if (Math.abs(augmented[row][col]) > Math.abs(augmented[maxRow][col])) {
|
||||
maxRow = row;
|
||||
}
|
||||
}
|
||||
|
||||
// Swap rows
|
||||
[augmented[col], augmented[maxRow]] = [augmented[maxRow], augmented[col]];
|
||||
|
||||
// Eliminate
|
||||
for (let row = col + 1; row < n; row++) {
|
||||
if (Math.abs(augmented[col][col]) < 1e-10) continue;
|
||||
|
||||
const factor = augmented[row][col] / augmented[col][col];
|
||||
for (let j = col; j <= n; j++) {
|
||||
augmented[row][j] -= factor * augmented[col][j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Back substitution
|
||||
const x = new Array(n).fill(0);
|
||||
for (let i = n - 1; i >= 0; i--) {
|
||||
x[i] = augmented[i][n];
|
||||
for (let j = i + 1; j < n; j++) {
|
||||
x[i] -= augmented[i][j] * x[j];
|
||||
}
|
||||
if (Math.abs(augmented[i][i]) > 1e-10) {
|
||||
x[i] /= augmented[i][i];
|
||||
}
|
||||
}
|
||||
|
||||
return x;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a 3x3 transform matrix to a point.
|
||||
*/
|
||||
function applyTransform(H: number[], x: number, y: number): Point {
|
||||
let w = H[6] * x + H[7] * y + H[8];
|
||||
if (Math.abs(w) < 1e-10) w = 1e-10;
|
||||
|
||||
return {
|
||||
x: (H[0] * x + H[1] * y + H[2]) / w,
|
||||
y: (H[3] * x + H[4] * y + H[5]) / w,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Invert a 3x3 matrix.
|
||||
*/
|
||||
function invertMatrix3x3(m: number[]): number[] {
|
||||
const det =
|
||||
m[0] * (m[4] * m[8] - m[5] * m[7]) -
|
||||
m[1] * (m[3] * m[8] - m[5] * m[6]) +
|
||||
m[2] * (m[3] * m[7] - m[4] * m[6]);
|
||||
|
||||
const invDet = Math.abs(det) < 1e-10 ? 1e10 : 1 / det;
|
||||
|
||||
return [
|
||||
(m[4] * m[8] - m[5] * m[7]) * invDet,
|
||||
(m[2] * m[7] - m[1] * m[8]) * invDet,
|
||||
(m[1] * m[5] - m[2] * m[4]) * invDet,
|
||||
(m[5] * m[6] - m[3] * m[8]) * invDet,
|
||||
(m[0] * m[8] - m[2] * m[6]) * invDet,
|
||||
(m[2] * m[3] - m[0] * m[5]) * invDet,
|
||||
(m[3] * m[7] - m[4] * m[6]) * invDet,
|
||||
(m[1] * m[6] - m[0] * m[7]) * invDet,
|
||||
(m[0] * m[4] - m[1] * m[3]) * invDet,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate distance between two points.
|
||||
*/
|
||||
function distance(a: Point, b: Point): number {
|
||||
const dx = b.x - a.x;
|
||||
const dy = b.y - a.y;
|
||||
return Math.sqrt(dx * dx + dy * dy);
|
||||
}
|
||||
313
lib/recognition/recognitionService.ts
Normal file
313
lib/recognition/recognitionService.ts
Normal file
|
|
@ -0,0 +1,313 @@
|
|||
/**
|
||||
* Card recognition service that orchestrates the full pipeline.
|
||||
*
|
||||
* Pipeline:
|
||||
* 1. Card detection (optional) - find card boundaries
|
||||
* 2. Perspective correction - warp to rectangle
|
||||
* 3. CLAHE preprocessing - normalize lighting
|
||||
* 4. Resize to 32x32
|
||||
* 5. Compute perceptual hash
|
||||
* 6. Match against database via Hamming distance
|
||||
*/
|
||||
|
||||
import { detectCard, CardDetectionResult } from "./cardDetector";
|
||||
import { warpPerspective } from "./perspectiveCorrection";
|
||||
import { applyCLAHE } from "./clahe";
|
||||
import {
|
||||
computeColorHash,
|
||||
hammingDistance,
|
||||
MATCH_THRESHOLD,
|
||||
HASH_BITS,
|
||||
} from "./perceptualHash";
|
||||
import { resizeImage, rotateImage } from "./imageUtils";
|
||||
|
||||
export interface RecognitionOptions {
|
||||
/** Enable card detection and perspective correction. */
|
||||
enableCardDetection?: boolean;
|
||||
/** Enable rotation matching (try 0°, 90°, 180°, 270°). */
|
||||
enableRotationMatching?: boolean;
|
||||
/** Minimum confidence to accept a match (0-1). */
|
||||
minConfidence?: number;
|
||||
/** Maximum Hamming distance to accept a match. */
|
||||
matchThreshold?: number;
|
||||
}
|
||||
|
||||
export interface CardMatch {
|
||||
/** Card ID from database. */
|
||||
cardId: string;
|
||||
/** Match confidence (0-1). */
|
||||
confidence: number;
|
||||
/** Hamming distance between hashes. */
|
||||
distance: number;
|
||||
/** Rotation used for match (0, 90, 180, or 270). */
|
||||
rotation: number;
|
||||
}
|
||||
|
||||
export interface RecognitionResult {
|
||||
success: boolean;
|
||||
match?: CardMatch;
|
||||
cardDetection?: CardDetectionResult;
|
||||
error?: string;
|
||||
processingTimeMs: number;
|
||||
}
|
||||
|
||||
export interface CardHashEntry {
|
||||
id: string;
|
||||
name: string;
|
||||
setCode: string;
|
||||
collectorNumber?: string;
|
||||
imageUri?: string;
|
||||
hash: Uint8Array;
|
||||
}
|
||||
|
||||
const DEFAULT_OPTIONS: Required<RecognitionOptions> = {
|
||||
enableCardDetection: true,
|
||||
enableRotationMatching: true,
|
||||
minConfidence: 0.85,
|
||||
matchThreshold: MATCH_THRESHOLD,
|
||||
};
|
||||
|
||||
/**
|
||||
* Calculate confidence from Hamming distance.
|
||||
*/
|
||||
export function calculateConfidence(distance: number, totalBits: number = HASH_BITS): number {
|
||||
return 1 - distance / totalBits;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recognize a card from RGBA pixel data.
|
||||
*
|
||||
* @param pixels - RGBA pixel data
|
||||
* @param width - Image width
|
||||
* @param height - Image height
|
||||
* @param cardHashes - Array of card hashes to match against
|
||||
* @param options - Recognition options
|
||||
* @returns Recognition result with best match
|
||||
*/
|
||||
export function recognizeCard(
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
width: number,
|
||||
height: number,
|
||||
cardHashes: CardHashEntry[],
|
||||
options: RecognitionOptions = {}
|
||||
): RecognitionResult {
|
||||
const startTime = performance.now();
|
||||
const opts = { ...DEFAULT_OPTIONS, ...options };
|
||||
|
||||
try {
|
||||
if (cardHashes.length === 0) {
|
||||
return {
|
||||
success: false,
|
||||
error: "No cards in database",
|
||||
processingTimeMs: performance.now() - startTime,
|
||||
};
|
||||
}
|
||||
|
||||
let cardPixels = pixels;
|
||||
let cardWidth = width;
|
||||
let cardHeight = height;
|
||||
let detection: CardDetectionResult | undefined;
|
||||
|
||||
// Step 1: Detect and extract card (if enabled)
|
||||
if (opts.enableCardDetection) {
|
||||
detection = detectCard(pixels, width, height);
|
||||
|
||||
if (detection.found) {
|
||||
const warped = warpPerspective(
|
||||
pixels,
|
||||
width,
|
||||
height,
|
||||
detection.corners
|
||||
);
|
||||
cardPixels = warped.pixels;
|
||||
cardWidth = warped.width;
|
||||
cardHeight = warped.height;
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: Find best match (with or without rotation)
|
||||
const match = opts.enableRotationMatching
|
||||
? findBestMatchWithRotations(cardPixels, cardWidth, cardHeight, cardHashes, opts)
|
||||
: findBestMatchSingle(cardPixels, cardWidth, cardHeight, cardHashes, opts);
|
||||
|
||||
const processingTimeMs = performance.now() - startTime;
|
||||
|
||||
if (!match) {
|
||||
return {
|
||||
success: false,
|
||||
cardDetection: detection,
|
||||
error: "No match found",
|
||||
processingTimeMs,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
match,
|
||||
cardDetection: detection,
|
||||
processingTimeMs,
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : "Unknown error",
|
||||
processingTimeMs: performance.now() - startTime,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute hash for an image with full preprocessing pipeline.
|
||||
*/
|
||||
export function computeImageHash(
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
width: number,
|
||||
height: number
|
||||
): Uint8Array {
|
||||
// Apply CLAHE
|
||||
const clahePixels = applyCLAHE(pixels, width, height);
|
||||
|
||||
// Resize to 32x32
|
||||
const resized = resizeImage(clahePixels, width, height, 32, 32);
|
||||
|
||||
// Compute hash
|
||||
return computeColorHash(resized);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find best match trying all 4 rotations.
|
||||
*/
|
||||
function findBestMatchWithRotations(
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
width: number,
|
||||
height: number,
|
||||
cardHashes: CardHashEntry[],
|
||||
opts: Required<RecognitionOptions>
|
||||
): CardMatch | null {
|
||||
let bestMatch: CardMatch | null = null;
|
||||
|
||||
const rotations: Array<0 | 90 | 180 | 270> = [0, 90, 180, 270];
|
||||
|
||||
for (const rotation of rotations) {
|
||||
const { pixels: rotatedPixels, width: rotatedWidth, height: rotatedHeight } =
|
||||
rotation === 0
|
||||
? { pixels, width, height }
|
||||
: rotateImage(pixels, width, height, rotation);
|
||||
|
||||
// Apply CLAHE
|
||||
const clahePixels = applyCLAHE(rotatedPixels, rotatedWidth, rotatedHeight);
|
||||
|
||||
// Resize to 32x32
|
||||
const resized = resizeImage(clahePixels, rotatedWidth, rotatedHeight, 32, 32);
|
||||
|
||||
// Compute hash
|
||||
const queryHash = computeColorHash(resized);
|
||||
|
||||
// Find best match for this rotation
|
||||
for (const card of cardHashes) {
|
||||
if (!card.hash || card.hash.length !== queryHash.length) continue;
|
||||
|
||||
const distance = hammingDistance(queryHash, card.hash);
|
||||
const confidence = calculateConfidence(distance);
|
||||
|
||||
if (distance <= opts.matchThreshold && confidence >= opts.minConfidence) {
|
||||
if (!bestMatch || distance < bestMatch.distance) {
|
||||
bestMatch = {
|
||||
cardId: card.id,
|
||||
confidence,
|
||||
distance,
|
||||
rotation,
|
||||
};
|
||||
}
|
||||
|
||||
// Early exit on perfect match
|
||||
if (distance === 0) {
|
||||
return bestMatch;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return bestMatch;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find best match without rotation.
|
||||
*/
|
||||
function findBestMatchSingle(
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
width: number,
|
||||
height: number,
|
||||
cardHashes: CardHashEntry[],
|
||||
opts: Required<RecognitionOptions>
|
||||
): CardMatch | null {
|
||||
// Apply CLAHE
|
||||
const clahePixels = applyCLAHE(pixels, width, height);
|
||||
|
||||
// Resize to 32x32
|
||||
const resized = resizeImage(clahePixels, width, height, 32, 32);
|
||||
|
||||
// Compute hash
|
||||
const queryHash = computeColorHash(resized);
|
||||
|
||||
let bestMatch: CardMatch | null = null;
|
||||
|
||||
for (const card of cardHashes) {
|
||||
if (!card.hash || card.hash.length !== queryHash.length) continue;
|
||||
|
||||
const distance = hammingDistance(queryHash, card.hash);
|
||||
const confidence = calculateConfidence(distance);
|
||||
|
||||
if (distance <= opts.matchThreshold && confidence >= opts.minConfidence) {
|
||||
if (!bestMatch || distance < bestMatch.distance) {
|
||||
bestMatch = {
|
||||
cardId: card.id,
|
||||
confidence,
|
||||
distance,
|
||||
rotation: 0,
|
||||
};
|
||||
}
|
||||
|
||||
if (distance === 0) {
|
||||
return bestMatch;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return bestMatch;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a recognition service instance with cached card hashes.
|
||||
*/
|
||||
export function createRecognitionService(cardHashes: CardHashEntry[]) {
|
||||
let cachedHashes = cardHashes;
|
||||
|
||||
return {
|
||||
/**
|
||||
* Recognize a card from RGBA pixel data.
|
||||
*/
|
||||
recognize(
|
||||
pixels: Uint8Array | Uint8ClampedArray,
|
||||
width: number,
|
||||
height: number,
|
||||
options?: RecognitionOptions
|
||||
): RecognitionResult {
|
||||
return recognizeCard(pixels, width, height, cachedHashes, options);
|
||||
},
|
||||
|
||||
/**
|
||||
* Update the cached card hashes.
|
||||
*/
|
||||
updateHashes(hashes: CardHashEntry[]) {
|
||||
cachedHashes = hashes;
|
||||
},
|
||||
|
||||
/**
|
||||
* Get the number of cached hashes.
|
||||
*/
|
||||
getHashCount(): number {
|
||||
return cachedHashes.length;
|
||||
},
|
||||
};
|
||||
}
|
||||
121
lib/recognition/skiaDecoder.ts
Normal file
121
lib/recognition/skiaDecoder.ts
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
/**
|
||||
* Skia-based image decoder for getting RGBA pixel data.
|
||||
* Uses react-native-skia to decode images and extract pixel buffers.
|
||||
*/
|
||||
|
||||
import {
|
||||
Skia,
|
||||
useImage,
|
||||
SkImage,
|
||||
} from "@shopify/react-native-skia";
|
||||
|
||||
/**
|
||||
* Decode a base64 PNG/JPEG image and return RGBA pixel data.
|
||||
*
|
||||
* @param base64 - Base64 encoded image data (without data URI prefix)
|
||||
* @returns RGBA pixel data with dimensions
|
||||
*/
|
||||
export function decodeImageBase64(base64: string): {
|
||||
pixels: Uint8Array;
|
||||
width: number;
|
||||
height: number;
|
||||
} | null {
|
||||
try {
|
||||
// Decode base64 to data
|
||||
const data = Skia.Data.fromBase64(base64);
|
||||
if (!data) return null;
|
||||
|
||||
// Create image from data
|
||||
const image = Skia.Image.MakeImageFromEncoded(data);
|
||||
if (!image) return null;
|
||||
|
||||
const width = image.width();
|
||||
const height = image.height();
|
||||
|
||||
// Read pixels from the image
|
||||
// Note: Skia images are in RGBA format
|
||||
const pixels = image.readPixels(0, 0, {
|
||||
width,
|
||||
height,
|
||||
colorType: 4, // RGBA_8888
|
||||
alphaType: 1, // Unpremultiplied
|
||||
});
|
||||
|
||||
if (!pixels) return null;
|
||||
|
||||
return {
|
||||
pixels: new Uint8Array(pixels),
|
||||
width,
|
||||
height,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("[SkiaDecoder] Failed to decode image:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode an image from a file URI and return RGBA pixel data.
|
||||
*
|
||||
* @param uri - File URI (e.g., file:///path/to/image.png)
|
||||
* @returns Promise with RGBA pixel data
|
||||
*/
|
||||
export async function decodeImageFromUri(uri: string): Promise<{
|
||||
pixels: Uint8Array;
|
||||
width: number;
|
||||
height: number;
|
||||
} | null> {
|
||||
try {
|
||||
// Fetch the image data
|
||||
const response = await fetch(uri);
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
const base64 = arrayBufferToBase64(arrayBuffer);
|
||||
|
||||
return decodeImageBase64(base64);
|
||||
} catch (error) {
|
||||
console.error("[SkiaDecoder] Failed to load image from URI:", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert ArrayBuffer to base64 string.
|
||||
*/
|
||||
function arrayBufferToBase64(buffer: ArrayBuffer): string {
|
||||
let binary = "";
|
||||
const bytes = new Uint8Array(buffer);
|
||||
const len = bytes.byteLength;
|
||||
for (let i = 0; i < len; i++) {
|
||||
binary += String.fromCharCode(bytes[i]);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
/**
|
||||
* React hook to decode an image from URI.
|
||||
* Uses Skia's useImage hook for caching.
|
||||
*/
|
||||
export function useDecodedImage(uri: string | null) {
|
||||
const skiaImage = useImage(uri);
|
||||
|
||||
if (!skiaImage) {
|
||||
return { loading: true, pixels: null, width: 0, height: 0 };
|
||||
}
|
||||
|
||||
const width = skiaImage.width();
|
||||
const height = skiaImage.height();
|
||||
|
||||
const pixels = skiaImage.readPixels(0, 0, {
|
||||
width,
|
||||
height,
|
||||
colorType: 4, // RGBA_8888
|
||||
alphaType: 1, // Unpremultiplied
|
||||
});
|
||||
|
||||
return {
|
||||
loading: false,
|
||||
pixels: pixels ? new Uint8Array(pixels) : null,
|
||||
width,
|
||||
height,
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue