diff --git a/.gitignore b/.gitignore index 6facb8c..29a138d 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,3 @@ Thumbs.db *.csv *.dlens *.apk -debug/ diff --git a/.justfile b/.justfile index 055de56..1f663ae 100644 --- a/.justfile +++ b/.justfile @@ -1,46 +1,24 @@ # Scry development commands - -set shell := ["C:/Program Files/Git/usr/bin/bash.exe", "-c"] -set unstable := true - # Android SDK paths -android_sdk := replace(env('LOCALAPPDATA'), '\', '/') / "Android/Sdk" +android_sdk := env_var('LOCALAPPDATA') / "Android/Sdk" adb := android_sdk / "platform-tools/adb.exe" emulator := android_sdk / "emulator/emulator.exe" camera_virtual := "-camera-back virtualscene -virtualscene-poster wall=\"" + (justfile_directory() / "TestImages/reference_alpha/serra_angel.jpg") + "\"" camera_webcam := "-camera-back webcam0 -camera-front webcam0" -[private] -@default: - just --list - -# Start emulator in background emu camera="virtual": - {{ emulator }} -avd Pixel_6 {{ if camera == "virtual" { camera_virtual } else { camera_webcam } }} -no-snapshot-load -gpu host + {{ emulator }} -avd Pixel_6 {{ if camera == "virtual" { camera_virtual } else { camera_webcam } }} -gpu host & # Kill the running emulator emu-kill: {{ adb }} emu kill -# Wait for emulator to fully boot (timeout after 2 minutes) -[script] +# Wait for emulator to fully boot emu-wait: - # Wait for Android emulator to boot with timeout - TIMEOUT=120 - - echo "Waiting for emulator to boot..." - - for ((i=TIMEOUT; i>0; i--)); do - if [ "$({{ adb }} shell getprop sys.boot_completed 2>/dev/null)" = "1" ]; then - echo "Emulator ready" - exit 0 - fi - sleep 1 - done - - echo "Emulator failed to boot within 2 minutes" - exit 1 + @echo "Waiting for emulator to boot..." + @while [ "$({{ adb }} shell getprop sys.boot_completed 2>/dev/null)" != "1" ]; do sleep 1; done + @echo "Emulator ready" # Build a project build project="src/Scry.App" target="net10.0-android": @@ -74,11 +52,11 @@ test: dotnet test test/Scry.Tests # Generate the card hash database from Scryfall -gen-db: (build "tools/DbGenerator" "net10.0") +gen-db *args: (build "tools/DbGenerator" "net10.0") @echo "Running Database generator (this takes a while)..." - dotnet run --project tools/DbGenerator --no-build -- src/Scry.App/Resources/Raw/card_hashes.db + dotnet run --project tools/DbGenerator --no-build -- src/Scry.App/Resources/Raw/card_hashes.db {{ args }} @echo "Completed generating the database" # Full workflow: start emulator, wait, run with hot reload -dev: +dev: emu emu-wait dotnet watch --project src/Scry.App -f net10.0-android diff --git a/AGENTS.md b/AGENTS.md index a04e7bf..64bdd14 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,291 +1,30 @@ # Agent Instructions -## Overview +## Build and Task Commands -Scry is a Magic: The Gathering card scanner app built with .NET MAUI. It uses perceptual hashing to match photographed cards against a database of known card images from Scryfall. +Prefer using `just` commands over raw dotnet/build commands: -**Key components:** -- Mobile scanning app (MAUI/Android) -- Card recognition via perceptual hashing (not OCR) -- SQLite database with pre-computed hashes -- Scryfall API integration for card data - -## Build Commands - -Use `just` commands (defined in `.justfile`): - -| Task | Command | Notes | -|------|---------|-------| -| Build project | `just build` | Builds Android debug | -| Run tests | `just test` | Runs xUnit tests | -| Generate card database | `just gen-db` | Downloads from Scryfall, computes hashes | -| Publish app | `just publish` | Creates release APK | -| Hot reload dev | `just dev` | Uses `dotnet watch` | -| Start emulator | `just emu` | Virtual camera with Serra Angel | -| Install to device | `just install` | Installs release APK | - -### Database Generator Options - -```bash -just gen-db # Default: 500 cards with test images -dotnet run --project tools/DbGenerator -- -c 1000 # More cards -dotnet run --project tools/DbGenerator -- --force # Rebuild from scratch -dotnet run --project tools/DbGenerator -- --no-test-cards # Skip priority test cards -``` +| Task | Command | +|------|---------| +| Build project | `just build` | +| Run tests | `just test` | +| Generate card database | `just gen-db` | +| Publish app | `just publish` | +| Run full dev workflow | `just dev` | ## Project Structure -``` -src/ -├── Scry.App/ # MAUI mobile app (Android target) -│ ├── Views/ # XAML pages (ScanPage, CollectionPage, etc.) -│ ├── ViewModels/ # MVVM ViewModels using CommunityToolkit.Mvvm -│ ├── Services/ # App-layer services (ICardRecognitionService, ICardRepository) -│ ├── Converters/ # XAML value converters -│ ├── Models/ # App-specific models (CollectionEntry) -│ └── Resources/Raw/ # Bundled card_hashes.db -│ -└── Scry.Core/ # Platform-independent core library - ├── Recognition/ # CardRecognitionService, RecognitionOptions - ├── Imaging/ # PerceptualHash, ImagePreprocessor, CardDetector - ├── Data/ # CardDatabase (SQLite) - ├── Models/ # Card, Oracle, Set, ScanResult - └── Scryfall/ # ScryfallClient for API/bulk data +- `src/Scry.App` - MAUI mobile app +- `src/Scry.Core` - Core library (recognition, hashing, database) +- `test/Scry.Tests` - Unit tests +- `tools/DbGenerator` - Card hash database generator -test/ -└── Scry.Tests/ # xUnit tests +## Database Generator -tools/ -└── DbGenerator/ # CLI tool to generate card_hashes.db +The `just gen-db` command: +- Builds the DbGenerator tool +- Runs it against `src/Scry.App/Resources/Raw/card_hashes.db` +- Supports incremental updates (only downloads missing cards) +- Prefers LEA/LEB (Alpha/Beta) sets for reference_alpha test cards -TestImages/ # Test images organized by category -├── reference_alpha/ # Alpha/Beta cards for testing -├── single_cards/ # Individual card photos -├── varying_quality/ # Different lighting/quality -├── hands/ # Cards held in hand -├── foil/ # Foil cards with glare -└── ... # More categories -``` - -## Architecture - -### Recognition Pipeline - -``` -Camera Image - │ - ▼ -┌─────────────────────┐ -│ CardDetector │ ← Edge detection, find card quad -│ (optional) │ -└─────────────────────┘ - │ - ▼ -┌─────────────────────┐ -│ PerspectiveCorrection│ ← Warp to rectangle -└─────────────────────┘ - │ - ▼ -┌─────────────────────┐ -│ ImagePreprocessor │ ← CLAHE for lighting normalization -│ (ApplyClahe) │ -└─────────────────────┘ - │ - ▼ -┌─────────────────────┐ -│ PerceptualHash │ ← Compute 192-bit color hash (24 bytes) -│ (ComputeColorHash) │ -└─────────────────────┘ - │ - ▼ -┌─────────────────────┐ -│ CardRecognitionService│ ← Hamming distance match against DB -└─────────────────────┘ -``` - -### Data Model - -Three-table schema mirroring Scryfall: - -- **oracles** - Abstract game cards (one per unique card name) -- **sets** - MTG sets with metadata -- **cards** - Printings with perceptual hashes (one per unique artwork) - -The `Card` model includes denormalized Oracle fields for convenience. - -### Key Classes - -| Class | Purpose | -|-------|---------| -| `CardRecognitionService` | Main recognition logic, caches DB, handles rotation matching | -| `PerceptualHash` | DCT-based color hash (192-bit = 8 bytes × 3 RGB channels) | -| `ImagePreprocessor` | CLAHE, resize, grayscale conversions | -| `CardDetector` | Edge detection + contour analysis to find card boundaries | -| `PerspectiveCorrection` | Warp detected quad to rectangle | -| `CardDatabase` | SQLite wrapper with batch insert, queries | -| `ScryfallClient` | Bulk data streaming, image downloads | - -## Code Conventions - -### General - -- **Target**: .NET 10.0 (net10.0-android for app, net10.0 for Core/tools) -- **Nullable**: Enabled everywhere (`enable`) -- **Warnings as errors**: `true` -- **Central package management**: Versions in `Directory.Packages.props` - -### C# Style - -- Records for data models (`record Card`, `record ScanResult`) -- `required` properties for non-nullable required fields -- Extension methods for conversions (`ScryfallCard.ToCard()`) -- Static classes for pure functions (`PerceptualHash`, `ImagePreprocessor`) -- `using` declarations (not `using` blocks) for disposables -- File-scoped namespaces -- Primary constructors where appropriate -- `CancellationToken` parameter on all async methods - -### MVVM (App layer) - -- `CommunityToolkit.Mvvm` for source generators -- `[ObservableProperty]` attributes for bindable properties -- `[RelayCommand]` for commands -- ViewModels in `Scry.ViewModels` namespace - -### Naming - -- Services: `ICardRecognitionService`, `CardRecognitionService` -- Database methods: `GetCardsWithHashAsync`, `InsertCardBatchAsync` -- Hash methods: `ComputeColorHash`, `HammingDistance` -- Test methods: `RecognizeAsync_ExactMatch_ReturnsSuccess` - -## Testing - -Tests are in `test/Scry.Tests` using xUnit. - -```bash -just test # Run all tests -dotnet test --filter "FullyQualifiedName~PerceptualHash" # Filter by name -``` - -### Test Categories - -| Test Class | Tests | -|------------|-------| -| `PerceptualHashTests` | Hash computation, Hamming distance | -| `CardRecognitionTests` | End-to-end recognition | -| `CardDatabaseTests` | SQLite CRUD operations | -| `ImagePreprocessorTests` | CLAHE, resize | -| `RobustnessAnalysisTests` | Multiple image variations | - -### Test Images - -TestImages directory contains categorized reference images: -- `reference_alpha/` - Alpha/Beta cards (matching DbGenerator priority cards) -- `single_cards/` - Clean single card photos -- `varying_quality/` - Different lighting/blur conditions - -## Key Algorithms - -### Perceptual Hash (pHash) - -Color-aware 192-bit hash: -1. Resize to 32×32 -2. For each RGB channel: - - Compute 2D DCT - - Extract 8×8 low-frequency coefficients (skip DC) - - Compare each to median → 63 bits per channel -3. Concatenate R, G, B hashes → 24 bytes (192 bits) - -Matching uses Hamming distance with threshold ≤25 bits and minimum confidence 85%. - -### CLAHE (Contrast Limited Adaptive Histogram Equalization) - -Applied in LAB color space to L channel only: -- Tile-based histogram equalization (8×8 tiles) -- Clip limit prevents over-amplification of noise -- Bilinear interpolation between tiles for smooth output - -### Card Detection - -Pure SkiaSharp implementation: -1. Grayscale → Gaussian blur → Canny edge detection -2. Contour tracing via flood fill -3. Douglas-Peucker simplification → Convex hull -4. Find best quadrilateral matching MTG aspect ratio (88/63 ≈ 1.397) -5. Order corners: top-left, top-right, bottom-right, bottom-left - -## Debug Mode - -Set `RecognitionOptions.DebugOutputDirectory` to save pipeline stages: -- `01_input.png` - Original image -- `02_detection.png` - Card detection visualization -- `03_perspective_corrected.png` - Warped card -- `05_clahe_*.png` - After CLAHE preprocessing - -On Android: `/sdcard/Download/scry-debug` (pull with `adb pull`) - -## Dependencies - -### Core Library (Scry.Core) - -- **SkiaSharp** - Image processing, DCT, edge detection -- **Microsoft.Data.Sqlite** - SQLite database -- **Microsoft.Extensions.Options** - Options pattern - -### App (Scry.App) - -- **CommunityToolkit.Maui** - MAUI extensions -- **CommunityToolkit.Maui.Camera** - Camera integration -- **CommunityToolkit.Mvvm** - MVVM source generators - -### DbGenerator Tool - -- **Spectre.Console** / **Spectre.Console.Cli** - Rich terminal UI - -## Common Tasks - -### Adding a New Card to Priority Test Set - -1. Add image to `TestImages/reference_alpha/` or appropriate folder -2. Add entry to `GenerateCommand.PriorityCardsWithSets` dictionary -3. Run `just gen-db` to regenerate database - -### Debugging Recognition Failures - -1. Enable debug output in `MauiProgram.cs`: - ```csharp - options.DebugOutputDirectory = "/sdcard/Download/scry-debug"; - ``` -2. Run recognition -3. Pull debug images: `adb pull /sdcard/Download/scry-debug` -4. Compare `05_clahe_*.png` with reference images in database - -### Modifying Hash Algorithm - -1. Update `PerceptualHash.ComputeColorHash()` -2. Update `CardRecognitionService.ColorHashBits` constant -3. Regenerate database: `just gen-db --force` -4. Run tests: `just test` - -## Gotchas - -1. **Hash size is 24 bytes (192 bits)** - 3 RGB channels × 8 bytes each -2. **Confidence threshold is 85%** - Configurable in `CardRecognitionService.MinConfidence` -3. **Card detection is optional** - Controlled by `RecognitionOptions.EnableCardDetection` -4. **Rotation matching tries 4 orientations** - Controlled by `RecognitionOptions.EnableRotationMatching` -5. **Database is bundled in APK** - Copied on first run to app data directory -6. **Multi-face cards** - Only front face image is used for hashing -7. **Rate limiting** - DbGenerator uses 50ms delay between Scryfall image downloads - -## CI/CD - -Forgejo Actions workflow (`.forgejo/workflows/release.yml`): -- Builds for win-x64, linux-x64, osx-x64 -- Creates "standard" and "embedded" (with APK) variants -- Publishes to Forgejo releases - -## External Resources - -- [Scryfall API](https://scryfall.com/docs/api) - Card data source -- [CARD_RECOGNITION.md](docs/CARD_RECOGNITION.md) - Detailed architecture doc +Use `--force` flag to rebuild from scratch if needed. diff --git a/Directory.Packages.props b/Directory.Packages.props index baa520b..880d82a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -17,9 +17,6 @@ /> - - - diff --git a/docs/CARD_RECOGNITION.md b/docs/CARD_RECOGNITION.md index b788b81..591293f 100644 --- a/docs/CARD_RECOGNITION.md +++ b/docs/CARD_RECOGNITION.md @@ -313,63 +313,22 @@ public MatchResult Match(byte[] queryHash, CardDatabase db) ### Database Schema -The schema mirrors Scryfall's data model with three main tables: - ```sql --- Abstract game cards (oracle) -CREATE TABLE oracles ( - id TEXT PRIMARY KEY, -- Scryfall oracle_id - name TEXT NOT NULL, - mana_cost TEXT, - cmc REAL, - type_line TEXT, - oracle_text TEXT, - colors TEXT, -- JSON array - color_identity TEXT, -- JSON array - keywords TEXT, -- JSON array - reserved INTEGER DEFAULT 0, - legalities TEXT, -- JSON object - power TEXT, - toughness TEXT -); - --- MTG sets -CREATE TABLE sets ( - id TEXT PRIMARY KEY, -- Scryfall set id - code TEXT NOT NULL UNIQUE, -- e.g., "lea", "mh2" - name TEXT NOT NULL, -- e.g., "Limited Edition Alpha" - set_type TEXT, -- e.g., "expansion", "core" - released_at TEXT, - card_count INTEGER, - icon_svg_uri TEXT, - digital INTEGER DEFAULT 0, - parent_set_code TEXT, - block TEXT -); - --- Card printings with perceptual hashes CREATE TABLE cards ( - id TEXT PRIMARY KEY, -- Scryfall card ID (printing) - oracle_id TEXT NOT NULL, -- FK to oracles - set_id TEXT NOT NULL, -- FK to sets - set_code TEXT, + id TEXT PRIMARY KEY, -- Scryfall ID + oracle_id TEXT NOT NULL, name TEXT NOT NULL, + set_code TEXT NOT NULL, collector_number TEXT, - rarity TEXT, - artist TEXT, - illustration_id TEXT, -- Same across printings with identical art - image_uri TEXT, - hash BLOB, -- Perceptual hash for matching - lang TEXT DEFAULT 'en', - prices_usd REAL, - prices_usd_foil REAL, - FOREIGN KEY (oracle_id) REFERENCES oracles(id), - FOREIGN KEY (set_id) REFERENCES sets(id) + illustration_id TEXT, + image_url TEXT, + art_hash BLOB, -- 256-bit hash of art region + full_hash BLOB, -- 256-bit hash of full card + color_hash BLOB -- 768-bit color-aware hash ); -CREATE INDEX idx_cards_oracle_id ON cards(oracle_id); -CREATE INDEX idx_cards_set_id ON cards(set_id); -CREATE INDEX idx_cards_name ON cards(name); +CREATE INDEX idx_cards_oracle ON cards(oracle_id); +CREATE INDEX idx_cards_illustration ON cards(illustration_id); ``` ### Phase 2: Enhanced (Add OCR Fallback) diff --git a/screen.png b/screen.png new file mode 100644 index 0000000..3d9ea9a Binary files /dev/null and b/screen.png differ diff --git a/src/Scry.App/MauiProgram.cs b/src/Scry.App/MauiProgram.cs index ebeca2a..0eb375e 100644 --- a/src/Scry.App/MauiProgram.cs +++ b/src/Scry.App/MauiProgram.cs @@ -24,22 +24,11 @@ public static class MauiProgram }); // Core Services (from Scry.Core) - builder.Services.AddSingleton(sp => + builder.Services.AddSingleton(sp => { var dbPath = Path.Combine(FileSystem.AppDataDirectory, "card_hashes.db"); EnsureDatabaseCopied(dbPath); - return new CardDatabase(dbPath); - }); - - // Recognition options - configure debug output in DEBUG builds - builder.Services.Configure(options => - { -#if DEBUG && ANDROID - // Use Download folder for easy adb pull access - options.DebugOutputDirectory = "/sdcard/Download/scry-debug"; -#elif DEBUG - options.DebugOutputDirectory = "./debug"; -#endif + return new CardHashDatabase(dbPath); }); builder.Services.AddSingleton(); @@ -68,25 +57,14 @@ public static class MauiProgram private static void EnsureDatabaseCopied(string targetPath) { + if (File.Exists(targetPath)) + return; + try { - using var bundledStream = FileSystem.OpenAppPackageFileAsync("card_hashes.db").GetAwaiter().GetResult(); - - if (File.Exists(targetPath)) - { - // Compare sizes - if bundled is larger, replace - var existingSize = new FileInfo(targetPath).Length; - var bundledSize = bundledStream.Length; - - if (bundledSize <= existingSize) - return; - - // Bundled db is larger, delete old and copy new - File.Delete(targetPath); - } - + using var stream = FileSystem.OpenAppPackageFileAsync("card_hashes.db").GetAwaiter().GetResult(); using var fileStream = File.Create(targetPath); - bundledStream.CopyTo(fileStream); + stream.CopyTo(fileStream); } catch { diff --git a/src/Scry.App/Resources/Raw/card_hashes.db b/src/Scry.App/Resources/Raw/card_hashes.db index 7f18094..27cc003 100644 Binary files a/src/Scry.App/Resources/Raw/card_hashes.db and b/src/Scry.App/Resources/Raw/card_hashes.db differ diff --git a/src/Scry.App/Scry.App.csproj b/src/Scry.App/Scry.App.csproj index 3a04d80..b0ae247 100644 --- a/src/Scry.App/Scry.App.csproj +++ b/src/Scry.App/Scry.App.csproj @@ -29,11 +29,6 @@ true - - - - - Scry diff --git a/src/Scry.App/Services/MockCardRecognitionService.cs b/src/Scry.App/Services/MockCardRecognitionService.cs index a2db310..16102ca 100644 --- a/src/Scry.App/Services/MockCardRecognitionService.cs +++ b/src/Scry.App/Services/MockCardRecognitionService.cs @@ -12,99 +12,93 @@ public class MockCardRecognitionService : ICardRecognitionService [ new Card { - Id = "4cbc6901-6a4a-4d0a-83ea-7eefa3b35021", - OracleId = "orb-sol-ring", - SetId = "set-c21", + Id = "sol-ring-c21", Name = "Sol Ring", SetCode = "C21", SetName = "Commander 2021", CollectorNumber = "263", + ScryfallId = "4cbc6901-6a4a-4d0a-83ea-7eefa3b35021", ImageUri = "https://cards.scryfall.io/normal/front/4/c/4cbc6901-6a4a-4d0a-83ea-7eefa3b35021.jpg", ManaCost = "{1}", TypeLine = "Artifact", OracleText = "{T}: Add {C}{C}.", Rarity = "uncommon", - PricesUsd = 1.50m + PriceUsd = 1.50m }, new Card { - Id = "e3285e6b-3e79-4d7c-bf96-d920f973b122", - OracleId = "orb-lightning-bolt", - SetId = "set-2xm", + Id = "lightning-bolt-2xm", Name = "Lightning Bolt", SetCode = "2XM", SetName = "Double Masters", CollectorNumber = "129", + ScryfallId = "e3285e6b-3e79-4d7c-bf96-d920f973b122", ImageUri = "https://cards.scryfall.io/normal/front/e/3/e3285e6b-3e79-4d7c-bf96-d920f973b122.jpg", ManaCost = "{R}", TypeLine = "Instant", OracleText = "Lightning Bolt deals 3 damage to any target.", Rarity = "uncommon", - PricesUsd = 2.00m + PriceUsd = 2.00m }, new Card { - Id = "ce30f926-bc06-46ee-9f35-0c32659a1b1c", - OracleId = "orb-counterspell", - SetId = "set-cmr", + Id = "counterspell-cmr", Name = "Counterspell", SetCode = "CMR", SetName = "Commander Legends", CollectorNumber = "395", + ScryfallId = "ce30f926-bc06-46ee-9f35-0c32659a1b1c", ImageUri = "https://cards.scryfall.io/normal/front/c/e/ce30f926-bc06-46ee-9f35-0c32659a1b1c.jpg", ManaCost = "{U}{U}", TypeLine = "Instant", OracleText = "Counter target spell.", Rarity = "uncommon", - PricesUsd = 1.25m + PriceUsd = 1.25m }, new Card { - Id = "73542c66-eb3a-46e8-a8f6-5f02087b28cf", - OracleId = "orb-llanowar-elves", - SetId = "set-m19", + Id = "llanowar-elves-m19", Name = "Llanowar Elves", SetCode = "M19", SetName = "Core Set 2019", CollectorNumber = "314", + ScryfallId = "73542c66-eb3a-46e8-a8f6-5f02087b28cf", ImageUri = "https://cards.scryfall.io/normal/front/7/3/73542c66-eb3a-46e8-a8f6-5f02087b28cf.jpg", ManaCost = "{G}", TypeLine = "Creature — Elf Druid", OracleText = "{T}: Add {G}.", Rarity = "common", - PricesUsd = 0.25m + PriceUsd = 0.25m }, new Card { - Id = "b8dcd4a6-5c0b-4e1e-8e1d-2e2e5c1d6a1e", - OracleId = "orb-swords-to-plowshares", - SetId = "set-cmr", + Id = "swords-to-plowshares-cmr", Name = "Swords to Plowshares", SetCode = "CMR", SetName = "Commander Legends", CollectorNumber = "387", + ScryfallId = "b8dcd4a6-5c0b-4e1e-8e1d-2e2e5c1d6a1e", ImageUri = "https://cards.scryfall.io/normal/front/b/8/b8dcd4a6-5c0b-4e1e-8e1d-2e2e5c1d6a1e.jpg", ManaCost = "{W}", TypeLine = "Instant", OracleText = "Exile target creature. Its controller gains life equal to its power.", Rarity = "uncommon", - PricesUsd = 3.50m + PriceUsd = 3.50m }, new Card { - Id = "bd8fa327-dd41-4737-8f19-2cf5eb1f7c2e", - OracleId = "orb-black-lotus", - SetId = "set-lea", + Id = "black-lotus-lea", Name = "Black Lotus", SetCode = "LEA", SetName = "Limited Edition Alpha", CollectorNumber = "232", + ScryfallId = "bd8fa327-dd41-4737-8f19-2cf5eb1f7c2e", ImageUri = "https://cards.scryfall.io/normal/front/b/d/bd8fa327-dd41-4737-8f19-2cf5eb1f7c2e.jpg", ManaCost = "{0}", TypeLine = "Artifact", OracleText = "{T}, Sacrifice Black Lotus: Add three mana of any one color.", Rarity = "rare", - PricesUsd = 500000.00m + PriceUsd = 500000.00m } ]; diff --git a/src/Scry.App/ViewModels/SettingsViewModel.cs b/src/Scry.App/ViewModels/SettingsViewModel.cs index 407a913..c84bc3c 100644 --- a/src/Scry.App/ViewModels/SettingsViewModel.cs +++ b/src/Scry.App/ViewModels/SettingsViewModel.cs @@ -6,21 +6,15 @@ namespace Scry.ViewModels; public partial class SettingsViewModel : ObservableObject { - private readonly CardDatabase _database; + private readonly CardHashDatabase _database; [ObservableProperty] private int _cardCount; - [ObservableProperty] - private int _oracleCount; - - [ObservableProperty] - private int _setCount; - [ObservableProperty] private string? _statusMessage; - public SettingsViewModel(CardDatabase database) + public SettingsViewModel(CardHashDatabase database) { _database = database; } @@ -28,9 +22,7 @@ public partial class SettingsViewModel : ObservableObject [RelayCommand] private async Task LoadAsync() { - CardCount = await _database.GetCardCountAsync(); - OracleCount = await _database.GetOracleCountAsync(); - SetCount = await _database.GetSetCountAsync(); - StatusMessage = $"Database ready: {CardCount:N0} cards, {OracleCount:N0} oracles, {SetCount:N0} sets"; + CardCount = await _database.GetHashCountAsync(); + StatusMessage = $"Database ready with {CardCount:N0} cards"; } } diff --git a/src/Scry.Core/Data/CardDatabase.cs b/src/Scry.Core/Data/CardDatabase.cs deleted file mode 100644 index 23a3dac..0000000 --- a/src/Scry.Core/Data/CardDatabase.cs +++ /dev/null @@ -1,739 +0,0 @@ -using Microsoft.Data.Sqlite; -using Scry.Core.Models; - -namespace Scry.Core.Data; - -/// -/// SQLite database for storing card data and perceptual hashes. -/// Schema mirrors Scryfall's data model: oracles (game cards), sets, and cards (printings). -/// -public class CardDatabase : IDisposable -{ - private readonly SqliteConnection _connection; - private readonly string _dbPath; - - public CardDatabase(string dbPath) - { - _dbPath = dbPath; - _connection = new SqliteConnection($"Data Source={dbPath}"); - _connection.Open(); - Initialize(); - } - - private void Initialize() - { - using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - -- Abstract game cards (oracle) - CREATE TABLE IF NOT EXISTS oracles ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - mana_cost TEXT, - cmc REAL, - type_line TEXT, - oracle_text TEXT, - colors TEXT, - color_identity TEXT, - keywords TEXT, - reserved INTEGER DEFAULT 0, - legalities TEXT, - power TEXT, - toughness TEXT - ); - - CREATE INDEX IF NOT EXISTS idx_oracles_name ON oracles(name); - - -- MTG sets - CREATE TABLE IF NOT EXISTS sets ( - id TEXT PRIMARY KEY, - code TEXT NOT NULL UNIQUE, - name TEXT NOT NULL, - set_type TEXT, - released_at TEXT, - card_count INTEGER, - icon_svg_uri TEXT, - digital INTEGER DEFAULT 0, - parent_set_code TEXT, - block TEXT - ); - - CREATE INDEX IF NOT EXISTS idx_sets_code ON sets(code); - - -- Card printings with hashes - CREATE TABLE IF NOT EXISTS cards ( - id TEXT PRIMARY KEY, - oracle_id TEXT NOT NULL, - set_id TEXT NOT NULL, - set_code TEXT, - name TEXT NOT NULL, - collector_number TEXT, - rarity TEXT, - artist TEXT, - illustration_id TEXT, - image_uri TEXT, - hash BLOB, - lang TEXT DEFAULT 'en', - prices_usd REAL, - prices_usd_foil REAL, - FOREIGN KEY (oracle_id) REFERENCES oracles(id), - FOREIGN KEY (set_id) REFERENCES sets(id) - ); - - CREATE INDEX IF NOT EXISTS idx_cards_oracle_id ON cards(oracle_id); - CREATE INDEX IF NOT EXISTS idx_cards_set_id ON cards(set_id); - CREATE INDEX IF NOT EXISTS idx_cards_name ON cards(name); - CREATE INDEX IF NOT EXISTS idx_cards_set_code ON cards(set_code); - - -- Metadata for tracking sync state - CREATE TABLE IF NOT EXISTS metadata ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - """; - cmd.ExecuteNonQuery(); - } - - #region Metadata - - public async Task GetMetadataAsync(string key, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "SELECT value FROM metadata WHERE key = $key"; - cmd.Parameters.AddWithValue("$key", key); - - var result = await cmd.ExecuteScalarAsync(ct); - return result as string; - } - - public async Task SetMetadataAsync(string key, string value, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - INSERT OR REPLACE INTO metadata (key, value) VALUES ($key, $value) - """; - cmd.Parameters.AddWithValue("$key", key); - cmd.Parameters.AddWithValue("$value", value); - await cmd.ExecuteNonQueryAsync(ct); - } - - #endregion - - #region Oracles - - public async Task InsertOracleAsync(Oracle oracle, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - INSERT OR REPLACE INTO oracles - (id, name, mana_cost, cmc, type_line, oracle_text, colors, color_identity, keywords, reserved, legalities, power, toughness) - VALUES ($id, $name, $mana_cost, $cmc, $type_line, $oracle_text, $colors, $color_identity, $keywords, $reserved, $legalities, $power, $toughness) - """; - cmd.Parameters.AddWithValue("$id", oracle.Id); - cmd.Parameters.AddWithValue("$name", oracle.Name); - cmd.Parameters.AddWithValue("$mana_cost", oracle.ManaCost ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$cmc", oracle.Cmc ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$type_line", oracle.TypeLine ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$oracle_text", oracle.OracleText ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$colors", oracle.Colors ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$color_identity", oracle.ColorIdentity ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$keywords", oracle.Keywords ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$reserved", oracle.Reserved ? 1 : 0); - cmd.Parameters.AddWithValue("$legalities", oracle.Legalities ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$power", oracle.Power ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$toughness", oracle.Toughness ?? (object)DBNull.Value); - - await cmd.ExecuteNonQueryAsync(ct); - } - - public async Task InsertOracleBatchAsync(IEnumerable oracles, CancellationToken ct = default) - { - await using var transaction = await _connection.BeginTransactionAsync(ct); - - try - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - INSERT OR REPLACE INTO oracles - (id, name, mana_cost, cmc, type_line, oracle_text, colors, color_identity, keywords, reserved, legalities, power, toughness) - VALUES ($id, $name, $mana_cost, $cmc, $type_line, $oracle_text, $colors, $color_identity, $keywords, $reserved, $legalities, $power, $toughness) - """; - - var idParam = cmd.Parameters.Add("$id", SqliteType.Text); - var nameParam = cmd.Parameters.Add("$name", SqliteType.Text); - var manaCostParam = cmd.Parameters.Add("$mana_cost", SqliteType.Text); - var cmcParam = cmd.Parameters.Add("$cmc", SqliteType.Real); - var typeLineParam = cmd.Parameters.Add("$type_line", SqliteType.Text); - var oracleTextParam = cmd.Parameters.Add("$oracle_text", SqliteType.Text); - var colorsParam = cmd.Parameters.Add("$colors", SqliteType.Text); - var colorIdentityParam = cmd.Parameters.Add("$color_identity", SqliteType.Text); - var keywordsParam = cmd.Parameters.Add("$keywords", SqliteType.Text); - var reservedParam = cmd.Parameters.Add("$reserved", SqliteType.Integer); - var legalitiesParam = cmd.Parameters.Add("$legalities", SqliteType.Text); - var powerParam = cmd.Parameters.Add("$power", SqliteType.Text); - var toughnessParam = cmd.Parameters.Add("$toughness", SqliteType.Text); - - foreach (var oracle in oracles) - { - ct.ThrowIfCancellationRequested(); - - idParam.Value = oracle.Id; - nameParam.Value = oracle.Name; - manaCostParam.Value = oracle.ManaCost ?? (object)DBNull.Value; - cmcParam.Value = oracle.Cmc ?? (object)DBNull.Value; - typeLineParam.Value = oracle.TypeLine ?? (object)DBNull.Value; - oracleTextParam.Value = oracle.OracleText ?? (object)DBNull.Value; - colorsParam.Value = oracle.Colors ?? (object)DBNull.Value; - colorIdentityParam.Value = oracle.ColorIdentity ?? (object)DBNull.Value; - keywordsParam.Value = oracle.Keywords ?? (object)DBNull.Value; - reservedParam.Value = oracle.Reserved ? 1 : 0; - legalitiesParam.Value = oracle.Legalities ?? (object)DBNull.Value; - powerParam.Value = oracle.Power ?? (object)DBNull.Value; - toughnessParam.Value = oracle.Toughness ?? (object)DBNull.Value; - - await cmd.ExecuteNonQueryAsync(ct); - } - - await transaction.CommitAsync(ct); - } - catch - { - await transaction.RollbackAsync(ct); - throw; - } - } - - public async Task GetOracleByIdAsync(string id, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - SELECT id, name, mana_cost, cmc, type_line, oracle_text, colors, color_identity, keywords, reserved, legalities, power, toughness - FROM oracles WHERE id = $id - """; - cmd.Parameters.AddWithValue("$id", id); - - await using var reader = await cmd.ExecuteReaderAsync(ct); - if (await reader.ReadAsync(ct)) - { - return ReadOracle(reader); - } - return null; - } - - public async Task GetOracleByNameAsync(string name, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - SELECT id, name, mana_cost, cmc, type_line, oracle_text, colors, color_identity, keywords, reserved, legalities, power, toughness - FROM oracles WHERE name = $name COLLATE NOCASE - """; - cmd.Parameters.AddWithValue("$name", name); - - await using var reader = await cmd.ExecuteReaderAsync(ct); - if (await reader.ReadAsync(ct)) - { - return ReadOracle(reader); - } - return null; - } - - public async Task> GetExistingOracleIdsAsync(CancellationToken ct = default) - { - var ids = new HashSet(); - - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "SELECT id FROM oracles"; - - await using var reader = await cmd.ExecuteReaderAsync(ct); - while (await reader.ReadAsync(ct)) - { - ids.Add(reader.GetString(0)); - } - - return ids; - } - - public async Task GetOracleCountAsync(CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "SELECT COUNT(*) FROM oracles"; - var result = await cmd.ExecuteScalarAsync(ct); - return Convert.ToInt32(result); - } - - private static Oracle ReadOracle(SqliteDataReader reader) => new() - { - Id = reader.GetString(0), - Name = reader.GetString(1), - ManaCost = reader.IsDBNull(2) ? null : reader.GetString(2), - Cmc = reader.IsDBNull(3) ? null : reader.GetDouble(3), - TypeLine = reader.IsDBNull(4) ? null : reader.GetString(4), - OracleText = reader.IsDBNull(5) ? null : reader.GetString(5), - Colors = reader.IsDBNull(6) ? null : reader.GetString(6), - ColorIdentity = reader.IsDBNull(7) ? null : reader.GetString(7), - Keywords = reader.IsDBNull(8) ? null : reader.GetString(8), - Reserved = reader.GetInt32(9) != 0, - Legalities = reader.IsDBNull(10) ? null : reader.GetString(10), - Power = reader.IsDBNull(11) ? null : reader.GetString(11), - Toughness = reader.IsDBNull(12) ? null : reader.GetString(12), - }; - - #endregion - - #region Sets - - public async Task InsertSetAsync(Set set, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - INSERT OR REPLACE INTO sets - (id, code, name, set_type, released_at, card_count, icon_svg_uri, digital, parent_set_code, block) - VALUES ($id, $code, $name, $set_type, $released_at, $card_count, $icon_svg_uri, $digital, $parent_set_code, $block) - """; - cmd.Parameters.AddWithValue("$id", set.Id); - cmd.Parameters.AddWithValue("$code", set.Code); - cmd.Parameters.AddWithValue("$name", set.Name); - cmd.Parameters.AddWithValue("$set_type", set.SetType ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$released_at", set.ReleasedAt ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$card_count", set.CardCount ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$icon_svg_uri", set.IconSvgUri ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$digital", set.Digital ? 1 : 0); - cmd.Parameters.AddWithValue("$parent_set_code", set.ParentSetCode ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$block", set.Block ?? (object)DBNull.Value); - - await cmd.ExecuteNonQueryAsync(ct); - } - - public async Task InsertSetBatchAsync(IEnumerable sets, CancellationToken ct = default) - { - await using var transaction = await _connection.BeginTransactionAsync(ct); - - try - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - INSERT OR REPLACE INTO sets - (id, code, name, set_type, released_at, card_count, icon_svg_uri, digital, parent_set_code, block) - VALUES ($id, $code, $name, $set_type, $released_at, $card_count, $icon_svg_uri, $digital, $parent_set_code, $block) - """; - - var idParam = cmd.Parameters.Add("$id", SqliteType.Text); - var codeParam = cmd.Parameters.Add("$code", SqliteType.Text); - var nameParam = cmd.Parameters.Add("$name", SqliteType.Text); - var setTypeParam = cmd.Parameters.Add("$set_type", SqliteType.Text); - var releasedAtParam = cmd.Parameters.Add("$released_at", SqliteType.Text); - var cardCountParam = cmd.Parameters.Add("$card_count", SqliteType.Integer); - var iconSvgUriParam = cmd.Parameters.Add("$icon_svg_uri", SqliteType.Text); - var digitalParam = cmd.Parameters.Add("$digital", SqliteType.Integer); - var parentSetCodeParam = cmd.Parameters.Add("$parent_set_code", SqliteType.Text); - var blockParam = cmd.Parameters.Add("$block", SqliteType.Text); - - foreach (var set in sets) - { - ct.ThrowIfCancellationRequested(); - - idParam.Value = set.Id; - codeParam.Value = set.Code; - nameParam.Value = set.Name; - setTypeParam.Value = set.SetType ?? (object)DBNull.Value; - releasedAtParam.Value = set.ReleasedAt ?? (object)DBNull.Value; - cardCountParam.Value = set.CardCount ?? (object)DBNull.Value; - iconSvgUriParam.Value = set.IconSvgUri ?? (object)DBNull.Value; - digitalParam.Value = set.Digital ? 1 : 0; - parentSetCodeParam.Value = set.ParentSetCode ?? (object)DBNull.Value; - blockParam.Value = set.Block ?? (object)DBNull.Value; - - await cmd.ExecuteNonQueryAsync(ct); - } - - await transaction.CommitAsync(ct); - } - catch - { - await transaction.RollbackAsync(ct); - throw; - } - } - - public async Task GetSetByIdAsync(string id, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - SELECT id, code, name, set_type, released_at, card_count, icon_svg_uri, digital, parent_set_code, block - FROM sets WHERE id = $id - """; - cmd.Parameters.AddWithValue("$id", id); - - await using var reader = await cmd.ExecuteReaderAsync(ct); - if (await reader.ReadAsync(ct)) - { - return ReadSet(reader); - } - return null; - } - - public async Task GetSetByCodeAsync(string code, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - SELECT id, code, name, set_type, released_at, card_count, icon_svg_uri, digital, parent_set_code, block - FROM sets WHERE code = $code COLLATE NOCASE - """; - cmd.Parameters.AddWithValue("$code", code); - - await using var reader = await cmd.ExecuteReaderAsync(ct); - if (await reader.ReadAsync(ct)) - { - return ReadSet(reader); - } - return null; - } - - public async Task> GetExistingSetIdsAsync(CancellationToken ct = default) - { - var ids = new HashSet(); - - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "SELECT id FROM sets"; - - await using var reader = await cmd.ExecuteReaderAsync(ct); - while (await reader.ReadAsync(ct)) - { - ids.Add(reader.GetString(0)); - } - - return ids; - } - - public async Task GetSetCountAsync(CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "SELECT COUNT(*) FROM sets"; - var result = await cmd.ExecuteScalarAsync(ct); - return Convert.ToInt32(result); - } - - private static Set ReadSet(SqliteDataReader reader) => new() - { - Id = reader.GetString(0), - Code = reader.GetString(1), - Name = reader.GetString(2), - SetType = reader.IsDBNull(3) ? null : reader.GetString(3), - ReleasedAt = reader.IsDBNull(4) ? null : reader.GetString(4), - CardCount = reader.IsDBNull(5) ? null : reader.GetInt32(5), - IconSvgUri = reader.IsDBNull(6) ? null : reader.GetString(6), - Digital = reader.GetInt32(7) != 0, - ParentSetCode = reader.IsDBNull(8) ? null : reader.GetString(8), - Block = reader.IsDBNull(9) ? null : reader.GetString(9), - }; - - #endregion - - #region Cards (Printings) - - public async Task InsertCardAsync(Card card, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - INSERT OR REPLACE INTO cards - (id, oracle_id, set_id, set_code, name, collector_number, rarity, artist, illustration_id, image_uri, hash, lang, prices_usd, prices_usd_foil) - VALUES ($id, $oracle_id, $set_id, $set_code, $name, $collector_number, $rarity, $artist, $illustration_id, $image_uri, $hash, $lang, $prices_usd, $prices_usd_foil) - """; - cmd.Parameters.AddWithValue("$id", card.Id); - cmd.Parameters.AddWithValue("$oracle_id", card.OracleId); - cmd.Parameters.AddWithValue("$set_id", card.SetId); - cmd.Parameters.AddWithValue("$set_code", card.SetCode ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$name", card.Name); - cmd.Parameters.AddWithValue("$collector_number", card.CollectorNumber ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$rarity", card.Rarity ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$artist", card.Artist ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$illustration_id", card.IllustrationId ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$image_uri", card.ImageUri ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$hash", card.Hash ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$lang", card.Lang ?? "en"); - cmd.Parameters.AddWithValue("$prices_usd", card.PricesUsd ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$prices_usd_foil", card.PricesUsdFoil ?? (object)DBNull.Value); - - await cmd.ExecuteNonQueryAsync(ct); - } - - public async Task InsertCardBatchAsync(IEnumerable cards, CancellationToken ct = default) - { - await using var transaction = await _connection.BeginTransactionAsync(ct); - - try - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - INSERT OR REPLACE INTO cards - (id, oracle_id, set_id, set_code, name, collector_number, rarity, artist, illustration_id, image_uri, hash, lang, prices_usd, prices_usd_foil) - VALUES ($id, $oracle_id, $set_id, $set_code, $name, $collector_number, $rarity, $artist, $illustration_id, $image_uri, $hash, $lang, $prices_usd, $prices_usd_foil) - """; - - var idParam = cmd.Parameters.Add("$id", SqliteType.Text); - var oracleIdParam = cmd.Parameters.Add("$oracle_id", SqliteType.Text); - var setIdParam = cmd.Parameters.Add("$set_id", SqliteType.Text); - var setCodeParam = cmd.Parameters.Add("$set_code", SqliteType.Text); - var nameParam = cmd.Parameters.Add("$name", SqliteType.Text); - var collectorNumberParam = cmd.Parameters.Add("$collector_number", SqliteType.Text); - var rarityParam = cmd.Parameters.Add("$rarity", SqliteType.Text); - var artistParam = cmd.Parameters.Add("$artist", SqliteType.Text); - var illustrationIdParam = cmd.Parameters.Add("$illustration_id", SqliteType.Text); - var imageUriParam = cmd.Parameters.Add("$image_uri", SqliteType.Text); - var hashParam = cmd.Parameters.Add("$hash", SqliteType.Blob); - var langParam = cmd.Parameters.Add("$lang", SqliteType.Text); - var pricesUsdParam = cmd.Parameters.Add("$prices_usd", SqliteType.Real); - var pricesUsdFoilParam = cmd.Parameters.Add("$prices_usd_foil", SqliteType.Real); - - foreach (var card in cards) - { - ct.ThrowIfCancellationRequested(); - - idParam.Value = card.Id; - oracleIdParam.Value = card.OracleId; - setIdParam.Value = card.SetId; - setCodeParam.Value = card.SetCode ?? (object)DBNull.Value; - nameParam.Value = card.Name; - collectorNumberParam.Value = card.CollectorNumber ?? (object)DBNull.Value; - rarityParam.Value = card.Rarity ?? (object)DBNull.Value; - artistParam.Value = card.Artist ?? (object)DBNull.Value; - illustrationIdParam.Value = card.IllustrationId ?? (object)DBNull.Value; - imageUriParam.Value = card.ImageUri ?? (object)DBNull.Value; - hashParam.Value = card.Hash ?? (object)DBNull.Value; - langParam.Value = card.Lang ?? "en"; - pricesUsdParam.Value = card.PricesUsd ?? (object)DBNull.Value; - pricesUsdFoilParam.Value = card.PricesUsdFoil ?? (object)DBNull.Value; - - await cmd.ExecuteNonQueryAsync(ct); - } - - await transaction.CommitAsync(ct); - } - catch - { - await transaction.RollbackAsync(ct); - throw; - } - } - - public async Task GetCardByIdAsync(string id, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - SELECT id, oracle_id, set_id, set_code, name, collector_number, rarity, artist, illustration_id, image_uri, hash, lang, prices_usd, prices_usd_foil - FROM cards WHERE id = $id - """; - cmd.Parameters.AddWithValue("$id", id); - - await using var reader = await cmd.ExecuteReaderAsync(ct); - if (await reader.ReadAsync(ct)) - { - return ReadCard(reader); - } - return null; - } - - public async Task> GetCardsByOracleIdAsync(string oracleId, CancellationToken ct = default) - { - var cards = new List(); - - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - SELECT id, oracle_id, set_id, set_code, name, collector_number, rarity, artist, illustration_id, image_uri, hash, lang, prices_usd, prices_usd_foil - FROM cards WHERE oracle_id = $oracle_id - """; - cmd.Parameters.AddWithValue("$oracle_id", oracleId); - - await using var reader = await cmd.ExecuteReaderAsync(ct); - while (await reader.ReadAsync(ct)) - { - cards.Add(ReadCard(reader)); - } - - return cards; - } - - public async Task> GetCardsByNameAsync(string name, CancellationToken ct = default) - { - var cards = new List(); - - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - SELECT id, oracle_id, set_id, set_code, name, collector_number, rarity, artist, illustration_id, image_uri, hash, lang, prices_usd, prices_usd_foil - FROM cards WHERE name = $name COLLATE NOCASE - """; - cmd.Parameters.AddWithValue("$name", name); - - await using var reader = await cmd.ExecuteReaderAsync(ct); - while (await reader.ReadAsync(ct)) - { - cards.Add(ReadCard(reader)); - } - - return cards; - } - - public async Task> GetAllCardsAsync(CancellationToken ct = default) - { - var cards = new List(); - - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - SELECT id, oracle_id, set_id, set_code, name, collector_number, rarity, artist, illustration_id, image_uri, hash, lang, prices_usd, prices_usd_foil - FROM cards - """; - - await using var reader = await cmd.ExecuteReaderAsync(ct); - while (await reader.ReadAsync(ct)) - { - cards.Add(ReadCard(reader)); - } - - return cards; - } - - public async Task> GetCardsWithHashAsync(CancellationToken ct = default) - { - var cards = new List(); - - await using var cmd = _connection.CreateCommand(); - // Join with oracles and sets to get denormalized data - cmd.CommandText = """ - SELECT c.id, c.oracle_id, c.set_id, c.set_code, c.name, c.collector_number, c.rarity, c.artist, - c.illustration_id, c.image_uri, c.hash, c.lang, c.prices_usd, c.prices_usd_foil, - o.mana_cost, o.type_line, o.oracle_text, o.power, o.toughness, - s.name as set_name - FROM cards c - LEFT JOIN oracles o ON c.oracle_id = o.id - LEFT JOIN sets s ON c.set_id = s.id - WHERE c.hash IS NOT NULL - """; - - await using var reader = await cmd.ExecuteReaderAsync(ct); - while (await reader.ReadAsync(ct)) - { - cards.Add(ReadCardWithOracle(reader)); - } - - return cards; - } - - public async Task> GetExistingCardIdsAsync(CancellationToken ct = default) - { - var ids = new HashSet(); - - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "SELECT id FROM cards"; - - await using var reader = await cmd.ExecuteReaderAsync(ct); - while (await reader.ReadAsync(ct)) - { - ids.Add(reader.GetString(0)); - } - - return ids; - } - - public async Task> GetExistingCardNamesAsync(CancellationToken ct = default) - { - var names = new HashSet(StringComparer.OrdinalIgnoreCase); - - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "SELECT DISTINCT name FROM cards"; - - await using var reader = await cmd.ExecuteReaderAsync(ct); - while (await reader.ReadAsync(ct)) - { - names.Add(reader.GetString(0)); - } - - return names; - } - - public async Task GetCardCountAsync(CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "SELECT COUNT(*) FROM cards"; - var result = await cmd.ExecuteScalarAsync(ct); - return Convert.ToInt32(result); - } - - public async Task DeleteCardByIdAsync(string id, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "DELETE FROM cards WHERE id = $id"; - cmd.Parameters.AddWithValue("$id", id); - await cmd.ExecuteNonQueryAsync(ct); - } - - public async Task ClearCardsAsync(CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "DELETE FROM cards"; - await cmd.ExecuteNonQueryAsync(ct); - } - - public async Task ClearAllAsync(CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - DELETE FROM cards; - DELETE FROM oracles; - DELETE FROM sets; - DELETE FROM metadata; - """; - await cmd.ExecuteNonQueryAsync(ct); - } - - private static Card ReadCard(SqliteDataReader reader) => new() - { - Id = reader.GetString(0), - OracleId = reader.GetString(1), - SetId = reader.GetString(2), - SetCode = reader.IsDBNull(3) ? null : reader.GetString(3), - Name = reader.GetString(4), - CollectorNumber = reader.IsDBNull(5) ? null : reader.GetString(5), - Rarity = reader.IsDBNull(6) ? null : reader.GetString(6), - Artist = reader.IsDBNull(7) ? null : reader.GetString(7), - IllustrationId = reader.IsDBNull(8) ? null : reader.GetString(8), - ImageUri = reader.IsDBNull(9) ? null : reader.GetString(9), - Hash = reader.IsDBNull(10) ? null : (byte[])reader.GetValue(10), - Lang = reader.IsDBNull(11) ? null : reader.GetString(11), - PricesUsd = reader.IsDBNull(12) ? null : (decimal)reader.GetDouble(12), - PricesUsdFoil = reader.IsDBNull(13) ? null : (decimal)reader.GetDouble(13), - }; - - /// - /// Reads a card with joined Oracle and Set data (columns 14-19). - /// - private static Card ReadCardWithOracle(SqliteDataReader reader) => new() - { - Id = reader.GetString(0), - OracleId = reader.GetString(1), - SetId = reader.GetString(2), - SetCode = reader.IsDBNull(3) ? null : reader.GetString(3), - Name = reader.GetString(4), - CollectorNumber = reader.IsDBNull(5) ? null : reader.GetString(5), - Rarity = reader.IsDBNull(6) ? null : reader.GetString(6), - Artist = reader.IsDBNull(7) ? null : reader.GetString(7), - IllustrationId = reader.IsDBNull(8) ? null : reader.GetString(8), - ImageUri = reader.IsDBNull(9) ? null : reader.GetString(9), - Hash = reader.IsDBNull(10) ? null : (byte[])reader.GetValue(10), - Lang = reader.IsDBNull(11) ? null : reader.GetString(11), - PricesUsd = reader.IsDBNull(12) ? null : (decimal)reader.GetDouble(12), - PricesUsdFoil = reader.IsDBNull(13) ? null : (decimal)reader.GetDouble(13), - // Denormalized Oracle fields (from JOIN) - ManaCost = reader.IsDBNull(14) ? null : reader.GetString(14), - TypeLine = reader.IsDBNull(15) ? null : reader.GetString(15), - OracleText = reader.IsDBNull(16) ? null : reader.GetString(16), - Power = reader.IsDBNull(17) ? null : reader.GetString(17), - Toughness = reader.IsDBNull(18) ? null : reader.GetString(18), - SetName = reader.IsDBNull(19) ? null : reader.GetString(19), - }; - - #endregion - - public void Dispose() - { - _connection.Dispose(); - } -} diff --git a/src/Scry.Core/Data/CardHashDatabase.cs b/src/Scry.Core/Data/CardHashDatabase.cs new file mode 100644 index 0000000..cc2629a --- /dev/null +++ b/src/Scry.Core/Data/CardHashDatabase.cs @@ -0,0 +1,234 @@ +using Microsoft.Data.Sqlite; +using Scry.Core.Models; + +namespace Scry.Core.Data; + +public class CardHashDatabase : IDisposable +{ + private readonly SqliteConnection _connection; + private readonly string _dbPath; + + public CardHashDatabase(string dbPath) + { + _dbPath = dbPath; + _connection = new SqliteConnection($"Data Source={dbPath}"); + _connection.Open(); + Initialize(); + } + + private void Initialize() + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + CREATE TABLE IF NOT EXISTS card_hashes ( + card_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + set_code TEXT NOT NULL, + collector_number TEXT, + hash BLOB NOT NULL, + image_uri TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_card_hashes_name ON card_hashes(name); + CREATE INDEX IF NOT EXISTS idx_card_hashes_set ON card_hashes(set_code); + + CREATE TABLE IF NOT EXISTS metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + """; + cmd.ExecuteNonQuery(); + } + + public async Task GetMetadataAsync(string key, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT value FROM metadata WHERE key = $key"; + cmd.Parameters.AddWithValue("$key", key); + + var result = await cmd.ExecuteScalarAsync(ct); + return result as string; + } + + public async Task SetMetadataAsync(string key, string value, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + INSERT OR REPLACE INTO metadata (key, value) VALUES ($key, $value) + """; + cmd.Parameters.AddWithValue("$key", key); + cmd.Parameters.AddWithValue("$value", value); + await cmd.ExecuteNonQueryAsync(ct); + } + + public async Task InsertHashAsync(CardHash hash, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + INSERT OR REPLACE INTO card_hashes + (card_id, name, set_code, collector_number, hash, image_uri) + VALUES ($card_id, $name, $set_code, $collector_number, $hash, $image_uri) + """; + cmd.Parameters.AddWithValue("$card_id", hash.CardId); + cmd.Parameters.AddWithValue("$name", hash.Name); + cmd.Parameters.AddWithValue("$set_code", hash.SetCode); + cmd.Parameters.AddWithValue("$collector_number", hash.CollectorNumber ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$hash", hash.Hash); + cmd.Parameters.AddWithValue("$image_uri", hash.ImageUri ?? (object)DBNull.Value); + + await cmd.ExecuteNonQueryAsync(ct); + } + + public async Task InsertHashBatchAsync(IEnumerable hashes, CancellationToken ct = default) + { + await using var transaction = await _connection.BeginTransactionAsync(ct); + + try + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + INSERT OR REPLACE INTO card_hashes + (card_id, name, set_code, collector_number, hash, image_uri) + VALUES ($card_id, $name, $set_code, $collector_number, $hash, $image_uri) + """; + + var cardIdParam = cmd.Parameters.Add("$card_id", SqliteType.Text); + var nameParam = cmd.Parameters.Add("$name", SqliteType.Text); + var setCodeParam = cmd.Parameters.Add("$set_code", SqliteType.Text); + var collectorNumberParam = cmd.Parameters.Add("$collector_number", SqliteType.Text); + var hashParam = cmd.Parameters.Add("$hash", SqliteType.Blob); + var imageUriParam = cmd.Parameters.Add("$image_uri", SqliteType.Text); + + foreach (var hash in hashes) + { + ct.ThrowIfCancellationRequested(); + + cardIdParam.Value = hash.CardId; + nameParam.Value = hash.Name; + setCodeParam.Value = hash.SetCode; + collectorNumberParam.Value = hash.CollectorNumber ?? (object)DBNull.Value; + hashParam.Value = hash.Hash; + imageUriParam.Value = hash.ImageUri ?? (object)DBNull.Value; + + await cmd.ExecuteNonQueryAsync(ct); + } + + await transaction.CommitAsync(ct); + } + catch + { + await transaction.RollbackAsync(ct); + throw; + } + } + + public async Task> GetAllHashesAsync(CancellationToken ct = default) + { + var hashes = new List(); + + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT card_id, name, set_code, collector_number, hash, image_uri FROM card_hashes"; + + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + hashes.Add(new CardHash + { + CardId = reader.GetString(0), + Name = reader.GetString(1), + SetCode = reader.GetString(2), + CollectorNumber = reader.IsDBNull(3) ? null : reader.GetString(3), + Hash = (byte[])reader.GetValue(4), + ImageUri = reader.IsDBNull(5) ? null : reader.GetString(5) + }); + } + + return hashes; + } + + public async Task GetHashCountAsync(CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM card_hashes"; + var result = await cmd.ExecuteScalarAsync(ct); + return Convert.ToInt32(result); + } + + public async Task GetHashByIdAsync(string cardId, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + SELECT card_id, name, set_code, collector_number, hash, image_uri + FROM card_hashes WHERE card_id = $card_id + """; + cmd.Parameters.AddWithValue("$card_id", cardId); + + await using var reader = await cmd.ExecuteReaderAsync(ct); + if (await reader.ReadAsync(ct)) + { + return new CardHash + { + CardId = reader.GetString(0), + Name = reader.GetString(1), + SetCode = reader.GetString(2), + CollectorNumber = reader.IsDBNull(3) ? null : reader.GetString(3), + Hash = (byte[])reader.GetValue(4), + ImageUri = reader.IsDBNull(5) ? null : reader.GetString(5) + }; + } + + return null; + } + + public async Task> GetExistingCardIdsAsync(CancellationToken ct = default) + { + var ids = new HashSet(); + + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT card_id FROM card_hashes"; + + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + ids.Add(reader.GetString(0)); + } + + return ids; + } + + public async Task> GetExistingCardNamesAsync(CancellationToken ct = default) + { + var names = new HashSet(StringComparer.OrdinalIgnoreCase); + + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT DISTINCT name FROM card_hashes"; + + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + names.Add(reader.GetString(0)); + } + + return names; + } + + public async Task DeleteByCardIdAsync(string cardId, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "DELETE FROM card_hashes WHERE card_id = $card_id"; + cmd.Parameters.AddWithValue("$card_id", cardId); + await cmd.ExecuteNonQueryAsync(ct); + } + + public async Task ClearAsync(CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "DELETE FROM card_hashes"; + await cmd.ExecuteNonQueryAsync(ct); + } + + public void Dispose() + { + _connection.Dispose(); + } +} diff --git a/src/Scry.Core/Models/Card.cs b/src/Scry.Core/Models/Card.cs index 50d3bbd..42944c6 100644 --- a/src/Scry.Core/Models/Card.cs +++ b/src/Scry.Core/Models/Card.cs @@ -1,143 +1,32 @@ namespace Scry.Core.Models; -/// -/// Represents a specific printing of a card in a set. -/// Maps to Scryfall's Card object (which is really a printing). -/// Contains the perceptual hash for image matching. -/// Includes denormalized Oracle data for convenience. -/// public record Card { - /// - /// Scryfall's unique card ID for this specific printing. - /// public required string Id { get; init; } - - /// - /// Oracle ID linking to the abstract game card. - /// - public required string OracleId { get; init; } - - /// - /// Set ID this printing belongs to. - /// - public required string SetId { get; init; } - - /// - /// Set code (e.g., "lea", "mh2") - denormalized for convenience. - /// - public string? SetCode { get; init; } - - /// - /// Set name - denormalized for convenience. - /// - public string? SetName { get; init; } - - /// - /// Card name - denormalized from Oracle for convenience. - /// public required string Name { get; init; } - - /// - /// Collector number within the set. - /// + public string? SetCode { get; init; } + public string? SetName { get; init; } public string? CollectorNumber { get; init; } - - /// - /// Rarity (common, uncommon, rare, mythic). - /// + public string? ScryfallId { get; init; } public string? Rarity { get; init; } - - /// - /// Artist name. - /// - public string? Artist { get; init; } - - /// - /// Illustration ID - same across printings with identical art. - /// - public string? IllustrationId { get; init; } - - /// - /// URI to the card image (normal size). - /// - public string? ImageUri { get; init; } - - /// - /// Perceptual hash for image matching. - /// - public byte[]? Hash { get; init; } - - /// - /// Language code (e.g., "en", "ja"). - /// - public string? Lang { get; init; } - - /// - /// USD price for non-foil. - /// - public decimal? PricesUsd { get; init; } - - /// - /// USD price for foil. - /// - public decimal? PricesUsdFoil { get; init; } - - #region Denormalized Oracle Fields (for App layer convenience) - - /// - /// Mana cost in Scryfall notation (e.g., "{2}{U}{U}"). - /// Denormalized from Oracle. - /// public string? ManaCost { get; init; } - - /// - /// Full type line (e.g., "Legendary Creature — Human Wizard"). - /// Denormalized from Oracle. - /// public string? TypeLine { get; init; } - - /// - /// Official Oracle rules text. - /// Denormalized from Oracle. - /// public string? OracleText { get; init; } + public string? ImageUri { get; init; } + public string? ImageUriSmall { get; init; } + public string? ImageUriLarge { get; init; } + public string? Artist { get; init; } + public string? Lang { get; init; } + public decimal? PriceUsd { get; init; } + public decimal? PriceUsdFoil { get; init; } /// - /// Power for creatures (may contain non-numeric values like "*"). - /// Denormalized from Oracle. - /// - public string? Power { get; init; } - - /// - /// Toughness for creatures (may contain non-numeric values like "*"). - /// Denormalized from Oracle. - /// - public string? Toughness { get; init; } - - #endregion - - #region Compatibility Aliases - - /// - /// Alias for ImageUri for compatibility. + /// Alias for ImageUri for compatibility with App layer /// public string? ImageUrl => ImageUri; /// - /// Alias for PricesUsd for compatibility. + /// Alias for PriceUsd for compatibility with App layer /// - public decimal? Price => PricesUsd; - - /// - /// Alias for Id (Scryfall ID) for compatibility. - /// - public string ScryfallId => Id; - - /// - /// Alias for PricesUsd for compatibility. - /// - public decimal? PriceUsd => PricesUsd; - - #endregion + public decimal? Price => PriceUsd; } diff --git a/src/Scry.Core/Models/CardHash.cs b/src/Scry.Core/Models/CardHash.cs new file mode 100644 index 0000000..e0724a5 --- /dev/null +++ b/src/Scry.Core/Models/CardHash.cs @@ -0,0 +1,11 @@ +namespace Scry.Core.Models; + +public record CardHash +{ + public required string CardId { get; init; } + public required string Name { get; init; } + public required string SetCode { get; init; } + public string? CollectorNumber { get; init; } + public required byte[] Hash { get; init; } + public string? ImageUri { get; init; } +} diff --git a/src/Scry.Core/Models/Oracle.cs b/src/Scry.Core/Models/Oracle.cs deleted file mode 100644 index 9b76120..0000000 --- a/src/Scry.Core/Models/Oracle.cs +++ /dev/null @@ -1,73 +0,0 @@ -namespace Scry.Core.Models; - -/// -/// Represents an abstract game card - the rules object shared across all printings. -/// Maps to Scryfall's oracle_id concept. -/// -public record Oracle -{ - /// - /// Scryfall's oracle_id - unique identifier for this game card across all printings. - /// - public required string Id { get; init; } - - /// - /// The card name (e.g., "Lightning Bolt"). - /// - public required string Name { get; init; } - - /// - /// Mana cost in Scryfall notation (e.g., "{2}{U}{U}"). - /// - public string? ManaCost { get; init; } - - /// - /// Mana value (converted mana cost). - /// - public double? Cmc { get; init; } - - /// - /// Full type line (e.g., "Legendary Creature — Human Wizard"). - /// - public string? TypeLine { get; init; } - - /// - /// Official Oracle rules text. - /// - public string? OracleText { get; init; } - - /// - /// Card colors as JSON array (e.g., ["U", "R"]). - /// - public string? Colors { get; init; } - - /// - /// Color identity for Commander as JSON array. - /// - public string? ColorIdentity { get; init; } - - /// - /// Keywords as JSON array (e.g., ["Flying", "Trample"]). - /// - public string? Keywords { get; init; } - - /// - /// Whether this card is on the Reserved List. - /// - public bool Reserved { get; init; } - - /// - /// Format legalities as JSON object. - /// - public string? Legalities { get; init; } - - /// - /// Power for creatures (may contain non-numeric values like "*"). - /// - public string? Power { get; init; } - - /// - /// Toughness for creatures (may contain non-numeric values like "*"). - /// - public string? Toughness { get; init; } -} diff --git a/src/Scry.Core/Models/Set.cs b/src/Scry.Core/Models/Set.cs deleted file mode 100644 index c2f1b19..0000000 --- a/src/Scry.Core/Models/Set.cs +++ /dev/null @@ -1,57 +0,0 @@ -namespace Scry.Core.Models; - -/// -/// Represents an MTG set. Maps to Scryfall's Set object. -/// -public record Set -{ - /// - /// Scryfall's unique set ID. - /// - public required string Id { get; init; } - - /// - /// Unique 3-6 letter set code (e.g., "lea", "mh2"). - /// - public required string Code { get; init; } - - /// - /// English name of the set (e.g., "Limited Edition Alpha"). - /// - public required string Name { get; init; } - - /// - /// Set classification (e.g., "expansion", "core", "masters", "commander"). - /// - public string? SetType { get; init; } - - /// - /// Release date in ISO 8601 format. - /// - public string? ReleasedAt { get; init; } - - /// - /// Number of cards in the set. - /// - public int? CardCount { get; init; } - - /// - /// URI to the set's icon SVG. - /// - public string? IconSvgUri { get; init; } - - /// - /// Whether this is a digital-only set. - /// - public bool Digital { get; init; } - - /// - /// Parent set code for promo/token sets. - /// - public string? ParentSetCode { get; init; } - - /// - /// Block name, if applicable. - /// - public string? Block { get; init; } -} diff --git a/src/Scry.Core/Recognition/CardRecognitionService.cs b/src/Scry.Core/Recognition/CardRecognitionService.cs index b560e14..a417aad 100644 --- a/src/Scry.Core/Recognition/CardRecognitionService.cs +++ b/src/Scry.Core/Recognition/CardRecognitionService.cs @@ -1,5 +1,4 @@ using System.Diagnostics; -using Microsoft.Extensions.Options; using Scry.Core.Data; using Scry.Core.Imaging; using Scry.Core.Models; @@ -9,23 +8,29 @@ namespace Scry.Core.Recognition; public class CardRecognitionService : IDisposable { - private readonly CardDatabase _database; - private readonly RecognitionOptions _options; - private List? _cardCache; + private readonly CardHashDatabase _database; + private List? _hashCache; private readonly SemaphoreSlim _cacheLock = new(1, 1); private const int ColorHashBits = 192; private const int MatchThreshold = 25; private const float MinConfidence = 0.85f; - public CardRecognitionService(CardDatabase database, IOptions options) + /// + /// Enable card detection and perspective correction. + /// When disabled, assumes the input image is already a cropped card. + /// + public bool EnableCardDetection { get; set; } = true; + + /// + /// Try multiple rotations (0°, 90°, 180°, 270°) when matching. + /// Useful when card orientation is unknown. + /// + public bool EnableRotationMatching { get; set; } = true; + + public CardRecognitionService(CardHashDatabase database) { _database = database; - _options = options.Value; - } - - public CardRecognitionService(CardDatabase database) : this(database, Options.Create(new RecognitionOptions())) - { } public async Task RecognizeAsync(Stream imageStream, CancellationToken ct = default) @@ -55,49 +60,29 @@ public class CardRecognitionService : IDisposable public async Task RecognizeAsync(SKBitmap bitmap, CancellationToken ct = default) { var stopwatch = Stopwatch.StartNew(); - var debugDir = _options.DebugOutputDirectory; - var debugEnabled = !string.IsNullOrEmpty(debugDir); - - if (debugEnabled) - { - Directory.CreateDirectory(debugDir!); - SaveDebugImage(bitmap, debugDir!, "01_input"); - } try { - var cards = await GetCardCacheAsync(ct); - Console.WriteLine($"[Scry] Database has {cards.Count} cards with hashes"); + var hashes = await GetHashCacheAsync(ct); + Console.WriteLine($"[Scry] Database has {hashes.Count} hashes"); - if (cards.Count == 0) + if (hashes.Count == 0) { - return ScanResult.Failed("No cards in database. Run sync first."); + return ScanResult.Failed("No card hashes in database. Run sync first."); } // Step 1: Detect and extract card from image (if enabled) SKBitmap cardImage; bool cardDetected = false; - if (_options.EnableCardDetection) + if (EnableCardDetection) { var detection = CardDetector.DetectCard(bitmap); - - if (debugEnabled) - { - // Save detection visualization - SaveDetectionDebugImage(bitmap, detection, debugDir!); - } - if (detection.Found) { cardImage = PerspectiveCorrection.WarpPerspective(bitmap, detection.Corners); cardDetected = true; Console.WriteLine($"[Scry] Card detected with confidence {detection.Confidence:P0}"); - - if (debugEnabled) - { - SaveDebugImage(cardImage, debugDir!, "03_perspective_corrected"); - } } else { @@ -114,9 +99,9 @@ public class CardRecognitionService : IDisposable try { // Step 2: Try matching with rotation variants (if enabled) - var bestMatch = _options.EnableRotationMatching - ? await FindBestMatchWithRotationsAsync(cardImage, cards, debugDir, ct) - : FindBestMatchSingle(cardImage, cards, debugDir); + var bestMatch = EnableRotationMatching + ? await FindBestMatchWithRotationsAsync(cardImage, hashes, ct) + : FindBestMatchSingle(cardImage, hashes); stopwatch.Stop(); @@ -125,16 +110,25 @@ public class CardRecognitionService : IDisposable return ScanResult.Failed($"No match found (detection={cardDetected})"); } - var (matchedCard, distance, rotation) = bestMatch.Value; + var (cardHash, distance, rotation) = bestMatch.Value; var confidence = PerceptualHash.CalculateConfidence(distance, ColorHashBits); - Console.WriteLine($"[Scry] Best match: {matchedCard.Name}, distance={distance}, confidence={confidence:P0}, rotation={rotation}°"); + Console.WriteLine($"[Scry] Best match: {cardHash.Name}, distance={distance}, confidence={confidence:P0}, rotation={rotation}°"); if (confidence < MinConfidence) { return ScanResult.Failed($"Match confidence too low: {confidence:P0}"); } - return ScanResult.Matched(matchedCard, confidence, distance, stopwatch.Elapsed); + var card = new Card + { + Id = cardHash.CardId, + Name = cardHash.Name, + SetCode = cardHash.SetCode, + CollectorNumber = cardHash.CollectorNumber, + ImageUri = cardHash.ImageUri + }; + + return ScanResult.Matched(card, confidence, distance, stopwatch.Elapsed); } finally { @@ -182,7 +176,7 @@ public class CardRecognitionService : IDisposable await _cacheLock.WaitAsync(); try { - _cardCache = null; + _hashCache = null; } finally { @@ -190,16 +184,16 @@ public class CardRecognitionService : IDisposable } } - private async Task> GetCardCacheAsync(CancellationToken ct) + private async Task> GetHashCacheAsync(CancellationToken ct) { - if (_cardCache != null) - return _cardCache; + if (_hashCache != null) + return _hashCache; await _cacheLock.WaitAsync(ct); try { - _cardCache ??= await _database.GetCardsWithHashAsync(ct); - return _cardCache; + _hashCache ??= await _database.GetAllHashesAsync(ct); + return _hashCache; } finally { @@ -210,18 +204,16 @@ public class CardRecognitionService : IDisposable /// /// Find best match trying all 4 rotations (0°, 90°, 180°, 270°). /// - private Task<(Card Card, int Distance, int Rotation)?> FindBestMatchWithRotationsAsync( + private Task<(CardHash Hash, int Distance, int Rotation)?> FindBestMatchWithRotationsAsync( SKBitmap cardImage, - List candidates, - string? debugDir, + List candidates, CancellationToken ct) { return Task.Run(() => { - Card? bestMatch = null; + CardHash? bestMatch = null; var bestDistance = int.MaxValue; var bestRotation = 0; - var debugEnabled = !string.IsNullOrEmpty(debugDir); var rotations = new[] { 0, 90, 180, 270 }; @@ -232,25 +224,14 @@ public class CardRecognitionService : IDisposable using var rotated = rotation == 0 ? null : RotateImage(cardImage, rotation); var imageToHash = rotated ?? cardImage; - if (debugEnabled && rotation != 0) - { - SaveDebugImage(imageToHash, debugDir!, $"04_rotated_{rotation}"); - } - // Apply CLAHE and compute hash using var preprocessed = ImagePreprocessor.ApplyClahe(imageToHash); - - if (debugEnabled) - { - SaveDebugImage(preprocessed, debugDir!, $"05_clahe_{rotation}"); - } - var queryHash = PerceptualHash.ComputeColorHash(preprocessed); // Find best match for this rotation foreach (var candidate in candidates) { - if (candidate.Hash == null || candidate.Hash.Length != queryHash.Length) + if (candidate.Hash.Length != queryHash.Length) continue; var distance = PerceptualHash.HammingDistance(queryHash, candidate.Hash); @@ -271,36 +252,27 @@ public class CardRecognitionService : IDisposable if (bestMatch == null || bestDistance > MatchThreshold) return null; - return ((Card Card, int Distance, int Rotation)?)(bestMatch, bestDistance, bestRotation); + return ((CardHash Hash, int Distance, int Rotation)?)(bestMatch, bestDistance, bestRotation); }, ct); } /// /// Find best match without rotation (single orientation). /// - private (Card Card, int Distance, int Rotation)? FindBestMatchSingle( + private (CardHash Hash, int Distance, int Rotation)? FindBestMatchSingle( SKBitmap cardImage, - List candidates, - string? debugDir) + List candidates) { - var debugEnabled = !string.IsNullOrEmpty(debugDir); - // Apply CLAHE and compute hash using var preprocessed = ImagePreprocessor.ApplyClahe(cardImage); - - if (debugEnabled) - { - SaveDebugImage(preprocessed, debugDir!, "05_clahe_0"); - } - var queryHash = PerceptualHash.ComputeColorHash(preprocessed); - Card? bestMatch = null; + CardHash? bestMatch = null; var bestDistance = int.MaxValue; foreach (var candidate in candidates) { - if (candidate.Hash == null || candidate.Hash.Length != queryHash.Length) + if (candidate.Hash.Length != queryHash.Length) continue; var distance = PerceptualHash.HammingDistance(queryHash, candidate.Hash); @@ -345,83 +317,6 @@ public class CardRecognitionService : IDisposable return rotated; } - /// - /// Save a debug image to disk. - /// - private static void SaveDebugImage(SKBitmap bitmap, string directory, string name) - { - var path = Path.Combine(directory, $"{name}.png"); - using var image = SKImage.FromBitmap(bitmap); - using var data = image.Encode(SKEncodedImageFormat.Png, 100); - using var stream = File.OpenWrite(path); - data.SaveTo(stream); - Console.WriteLine($"[Scry Debug] Saved: {path}"); - } - - /// - /// Save a debug image showing the card detection result. - /// - private static void SaveDetectionDebugImage(SKBitmap original, CardDetector.CardDetectionResult detection, string directory) - { - using var annotated = new SKBitmap(original.Width, original.Height, original.ColorType, original.AlphaType); - using var canvas = new SKCanvas(annotated); - - canvas.DrawBitmap(original, 0, 0); - - if (detection.Found && detection.Corners.Length == 4) - { - // Draw detected corners and edges - using var cornerPaint = new SKPaint - { - Color = SKColors.Red, - Style = SKPaintStyle.Fill, - IsAntialias = true - }; - - using var edgePaint = new SKPaint - { - Color = SKColors.Lime, - Style = SKPaintStyle.Stroke, - StrokeWidth = 3, - IsAntialias = true - }; - - var corners = detection.Corners; - - // Draw edges - for (int i = 0; i < 4; i++) - { - var p1 = corners[i]; - var p2 = corners[(i + 1) % 4]; - canvas.DrawLine(p1.X, p1.Y, p2.X, p2.Y, edgePaint); - } - - // Draw corners - foreach (var corner in corners) - { - canvas.DrawCircle(corner.X, corner.Y, 8, cornerPaint); - } - } - - // Add debug text - using var textPaint = new SKPaint - { - Color = detection.Found ? SKColors.Lime : SKColors.Red, - IsAntialias = true - }; - using var font = new SKFont - { - Size = 24 - }; - - var message = detection.Found - ? $"Detected: {detection.Confidence:P0}" - : $"Not found: {detection.DebugMessage}"; - canvas.DrawText(message, 10, 30, SKTextAlign.Left, font, textPaint); - - SaveDebugImage(annotated, directory, "02_detection"); - } - public void Dispose() { _cacheLock.Dispose(); diff --git a/src/Scry.Core/Recognition/HashDatabaseSyncService.cs b/src/Scry.Core/Recognition/HashDatabaseSyncService.cs index 80402ae..ee7a5c1 100644 --- a/src/Scry.Core/Recognition/HashDatabaseSyncService.cs +++ b/src/Scry.Core/Recognition/HashDatabaseSyncService.cs @@ -9,12 +9,12 @@ namespace Scry.Core.Recognition; public class HashDatabaseSyncService { private readonly ScryfallClient _scryfallClient; - private readonly CardDatabase _database; + private readonly CardHashDatabase _database; private readonly HttpClient _imageClient; public event Action? OnProgress; - public HashDatabaseSyncService(ScryfallClient scryfallClient, CardDatabase database, HttpClient? imageClient = null) + public HashDatabaseSyncService(ScryfallClient scryfallClient, CardHashDatabase database, HttpClient? imageClient = null) { _scryfallClient = scryfallClient; _database = database; @@ -29,21 +29,6 @@ public class HashDatabaseSyncService try { - // Fetch all sets first - ReportProgress(new SyncProgress { Stage = SyncStage.Initializing, Message = "Fetching sets..." }); - var scryfallSets = await _scryfallClient.GetAllSetsAsync(ct); - var existingSetIds = await _database.GetExistingSetIdsAsync(ct); - - var newSets = scryfallSets - .Where(s => s.Id != null && !existingSetIds.Contains(s.Id)) - .Select(s => s.ToSet()) - .ToList(); - - if (newSets.Count > 0) - { - await _database.InsertSetBatchAsync(newSets, ct); - } - var bulkInfo = await _scryfallClient.GetBulkDataInfoAsync(options.BulkDataType, ct); if (bulkInfo?.DownloadUri == null) { @@ -65,25 +50,21 @@ public class HashDatabaseSyncService ReportProgress(new SyncProgress { Stage = SyncStage.Downloading, Message = "Downloading card data..." }); - var existingOracleIds = await _database.GetExistingOracleIdsAsync(ct); - var cardBatch = new List(); - var oracleBatch = new Dictionary(); + var batch = new List(); var processed = 0; var errors = 0; - await foreach (var scryfallCard in _scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadUri, ct)) + await foreach (var card in _scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadUri, ct)) { ct.ThrowIfCancellationRequested(); - if (scryfallCard.Lang != "en" && !options.IncludeNonEnglish) + if (card.Lang != "en" && !options.IncludeNonEnglish) continue; - var imageUri = scryfallCard.GetImageUri(options.ImageSize); + var imageUri = card.GetImageUri(options.ImageSize); if (string.IsNullOrEmpty(imageUri)) continue; - var oracleId = scryfallCard.OracleId ?? scryfallCard.Id ?? ""; - try { var imageBytes = await DownloadWithRetryAsync(imageUri, options.MaxRetries, ct); @@ -103,34 +84,23 @@ public class HashDatabaseSyncService using var preprocessed = ImagePreprocessor.ApplyClahe(bitmap); var hash = PerceptualHash.ComputeColorHash(preprocessed); - // Track oracle if new - if (!existingOracleIds.Contains(oracleId) && !oracleBatch.ContainsKey(oracleId)) + batch.Add(new CardHash { - oracleBatch[oracleId] = scryfallCard.ToOracle(); - } - - // Create card with hash - var card = scryfallCard.ToCard() with { Hash = hash }; - cardBatch.Add(card); + CardId = card.Id ?? Guid.NewGuid().ToString(), + Name = card.Name ?? "Unknown", + SetCode = card.Set ?? "???", + CollectorNumber = card.CollectorNumber, + Hash = hash, + ImageUri = imageUri + }); processed++; - if (cardBatch.Count >= options.BatchSize) + if (batch.Count >= options.BatchSize) { - // Insert oracles first - if (oracleBatch.Count > 0) - { - await _database.InsertOracleBatchAsync(oracleBatch.Values, ct); - foreach (var id in oracleBatch.Keys) - { - existingOracleIds.Add(id); - } - oracleBatch.Clear(); - } - - await _database.InsertCardBatchAsync(cardBatch, ct); - result.ProcessedCards += cardBatch.Count; - cardBatch.Clear(); + await _database.InsertHashBatchAsync(batch, ct); + result.ProcessedCards += batch.Count; + batch.Clear(); ReportProgress(new SyncProgress { @@ -151,20 +121,14 @@ public class HashDatabaseSyncService if (options.StopOnError) throw; - result.Errors.Add($"{scryfallCard.Name}: {ex.Message}"); + result.Errors.Add($"{card.Name}: {ex.Message}"); } } - // Insert remaining batches - if (oracleBatch.Count > 0) + if (batch.Count > 0) { - await _database.InsertOracleBatchAsync(oracleBatch.Values, ct); - } - - if (cardBatch.Count > 0) - { - await _database.InsertCardBatchAsync(cardBatch, ct); - result.ProcessedCards += cardBatch.Count; + await _database.InsertHashBatchAsync(batch, ct); + result.ProcessedCards += batch.Count; } await _database.SetMetadataAsync("last_sync_date", DateTime.UtcNow.ToString("O"), ct); diff --git a/src/Scry.Core/Recognition/RecognitionOptions.cs b/src/Scry.Core/Recognition/RecognitionOptions.cs deleted file mode 100644 index 8a9f4ee..0000000 --- a/src/Scry.Core/Recognition/RecognitionOptions.cs +++ /dev/null @@ -1,24 +0,0 @@ -namespace Scry.Core.Recognition; - -/// -/// Configuration options for card recognition. -/// -public class RecognitionOptions -{ - /// - /// When set, saves debug images of each pipeline step to this directory. - /// - public string? DebugOutputDirectory { get; set; } - - /// - /// Enable card detection and perspective correction. - /// When disabled, assumes the input image is already a cropped card. - /// - public bool EnableCardDetection { get; set; } = true; - - /// - /// Try multiple rotations (0°, 90°, 180°, 270°) when matching. - /// Useful when card orientation is unknown. - /// - public bool EnableRotationMatching { get; set; } = true; -} diff --git a/src/Scry.Core/Scry.Core.csproj b/src/Scry.Core/Scry.Core.csproj index 9464c4d..e2e0044 100644 --- a/src/Scry.Core/Scry.Core.csproj +++ b/src/Scry.Core/Scry.Core.csproj @@ -10,7 +10,6 @@ - diff --git a/src/Scry.Core/Scryfall/ScryfallClient.cs b/src/Scry.Core/Scryfall/ScryfallClient.cs index 58cf64c..62e8551 100644 --- a/src/Scry.Core/Scryfall/ScryfallClient.cs +++ b/src/Scry.Core/Scryfall/ScryfallClient.cs @@ -1,6 +1,5 @@ using System.IO.Compression; using System.Text.Json; -using System.Text.Json.Serialization; using Scry.Core.Models; namespace Scry.Core.Scryfall; @@ -9,7 +8,6 @@ public class ScryfallClient : IDisposable { private readonly HttpClient _httpClient; private const string BulkDataUrl = "https://api.scryfall.com/bulk-data"; - private const string SetsUrl = "https://api.scryfall.com/sets"; public ScryfallClient(HttpClient? httpClient = null) { @@ -26,27 +24,6 @@ public class ScryfallClient : IDisposable d.Type?.Equals(type, StringComparison.OrdinalIgnoreCase) == true); } - public async Task> GetAllSetsAsync(CancellationToken ct = default) - { - var sets = new List(); - var url = SetsUrl; - - while (!string.IsNullOrEmpty(url)) - { - var response = await _httpClient.GetStringAsync(url, ct); - var setsResponse = JsonSerializer.Deserialize(response, JsonOptions); - - if (setsResponse?.Data != null) - { - sets.AddRange(setsResponse.Data); - } - - url = setsResponse?.NextPage; - } - - return sets; - } - public async IAsyncEnumerable StreamBulkDataAsync( string downloadUri, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct = default) @@ -95,8 +72,6 @@ public class ScryfallClient : IDisposable }; } -#region API Response Models - public record BulkDataResponse { public List? Data { get; init; } @@ -112,62 +87,21 @@ public record BulkDataInfo public long? Size { get; init; } } -public record SetsResponse -{ - public List? Data { get; init; } - public bool HasMore { get; init; } - public string? NextPage { get; init; } -} - -public record ScryfallSet -{ - public string? Id { get; init; } - public string? Code { get; init; } - public string? Name { get; init; } - public string? SetType { get; init; } - public string? ReleasedAt { get; init; } - public int? CardCount { get; init; } - public string? IconSvgUri { get; init; } - public bool Digital { get; init; } - public string? ParentSetCode { get; init; } - public string? Block { get; init; } -} - public record ScryfallCard { - // Core identifiers public string? Id { get; init; } - public string? OracleId { get; init; } - - // Oracle/game card fields public string? Name { get; init; } - public string? ManaCost { get; init; } - public double? Cmc { get; init; } - public string? TypeLine { get; init; } - public string? OracleText { get; init; } - public List? Colors { get; init; } - public List? ColorIdentity { get; init; } - public List? Keywords { get; init; } - public bool Reserved { get; init; } - public Dictionary? Legalities { get; init; } - public string? Power { get; init; } - public string? Toughness { get; init; } - - // Printing-specific fields public string? Set { get; init; } - public string? SetId { get; init; } public string? SetName { get; init; } public string? CollectorNumber { get; init; } public string? Rarity { get; init; } - public string? Artist { get; init; } - public string? IllustrationId { get; init; } + public string? ManaCost { get; init; } + public string? TypeLine { get; init; } + public string? OracleText { get; init; } public string? Lang { get; init; } - - // Images and prices + public string? Artist { get; init; } public ImageUris? ImageUris { get; init; } public Prices? Prices { get; init; } - - // Multi-face cards public List? CardFaces { get; init; } } @@ -194,45 +128,11 @@ public record CardFace public string? ManaCost { get; init; } public string? TypeLine { get; init; } public string? OracleText { get; init; } - public List? Colors { get; init; } - public string? Power { get; init; } - public string? Toughness { get; init; } public ImageUris? ImageUris { get; init; } } -#endregion - -#region Extension Methods - public static class ScryfallCardExtensions { - /// - /// Extracts the Oracle (abstract game card) from a Scryfall card. - /// - public static Oracle ToOracle(this ScryfallCard scryfall) - { - return new Oracle - { - Id = scryfall.OracleId ?? scryfall.Id ?? Guid.NewGuid().ToString(), - Name = scryfall.Name ?? "Unknown", - ManaCost = scryfall.ManaCost, - Cmc = scryfall.Cmc, - TypeLine = scryfall.TypeLine, - OracleText = scryfall.OracleText, - Colors = scryfall.Colors != null ? JsonSerializer.Serialize(scryfall.Colors) : null, - ColorIdentity = scryfall.ColorIdentity != null ? JsonSerializer.Serialize(scryfall.ColorIdentity) : null, - Keywords = scryfall.Keywords != null ? JsonSerializer.Serialize(scryfall.Keywords) : null, - Reserved = scryfall.Reserved, - Legalities = scryfall.Legalities != null ? JsonSerializer.Serialize(scryfall.Legalities) : null, - Power = scryfall.Power, - Toughness = scryfall.Toughness, - }; - } - - /// - /// Converts a Scryfall card to a Card (printing) model. - /// Note: Hash must be computed separately and set on the returned Card. - /// public static Card ToCard(this ScryfallCard scryfall) { var imageUris = scryfall.ImageUris ?? scryfall.CardFaces?.FirstOrDefault()?.ImageUris; @@ -240,46 +140,21 @@ public static class ScryfallCardExtensions return new Card { Id = scryfall.Id ?? Guid.NewGuid().ToString(), - OracleId = scryfall.OracleId ?? scryfall.Id ?? Guid.NewGuid().ToString(), - SetId = scryfall.SetId ?? "", + Name = scryfall.Name ?? "Unknown", SetCode = scryfall.Set, SetName = scryfall.SetName, - Name = scryfall.Name ?? "Unknown", CollectorNumber = scryfall.CollectorNumber, Rarity = scryfall.Rarity, - Artist = scryfall.Artist, - IllustrationId = scryfall.IllustrationId, - ImageUri = imageUris?.Normal, - Lang = scryfall.Lang, - PricesUsd = decimal.TryParse(scryfall.Prices?.Usd, out var usd) ? usd : null, - PricesUsdFoil = decimal.TryParse(scryfall.Prices?.UsdFoil, out var foil) ? foil : null, - Hash = null, // Must be computed separately - // Denormalized Oracle fields ManaCost = scryfall.ManaCost, TypeLine = scryfall.TypeLine, OracleText = scryfall.OracleText, - Power = scryfall.Power, - Toughness = scryfall.Toughness, - }; - } - - /// - /// Converts a Scryfall set to a Set model. - /// - public static Set ToSet(this ScryfallSet scryfall) - { - return new Set - { - Id = scryfall.Id ?? Guid.NewGuid().ToString(), - Code = scryfall.Code ?? "???", - Name = scryfall.Name ?? "Unknown", - SetType = scryfall.SetType, - ReleasedAt = scryfall.ReleasedAt, - CardCount = scryfall.CardCount, - IconSvgUri = scryfall.IconSvgUri, - Digital = scryfall.Digital, - ParentSetCode = scryfall.ParentSetCode, - Block = scryfall.Block, + ImageUri = imageUris?.Normal, + ImageUriSmall = imageUris?.Small, + ImageUriLarge = imageUris?.Large ?? imageUris?.Png, + Artist = scryfall.Artist, + Lang = scryfall.Lang, + PriceUsd = decimal.TryParse(scryfall.Prices?.Usd, out var usd) ? usd : null, + PriceUsdFoil = decimal.TryParse(scryfall.Prices?.UsdFoil, out var foil) ? foil : null }; } @@ -298,5 +173,3 @@ public static class ScryfallCardExtensions }; } } - -#endregion diff --git a/test/Scry.Tests/CardDatabaseTests.cs b/test/Scry.Tests/CardDatabaseTests.cs deleted file mode 100644 index 090998e..0000000 --- a/test/Scry.Tests/CardDatabaseTests.cs +++ /dev/null @@ -1,304 +0,0 @@ -using Microsoft.Data.Sqlite; -using Scry.Core.Data; -using Scry.Core.Models; -using Xunit; - -namespace Scry.Tests; - -public class CardDatabaseTests : IDisposable -{ - private readonly string _dbPath; - private readonly CardDatabase _database; - - public CardDatabaseTests() - { - _dbPath = Path.Combine(Path.GetTempPath(), $"scry_test_{Guid.NewGuid()}.db"); - _database = new CardDatabase(_dbPath); - } - - [Fact] - public async Task InsertCard_ThenRetrieve_ReturnsMatch() - { - // First insert oracle and set (foreign keys) - var oracle = new Oracle - { - Id = "oracle-1", - Name = "Test Card", - ManaCost = "{1}{U}", - TypeLine = "Creature" - }; - await _database.InsertOracleAsync(oracle); - - var set = new Set - { - Id = "set-1", - Code = "TST", - Name = "Test Set" - }; - await _database.InsertSetAsync(set); - - var card = new Card - { - Id = "test-id", - OracleId = "oracle-1", - SetId = "set-1", - SetCode = "TST", - Name = "Test Card", - CollectorNumber = "1", - Hash = new byte[] { 0x01, 0x02, 0x03 }, - ImageUri = "https://example.com/image.jpg" - }; - - await _database.InsertCardAsync(card); - var retrieved = await _database.GetCardByIdAsync("test-id"); - - Assert.NotNull(retrieved); - Assert.Equal("Test Card", retrieved.Name); - Assert.Equal("TST", retrieved.SetCode); - Assert.Equal(card.Hash, retrieved.Hash); - } - - [Fact] - public async Task InsertCardBatch_InsertsAllCards() - { - // Insert oracle first - var oracle = new Oracle { Id = "oracle-batch", Name = "Batch Card" }; - await _database.InsertOracleAsync(oracle); - - var set = new Set { Id = "set-batch", Code = "TST", Name = "Test Set" }; - await _database.InsertSetAsync(set); - - var cards = Enumerable.Range(0, 100).Select(i => new Card - { - Id = $"card-{i}", - OracleId = "oracle-batch", - SetId = "set-batch", - SetCode = "TST", - Name = $"Card {i}", - Hash = new byte[] { (byte)i } - }).ToList(); - - await _database.InsertCardBatchAsync(cards); - var count = await _database.GetCardCountAsync(); - - Assert.Equal(100, count); - } - - [Fact] - public async Task GetAllCards_ReturnsAllCards() - { - var oracle = new Oracle { Id = "oracle-all", Name = "All Card" }; - await _database.InsertOracleAsync(oracle); - - var set = new Set { Id = "set-all", Code = "TST", Name = "Test Set" }; - await _database.InsertSetAsync(set); - - var cards = Enumerable.Range(0, 10).Select(i => new Card - { - Id = $"card-{i}", - OracleId = "oracle-all", - SetId = "set-all", - SetCode = "TST", - Name = $"Card {i}", - Hash = new byte[] { (byte)i } - }).ToList(); - - await _database.InsertCardBatchAsync(cards); - var all = await _database.GetAllCardsAsync(); - - Assert.Equal(10, all.Count); - } - - [Fact] - public async Task GetCardsByOracleId_ReturnsAllPrintings() - { - var oracle = new Oracle { Id = "oracle-multi", Name = "Multi Print Card" }; - await _database.InsertOracleAsync(oracle); - - var set1 = new Set { Id = "set-1", Code = "S1", Name = "Set 1" }; - var set2 = new Set { Id = "set-2", Code = "S2", Name = "Set 2" }; - await _database.InsertSetAsync(set1); - await _database.InsertSetAsync(set2); - - var cards = new[] - { - new Card { Id = "print-1", OracleId = "oracle-multi", SetId = "set-1", SetCode = "S1", Name = "Multi Print Card", Hash = new byte[] { 0x01 } }, - new Card { Id = "print-2", OracleId = "oracle-multi", SetId = "set-2", SetCode = "S2", Name = "Multi Print Card", Hash = new byte[] { 0x02 } }, - }; - - await _database.InsertCardBatchAsync(cards); - var printings = await _database.GetCardsByOracleIdAsync("oracle-multi"); - - Assert.Equal(2, printings.Count); - } - - [Fact] - public async Task Metadata_SetAndGet() - { - await _database.SetMetadataAsync("test_key", "test_value"); - var value = await _database.GetMetadataAsync("test_key"); - - Assert.Equal("test_value", value); - } - - [Fact] - public async Task ClearCards_RemovesAllCards() - { - var oracle = new Oracle { Id = "oracle-clear", Name = "Clear Card" }; - await _database.InsertOracleAsync(oracle); - - var set = new Set { Id = "set-clear", Code = "TST", Name = "Test Set" }; - await _database.InsertSetAsync(set); - - var cards = Enumerable.Range(0, 10).Select(i => new Card - { - Id = $"card-{i}", - OracleId = "oracle-clear", - SetId = "set-clear", - SetCode = "TST", - Name = $"Card {i}", - Hash = new byte[] { (byte)i } - }).ToList(); - - await _database.InsertCardBatchAsync(cards); - await _database.ClearCardsAsync(); - var count = await _database.GetCardCountAsync(); - - Assert.Equal(0, count); - } - - [Fact] - public async Task InsertCard_DuplicateId_Updates() - { - var oracle = new Oracle { Id = "oracle-dup", Name = "Dup Card" }; - await _database.InsertOracleAsync(oracle); - - var set = new Set { Id = "set-dup", Code = "TST", Name = "Test Set" }; - await _database.InsertSetAsync(set); - - var card1 = new Card - { - Id = "duplicate-id", - OracleId = "oracle-dup", - SetId = "set-dup", - SetCode = "TST", - Name = "Original Name", - Hash = new byte[] { 0x01 } - }; - - var card2 = new Card - { - Id = "duplicate-id", - OracleId = "oracle-dup", - SetId = "set-dup", - SetCode = "TST", - Name = "Updated Name", - Hash = new byte[] { 0x02 } - }; - - await _database.InsertCardAsync(card1); - await _database.InsertCardAsync(card2); - - var retrieved = await _database.GetCardByIdAsync("duplicate-id"); - - Assert.NotNull(retrieved); - Assert.Equal("Updated Name", retrieved.Name); - Assert.Equal(new byte[] { 0x02 }, retrieved.Hash); - } - - [Fact] - public async Task InsertOracle_ThenRetrieveByName() - { - var oracle = new Oracle - { - Id = "oracle-name", - Name = "Lightning Bolt", - ManaCost = "{R}", - Cmc = 1, - TypeLine = "Instant", - OracleText = "Lightning Bolt deals 3 damage to any target." - }; - - await _database.InsertOracleAsync(oracle); - var retrieved = await _database.GetOracleByNameAsync("Lightning Bolt"); - - Assert.NotNull(retrieved); - Assert.Equal("{R}", retrieved.ManaCost); - Assert.Equal(1, retrieved.Cmc); - } - - [Fact] - public async Task InsertSet_ThenRetrieveByCode() - { - var set = new Set - { - Id = "set-lea", - Code = "lea", - Name = "Limited Edition Alpha", - SetType = "expansion", - ReleasedAt = "1993-08-05", - CardCount = 295 - }; - - await _database.InsertSetAsync(set); - var retrieved = await _database.GetSetByCodeAsync("lea"); - - Assert.NotNull(retrieved); - Assert.Equal("Limited Edition Alpha", retrieved.Name); - Assert.Equal(295, retrieved.CardCount); - } - - [Fact] - public async Task GetCardsWithHash_OnlyReturnsCardsWithHash() - { - var oracle = new Oracle { Id = "oracle-hash", Name = "Hash Card" }; - await _database.InsertOracleAsync(oracle); - - var set = new Set { Id = "set-hash", Code = "TST", Name = "Test Set" }; - await _database.InsertSetAsync(set); - - var cardWithHash = new Card - { - Id = "card-with-hash", - OracleId = "oracle-hash", - SetId = "set-hash", - SetCode = "TST", - Name = "Has Hash", - Hash = new byte[] { 0x01 } - }; - - var cardWithoutHash = new Card - { - Id = "card-no-hash", - OracleId = "oracle-hash", - SetId = "set-hash", - SetCode = "TST", - Name = "No Hash", - Hash = null - }; - - await _database.InsertCardAsync(cardWithHash); - await _database.InsertCardAsync(cardWithoutHash); - - var cardsWithHash = await _database.GetCardsWithHashAsync(); - - Assert.Single(cardsWithHash); - Assert.Equal("card-with-hash", cardsWithHash[0].Id); - } - - public void Dispose() - { - _database.Dispose(); - SqliteConnection.ClearAllPools(); - try - { - if (File.Exists(_dbPath)) - { - File.Delete(_dbPath); - } - } - catch (IOException) - { - } - } -} diff --git a/test/Scry.Tests/CardHashDatabaseTests.cs b/test/Scry.Tests/CardHashDatabaseTests.cs new file mode 100644 index 0000000..e6d8944 --- /dev/null +++ b/test/Scry.Tests/CardHashDatabaseTests.cs @@ -0,0 +1,146 @@ +using Microsoft.Data.Sqlite; +using Scry.Core.Data; +using Scry.Core.Models; +using Xunit; + +namespace Scry.Tests; + +public class CardHashDatabaseTests : IDisposable +{ + private readonly string _dbPath; + private readonly CardHashDatabase _database; + + public CardHashDatabaseTests() + { + _dbPath = Path.Combine(Path.GetTempPath(), $"scry_test_{Guid.NewGuid()}.db"); + _database = new CardHashDatabase(_dbPath); + } + + [Fact] + public async Task InsertHash_ThenRetrieve_ReturnsMatch() + { + var hash = new CardHash + { + CardId = "test-id", + Name = "Test Card", + SetCode = "TST", + CollectorNumber = "1", + Hash = new byte[] { 0x01, 0x02, 0x03 }, + ImageUri = "https://example.com/image.jpg" + }; + + await _database.InsertHashAsync(hash); + var retrieved = await _database.GetHashByIdAsync("test-id"); + + Assert.NotNull(retrieved); + Assert.Equal("Test Card", retrieved.Name); + Assert.Equal("TST", retrieved.SetCode); + Assert.Equal(hash.Hash, retrieved.Hash); + } + + [Fact] + public async Task InsertHashBatch_InsertsAllHashes() + { + var hashes = Enumerable.Range(0, 100).Select(i => new CardHash + { + CardId = $"card-{i}", + Name = $"Card {i}", + SetCode = "TST", + Hash = new byte[] { (byte)i } + }).ToList(); + + await _database.InsertHashBatchAsync(hashes); + var count = await _database.GetHashCountAsync(); + + Assert.Equal(100, count); + } + + [Fact] + public async Task GetAllHashes_ReturnsAllHashes() + { + var hashes = Enumerable.Range(0, 10).Select(i => new CardHash + { + CardId = $"card-{i}", + Name = $"Card {i}", + SetCode = "TST", + Hash = new byte[] { (byte)i } + }).ToList(); + + await _database.InsertHashBatchAsync(hashes); + var all = await _database.GetAllHashesAsync(); + + Assert.Equal(10, all.Count); + } + + [Fact] + public async Task Metadata_SetAndGet() + { + await _database.SetMetadataAsync("test_key", "test_value"); + var value = await _database.GetMetadataAsync("test_key"); + + Assert.Equal("test_value", value); + } + + [Fact] + public async Task Clear_RemovesAllHashes() + { + var hashes = Enumerable.Range(0, 10).Select(i => new CardHash + { + CardId = $"card-{i}", + Name = $"Card {i}", + SetCode = "TST", + Hash = new byte[] { (byte)i } + }).ToList(); + + await _database.InsertHashBatchAsync(hashes); + await _database.ClearAsync(); + var count = await _database.GetHashCountAsync(); + + Assert.Equal(0, count); + } + + [Fact] + public async Task InsertHash_DuplicateId_Updates() + { + var hash1 = new CardHash + { + CardId = "duplicate-id", + Name = "Original Name", + SetCode = "TST", + Hash = new byte[] { 0x01 } + }; + + var hash2 = new CardHash + { + CardId = "duplicate-id", + Name = "Updated Name", + SetCode = "TST", + Hash = new byte[] { 0x02 } + }; + + await _database.InsertHashAsync(hash1); + await _database.InsertHashAsync(hash2); + + var retrieved = await _database.GetHashByIdAsync("duplicate-id"); + + Assert.NotNull(retrieved); + Assert.Equal("Updated Name", retrieved.Name); + Assert.Equal(new byte[] { 0x02 }, retrieved.Hash); + } + + public void Dispose() + { + _database.Dispose(); + SqliteConnection.ClearAllPools(); + try + { + if (File.Exists(_dbPath)) + { + File.Delete(_dbPath); + } + } + catch (IOException) + { + } + } +} diff --git a/test/Scry.Tests/CardRecognitionTests.cs b/test/Scry.Tests/CardRecognitionTests.cs index 005abbe..35130fb 100644 --- a/test/Scry.Tests/CardRecognitionTests.cs +++ b/test/Scry.Tests/CardRecognitionTests.cs @@ -13,14 +13,14 @@ public class CardRecognitionTests : IDisposable { private readonly ITestOutputHelper _output; private readonly string _dbPath; - private readonly CardDatabase _database; + private readonly CardHashDatabase _database; private readonly CardRecognitionService _recognitionService; public CardRecognitionTests(ITestOutputHelper output) { _output = output; _dbPath = Path.Combine(Path.GetTempPath(), $"scry_recognition_test_{Guid.NewGuid()}.db"); - _database = new CardDatabase(_dbPath); + _database = new CardHashDatabase(_dbPath); _recognitionService = new CardRecognitionService(_database); } @@ -32,7 +32,7 @@ public class CardRecognitionTests : IDisposable var result = await _recognitionService.RecognizeAsync(bitmap); Assert.False(result.Success); - Assert.Contains("No cards", result.ErrorMessage); + Assert.Contains("No card hashes", result.ErrorMessage); } [Fact] @@ -41,17 +41,11 @@ public class CardRecognitionTests : IDisposable using var bitmap = CreateTestBitmap(100, 100); var hash = _recognitionService.ComputeHash(bitmap); - // Insert oracle and set first - await _database.InsertOracleAsync(new Oracle { Id = "oracle-test", Name = "Test Card" }); - await _database.InsertSetAsync(new Set { Id = "set-test", Code = "TST", Name = "Test Set" }); - - await _database.InsertCardAsync(new Card + await _database.InsertHashAsync(new CardHash { - Id = "test-card", - OracleId = "oracle-test", - SetId = "set-test", - SetCode = "TST", + CardId = "test-card", Name = "Test Card", + SetCode = "TST", Hash = hash, ImageUri = "https://example.com/test.jpg" }); @@ -84,16 +78,11 @@ public class CardRecognitionTests : IDisposable var hash = _recognitionService.ComputeHash(bitmap); var cardName = Path.GetFileNameWithoutExtension(imagePath); - await _database.InsertOracleAsync(new Oracle { Id = $"oracle-{cardName}", Name = cardName }); - await _database.InsertSetAsync(new Set { Id = "set-ref", Code = "REF", Name = "Reference Set" }); - - await _database.InsertCardAsync(new Card + await _database.InsertHashAsync(new CardHash { - Id = cardName, - OracleId = $"oracle-{cardName}", - SetId = "set-ref", - SetCode = "REF", + CardId = cardName, Name = cardName, + SetCode = "REF", Hash = hash }); await _recognitionService.InvalidateCacheAsync(); @@ -132,7 +121,7 @@ public class CardRecognitionTests : IDisposable return; } - using var testDb = new CardDatabase(dbPath); + using var testDb = new CardHashDatabase(dbPath); using var testRecognition = new CardRecognitionService(testDb); using var bitmap = SKBitmap.Decode(imagePath); @@ -140,31 +129,31 @@ public class CardRecognitionTests : IDisposable // First, just compute hash and check distance manually var queryHash = testRecognition.ComputeHash(bitmap); - var allCards = await testDb.GetCardsWithHashAsync(); + var allHashes = await testDb.GetAllHashesAsync(); _output.WriteLine($"Query hash length: {queryHash.Length} bytes"); - _output.WriteLine($"Database has {allCards.Count} cards with hashes"); + _output.WriteLine($"Database has {allHashes.Count} cards"); // Find Serra Angel and compute distance - var serraCard = allCards.FirstOrDefault(c => c.Name == "Serra Angel"); - if (serraCard?.Hash != null) + var serraHash = allHashes.FirstOrDefault(h => h.Name == "Serra Angel"); + if (serraHash != null) { - var distance = PerceptualHash.HammingDistance(queryHash, serraCard.Hash); - _output.WriteLine($"Serra Angel hash length: {serraCard.Hash.Length} bytes"); + var distance = PerceptualHash.HammingDistance(queryHash, serraHash.Hash); + _output.WriteLine($"Serra Angel hash length: {serraHash.Hash.Length} bytes"); _output.WriteLine($"Distance to Serra Angel: {distance}"); } // Find the actual best match int bestDistance = int.MaxValue; string? bestName = null; - foreach (var card in allCards) + foreach (var hash in allHashes) { - if (card.Hash == null || card.Hash.Length != queryHash.Length) continue; - var dist = PerceptualHash.HammingDistance(queryHash, card.Hash); + if (hash.Hash.Length != queryHash.Length) continue; + var dist = PerceptualHash.HammingDistance(queryHash, hash.Hash); if (dist < bestDistance) { bestDistance = dist; - bestName = card.Name; + bestName = hash.Name; } } _output.WriteLine($"Best match: {bestName}, distance: {bestDistance}"); @@ -191,16 +180,11 @@ public class CardRecognitionTests : IDisposable using var bitmap = CreateTestBitmap(200, 300); var hash = _recognitionService.ComputeHash(bitmap); - await _database.InsertOracleAsync(new Oracle { Id = "oracle-timing", Name = "Timing Test Card" }); - await _database.InsertSetAsync(new Set { Id = "set-timing", Code = "TST", Name = "Test Set" }); - - await _database.InsertCardAsync(new Card + await _database.InsertHashAsync(new CardHash { - Id = "timing-test", - OracleId = "oracle-timing", - SetId = "set-timing", - SetCode = "TST", + CardId = "timing-test", Name = "Timing Test Card", + SetCode = "TST", Hash = hash }); await _recognitionService.InvalidateCacheAsync(); diff --git a/test/Scry.Tests/RobustnessAnalysisTests.cs b/test/Scry.Tests/RobustnessAnalysisTests.cs index c335347..faa6810 100644 --- a/test/Scry.Tests/RobustnessAnalysisTests.cs +++ b/test/Scry.Tests/RobustnessAnalysisTests.cs @@ -15,14 +15,14 @@ public class RobustnessAnalysisTests : IDisposable { private readonly ITestOutputHelper _output; private readonly string _dbPath; - private readonly CardDatabase _database; + private readonly CardHashDatabase _database; private readonly CardRecognitionService _recognitionService; public RobustnessAnalysisTests(ITestOutputHelper output) { _output = output; _dbPath = Path.Combine(Path.GetTempPath(), $"scry_robustness_test_{Guid.NewGuid()}.db"); - _database = new CardDatabase(_dbPath); + _database = new CardHashDatabase(_dbPath); _recognitionService = new CardRecognitionService(_database); } @@ -52,16 +52,11 @@ public class RobustnessAnalysisTests : IDisposable var originalHash = _recognitionService.ComputeHash(original); // Register original in database - await _database.InsertOracleAsync(new Oracle { Id = "oracle-serra", Name = "Serra Angel" }); - await _database.InsertSetAsync(new Set { Id = "set-lea", Code = "LEA", Name = "Alpha" }); - - await _database.InsertCardAsync(new Card + await _database.InsertHashAsync(new CardHash { - Id = "serra-angel", - OracleId = "oracle-serra", - SetId = "set-lea", - SetCode = "LEA", + CardId = "serra-angel", Name = "Serra Angel", + SetCode = "LEA", Hash = originalHash }); await _recognitionService.InvalidateCacheAsync(); @@ -118,16 +113,11 @@ public class RobustnessAnalysisTests : IDisposable var originalHash = _recognitionService.ComputeHash(original); - await _database.InsertOracleAsync(new Oracle { Id = "oracle-serra", Name = "Serra Angel" }); - await _database.InsertSetAsync(new Set { Id = "set-lea", Code = "LEA", Name = "Alpha" }); - - await _database.InsertCardAsync(new Card + await _database.InsertHashAsync(new CardHash { - Id = "serra-angel", - OracleId = "oracle-serra", - SetId = "set-lea", - SetCode = "LEA", + CardId = "serra-angel", Name = "Serra Angel", + SetCode = "LEA", Hash = originalHash }); await _recognitionService.InvalidateCacheAsync(); @@ -177,16 +167,11 @@ public class RobustnessAnalysisTests : IDisposable var originalHash = _recognitionService.ComputeHash(original); - await _database.InsertOracleAsync(new Oracle { Id = "oracle-serra", Name = "Serra Angel" }); - await _database.InsertSetAsync(new Set { Id = "set-lea", Code = "LEA", Name = "Alpha" }); - - await _database.InsertCardAsync(new Card + await _database.InsertHashAsync(new CardHash { - Id = "serra-angel", - OracleId = "oracle-serra", - SetId = "set-lea", - SetCode = "LEA", + CardId = "serra-angel", Name = "Serra Angel", + SetCode = "LEA", Hash = originalHash }); await _recognitionService.InvalidateCacheAsync(); @@ -233,16 +218,11 @@ public class RobustnessAnalysisTests : IDisposable var originalHash = _recognitionService.ComputeHash(original); - await _database.InsertOracleAsync(new Oracle { Id = "oracle-serra", Name = "Serra Angel" }); - await _database.InsertSetAsync(new Set { Id = "set-lea", Code = "LEA", Name = "Alpha" }); - - await _database.InsertCardAsync(new Card + await _database.InsertHashAsync(new CardHash { - Id = "serra-angel", - OracleId = "oracle-serra", - SetId = "set-lea", - SetCode = "LEA", + CardId = "serra-angel", Name = "Serra Angel", + SetCode = "LEA", Hash = originalHash }); await _recognitionService.InvalidateCacheAsync(); @@ -285,16 +265,11 @@ public class RobustnessAnalysisTests : IDisposable var originalHash = _recognitionService.ComputeHash(original); - await _database.InsertOracleAsync(new Oracle { Id = "oracle-serra", Name = "Serra Angel" }); - await _database.InsertSetAsync(new Set { Id = "set-lea", Code = "LEA", Name = "Alpha" }); - - await _database.InsertCardAsync(new Card + await _database.InsertHashAsync(new CardHash { - Id = "serra-angel", - OracleId = "oracle-serra", - SetId = "set-lea", - SetCode = "LEA", + CardId = "serra-angel", Name = "Serra Angel", + SetCode = "LEA", Hash = originalHash }); await _recognitionService.InvalidateCacheAsync(); @@ -337,14 +312,14 @@ public class RobustnessAnalysisTests : IDisposable return; } - using var prodDb = new CardDatabase(dbPath); + using var prodDb = new CardHashDatabase(dbPath); using var prodRecognition = new CardRecognitionService(prodDb); var testImagesDir = Path.Combine(rootDir, "TestImages"); var categoriesToTest = new[] { "real_photos", "varying_quality", "angled", "low_light" }; _output.WriteLine("=== Real-World Recognition Test ==="); - _output.WriteLine($"Database cards: {(await prodDb.GetCardsWithHashAsync()).Count}"); + _output.WriteLine($"Database cards: {(await prodDb.GetAllHashesAsync()).Count}"); _output.WriteLine(""); foreach (var category in categoriesToTest) diff --git a/tools/DbGenerator/DbGenerator.csproj b/tools/DbGenerator/DbGenerator.csproj index 0b7c3bc..4884da9 100644 --- a/tools/DbGenerator/DbGenerator.csproj +++ b/tools/DbGenerator/DbGenerator.csproj @@ -9,8 +9,6 @@ - - diff --git a/tools/DbGenerator/GenerateCommand.cs b/tools/DbGenerator/GenerateCommand.cs deleted file mode 100644 index 2c9cfb5..0000000 --- a/tools/DbGenerator/GenerateCommand.cs +++ /dev/null @@ -1,495 +0,0 @@ -using System.ComponentModel; -using Scry.Core.Data; -using Scry.Core.Imaging; -using Scry.Core.Models; -using Scry.Core.Scryfall; -using SkiaSharp; -using Spectre.Console; -using Spectre.Console.Cli; - -namespace DbGenerator; - -public sealed class GenerateSettings : CommandSettings -{ - [CommandArgument(0, "[output]")] - [Description("Output database file path")] - [DefaultValue("card_hashes.db")] - public string Output { get; set; } = "card_hashes.db"; - - [CommandOption("-c|--count")] - [Description("Maximum number of cards to include")] - [DefaultValue(500)] - public int Count { get; set; } = 500; - - [CommandOption("--include-test-cards")] - [Description("Include priority test cards (default: true)")] - [DefaultValue(true)] - public bool IncludeTestCards { get; set; } = true; - - [CommandOption("--no-test-cards")] - [Description("Exclude priority test cards")] - public bool NoTestCards { get; set; } - - [CommandOption("-f|--force")] - [Description("Force rebuild from scratch")] - public bool Force { get; set; } -} - -public sealed class GenerateCommand : AsyncCommand -{ - // Cards that should be included for testing with preferred sets - private static readonly Dictionary PriorityCardsWithSets = new(StringComparer.OrdinalIgnoreCase) - { - // From reference_alpha/ - prefer LEA (Alpha) or LEB (Beta) for classic look - ["Ancestral Recall"] = ["lea", "leb"], - ["Badlands"] = ["lea", "leb"], - ["Balance"] = ["lea", "leb"], - ["Bayou"] = ["lea", "leb"], - ["Birds of Paradise"] = ["lea", "leb"], - ["Black Lotus"] = ["lea", "leb"], - ["Channel"] = ["lea", "leb"], - ["Chaos Orb"] = ["lea", "leb"], - ["Clone"] = ["lea", "leb"], - ["Control Magic"] = ["lea", "leb"], - ["Counterspell"] = ["lea", "leb"], - ["Dark Ritual"] = ["lea", "leb"], - ["Demonic Tutor"] = ["lea", "leb"], - ["Disenchant"] = ["lea", "leb"], - ["Fireball"] = ["lea", "leb"], - ["Force of Nature"] = ["lea", "leb"], - ["Fork"] = ["lea", "leb"], - ["Giant Growth"] = ["lea", "leb"], - ["Hypnotic Specter"] = ["lea", "leb"], - ["Lightning Bolt"] = ["lea", "leb"], - ["Llanowar Elves"] = ["lea", "leb"], - ["Mahamoti Djinn"] = ["lea", "leb"], - ["Mind Twist"] = ["lea", "leb"], - ["Mox Emerald"] = ["lea", "leb"], - ["Mox Jet"] = ["lea", "leb"], - ["Mox Pearl"] = ["lea", "leb"], - ["Mox Ruby"] = ["lea", "leb"], - ["Mox Sapphire"] = ["lea", "leb"], - ["Nightmare"] = ["lea", "leb"], - ["Plateau"] = ["lea", "leb"], - ["Regrowth"] = ["lea", "leb"], - ["Rock Hydra"] = ["lea", "leb"], - ["Royal Assassin"] = ["lea", "leb"], - ["Savannah"] = ["lea", "leb"], - ["Scrubland"] = ["lea", "leb"], - ["Serra Angel"] = ["lea", "leb"], - ["Shivan Dragon"] = ["lea", "leb"], - ["Sol Ring"] = ["lea", "leb"], - ["Swords to Plowshares"] = ["lea", "leb"], - ["Taiga"] = ["lea", "leb"], - ["Time Walk"] = ["lea", "leb"], - ["Timetwister"] = ["lea", "leb"], - ["Tropical Island"] = ["lea", "leb"], - ["Tundra"] = ["lea", "leb"], - ["Underground Sea"] = ["lea", "leb"], - ["Wheel of Fortune"] = ["lea", "leb"], - ["Wrath of God"] = ["lea", "leb"], - - // From reference/ - any set is fine - ["Brainstorm"] = [], - ["Force of Will"] = [], - ["Griselbrand"] = [], - ["Lotus Petal"] = [], - ["Ponder"] = [], - ["Show and Tell"] = [], - ["Volcanic Island"] = [], - ["Wasteland"] = [], - - // From single_cards/ - any set is fine - ["Adanto Vanguard"] = [], - ["Angel of Sanctions"] = [], - ["Attunement"] = [], - ["Avaricious Dragon"] = [], - ["Burgeoning"] = [], - ["Jarad, Golgari Lich Lord"] = [], - ["Meletis Charlatan"] = [], - ["Mindstab Thrull"] = [], - ["Pacifism"] = [], - ["Platinum Angel"] = [], - ["Queen Marchesa"] = [], - ["Spellseeker"] = [], - ["Tarmogoyf"] = [], - ["Thought Reflection"] = [], - ["Unsummon"] = [], - - // From varying_quality - prefer older sets - ["Dragon Whelp"] = ["lea", "leb"], - ["Evil Eye of Orms-by-Gore"] = [], - ["Instill Energy"] = ["lea", "leb"], - - // Popular cards for general testing - ["Lightning Helix"] = [], - ["Path to Exile"] = [], - ["Thoughtseize"] = [], - ["Fatal Push"] = [], - ["Snapcaster Mage"] = [], - ["Jace, the Mind Sculptor"] = [], - ["Liliana of the Veil"] = [], - ["Noble Hierarch"] = [], - ["Goblin Guide"] = [], - ["Eidolon of the Great Revel"] = [], - }; - - public override async Task ExecuteAsync(CommandContext context, GenerateSettings settings) - { - var outputDb = settings.Output; - var maxCards = settings.Count; - var includeTestCards = settings.IncludeTestCards && !settings.NoTestCards; - var forceRebuild = settings.Force; - - // Header - AnsiConsole.Write(new FigletText("Scry DB Gen").Color(Color.Blue)); - - var configTable = new Table() - .Border(TableBorder.Rounded) - .AddColumn("Setting") - .AddColumn("Value"); - - configTable.AddRow("Output", outputDb); - configTable.AddRow("Max Cards", maxCards.ToString()); - configTable.AddRow("Test Cards", includeTestCards ? "[green]Yes[/]" : "[grey]No[/]"); - configTable.AddRow("Force Rebuild", forceRebuild ? "[yellow]Yes[/]" : "[grey]No[/]"); - - AnsiConsole.Write(configTable); - AnsiConsole.WriteLine(); - - var priorityCards = new HashSet(PriorityCardsWithSets.Keys, StringComparer.OrdinalIgnoreCase); - - // Force rebuild if requested - if (forceRebuild && File.Exists(outputDb)) - { - AnsiConsole.MarkupLine("[yellow]Force rebuild requested, removing existing database...[/]"); - File.Delete(outputDb); - } - - using var httpClient = new HttpClient(); - httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("Scry/1.0 (MTG Card Scanner - Database Generator)"); - - using var scryfallClient = new ScryfallClient(httpClient); - using var db = new CardDatabase(outputDb); - - // Check existing database state - var existingCardIds = await db.GetExistingCardIdsAsync(); - var existingCardNames = await db.GetExistingCardNamesAsync(); - var existingOracleIds = await db.GetExistingOracleIdsAsync(); - var existingSetIds = await db.GetExistingSetIdsAsync(); - var existingCount = await db.GetCardCountAsync(); - var storedScryfallDate = await db.GetMetadataAsync("scryfall_updated_at"); - - var dbStateTable = new Table() - .Border(TableBorder.Rounded) - .Title("[blue]Current Database State[/]") - .AddColumn("Metric") - .AddColumn("Count", c => c.RightAligned()); - - dbStateTable.AddRow("Cards", existingCount.ToString()); - dbStateTable.AddRow("Oracles", existingOracleIds.Count.ToString()); - dbStateTable.AddRow("Sets", existingSetIds.Count.ToString()); - - AnsiConsole.Write(dbStateTable); - AnsiConsole.WriteLine(); - - // Fetch all sets - List scryfallSets = []; - await AnsiConsole.Status() - .Spinner(Spinner.Known.Dots) - .StartAsync("Fetching sets from Scryfall...", async ctx => - { - scryfallSets = await scryfallClient.GetAllSetsAsync(); - }); - - AnsiConsole.MarkupLine($"[green]✓[/] Found [blue]{scryfallSets.Count}[/] sets"); - - var setsById = scryfallSets.ToDictionary(s => s.Id ?? "", s => s); - var setsByCode = scryfallSets.ToDictionary(s => s.Code ?? "", s => s, StringComparer.OrdinalIgnoreCase); - - // Insert any new sets - var newSets = scryfallSets - .Where(s => s.Id != null && !existingSetIds.Contains(s.Id)) - .Select(s => s.ToSet()) - .ToList(); - - if (newSets.Count > 0) - { - AnsiConsole.MarkupLine($"[green]✓[/] Inserting [blue]{newSets.Count}[/] new sets"); - await db.InsertSetBatchAsync(newSets); - } - - // Fetch bulk data info - BulkDataInfo? bulkInfo = null; - await AnsiConsole.Status() - .Spinner(Spinner.Known.Dots) - .StartAsync("Fetching bulk data info...", async ctx => - { - bulkInfo = await scryfallClient.GetBulkDataInfoAsync("unique_artwork"); - }); - - if (bulkInfo?.DownloadUri == null) - { - AnsiConsole.MarkupLine("[red]✗ Failed to get bulk data info from Scryfall[/]"); - return 1; - } - - AnsiConsole.MarkupLine($"[green]✓[/] Scryfall data last updated: [blue]{bulkInfo.UpdatedAt:yyyy-MM-dd HH:mm}[/]"); - - // Check if we need to update at all - var scryfallDateStr = bulkInfo.UpdatedAt?.ToString("O") ?? ""; - var needsUpdate = existingCount == 0 || - storedScryfallDate != scryfallDateStr || - existingCount < maxCards; - - // Also check if all priority cards exist - var missingPriorityCards = includeTestCards - ? priorityCards.Where(c => !existingCardNames.Contains(c)).ToList() - : []; - - if (missingPriorityCards is not []) - { - AnsiConsole.MarkupLine($"[yellow]![/] Missing [blue]{missingPriorityCards.Count}[/] priority cards"); - needsUpdate = true; - } - - if (!needsUpdate) - { - AnsiConsole.MarkupLine("[green]✓ Database is up-to-date, no changes needed[/]"); - return 0; - } - - AnsiConsole.WriteLine(); - - var newCards = new List(); - var newOracles = new Dictionary(); - var processed = 0; - var errors = 0; - var skipped = 0; - var priorityFound = 0; - var priorityNeeded = includeTestCards ? priorityCards.Count : 0; - - // Track which priority cards we've already found with their set - var foundPriorityWithSet = new Dictionary(StringComparer.OrdinalIgnoreCase); - - // Helper to check if a set is preferred for a priority card - static bool IsPreferredSet(string cardName, string setCode) - { - if (!PriorityCardsWithSets.TryGetValue(cardName, out var preferredSets)) - return false; - - return preferredSets.Length == 0 || preferredSets.Contains(setCode, StringComparer.OrdinalIgnoreCase); - } - - await AnsiConsole.Progress() - .AutoClear(false) - .HideCompleted(false) - .Columns( - new RemainingTimeColumn(), - new SpinnerColumn(), - new ProgressBarColumn(), - new PercentageColumn(), - new TaskDescriptionColumn() - { - Alignment = Justify.Left, - } - ) - .StartAsync(async ctx => - { - var downloadTask = ctx.AddTask("[blue]Downloading & processing cards[/]", maxValue: maxCards); - var priorityTask = ctx.AddTask("[green]Priority cards[/]", maxValue: priorityNeeded); - - await foreach (var scryfallCard in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadUri)) - { - // Skip non-English cards - if (scryfallCard.Lang != "en") - continue; - - var imageUri = scryfallCard.GetImageUri("normal"); - if (string.IsNullOrEmpty(imageUri)) - continue; - - var cardId = scryfallCard.Id ?? Guid.NewGuid().ToString(); - var cardName = scryfallCard.Name ?? "Unknown"; - var setCode = scryfallCard.Set ?? "???"; - var oracleId = scryfallCard.OracleId ?? cardId; - var setId = scryfallCard.SetId ?? ""; - - // Check if this card already exists in the database - if (existingCardIds.Contains(cardId)) - { - skipped++; - continue; - } - - // Check if this is a priority card we might need - var isPriorityCard = includeTestCards && priorityCards.Contains(cardName); - var isPreferred = isPriorityCard && IsPreferredSet(cardName, setCode); - - // If this priority card already found with preferred set, skip - if (isPriorityCard && foundPriorityWithSet.TryGetValue(cardName, out var existingSet)) - { - if (IsPreferredSet(cardName, existingSet)) - continue; - if (!isPreferred) - continue; - } - - // Calculate how many slots we have left - var totalCards = existingCount + newCards.Count; - var priorityRemaining = priorityNeeded - foundPriorityWithSet.Count; - var slotsForNonPriority = maxCards - priorityRemaining; - - // Skip if we have enough non-priority cards and this isn't priority - if (!isPriorityCard && totalCards >= slotsForNonPriority) - continue; - - // Download and process image - try - { - downloadTask.Description = $"[blue]{Markup.Escape(cardName.Length > 30 ? cardName[..27] + "..." : cardName)}[/]"; - - var imageBytes = await httpClient.GetByteArrayAsync(imageUri); - using var bitmap = SKBitmap.Decode(imageBytes); - - if (bitmap == null) - { - errors++; - continue; - } - - // Apply CLAHE preprocessing and compute hash - using var preprocessed = ImagePreprocessor.ApplyClahe(bitmap); - var hash = PerceptualHash.ComputeColorHash(preprocessed); - - // Create Card (printing) with hash - var card = scryfallCard.ToCard() with { Hash = hash }; - newCards.Add(card); - - // Track Oracle if we haven't seen it - if (!existingOracleIds.Contains(oracleId) && !newOracles.ContainsKey(oracleId)) - { - newOracles[oracleId] = scryfallCard.ToOracle(); - } - - if (isPriorityCard) - { - foundPriorityWithSet[cardName] = setCode; - priorityFound++; - priorityTask.Increment(1); - } - - processed++; - downloadTask.Increment(1); - - // Check if we have enough cards - var foundAllPriority = foundPriorityWithSet.Count >= priorityNeeded; - if (existingCount + newCards.Count >= maxCards && foundAllPriority) - break; - - // Rate limit to be nice to Scryfall - await Task.Delay(50); - } - catch - { - errors++; - } - } - - downloadTask.Value = downloadTask.MaxValue; - priorityTask.Value = priorityTask.MaxValue; - }); - - AnsiConsole.WriteLine(); - - // Summary table - var summaryTable = new Table() - .Border(TableBorder.Rounded) - .Title("[blue]Processing Summary[/]") - .AddColumn("Metric") - .AddColumn("Count", c => c.RightAligned()); - - summaryTable.AddRow("Skipped (already in DB)", skipped.ToString()); - summaryTable.AddRow("Newly processed", $"[green]{processed}[/]"); - summaryTable.AddRow("New oracles", newOracles.Count.ToString()); - summaryTable.AddRow("Priority cards found", $"{priorityFound}/{priorityNeeded}"); - summaryTable.AddRow("Errors", errors > 0 ? $"[red]{errors}[/]" : "0"); - - AnsiConsole.Write(summaryTable); - AnsiConsole.WriteLine(); - - // Insert oracles first (cards reference them) - if (newOracles.Count > 0) - { - await AnsiConsole.Status() - .Spinner(Spinner.Known.Dots) - .StartAsync($"Inserting {newOracles.Count} new oracles...", async ctx => - { - await db.InsertOracleBatchAsync(newOracles.Values); - }); - AnsiConsole.MarkupLine($"[green]✓[/] Inserted [blue]{newOracles.Count}[/] oracles"); - } - - if (newCards.Count > 0) - { - await AnsiConsole.Status() - .Spinner(Spinner.Known.Dots) - .StartAsync($"Inserting {newCards.Count} new cards...", async ctx => - { - await db.InsertCardBatchAsync(newCards); - }); - AnsiConsole.MarkupLine($"[green]✓[/] Inserted [blue]{newCards.Count}[/] cards"); - } - - await db.SetMetadataAsync("generated_at", DateTime.UtcNow.ToString("O")); - await db.SetMetadataAsync("scryfall_updated_at", scryfallDateStr); - - var finalCardCount = await db.GetCardCountAsync(); - var finalOracleCount = await db.GetOracleCountAsync(); - var finalSetCount = await db.GetSetCountAsync(); - - await db.SetMetadataAsync("card_count", finalCardCount.ToString()); - await db.SetMetadataAsync("oracle_count", finalOracleCount.ToString()); - await db.SetMetadataAsync("set_count", finalSetCount.ToString()); - - AnsiConsole.WriteLine(); - - var finalTable = new Table() - .Border(TableBorder.Double) - .Title("[green]Final Database State[/]") - .AddColumn("Metric") - .AddColumn("Count", c => c.RightAligned()); - - finalTable.AddRow("Cards", $"[green]{finalCardCount}[/]"); - finalTable.AddRow("Oracles", $"[green]{finalOracleCount}[/]"); - finalTable.AddRow("Sets", $"[green]{finalSetCount}[/]"); - finalTable.AddRow("Output", $"[blue]{outputDb}[/]"); - - AnsiConsole.Write(finalTable); - - // Report missing priority cards - if (includeTestCards) - { - var missing = priorityCards.Where(c => !foundPriorityWithSet.ContainsKey(c)).ToList(); - - if (missing.Count > 0) - { - AnsiConsole.WriteLine(); - AnsiConsole.MarkupLine($"[yellow]Missing priority cards ({missing.Count}):[/]"); - - var tree = new Tree("[yellow]Missing Cards[/]"); - foreach (var name in missing.Take(20)) - { - tree.AddNode($"[grey]{Markup.Escape(name)}[/]"); - } - if (missing.Count > 20) - { - tree.AddNode($"[grey]... and {missing.Count - 20} more[/]"); - } - AnsiConsole.Write(tree); - } - } - - return 0; - } -} diff --git a/tools/DbGenerator/Program.cs b/tools/DbGenerator/Program.cs index 99d841a..7857845 100644 --- a/tools/DbGenerator/Program.cs +++ b/tools/DbGenerator/Program.cs @@ -1,12 +1,377 @@ -using DbGenerator; -using Spectre.Console.Cli; +using Scry.Core.Data; +using Scry.Core.Imaging; +using Scry.Core.Models; +using Scry.Core.Scryfall; +using SkiaSharp; -var app = new CommandApp(); +// Generate a card hash database from Scryfall images +// Usage: dotnet run -- [--count N] [--include-test-cards] [--force] -app.Configure(config => +var outputDb = args.Length > 0 ? args[0] : "card_hashes.db"; +var maxCards = 500; +var includeTestCards = true; +var forceRebuild = false; + +// Parse arguments +for (var i = 0; i < args.Length; i++) { - config.SetApplicationName("dbgen"); - config.SetApplicationVersion("1.0.0"); -}); + if (args[i] == "--count" && i + 1 < args.Length && int.TryParse(args[i + 1], out var parsedCount)) + { + maxCards = parsedCount; + i++; + } + else if (args[i] == "--include-test-cards") + { + includeTestCards = true; + } + else if (args[i] == "--no-test-cards") + { + includeTestCards = false; + } + else if (args[i] == "--force") + { + forceRebuild = true; + } +} -return await app.RunAsync(args); +Console.WriteLine($"Generating hash database with up to {maxCards} cards"); +Console.WriteLine($"Output: {outputDb}"); +Console.WriteLine($"Include test cards: {includeTestCards}"); +Console.WriteLine($"Force rebuild: {forceRebuild}"); +Console.WriteLine(); + +// Cards that should be included for testing with preferred sets +// Key: card name, Value: preferred set codes (first match wins) or empty for any +var priorityCardsWithSets = new Dictionary(StringComparer.OrdinalIgnoreCase) +{ + // From reference_alpha/ - prefer LEA (Alpha) or LEB (Beta) for classic look + ["Ancestral Recall"] = ["lea", "leb"], + ["Badlands"] = ["lea", "leb"], + ["Balance"] = ["lea", "leb"], + ["Bayou"] = ["lea", "leb"], + ["Birds of Paradise"] = ["lea", "leb"], + ["Black Lotus"] = ["lea", "leb"], + ["Channel"] = ["lea", "leb"], + ["Chaos Orb"] = ["lea", "leb"], + ["Clone"] = ["lea", "leb"], + ["Control Magic"] = ["lea", "leb"], + ["Counterspell"] = ["lea", "leb"], + ["Dark Ritual"] = ["lea", "leb"], + ["Demonic Tutor"] = ["lea", "leb"], + ["Disenchant"] = ["lea", "leb"], + ["Fireball"] = ["lea", "leb"], + ["Force of Nature"] = ["lea", "leb"], + ["Fork"] = ["lea", "leb"], + ["Giant Growth"] = ["lea", "leb"], + ["Hypnotic Specter"] = ["lea", "leb"], + ["Lightning Bolt"] = ["lea", "leb"], + ["Llanowar Elves"] = ["lea", "leb"], + ["Mahamoti Djinn"] = ["lea", "leb"], + ["Mind Twist"] = ["lea", "leb"], + ["Mox Emerald"] = ["lea", "leb"], + ["Mox Jet"] = ["lea", "leb"], + ["Mox Pearl"] = ["lea", "leb"], + ["Mox Ruby"] = ["lea", "leb"], + ["Mox Sapphire"] = ["lea", "leb"], + ["Nightmare"] = ["lea", "leb"], + ["Plateau"] = ["lea", "leb"], + ["Regrowth"] = ["lea", "leb"], + ["Rock Hydra"] = ["lea", "leb"], + ["Royal Assassin"] = ["lea", "leb"], + ["Savannah"] = ["lea", "leb"], + ["Scrubland"] = ["lea", "leb"], + ["Serra Angel"] = ["lea", "leb"], + ["Shivan Dragon"] = ["lea", "leb"], + ["Sol Ring"] = ["lea", "leb"], + ["Swords to Plowshares"] = ["lea", "leb"], + ["Taiga"] = ["lea", "leb"], + ["Time Walk"] = ["lea", "leb"], + ["Timetwister"] = ["lea", "leb"], + ["Tropical Island"] = ["lea", "leb"], + ["Tundra"] = ["lea", "leb"], + ["Underground Sea"] = ["lea", "leb"], + ["Wheel of Fortune"] = ["lea", "leb"], + ["Wrath of God"] = ["lea", "leb"], + + // From reference/ - any set is fine + ["Brainstorm"] = [], + ["Force of Will"] = [], + ["Griselbrand"] = [], + ["Lotus Petal"] = [], + ["Ponder"] = [], + ["Show and Tell"] = [], + ["Volcanic Island"] = [], + ["Wasteland"] = [], + + // From single_cards/ - any set is fine + ["Adanto Vanguard"] = [], + ["Angel of Sanctions"] = [], + ["Attunement"] = [], + ["Avaricious Dragon"] = [], + ["Burgeoning"] = [], + ["Jarad, Golgari Lich Lord"] = [], + ["Meletis Charlatan"] = [], + ["Mindstab Thrull"] = [], + ["Pacifism"] = [], + ["Platinum Angel"] = [], + ["Queen Marchesa"] = [], + ["Spellseeker"] = [], + ["Tarmogoyf"] = [], + ["Thought Reflection"] = [], + ["Unsummon"] = [], + + // From varying_quality - prefer older sets + ["Dragon Whelp"] = ["lea", "leb"], + ["Evil Eye of Orms-by-Gore"] = [], + ["Instill Energy"] = ["lea", "leb"], + + // Popular cards for general testing + ["Lightning Helix"] = [], + ["Path to Exile"] = [], + ["Thoughtseize"] = [], + ["Fatal Push"] = [], + ["Snapcaster Mage"] = [], + ["Jace, the Mind Sculptor"] = [], + ["Liliana of the Veil"] = [], + ["Noble Hierarch"] = [], + ["Goblin Guide"] = [], + ["Eidolon of the Great Revel"] = [], +}; + +var priorityCards = new HashSet(priorityCardsWithSets.Keys, StringComparer.OrdinalIgnoreCase); + +// Force rebuild if requested +if (forceRebuild && File.Exists(outputDb)) +{ + Console.WriteLine("Force rebuild requested, removing existing database..."); + File.Delete(outputDb); +} + +using var httpClient = new HttpClient(); +httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("Scry/1.0 (MTG Card Scanner - Database Generator)"); + +using var scryfallClient = new ScryfallClient(httpClient); +using var db = new CardHashDatabase(outputDb); + +// Check existing database state +var existingCardIds = await db.GetExistingCardIdsAsync(); +var existingCardNames = await db.GetExistingCardNamesAsync(); +var existingCount = await db.GetHashCountAsync(); +var storedScryfallDate = await db.GetMetadataAsync("scryfall_updated_at"); + +Console.WriteLine($"Existing database has {existingCount} cards"); + +Console.WriteLine("Fetching bulk data info from Scryfall..."); +var bulkInfo = await scryfallClient.GetBulkDataInfoAsync("unique_artwork"); + +if (bulkInfo?.DownloadUri == null) +{ + Console.WriteLine("Failed to get bulk data info from Scryfall"); + return 1; +} + +Console.WriteLine($"Scryfall data last updated: {bulkInfo.UpdatedAt}"); + +// Check if we need to update at all +var scryfallDateStr = bulkInfo.UpdatedAt?.ToString("O") ?? ""; +var needsUpdate = existingCount == 0 || + storedScryfallDate != scryfallDateStr || + existingCount < maxCards; + +// Also check if all priority cards exist +var missingPriorityCards = includeTestCards + ? priorityCards.Where(c => !existingCardNames.Contains(c)).ToList() + : new List(); + +if (missingPriorityCards.Count > 0) +{ + Console.WriteLine($"Missing {missingPriorityCards.Count} priority cards"); + needsUpdate = true; +} + +if (!needsUpdate) +{ + Console.WriteLine("Database is up-to-date, no changes needed"); + return 0; +} + +Console.WriteLine($"Downloading card data from: {bulkInfo.DownloadUri}"); +Console.WriteLine(); + +var newHashes = new List(); +var processed = 0; +var errors = 0; +var skipped = 0; +var priorityFound = 0; +var priorityNeeded = includeTestCards ? priorityCards.Count : 0; + +// Track which priority cards we've already found with their set +// Key: card name, Value: set code +var foundPriorityWithSet = new Dictionary(StringComparer.OrdinalIgnoreCase); + +// Track deferred priority cards that might get a better set later +// Key: card name, Value: list of (cardHash, setCode) candidates +var deferredPriority = new Dictionary>(StringComparer.OrdinalIgnoreCase); + +// Helper to check if a set is preferred for a priority card +bool IsPreferredSet(string cardName, string setCode) +{ + if (!priorityCardsWithSets.TryGetValue(cardName, out var preferredSets)) + return false; + return preferredSets.Length == 0 || preferredSets.Contains(setCode, StringComparer.OrdinalIgnoreCase); +} + +await foreach (var card in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadUri)) +{ + // Skip non-English cards + if (card.Lang != "en") + continue; + + var imageUri = card.GetImageUri("normal"); + if (string.IsNullOrEmpty(imageUri)) + continue; + + var cardId = card.Id ?? Guid.NewGuid().ToString(); + var cardName = card.Name ?? "Unknown"; + var setCode = card.Set ?? "???"; + + // Check if this card already exists in the database + if (existingCardIds.Contains(cardId)) + { + skipped++; + continue; + } + + // Check if this is a priority card we might need + var isPriorityCard = includeTestCards && priorityCards.Contains(cardName); + var isPreferred = isPriorityCard && IsPreferredSet(cardName, setCode); + + // If this priority card already found with preferred set, skip + if (isPriorityCard && foundPriorityWithSet.TryGetValue(cardName, out var existingSet)) + { + if (IsPreferredSet(cardName, existingSet)) + { + // Already have preferred version + continue; + } + // We have a non-preferred version; if this is preferred, we'll replace + if (!isPreferred) + { + continue; + } + } + + // Calculate how many slots we have left + var totalCards = existingCount + newHashes.Count; + var priorityRemaining = priorityNeeded - foundPriorityWithSet.Count; + var slotsForNonPriority = maxCards - priorityRemaining; + + // Skip if we have enough non-priority cards and this isn't priority + if (!isPriorityCard && totalCards >= slotsForNonPriority) + continue; + + // Download and process image + try + { + Console.Write($"[{processed + 1}] {cardName}... "); + + var imageBytes = await httpClient.GetByteArrayAsync(imageUri); + using var bitmap = SKBitmap.Decode(imageBytes); + + if (bitmap == null) + { + Console.WriteLine("decode failed"); + errors++; + continue; + } + + // Apply CLAHE preprocessing and compute hash + using var preprocessed = ImagePreprocessor.ApplyClahe(bitmap); + var hash = PerceptualHash.ComputeColorHash(preprocessed); + + var cardHash = new CardHash + { + CardId = cardId, + Name = cardName, + SetCode = setCode, + CollectorNumber = card.CollectorNumber, + Hash = hash, + ImageUri = imageUri + }; + + newHashes.Add(cardHash); + + if (isPriorityCard) + { + foundPriorityWithSet[cardName] = setCode; + priorityFound++; + Console.WriteLine($"OK (priority, {setCode})"); + } + else + { + Console.WriteLine($"OK ({setCode})"); + } + + processed++; + + // Check if we have enough cards + var foundAllPriority = foundPriorityWithSet.Count >= priorityNeeded; + if (existingCount + newHashes.Count >= maxCards && foundAllPriority) + { + Console.WriteLine($"\nReached {maxCards} cards limit with all priority cards"); + break; + } + + // Rate limit to be nice to Scryfall + await Task.Delay(50); + } + catch (Exception ex) + { + Console.WriteLine($"error: {ex.Message}"); + errors++; + } +} + +Console.WriteLine(); +Console.WriteLine($"Skipped (already in DB): {skipped}"); +Console.WriteLine($"Newly processed: {processed} cards"); +Console.WriteLine($"New priority cards found: {priorityFound}"); +Console.WriteLine($"Total priority cards: {foundPriorityWithSet.Count}/{priorityNeeded}"); +Console.WriteLine($"Errors: {errors}"); +Console.WriteLine(); + +if (newHashes.Count > 0) +{ + Console.WriteLine($"Inserting {newHashes.Count} new hashes into database..."); + await db.InsertHashBatchAsync(newHashes); +} + +await db.SetMetadataAsync("generated_at", DateTime.UtcNow.ToString("O")); +await db.SetMetadataAsync("scryfall_updated_at", scryfallDateStr); + +var finalCount = await db.GetHashCountAsync(); +await db.SetMetadataAsync("card_count", finalCount.ToString()); + +Console.WriteLine($"Database now has {finalCount} cards: {outputDb}"); + +// Report missing priority cards +if (includeTestCards) +{ + var missing = priorityCards.Where(c => !foundPriorityWithSet.ContainsKey(c)).ToList(); + + if (missing.Count > 0) + { + Console.WriteLine(); + Console.WriteLine($"Missing priority cards ({missing.Count}):"); + foreach (var name in missing.Take(20)) + { + Console.WriteLine($" - {name}"); + } + if (missing.Count > 20) + { + Console.WriteLine($" ... and {missing.Count - 20} more"); + } + } +} + +return 0;