diff --git a/.gitignore b/.gitignore index 29a138d..6facb8c 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ Thumbs.db *.csv *.dlens *.apk +debug/ diff --git a/.justfile b/.justfile index 1f663ae..055de56 100644 --- a/.justfile +++ b/.justfile @@ -1,24 +1,46 @@ # Scry development commands + +set shell := ["C:/Program Files/Git/usr/bin/bash.exe", "-c"] +set unstable := true + # Android SDK paths -android_sdk := env_var('LOCALAPPDATA') / "Android/Sdk" +android_sdk := replace(env('LOCALAPPDATA'), '\', '/') / "Android/Sdk" adb := android_sdk / "platform-tools/adb.exe" emulator := android_sdk / "emulator/emulator.exe" camera_virtual := "-camera-back virtualscene -virtualscene-poster wall=\"" + (justfile_directory() / "TestImages/reference_alpha/serra_angel.jpg") + "\"" camera_webcam := "-camera-back webcam0 -camera-front webcam0" +[private] +@default: + just --list + +# Start emulator in background emu camera="virtual": - {{ emulator }} -avd Pixel_6 {{ if camera == "virtual" { camera_virtual } else { camera_webcam } }} -gpu host & + {{ emulator }} -avd Pixel_6 {{ if camera == "virtual" { camera_virtual } else { camera_webcam } }} -no-snapshot-load -gpu host # Kill the running emulator emu-kill: {{ adb }} emu kill -# Wait for emulator to fully boot +# Wait for emulator to fully boot (timeout after 2 minutes) +[script] emu-wait: - @echo "Waiting for emulator to boot..." - @while [ "$({{ adb }} shell getprop sys.boot_completed 2>/dev/null)" != "1" ]; do sleep 1; done - @echo "Emulator ready" + # Wait for Android emulator to boot with timeout + TIMEOUT=120 + + echo "Waiting for emulator to boot..." + + for ((i=TIMEOUT; i>0; i--)); do + if [ "$({{ adb }} shell getprop sys.boot_completed 2>/dev/null)" = "1" ]; then + echo "Emulator ready" + exit 0 + fi + sleep 1 + done + + echo "Emulator failed to boot within 2 minutes" + exit 1 # Build a project build project="src/Scry.App" target="net10.0-android": @@ -52,11 +74,11 @@ test: dotnet test test/Scry.Tests # Generate the card hash database from Scryfall -gen-db *args: (build "tools/DbGenerator" "net10.0") +gen-db: (build "tools/DbGenerator" "net10.0") @echo "Running Database generator (this takes a while)..." - dotnet run --project tools/DbGenerator --no-build -- src/Scry.App/Resources/Raw/card_hashes.db {{ args }} + dotnet run --project tools/DbGenerator --no-build -- src/Scry.App/Resources/Raw/card_hashes.db @echo "Completed generating the database" # Full workflow: start emulator, wait, run with hot reload -dev: emu emu-wait +dev: dotnet watch --project src/Scry.App -f net10.0-android diff --git a/AGENTS.md b/AGENTS.md index 64bdd14..a04e7bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,30 +1,291 @@ # Agent Instructions -## Build and Task Commands +## Overview -Prefer using `just` commands over raw dotnet/build commands: +Scry is a Magic: The Gathering card scanner app built with .NET MAUI. It uses perceptual hashing to match photographed cards against a database of known card images from Scryfall. -| Task | Command | -|------|---------| -| Build project | `just build` | -| Run tests | `just test` | -| Generate card database | `just gen-db` | -| Publish app | `just publish` | -| Run full dev workflow | `just dev` | +**Key components:** +- Mobile scanning app (MAUI/Android) +- Card recognition via perceptual hashing (not OCR) +- SQLite database with pre-computed hashes +- Scryfall API integration for card data + +## Build Commands + +Use `just` commands (defined in `.justfile`): + +| Task | Command | Notes | +|------|---------|-------| +| Build project | `just build` | Builds Android debug | +| Run tests | `just test` | Runs xUnit tests | +| Generate card database | `just gen-db` | Downloads from Scryfall, computes hashes | +| Publish app | `just publish` | Creates release APK | +| Hot reload dev | `just dev` | Uses `dotnet watch` | +| Start emulator | `just emu` | Virtual camera with Serra Angel | +| Install to device | `just install` | Installs release APK | + +### Database Generator Options + +```bash +just gen-db # Default: 500 cards with test images +dotnet run --project tools/DbGenerator -- -c 1000 # More cards +dotnet run --project tools/DbGenerator -- --force # Rebuild from scratch +dotnet run --project tools/DbGenerator -- --no-test-cards # Skip priority test cards +``` ## Project Structure -- `src/Scry.App` - MAUI mobile app -- `src/Scry.Core` - Core library (recognition, hashing, database) -- `test/Scry.Tests` - Unit tests -- `tools/DbGenerator` - Card hash database generator +``` +src/ +├── Scry.App/ # MAUI mobile app (Android target) +│ ├── Views/ # XAML pages (ScanPage, CollectionPage, etc.) +│ ├── ViewModels/ # MVVM ViewModels using CommunityToolkit.Mvvm +│ ├── Services/ # App-layer services (ICardRecognitionService, ICardRepository) +│ ├── Converters/ # XAML value converters +│ ├── Models/ # App-specific models (CollectionEntry) +│ └── Resources/Raw/ # Bundled card_hashes.db +│ +└── Scry.Core/ # Platform-independent core library + ├── Recognition/ # CardRecognitionService, RecognitionOptions + ├── Imaging/ # PerceptualHash, ImagePreprocessor, CardDetector + ├── Data/ # CardDatabase (SQLite) + ├── Models/ # Card, Oracle, Set, ScanResult + └── Scryfall/ # ScryfallClient for API/bulk data -## Database Generator +test/ +└── Scry.Tests/ # xUnit tests -The `just gen-db` command: -- Builds the DbGenerator tool -- Runs it against `src/Scry.App/Resources/Raw/card_hashes.db` -- Supports incremental updates (only downloads missing cards) -- Prefers LEA/LEB (Alpha/Beta) sets for reference_alpha test cards +tools/ +└── DbGenerator/ # CLI tool to generate card_hashes.db -Use `--force` flag to rebuild from scratch if needed. +TestImages/ # Test images organized by category +├── reference_alpha/ # Alpha/Beta cards for testing +├── single_cards/ # Individual card photos +├── varying_quality/ # Different lighting/quality +├── hands/ # Cards held in hand +├── foil/ # Foil cards with glare +└── ... # More categories +``` + +## Architecture + +### Recognition Pipeline + +``` +Camera Image + │ + ▼ +┌─────────────────────┐ +│ CardDetector │ ← Edge detection, find card quad +│ (optional) │ +└─────────────────────┘ + │ + ▼ +┌─────────────────────┐ +│ PerspectiveCorrection│ ← Warp to rectangle +└─────────────────────┘ + │ + ▼ +┌─────────────────────┐ +│ ImagePreprocessor │ ← CLAHE for lighting normalization +│ (ApplyClahe) │ +└─────────────────────┘ + │ + ▼ +┌─────────────────────┐ +│ PerceptualHash │ ← Compute 192-bit color hash (24 bytes) +│ (ComputeColorHash) │ +└─────────────────────┘ + │ + ▼ +┌─────────────────────┐ +│ CardRecognitionService│ ← Hamming distance match against DB +└─────────────────────┘ +``` + +### Data Model + +Three-table schema mirroring Scryfall: + +- **oracles** - Abstract game cards (one per unique card name) +- **sets** - MTG sets with metadata +- **cards** - Printings with perceptual hashes (one per unique artwork) + +The `Card` model includes denormalized Oracle fields for convenience. + +### Key Classes + +| Class | Purpose | +|-------|---------| +| `CardRecognitionService` | Main recognition logic, caches DB, handles rotation matching | +| `PerceptualHash` | DCT-based color hash (192-bit = 8 bytes × 3 RGB channels) | +| `ImagePreprocessor` | CLAHE, resize, grayscale conversions | +| `CardDetector` | Edge detection + contour analysis to find card boundaries | +| `PerspectiveCorrection` | Warp detected quad to rectangle | +| `CardDatabase` | SQLite wrapper with batch insert, queries | +| `ScryfallClient` | Bulk data streaming, image downloads | + +## Code Conventions + +### General + +- **Target**: .NET 10.0 (net10.0-android for app, net10.0 for Core/tools) +- **Nullable**: Enabled everywhere (`enable`) +- **Warnings as errors**: `true` +- **Central package management**: Versions in `Directory.Packages.props` + +### C# Style + +- Records for data models (`record Card`, `record ScanResult`) +- `required` properties for non-nullable required fields +- Extension methods for conversions (`ScryfallCard.ToCard()`) +- Static classes for pure functions (`PerceptualHash`, `ImagePreprocessor`) +- `using` declarations (not `using` blocks) for disposables +- File-scoped namespaces +- Primary constructors where appropriate +- `CancellationToken` parameter on all async methods + +### MVVM (App layer) + +- `CommunityToolkit.Mvvm` for source generators +- `[ObservableProperty]` attributes for bindable properties +- `[RelayCommand]` for commands +- ViewModels in `Scry.ViewModels` namespace + +### Naming + +- Services: `ICardRecognitionService`, `CardRecognitionService` +- Database methods: `GetCardsWithHashAsync`, `InsertCardBatchAsync` +- Hash methods: `ComputeColorHash`, `HammingDistance` +- Test methods: `RecognizeAsync_ExactMatch_ReturnsSuccess` + +## Testing + +Tests are in `test/Scry.Tests` using xUnit. + +```bash +just test # Run all tests +dotnet test --filter "FullyQualifiedName~PerceptualHash" # Filter by name +``` + +### Test Categories + +| Test Class | Tests | +|------------|-------| +| `PerceptualHashTests` | Hash computation, Hamming distance | +| `CardRecognitionTests` | End-to-end recognition | +| `CardDatabaseTests` | SQLite CRUD operations | +| `ImagePreprocessorTests` | CLAHE, resize | +| `RobustnessAnalysisTests` | Multiple image variations | + +### Test Images + +TestImages directory contains categorized reference images: +- `reference_alpha/` - Alpha/Beta cards (matching DbGenerator priority cards) +- `single_cards/` - Clean single card photos +- `varying_quality/` - Different lighting/blur conditions + +## Key Algorithms + +### Perceptual Hash (pHash) + +Color-aware 192-bit hash: +1. Resize to 32×32 +2. For each RGB channel: + - Compute 2D DCT + - Extract 8×8 low-frequency coefficients (skip DC) + - Compare each to median → 63 bits per channel +3. Concatenate R, G, B hashes → 24 bytes (192 bits) + +Matching uses Hamming distance with threshold ≤25 bits and minimum confidence 85%. + +### CLAHE (Contrast Limited Adaptive Histogram Equalization) + +Applied in LAB color space to L channel only: +- Tile-based histogram equalization (8×8 tiles) +- Clip limit prevents over-amplification of noise +- Bilinear interpolation between tiles for smooth output + +### Card Detection + +Pure SkiaSharp implementation: +1. Grayscale → Gaussian blur → Canny edge detection +2. Contour tracing via flood fill +3. Douglas-Peucker simplification → Convex hull +4. Find best quadrilateral matching MTG aspect ratio (88/63 ≈ 1.397) +5. Order corners: top-left, top-right, bottom-right, bottom-left + +## Debug Mode + +Set `RecognitionOptions.DebugOutputDirectory` to save pipeline stages: +- `01_input.png` - Original image +- `02_detection.png` - Card detection visualization +- `03_perspective_corrected.png` - Warped card +- `05_clahe_*.png` - After CLAHE preprocessing + +On Android: `/sdcard/Download/scry-debug` (pull with `adb pull`) + +## Dependencies + +### Core Library (Scry.Core) + +- **SkiaSharp** - Image processing, DCT, edge detection +- **Microsoft.Data.Sqlite** - SQLite database +- **Microsoft.Extensions.Options** - Options pattern + +### App (Scry.App) + +- **CommunityToolkit.Maui** - MAUI extensions +- **CommunityToolkit.Maui.Camera** - Camera integration +- **CommunityToolkit.Mvvm** - MVVM source generators + +### DbGenerator Tool + +- **Spectre.Console** / **Spectre.Console.Cli** - Rich terminal UI + +## Common Tasks + +### Adding a New Card to Priority Test Set + +1. Add image to `TestImages/reference_alpha/` or appropriate folder +2. Add entry to `GenerateCommand.PriorityCardsWithSets` dictionary +3. Run `just gen-db` to regenerate database + +### Debugging Recognition Failures + +1. Enable debug output in `MauiProgram.cs`: + ```csharp + options.DebugOutputDirectory = "/sdcard/Download/scry-debug"; + ``` +2. Run recognition +3. Pull debug images: `adb pull /sdcard/Download/scry-debug` +4. Compare `05_clahe_*.png` with reference images in database + +### Modifying Hash Algorithm + +1. Update `PerceptualHash.ComputeColorHash()` +2. Update `CardRecognitionService.ColorHashBits` constant +3. Regenerate database: `just gen-db --force` +4. Run tests: `just test` + +## Gotchas + +1. **Hash size is 24 bytes (192 bits)** - 3 RGB channels × 8 bytes each +2. **Confidence threshold is 85%** - Configurable in `CardRecognitionService.MinConfidence` +3. **Card detection is optional** - Controlled by `RecognitionOptions.EnableCardDetection` +4. **Rotation matching tries 4 orientations** - Controlled by `RecognitionOptions.EnableRotationMatching` +5. **Database is bundled in APK** - Copied on first run to app data directory +6. **Multi-face cards** - Only front face image is used for hashing +7. **Rate limiting** - DbGenerator uses 50ms delay between Scryfall image downloads + +## CI/CD + +Forgejo Actions workflow (`.forgejo/workflows/release.yml`): +- Builds for win-x64, linux-x64, osx-x64 +- Creates "standard" and "embedded" (with APK) variants +- Publishes to Forgejo releases + +## External Resources + +- [Scryfall API](https://scryfall.com/docs/api) - Card data source +- [CARD_RECOGNITION.md](docs/CARD_RECOGNITION.md) - Detailed architecture doc diff --git a/Directory.Packages.props b/Directory.Packages.props index 880d82a..baa520b 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -17,6 +17,9 @@ /> + + + diff --git a/screen.png b/screen.png deleted file mode 100644 index 3d9ea9a..0000000 Binary files a/screen.png and /dev/null differ diff --git a/src/Scry.App/MauiProgram.cs b/src/Scry.App/MauiProgram.cs index 7b6252c..ebeca2a 100644 --- a/src/Scry.App/MauiProgram.cs +++ b/src/Scry.App/MauiProgram.cs @@ -30,6 +30,17 @@ public static class MauiProgram EnsureDatabaseCopied(dbPath); return new CardDatabase(dbPath); }); + + // Recognition options - configure debug output in DEBUG builds + builder.Services.Configure(options => + { +#if DEBUG && ANDROID + // Use Download folder for easy adb pull access + options.DebugOutputDirectory = "/sdcard/Download/scry-debug"; +#elif DEBUG + options.DebugOutputDirectory = "./debug"; +#endif + }); builder.Services.AddSingleton(); // App Services @@ -57,14 +68,25 @@ public static class MauiProgram private static void EnsureDatabaseCopied(string targetPath) { - if (File.Exists(targetPath)) - return; - try { - using var stream = FileSystem.OpenAppPackageFileAsync("card_hashes.db").GetAwaiter().GetResult(); + using var bundledStream = FileSystem.OpenAppPackageFileAsync("card_hashes.db").GetAwaiter().GetResult(); + + if (File.Exists(targetPath)) + { + // Compare sizes - if bundled is larger, replace + var existingSize = new FileInfo(targetPath).Length; + var bundledSize = bundledStream.Length; + + if (bundledSize <= existingSize) + return; + + // Bundled db is larger, delete old and copy new + File.Delete(targetPath); + } + using var fileStream = File.Create(targetPath); - stream.CopyTo(fileStream); + bundledStream.CopyTo(fileStream); } catch { diff --git a/src/Scry.App/Resources/Raw/card_hashes.db b/src/Scry.App/Resources/Raw/card_hashes.db index 793bbc4..7f18094 100644 Binary files a/src/Scry.App/Resources/Raw/card_hashes.db and b/src/Scry.App/Resources/Raw/card_hashes.db differ diff --git a/src/Scry.App/Scry.App.csproj b/src/Scry.App/Scry.App.csproj index b0ae247..3a04d80 100644 --- a/src/Scry.App/Scry.App.csproj +++ b/src/Scry.App/Scry.App.csproj @@ -29,6 +29,11 @@ true + + + + + Scry diff --git a/src/Scry.Core/Recognition/CardRecognitionService.cs b/src/Scry.Core/Recognition/CardRecognitionService.cs index fd00568..b560e14 100644 --- a/src/Scry.Core/Recognition/CardRecognitionService.cs +++ b/src/Scry.Core/Recognition/CardRecognitionService.cs @@ -1,4 +1,5 @@ using System.Diagnostics; +using Microsoft.Extensions.Options; using Scry.Core.Data; using Scry.Core.Imaging; using Scry.Core.Models; @@ -9,6 +10,7 @@ namespace Scry.Core.Recognition; public class CardRecognitionService : IDisposable { private readonly CardDatabase _database; + private readonly RecognitionOptions _options; private List? _cardCache; private readonly SemaphoreSlim _cacheLock = new(1, 1); @@ -16,21 +18,14 @@ public class CardRecognitionService : IDisposable private const int MatchThreshold = 25; private const float MinConfidence = 0.85f; - /// - /// Enable card detection and perspective correction. - /// When disabled, assumes the input image is already a cropped card. - /// - public bool EnableCardDetection { get; set; } = true; - - /// - /// Try multiple rotations (0°, 90°, 180°, 270°) when matching. - /// Useful when card orientation is unknown. - /// - public bool EnableRotationMatching { get; set; } = true; - - public CardRecognitionService(CardDatabase database) + public CardRecognitionService(CardDatabase database, IOptions options) { _database = database; + _options = options.Value; + } + + public CardRecognitionService(CardDatabase database) : this(database, Options.Create(new RecognitionOptions())) + { } public async Task RecognizeAsync(Stream imageStream, CancellationToken ct = default) @@ -60,6 +55,14 @@ public class CardRecognitionService : IDisposable public async Task RecognizeAsync(SKBitmap bitmap, CancellationToken ct = default) { var stopwatch = Stopwatch.StartNew(); + var debugDir = _options.DebugOutputDirectory; + var debugEnabled = !string.IsNullOrEmpty(debugDir); + + if (debugEnabled) + { + Directory.CreateDirectory(debugDir!); + SaveDebugImage(bitmap, debugDir!, "01_input"); + } try { @@ -75,14 +78,26 @@ public class CardRecognitionService : IDisposable SKBitmap cardImage; bool cardDetected = false; - if (EnableCardDetection) + if (_options.EnableCardDetection) { var detection = CardDetector.DetectCard(bitmap); + + if (debugEnabled) + { + // Save detection visualization + SaveDetectionDebugImage(bitmap, detection, debugDir!); + } + if (detection.Found) { cardImage = PerspectiveCorrection.WarpPerspective(bitmap, detection.Corners); cardDetected = true; Console.WriteLine($"[Scry] Card detected with confidence {detection.Confidence:P0}"); + + if (debugEnabled) + { + SaveDebugImage(cardImage, debugDir!, "03_perspective_corrected"); + } } else { @@ -99,9 +114,9 @@ public class CardRecognitionService : IDisposable try { // Step 2: Try matching with rotation variants (if enabled) - var bestMatch = EnableRotationMatching - ? await FindBestMatchWithRotationsAsync(cardImage, cards, ct) - : FindBestMatchSingle(cardImage, cards); + var bestMatch = _options.EnableRotationMatching + ? await FindBestMatchWithRotationsAsync(cardImage, cards, debugDir, ct) + : FindBestMatchSingle(cardImage, cards, debugDir); stopwatch.Stop(); @@ -198,6 +213,7 @@ public class CardRecognitionService : IDisposable private Task<(Card Card, int Distance, int Rotation)?> FindBestMatchWithRotationsAsync( SKBitmap cardImage, List candidates, + string? debugDir, CancellationToken ct) { return Task.Run(() => @@ -205,6 +221,7 @@ public class CardRecognitionService : IDisposable Card? bestMatch = null; var bestDistance = int.MaxValue; var bestRotation = 0; + var debugEnabled = !string.IsNullOrEmpty(debugDir); var rotations = new[] { 0, 90, 180, 270 }; @@ -215,8 +232,19 @@ public class CardRecognitionService : IDisposable using var rotated = rotation == 0 ? null : RotateImage(cardImage, rotation); var imageToHash = rotated ?? cardImage; + if (debugEnabled && rotation != 0) + { + SaveDebugImage(imageToHash, debugDir!, $"04_rotated_{rotation}"); + } + // Apply CLAHE and compute hash using var preprocessed = ImagePreprocessor.ApplyClahe(imageToHash); + + if (debugEnabled) + { + SaveDebugImage(preprocessed, debugDir!, $"05_clahe_{rotation}"); + } + var queryHash = PerceptualHash.ComputeColorHash(preprocessed); // Find best match for this rotation @@ -252,10 +280,19 @@ public class CardRecognitionService : IDisposable /// private (Card Card, int Distance, int Rotation)? FindBestMatchSingle( SKBitmap cardImage, - List candidates) + List candidates, + string? debugDir) { + var debugEnabled = !string.IsNullOrEmpty(debugDir); + // Apply CLAHE and compute hash using var preprocessed = ImagePreprocessor.ApplyClahe(cardImage); + + if (debugEnabled) + { + SaveDebugImage(preprocessed, debugDir!, "05_clahe_0"); + } + var queryHash = PerceptualHash.ComputeColorHash(preprocessed); Card? bestMatch = null; @@ -308,6 +345,83 @@ public class CardRecognitionService : IDisposable return rotated; } + /// + /// Save a debug image to disk. + /// + private static void SaveDebugImage(SKBitmap bitmap, string directory, string name) + { + var path = Path.Combine(directory, $"{name}.png"); + using var image = SKImage.FromBitmap(bitmap); + using var data = image.Encode(SKEncodedImageFormat.Png, 100); + using var stream = File.OpenWrite(path); + data.SaveTo(stream); + Console.WriteLine($"[Scry Debug] Saved: {path}"); + } + + /// + /// Save a debug image showing the card detection result. + /// + private static void SaveDetectionDebugImage(SKBitmap original, CardDetector.CardDetectionResult detection, string directory) + { + using var annotated = new SKBitmap(original.Width, original.Height, original.ColorType, original.AlphaType); + using var canvas = new SKCanvas(annotated); + + canvas.DrawBitmap(original, 0, 0); + + if (detection.Found && detection.Corners.Length == 4) + { + // Draw detected corners and edges + using var cornerPaint = new SKPaint + { + Color = SKColors.Red, + Style = SKPaintStyle.Fill, + IsAntialias = true + }; + + using var edgePaint = new SKPaint + { + Color = SKColors.Lime, + Style = SKPaintStyle.Stroke, + StrokeWidth = 3, + IsAntialias = true + }; + + var corners = detection.Corners; + + // Draw edges + for (int i = 0; i < 4; i++) + { + var p1 = corners[i]; + var p2 = corners[(i + 1) % 4]; + canvas.DrawLine(p1.X, p1.Y, p2.X, p2.Y, edgePaint); + } + + // Draw corners + foreach (var corner in corners) + { + canvas.DrawCircle(corner.X, corner.Y, 8, cornerPaint); + } + } + + // Add debug text + using var textPaint = new SKPaint + { + Color = detection.Found ? SKColors.Lime : SKColors.Red, + IsAntialias = true + }; + using var font = new SKFont + { + Size = 24 + }; + + var message = detection.Found + ? $"Detected: {detection.Confidence:P0}" + : $"Not found: {detection.DebugMessage}"; + canvas.DrawText(message, 10, 30, SKTextAlign.Left, font, textPaint); + + SaveDebugImage(annotated, directory, "02_detection"); + } + public void Dispose() { _cacheLock.Dispose(); diff --git a/src/Scry.Core/Recognition/RecognitionOptions.cs b/src/Scry.Core/Recognition/RecognitionOptions.cs new file mode 100644 index 0000000..8a9f4ee --- /dev/null +++ b/src/Scry.Core/Recognition/RecognitionOptions.cs @@ -0,0 +1,24 @@ +namespace Scry.Core.Recognition; + +/// +/// Configuration options for card recognition. +/// +public class RecognitionOptions +{ + /// + /// When set, saves debug images of each pipeline step to this directory. + /// + public string? DebugOutputDirectory { get; set; } + + /// + /// Enable card detection and perspective correction. + /// When disabled, assumes the input image is already a cropped card. + /// + public bool EnableCardDetection { get; set; } = true; + + /// + /// Try multiple rotations (0°, 90°, 180°, 270°) when matching. + /// Useful when card orientation is unknown. + /// + public bool EnableRotationMatching { get; set; } = true; +} diff --git a/src/Scry.Core/Scry.Core.csproj b/src/Scry.Core/Scry.Core.csproj index e2e0044..9464c4d 100644 --- a/src/Scry.Core/Scry.Core.csproj +++ b/src/Scry.Core/Scry.Core.csproj @@ -10,6 +10,7 @@ + diff --git a/tools/DbGenerator/DbGenerator.csproj b/tools/DbGenerator/DbGenerator.csproj index 4884da9..0b7c3bc 100644 --- a/tools/DbGenerator/DbGenerator.csproj +++ b/tools/DbGenerator/DbGenerator.csproj @@ -9,6 +9,8 @@ + + diff --git a/tools/DbGenerator/GenerateCommand.cs b/tools/DbGenerator/GenerateCommand.cs new file mode 100644 index 0000000..2c9cfb5 --- /dev/null +++ b/tools/DbGenerator/GenerateCommand.cs @@ -0,0 +1,495 @@ +using System.ComponentModel; +using Scry.Core.Data; +using Scry.Core.Imaging; +using Scry.Core.Models; +using Scry.Core.Scryfall; +using SkiaSharp; +using Spectre.Console; +using Spectre.Console.Cli; + +namespace DbGenerator; + +public sealed class GenerateSettings : CommandSettings +{ + [CommandArgument(0, "[output]")] + [Description("Output database file path")] + [DefaultValue("card_hashes.db")] + public string Output { get; set; } = "card_hashes.db"; + + [CommandOption("-c|--count")] + [Description("Maximum number of cards to include")] + [DefaultValue(500)] + public int Count { get; set; } = 500; + + [CommandOption("--include-test-cards")] + [Description("Include priority test cards (default: true)")] + [DefaultValue(true)] + public bool IncludeTestCards { get; set; } = true; + + [CommandOption("--no-test-cards")] + [Description("Exclude priority test cards")] + public bool NoTestCards { get; set; } + + [CommandOption("-f|--force")] + [Description("Force rebuild from scratch")] + public bool Force { get; set; } +} + +public sealed class GenerateCommand : AsyncCommand +{ + // Cards that should be included for testing with preferred sets + private static readonly Dictionary PriorityCardsWithSets = new(StringComparer.OrdinalIgnoreCase) + { + // From reference_alpha/ - prefer LEA (Alpha) or LEB (Beta) for classic look + ["Ancestral Recall"] = ["lea", "leb"], + ["Badlands"] = ["lea", "leb"], + ["Balance"] = ["lea", "leb"], + ["Bayou"] = ["lea", "leb"], + ["Birds of Paradise"] = ["lea", "leb"], + ["Black Lotus"] = ["lea", "leb"], + ["Channel"] = ["lea", "leb"], + ["Chaos Orb"] = ["lea", "leb"], + ["Clone"] = ["lea", "leb"], + ["Control Magic"] = ["lea", "leb"], + ["Counterspell"] = ["lea", "leb"], + ["Dark Ritual"] = ["lea", "leb"], + ["Demonic Tutor"] = ["lea", "leb"], + ["Disenchant"] = ["lea", "leb"], + ["Fireball"] = ["lea", "leb"], + ["Force of Nature"] = ["lea", "leb"], + ["Fork"] = ["lea", "leb"], + ["Giant Growth"] = ["lea", "leb"], + ["Hypnotic Specter"] = ["lea", "leb"], + ["Lightning Bolt"] = ["lea", "leb"], + ["Llanowar Elves"] = ["lea", "leb"], + ["Mahamoti Djinn"] = ["lea", "leb"], + ["Mind Twist"] = ["lea", "leb"], + ["Mox Emerald"] = ["lea", "leb"], + ["Mox Jet"] = ["lea", "leb"], + ["Mox Pearl"] = ["lea", "leb"], + ["Mox Ruby"] = ["lea", "leb"], + ["Mox Sapphire"] = ["lea", "leb"], + ["Nightmare"] = ["lea", "leb"], + ["Plateau"] = ["lea", "leb"], + ["Regrowth"] = ["lea", "leb"], + ["Rock Hydra"] = ["lea", "leb"], + ["Royal Assassin"] = ["lea", "leb"], + ["Savannah"] = ["lea", "leb"], + ["Scrubland"] = ["lea", "leb"], + ["Serra Angel"] = ["lea", "leb"], + ["Shivan Dragon"] = ["lea", "leb"], + ["Sol Ring"] = ["lea", "leb"], + ["Swords to Plowshares"] = ["lea", "leb"], + ["Taiga"] = ["lea", "leb"], + ["Time Walk"] = ["lea", "leb"], + ["Timetwister"] = ["lea", "leb"], + ["Tropical Island"] = ["lea", "leb"], + ["Tundra"] = ["lea", "leb"], + ["Underground Sea"] = ["lea", "leb"], + ["Wheel of Fortune"] = ["lea", "leb"], + ["Wrath of God"] = ["lea", "leb"], + + // From reference/ - any set is fine + ["Brainstorm"] = [], + ["Force of Will"] = [], + ["Griselbrand"] = [], + ["Lotus Petal"] = [], + ["Ponder"] = [], + ["Show and Tell"] = [], + ["Volcanic Island"] = [], + ["Wasteland"] = [], + + // From single_cards/ - any set is fine + ["Adanto Vanguard"] = [], + ["Angel of Sanctions"] = [], + ["Attunement"] = [], + ["Avaricious Dragon"] = [], + ["Burgeoning"] = [], + ["Jarad, Golgari Lich Lord"] = [], + ["Meletis Charlatan"] = [], + ["Mindstab Thrull"] = [], + ["Pacifism"] = [], + ["Platinum Angel"] = [], + ["Queen Marchesa"] = [], + ["Spellseeker"] = [], + ["Tarmogoyf"] = [], + ["Thought Reflection"] = [], + ["Unsummon"] = [], + + // From varying_quality - prefer older sets + ["Dragon Whelp"] = ["lea", "leb"], + ["Evil Eye of Orms-by-Gore"] = [], + ["Instill Energy"] = ["lea", "leb"], + + // Popular cards for general testing + ["Lightning Helix"] = [], + ["Path to Exile"] = [], + ["Thoughtseize"] = [], + ["Fatal Push"] = [], + ["Snapcaster Mage"] = [], + ["Jace, the Mind Sculptor"] = [], + ["Liliana of the Veil"] = [], + ["Noble Hierarch"] = [], + ["Goblin Guide"] = [], + ["Eidolon of the Great Revel"] = [], + }; + + public override async Task ExecuteAsync(CommandContext context, GenerateSettings settings) + { + var outputDb = settings.Output; + var maxCards = settings.Count; + var includeTestCards = settings.IncludeTestCards && !settings.NoTestCards; + var forceRebuild = settings.Force; + + // Header + AnsiConsole.Write(new FigletText("Scry DB Gen").Color(Color.Blue)); + + var configTable = new Table() + .Border(TableBorder.Rounded) + .AddColumn("Setting") + .AddColumn("Value"); + + configTable.AddRow("Output", outputDb); + configTable.AddRow("Max Cards", maxCards.ToString()); + configTable.AddRow("Test Cards", includeTestCards ? "[green]Yes[/]" : "[grey]No[/]"); + configTable.AddRow("Force Rebuild", forceRebuild ? "[yellow]Yes[/]" : "[grey]No[/]"); + + AnsiConsole.Write(configTable); + AnsiConsole.WriteLine(); + + var priorityCards = new HashSet(PriorityCardsWithSets.Keys, StringComparer.OrdinalIgnoreCase); + + // Force rebuild if requested + if (forceRebuild && File.Exists(outputDb)) + { + AnsiConsole.MarkupLine("[yellow]Force rebuild requested, removing existing database...[/]"); + File.Delete(outputDb); + } + + using var httpClient = new HttpClient(); + httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("Scry/1.0 (MTG Card Scanner - Database Generator)"); + + using var scryfallClient = new ScryfallClient(httpClient); + using var db = new CardDatabase(outputDb); + + // Check existing database state + var existingCardIds = await db.GetExistingCardIdsAsync(); + var existingCardNames = await db.GetExistingCardNamesAsync(); + var existingOracleIds = await db.GetExistingOracleIdsAsync(); + var existingSetIds = await db.GetExistingSetIdsAsync(); + var existingCount = await db.GetCardCountAsync(); + var storedScryfallDate = await db.GetMetadataAsync("scryfall_updated_at"); + + var dbStateTable = new Table() + .Border(TableBorder.Rounded) + .Title("[blue]Current Database State[/]") + .AddColumn("Metric") + .AddColumn("Count", c => c.RightAligned()); + + dbStateTable.AddRow("Cards", existingCount.ToString()); + dbStateTable.AddRow("Oracles", existingOracleIds.Count.ToString()); + dbStateTable.AddRow("Sets", existingSetIds.Count.ToString()); + + AnsiConsole.Write(dbStateTable); + AnsiConsole.WriteLine(); + + // Fetch all sets + List scryfallSets = []; + await AnsiConsole.Status() + .Spinner(Spinner.Known.Dots) + .StartAsync("Fetching sets from Scryfall...", async ctx => + { + scryfallSets = await scryfallClient.GetAllSetsAsync(); + }); + + AnsiConsole.MarkupLine($"[green]✓[/] Found [blue]{scryfallSets.Count}[/] sets"); + + var setsById = scryfallSets.ToDictionary(s => s.Id ?? "", s => s); + var setsByCode = scryfallSets.ToDictionary(s => s.Code ?? "", s => s, StringComparer.OrdinalIgnoreCase); + + // Insert any new sets + var newSets = scryfallSets + .Where(s => s.Id != null && !existingSetIds.Contains(s.Id)) + .Select(s => s.ToSet()) + .ToList(); + + if (newSets.Count > 0) + { + AnsiConsole.MarkupLine($"[green]✓[/] Inserting [blue]{newSets.Count}[/] new sets"); + await db.InsertSetBatchAsync(newSets); + } + + // Fetch bulk data info + BulkDataInfo? bulkInfo = null; + await AnsiConsole.Status() + .Spinner(Spinner.Known.Dots) + .StartAsync("Fetching bulk data info...", async ctx => + { + bulkInfo = await scryfallClient.GetBulkDataInfoAsync("unique_artwork"); + }); + + if (bulkInfo?.DownloadUri == null) + { + AnsiConsole.MarkupLine("[red]✗ Failed to get bulk data info from Scryfall[/]"); + return 1; + } + + AnsiConsole.MarkupLine($"[green]✓[/] Scryfall data last updated: [blue]{bulkInfo.UpdatedAt:yyyy-MM-dd HH:mm}[/]"); + + // Check if we need to update at all + var scryfallDateStr = bulkInfo.UpdatedAt?.ToString("O") ?? ""; + var needsUpdate = existingCount == 0 || + storedScryfallDate != scryfallDateStr || + existingCount < maxCards; + + // Also check if all priority cards exist + var missingPriorityCards = includeTestCards + ? priorityCards.Where(c => !existingCardNames.Contains(c)).ToList() + : []; + + if (missingPriorityCards is not []) + { + AnsiConsole.MarkupLine($"[yellow]![/] Missing [blue]{missingPriorityCards.Count}[/] priority cards"); + needsUpdate = true; + } + + if (!needsUpdate) + { + AnsiConsole.MarkupLine("[green]✓ Database is up-to-date, no changes needed[/]"); + return 0; + } + + AnsiConsole.WriteLine(); + + var newCards = new List(); + var newOracles = new Dictionary(); + var processed = 0; + var errors = 0; + var skipped = 0; + var priorityFound = 0; + var priorityNeeded = includeTestCards ? priorityCards.Count : 0; + + // Track which priority cards we've already found with their set + var foundPriorityWithSet = new Dictionary(StringComparer.OrdinalIgnoreCase); + + // Helper to check if a set is preferred for a priority card + static bool IsPreferredSet(string cardName, string setCode) + { + if (!PriorityCardsWithSets.TryGetValue(cardName, out var preferredSets)) + return false; + + return preferredSets.Length == 0 || preferredSets.Contains(setCode, StringComparer.OrdinalIgnoreCase); + } + + await AnsiConsole.Progress() + .AutoClear(false) + .HideCompleted(false) + .Columns( + new RemainingTimeColumn(), + new SpinnerColumn(), + new ProgressBarColumn(), + new PercentageColumn(), + new TaskDescriptionColumn() + { + Alignment = Justify.Left, + } + ) + .StartAsync(async ctx => + { + var downloadTask = ctx.AddTask("[blue]Downloading & processing cards[/]", maxValue: maxCards); + var priorityTask = ctx.AddTask("[green]Priority cards[/]", maxValue: priorityNeeded); + + await foreach (var scryfallCard in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadUri)) + { + // Skip non-English cards + if (scryfallCard.Lang != "en") + continue; + + var imageUri = scryfallCard.GetImageUri("normal"); + if (string.IsNullOrEmpty(imageUri)) + continue; + + var cardId = scryfallCard.Id ?? Guid.NewGuid().ToString(); + var cardName = scryfallCard.Name ?? "Unknown"; + var setCode = scryfallCard.Set ?? "???"; + var oracleId = scryfallCard.OracleId ?? cardId; + var setId = scryfallCard.SetId ?? ""; + + // Check if this card already exists in the database + if (existingCardIds.Contains(cardId)) + { + skipped++; + continue; + } + + // Check if this is a priority card we might need + var isPriorityCard = includeTestCards && priorityCards.Contains(cardName); + var isPreferred = isPriorityCard && IsPreferredSet(cardName, setCode); + + // If this priority card already found with preferred set, skip + if (isPriorityCard && foundPriorityWithSet.TryGetValue(cardName, out var existingSet)) + { + if (IsPreferredSet(cardName, existingSet)) + continue; + if (!isPreferred) + continue; + } + + // Calculate how many slots we have left + var totalCards = existingCount + newCards.Count; + var priorityRemaining = priorityNeeded - foundPriorityWithSet.Count; + var slotsForNonPriority = maxCards - priorityRemaining; + + // Skip if we have enough non-priority cards and this isn't priority + if (!isPriorityCard && totalCards >= slotsForNonPriority) + continue; + + // Download and process image + try + { + downloadTask.Description = $"[blue]{Markup.Escape(cardName.Length > 30 ? cardName[..27] + "..." : cardName)}[/]"; + + var imageBytes = await httpClient.GetByteArrayAsync(imageUri); + using var bitmap = SKBitmap.Decode(imageBytes); + + if (bitmap == null) + { + errors++; + continue; + } + + // Apply CLAHE preprocessing and compute hash + using var preprocessed = ImagePreprocessor.ApplyClahe(bitmap); + var hash = PerceptualHash.ComputeColorHash(preprocessed); + + // Create Card (printing) with hash + var card = scryfallCard.ToCard() with { Hash = hash }; + newCards.Add(card); + + // Track Oracle if we haven't seen it + if (!existingOracleIds.Contains(oracleId) && !newOracles.ContainsKey(oracleId)) + { + newOracles[oracleId] = scryfallCard.ToOracle(); + } + + if (isPriorityCard) + { + foundPriorityWithSet[cardName] = setCode; + priorityFound++; + priorityTask.Increment(1); + } + + processed++; + downloadTask.Increment(1); + + // Check if we have enough cards + var foundAllPriority = foundPriorityWithSet.Count >= priorityNeeded; + if (existingCount + newCards.Count >= maxCards && foundAllPriority) + break; + + // Rate limit to be nice to Scryfall + await Task.Delay(50); + } + catch + { + errors++; + } + } + + downloadTask.Value = downloadTask.MaxValue; + priorityTask.Value = priorityTask.MaxValue; + }); + + AnsiConsole.WriteLine(); + + // Summary table + var summaryTable = new Table() + .Border(TableBorder.Rounded) + .Title("[blue]Processing Summary[/]") + .AddColumn("Metric") + .AddColumn("Count", c => c.RightAligned()); + + summaryTable.AddRow("Skipped (already in DB)", skipped.ToString()); + summaryTable.AddRow("Newly processed", $"[green]{processed}[/]"); + summaryTable.AddRow("New oracles", newOracles.Count.ToString()); + summaryTable.AddRow("Priority cards found", $"{priorityFound}/{priorityNeeded}"); + summaryTable.AddRow("Errors", errors > 0 ? $"[red]{errors}[/]" : "0"); + + AnsiConsole.Write(summaryTable); + AnsiConsole.WriteLine(); + + // Insert oracles first (cards reference them) + if (newOracles.Count > 0) + { + await AnsiConsole.Status() + .Spinner(Spinner.Known.Dots) + .StartAsync($"Inserting {newOracles.Count} new oracles...", async ctx => + { + await db.InsertOracleBatchAsync(newOracles.Values); + }); + AnsiConsole.MarkupLine($"[green]✓[/] Inserted [blue]{newOracles.Count}[/] oracles"); + } + + if (newCards.Count > 0) + { + await AnsiConsole.Status() + .Spinner(Spinner.Known.Dots) + .StartAsync($"Inserting {newCards.Count} new cards...", async ctx => + { + await db.InsertCardBatchAsync(newCards); + }); + AnsiConsole.MarkupLine($"[green]✓[/] Inserted [blue]{newCards.Count}[/] cards"); + } + + await db.SetMetadataAsync("generated_at", DateTime.UtcNow.ToString("O")); + await db.SetMetadataAsync("scryfall_updated_at", scryfallDateStr); + + var finalCardCount = await db.GetCardCountAsync(); + var finalOracleCount = await db.GetOracleCountAsync(); + var finalSetCount = await db.GetSetCountAsync(); + + await db.SetMetadataAsync("card_count", finalCardCount.ToString()); + await db.SetMetadataAsync("oracle_count", finalOracleCount.ToString()); + await db.SetMetadataAsync("set_count", finalSetCount.ToString()); + + AnsiConsole.WriteLine(); + + var finalTable = new Table() + .Border(TableBorder.Double) + .Title("[green]Final Database State[/]") + .AddColumn("Metric") + .AddColumn("Count", c => c.RightAligned()); + + finalTable.AddRow("Cards", $"[green]{finalCardCount}[/]"); + finalTable.AddRow("Oracles", $"[green]{finalOracleCount}[/]"); + finalTable.AddRow("Sets", $"[green]{finalSetCount}[/]"); + finalTable.AddRow("Output", $"[blue]{outputDb}[/]"); + + AnsiConsole.Write(finalTable); + + // Report missing priority cards + if (includeTestCards) + { + var missing = priorityCards.Where(c => !foundPriorityWithSet.ContainsKey(c)).ToList(); + + if (missing.Count > 0) + { + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine($"[yellow]Missing priority cards ({missing.Count}):[/]"); + + var tree = new Tree("[yellow]Missing Cards[/]"); + foreach (var name in missing.Take(20)) + { + tree.AddNode($"[grey]{Markup.Escape(name)}[/]"); + } + if (missing.Count > 20) + { + tree.AddNode($"[grey]... and {missing.Count - 20} more[/]"); + } + AnsiConsole.Write(tree); + } + } + + return 0; + } +} diff --git a/tools/DbGenerator/Program.cs b/tools/DbGenerator/Program.cs index 5717926..99d841a 100644 --- a/tools/DbGenerator/Program.cs +++ b/tools/DbGenerator/Program.cs @@ -1,408 +1,12 @@ -using Scry.Core.Data; -using Scry.Core.Imaging; -using Scry.Core.Models; -using Scry.Core.Scryfall; -using SkiaSharp; +using DbGenerator; +using Spectre.Console.Cli; -// Generate a card hash database from Scryfall images -// Usage: dotnet run -- [--count N] [--include-test-cards] [--force] +var app = new CommandApp(); -var outputDb = args.Length > 0 ? args[0] : "card_hashes.db"; -var maxCards = 500; -var includeTestCards = true; -var forceRebuild = false; - -// Parse arguments -for (var i = 0; i < args.Length; i++) +app.Configure(config => { - if (args[i] == "--count" && i + 1 < args.Length && int.TryParse(args[i + 1], out var parsedCount)) - { - maxCards = parsedCount; - i++; - } - else if (args[i] == "--include-test-cards") - { - includeTestCards = true; - } - else if (args[i] == "--no-test-cards") - { - includeTestCards = false; - } - else if (args[i] == "--force") - { - forceRebuild = true; - } -} + config.SetApplicationName("dbgen"); + config.SetApplicationVersion("1.0.0"); +}); -Console.WriteLine($"Generating hash database with up to {maxCards} cards"); -Console.WriteLine($"Output: {outputDb}"); -Console.WriteLine($"Include test cards: {includeTestCards}"); -Console.WriteLine($"Force rebuild: {forceRebuild}"); -Console.WriteLine(); - -// Cards that should be included for testing with preferred sets -// Key: card name, Value: preferred set codes (first match wins) or empty for any -var priorityCardsWithSets = new Dictionary(StringComparer.OrdinalIgnoreCase) -{ - // From reference_alpha/ - prefer LEA (Alpha) or LEB (Beta) for classic look - ["Ancestral Recall"] = ["lea", "leb"], - ["Badlands"] = ["lea", "leb"], - ["Balance"] = ["lea", "leb"], - ["Bayou"] = ["lea", "leb"], - ["Birds of Paradise"] = ["lea", "leb"], - ["Black Lotus"] = ["lea", "leb"], - ["Channel"] = ["lea", "leb"], - ["Chaos Orb"] = ["lea", "leb"], - ["Clone"] = ["lea", "leb"], - ["Control Magic"] = ["lea", "leb"], - ["Counterspell"] = ["lea", "leb"], - ["Dark Ritual"] = ["lea", "leb"], - ["Demonic Tutor"] = ["lea", "leb"], - ["Disenchant"] = ["lea", "leb"], - ["Fireball"] = ["lea", "leb"], - ["Force of Nature"] = ["lea", "leb"], - ["Fork"] = ["lea", "leb"], - ["Giant Growth"] = ["lea", "leb"], - ["Hypnotic Specter"] = ["lea", "leb"], - ["Lightning Bolt"] = ["lea", "leb"], - ["Llanowar Elves"] = ["lea", "leb"], - ["Mahamoti Djinn"] = ["lea", "leb"], - ["Mind Twist"] = ["lea", "leb"], - ["Mox Emerald"] = ["lea", "leb"], - ["Mox Jet"] = ["lea", "leb"], - ["Mox Pearl"] = ["lea", "leb"], - ["Mox Ruby"] = ["lea", "leb"], - ["Mox Sapphire"] = ["lea", "leb"], - ["Nightmare"] = ["lea", "leb"], - ["Plateau"] = ["lea", "leb"], - ["Regrowth"] = ["lea", "leb"], - ["Rock Hydra"] = ["lea", "leb"], - ["Royal Assassin"] = ["lea", "leb"], - ["Savannah"] = ["lea", "leb"], - ["Scrubland"] = ["lea", "leb"], - ["Serra Angel"] = ["lea", "leb"], - ["Shivan Dragon"] = ["lea", "leb"], - ["Sol Ring"] = ["lea", "leb"], - ["Swords to Plowshares"] = ["lea", "leb"], - ["Taiga"] = ["lea", "leb"], - ["Time Walk"] = ["lea", "leb"], - ["Timetwister"] = ["lea", "leb"], - ["Tropical Island"] = ["lea", "leb"], - ["Tundra"] = ["lea", "leb"], - ["Underground Sea"] = ["lea", "leb"], - ["Wheel of Fortune"] = ["lea", "leb"], - ["Wrath of God"] = ["lea", "leb"], - - // From reference/ - any set is fine - ["Brainstorm"] = [], - ["Force of Will"] = [], - ["Griselbrand"] = [], - ["Lotus Petal"] = [], - ["Ponder"] = [], - ["Show and Tell"] = [], - ["Volcanic Island"] = [], - ["Wasteland"] = [], - - // From single_cards/ - any set is fine - ["Adanto Vanguard"] = [], - ["Angel of Sanctions"] = [], - ["Attunement"] = [], - ["Avaricious Dragon"] = [], - ["Burgeoning"] = [], - ["Jarad, Golgari Lich Lord"] = [], - ["Meletis Charlatan"] = [], - ["Mindstab Thrull"] = [], - ["Pacifism"] = [], - ["Platinum Angel"] = [], - ["Queen Marchesa"] = [], - ["Spellseeker"] = [], - ["Tarmogoyf"] = [], - ["Thought Reflection"] = [], - ["Unsummon"] = [], - - // From varying_quality - prefer older sets - ["Dragon Whelp"] = ["lea", "leb"], - ["Evil Eye of Orms-by-Gore"] = [], - ["Instill Energy"] = ["lea", "leb"], - - // Popular cards for general testing - ["Lightning Helix"] = [], - ["Path to Exile"] = [], - ["Thoughtseize"] = [], - ["Fatal Push"] = [], - ["Snapcaster Mage"] = [], - ["Jace, the Mind Sculptor"] = [], - ["Liliana of the Veil"] = [], - ["Noble Hierarch"] = [], - ["Goblin Guide"] = [], - ["Eidolon of the Great Revel"] = [], -}; - -var priorityCards = new HashSet(priorityCardsWithSets.Keys, StringComparer.OrdinalIgnoreCase); - -// Force rebuild if requested -if (forceRebuild && File.Exists(outputDb)) -{ - Console.WriteLine("Force rebuild requested, removing existing database..."); - File.Delete(outputDb); -} - -using var httpClient = new HttpClient(); -httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("Scry/1.0 (MTG Card Scanner - Database Generator)"); - -using var scryfallClient = new ScryfallClient(httpClient); -using var db = new CardDatabase(outputDb); - -// Check existing database state -var existingCardIds = await db.GetExistingCardIdsAsync(); -var existingCardNames = await db.GetExistingCardNamesAsync(); -var existingOracleIds = await db.GetExistingOracleIdsAsync(); -var existingSetIds = await db.GetExistingSetIdsAsync(); -var existingCount = await db.GetCardCountAsync(); -var storedScryfallDate = await db.GetMetadataAsync("scryfall_updated_at"); - -Console.WriteLine($"Existing database has {existingCount} cards, {existingOracleIds.Count} oracles, {existingSetIds.Count} sets"); - -// Fetch all sets first -Console.WriteLine("Fetching sets from Scryfall..."); -var scryfallSets = await scryfallClient.GetAllSetsAsync(); -var setsById = scryfallSets.ToDictionary(s => s.Id ?? "", s => s); -var setsByCode = scryfallSets.ToDictionary(s => s.Code ?? "", s => s, StringComparer.OrdinalIgnoreCase); -Console.WriteLine($"Found {scryfallSets.Count} sets"); - -// Insert any new sets -var newSets = scryfallSets - .Where(s => s.Id != null && !existingSetIds.Contains(s.Id)) - .Select(s => s.ToSet()) - .ToList(); - -if (newSets.Count > 0) -{ - Console.WriteLine($"Inserting {newSets.Count} new sets..."); - await db.InsertSetBatchAsync(newSets); -} - -Console.WriteLine("Fetching bulk data info from Scryfall..."); -var bulkInfo = await scryfallClient.GetBulkDataInfoAsync("unique_artwork"); - -if (bulkInfo?.DownloadUri == null) -{ - Console.WriteLine("Failed to get bulk data info from Scryfall"); - return 1; -} - -Console.WriteLine($"Scryfall data last updated: {bulkInfo.UpdatedAt}"); - -// Check if we need to update at all -var scryfallDateStr = bulkInfo.UpdatedAt?.ToString("O") ?? ""; -var needsUpdate = existingCount == 0 || - storedScryfallDate != scryfallDateStr || - existingCount < maxCards; - -// Also check if all priority cards exist -var missingPriorityCards = includeTestCards - ? priorityCards.Where(c => !existingCardNames.Contains(c)).ToList() - : new List(); - -if (missingPriorityCards.Count > 0) -{ - Console.WriteLine($"Missing {missingPriorityCards.Count} priority cards"); - needsUpdate = true; -} - -if (!needsUpdate) -{ - Console.WriteLine("Database is up-to-date, no changes needed"); - return 0; -} - -Console.WriteLine($"Downloading card data from: {bulkInfo.DownloadUri}"); -Console.WriteLine(); - -var newCards = new List(); -var newOracles = new Dictionary(); -var processed = 0; -var errors = 0; -var skipped = 0; -var priorityFound = 0; -var priorityNeeded = includeTestCards ? priorityCards.Count : 0; - -// Track which priority cards we've already found with their set -// Key: card name, Value: set code -var foundPriorityWithSet = new Dictionary(StringComparer.OrdinalIgnoreCase); - -// Helper to check if a set is preferred for a priority card -bool IsPreferredSet(string cardName, string setCode) -{ - if (!priorityCardsWithSets.TryGetValue(cardName, out var preferredSets)) - return false; - return preferredSets.Length == 0 || preferredSets.Contains(setCode, StringComparer.OrdinalIgnoreCase); -} - -await foreach (var scryfallCard in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadUri)) -{ - // Skip non-English cards - if (scryfallCard.Lang != "en") - continue; - - var imageUri = scryfallCard.GetImageUri("normal"); - if (string.IsNullOrEmpty(imageUri)) - continue; - - var cardId = scryfallCard.Id ?? Guid.NewGuid().ToString(); - var cardName = scryfallCard.Name ?? "Unknown"; - var setCode = scryfallCard.Set ?? "???"; - var oracleId = scryfallCard.OracleId ?? cardId; - var setId = scryfallCard.SetId ?? ""; - - // Check if this card already exists in the database - if (existingCardIds.Contains(cardId)) - { - skipped++; - continue; - } - - // Check if this is a priority card we might need - var isPriorityCard = includeTestCards && priorityCards.Contains(cardName); - var isPreferred = isPriorityCard && IsPreferredSet(cardName, setCode); - - // If this priority card already found with preferred set, skip - if (isPriorityCard && foundPriorityWithSet.TryGetValue(cardName, out var existingSet)) - { - if (IsPreferredSet(cardName, existingSet)) - { - // Already have preferred version - continue; - } - // We have a non-preferred version; if this is preferred, we'll replace - if (!isPreferred) - { - continue; - } - } - - // Calculate how many slots we have left - var totalCards = existingCount + newCards.Count; - var priorityRemaining = priorityNeeded - foundPriorityWithSet.Count; - var slotsForNonPriority = maxCards - priorityRemaining; - - // Skip if we have enough non-priority cards and this isn't priority - if (!isPriorityCard && totalCards >= slotsForNonPriority) - continue; - - // Download and process image - try - { - Console.Write($"[{processed + 1}] {cardName}... "); - - var imageBytes = await httpClient.GetByteArrayAsync(imageUri); - using var bitmap = SKBitmap.Decode(imageBytes); - - if (bitmap == null) - { - Console.WriteLine("decode failed"); - errors++; - continue; - } - - // Apply CLAHE preprocessing and compute hash - using var preprocessed = ImagePreprocessor.ApplyClahe(bitmap); - var hash = PerceptualHash.ComputeColorHash(preprocessed); - - // Create Card (printing) with hash - var card = scryfallCard.ToCard() with { Hash = hash }; - newCards.Add(card); - - // Track Oracle if we haven't seen it - if (!existingOracleIds.Contains(oracleId) && !newOracles.ContainsKey(oracleId)) - { - newOracles[oracleId] = scryfallCard.ToOracle(); - } - - if (isPriorityCard) - { - foundPriorityWithSet[cardName] = setCode; - priorityFound++; - Console.WriteLine($"OK (priority, {setCode})"); - } - else - { - Console.WriteLine($"OK ({setCode})"); - } - - processed++; - - // Check if we have enough cards - var foundAllPriority = foundPriorityWithSet.Count >= priorityNeeded; - if (existingCount + newCards.Count >= maxCards && foundAllPriority) - { - Console.WriteLine($"\nReached {maxCards} cards limit with all priority cards"); - break; - } - - // Rate limit to be nice to Scryfall - await Task.Delay(50); - } - catch (Exception ex) - { - Console.WriteLine($"error: {ex.Message}"); - errors++; - } -} - -Console.WriteLine(); -Console.WriteLine($"Skipped (already in DB): {skipped}"); -Console.WriteLine($"Newly processed: {processed} cards"); -Console.WriteLine($"New oracles: {newOracles.Count}"); -Console.WriteLine($"New priority cards found: {priorityFound}"); -Console.WriteLine($"Total priority cards: {foundPriorityWithSet.Count}/{priorityNeeded}"); -Console.WriteLine($"Errors: {errors}"); -Console.WriteLine(); - -// Insert oracles first (cards reference them) -if (newOracles.Count > 0) -{ - Console.WriteLine($"Inserting {newOracles.Count} new oracles..."); - await db.InsertOracleBatchAsync(newOracles.Values); -} - -if (newCards.Count > 0) -{ - Console.WriteLine($"Inserting {newCards.Count} new cards..."); - await db.InsertCardBatchAsync(newCards); -} - -await db.SetMetadataAsync("generated_at", DateTime.UtcNow.ToString("O")); -await db.SetMetadataAsync("scryfall_updated_at", scryfallDateStr); - -var finalCardCount = await db.GetCardCountAsync(); -var finalOracleCount = await db.GetOracleCountAsync(); -var finalSetCount = await db.GetSetCountAsync(); - -await db.SetMetadataAsync("card_count", finalCardCount.ToString()); -await db.SetMetadataAsync("oracle_count", finalOracleCount.ToString()); -await db.SetMetadataAsync("set_count", finalSetCount.ToString()); - -Console.WriteLine($"Database now has {finalCardCount} cards, {finalOracleCount} oracles, {finalSetCount} sets: {outputDb}"); - -// Report missing priority cards -if (includeTestCards) -{ - var missing = priorityCards.Where(c => !foundPriorityWithSet.ContainsKey(c)).ToList(); - - if (missing.Count > 0) - { - Console.WriteLine(); - Console.WriteLine($"Missing priority cards ({missing.Count}):"); - foreach (var name in missing.Take(20)) - { - Console.WriteLine($" - {name}"); - } - if (missing.Count > 20) - { - Console.WriteLine($" ... and {missing.Count - 20} more"); - } - } -} - -return 0; +return await app.RunAsync(args);