Add debug output support and refactor DbGenerator CLI
- Add RecognitionOptions with DebugOutputDirectory for saving pipeline
stages (input, detection, perspective correction, CLAHE preprocessing)
- Wire up IOptions<RecognitionOptions> via DI in MauiProgram
- Extract GenerateCommand from Program.cs using Spectre.Console.Cli
- Add priority card support with preferred set matching (Alpha/Beta)
- Expand card_hashes.db with more cards for better recognition coverage
- Update AGENTS.md with comprehensive project documentation
💘 Generated with Crush
Assisted-by: Claude Opus 4.5 via Crush <crush@charm.land>
This commit is contained in:
Chris Kruining
parent
54ba7496c6
commit
56499d5af9
14 changed files with 1010 additions and 456 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -21,3 +21,4 @@ Thumbs.db
|
|||
*.csv
|
||||
*.dlens
|
||||
*.apk
|
||||
debug/
|
||||
|
|
|
|||
40
.justfile
40
.justfile
|
|
@ -1,24 +1,46 @@
|
|||
# Scry development commands
|
||||
|
||||
set shell := ["C:/Program Files/Git/usr/bin/bash.exe", "-c"]
|
||||
set unstable := true
|
||||
|
||||
# Android SDK paths
|
||||
|
||||
android_sdk := env_var('LOCALAPPDATA') / "Android/Sdk"
|
||||
android_sdk := replace(env('LOCALAPPDATA'), '\', '/') / "Android/Sdk"
|
||||
adb := android_sdk / "platform-tools/adb.exe"
|
||||
emulator := android_sdk / "emulator/emulator.exe"
|
||||
camera_virtual := "-camera-back virtualscene -virtualscene-poster wall=\"" + (justfile_directory() / "TestImages/reference_alpha/serra_angel.jpg") + "\""
|
||||
camera_webcam := "-camera-back webcam0 -camera-front webcam0"
|
||||
|
||||
[private]
|
||||
@default:
|
||||
just --list
|
||||
|
||||
# Start emulator in background
|
||||
emu camera="virtual":
|
||||
{{ emulator }} -avd Pixel_6 {{ if camera == "virtual" { camera_virtual } else { camera_webcam } }} -gpu host &
|
||||
{{ emulator }} -avd Pixel_6 {{ if camera == "virtual" { camera_virtual } else { camera_webcam } }} -no-snapshot-load -gpu host
|
||||
|
||||
# Kill the running emulator
|
||||
emu-kill:
|
||||
{{ adb }} emu kill
|
||||
|
||||
# Wait for emulator to fully boot
|
||||
# Wait for emulator to fully boot (timeout after 2 minutes)
|
||||
[script]
|
||||
emu-wait:
|
||||
@echo "Waiting for emulator to boot..."
|
||||
@while [ "$({{ adb }} shell getprop sys.boot_completed 2>/dev/null)" != "1" ]; do sleep 1; done
|
||||
@echo "Emulator ready"
|
||||
# Wait for Android emulator to boot with timeout
|
||||
TIMEOUT=120
|
||||
|
||||
echo "Waiting for emulator to boot..."
|
||||
|
||||
for ((i=TIMEOUT; i>0; i--)); do
|
||||
if [ "$({{ adb }} shell getprop sys.boot_completed 2>/dev/null)" = "1" ]; then
|
||||
echo "Emulator ready"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "Emulator failed to boot within 2 minutes"
|
||||
exit 1
|
||||
|
||||
# Build a project
|
||||
build project="src/Scry.App" target="net10.0-android":
|
||||
|
|
@ -52,11 +74,11 @@ test:
|
|||
dotnet test test/Scry.Tests
|
||||
|
||||
# Generate the card hash database from Scryfall
|
||||
gen-db *args: (build "tools/DbGenerator" "net10.0")
|
||||
gen-db: (build "tools/DbGenerator" "net10.0")
|
||||
@echo "Running Database generator (this takes a while)..."
|
||||
dotnet run --project tools/DbGenerator --no-build -- src/Scry.App/Resources/Raw/card_hashes.db {{ args }}
|
||||
dotnet run --project tools/DbGenerator --no-build -- src/Scry.App/Resources/Raw/card_hashes.db
|
||||
@echo "Completed generating the database"
|
||||
|
||||
# Full workflow: start emulator, wait, run with hot reload
|
||||
dev: emu emu-wait
|
||||
dev:
|
||||
dotnet watch --project src/Scry.App -f net10.0-android
|
||||
|
|
|
|||
301
AGENTS.md
301
AGENTS.md
|
|
@ -1,30 +1,291 @@
|
|||
# Agent Instructions
|
||||
|
||||
## Build and Task Commands
|
||||
## Overview
|
||||
|
||||
Prefer using `just` commands over raw dotnet/build commands:
|
||||
Scry is a Magic: The Gathering card scanner app built with .NET MAUI. It uses perceptual hashing to match photographed cards against a database of known card images from Scryfall.
|
||||
|
||||
| Task | Command |
|
||||
|------|---------|
|
||||
| Build project | `just build` |
|
||||
| Run tests | `just test` |
|
||||
| Generate card database | `just gen-db` |
|
||||
| Publish app | `just publish` |
|
||||
| Run full dev workflow | `just dev` |
|
||||
**Key components:**
|
||||
- Mobile scanning app (MAUI/Android)
|
||||
- Card recognition via perceptual hashing (not OCR)
|
||||
- SQLite database with pre-computed hashes
|
||||
- Scryfall API integration for card data
|
||||
|
||||
## Build Commands
|
||||
|
||||
Use `just` commands (defined in `.justfile`):
|
||||
|
||||
| Task | Command | Notes |
|
||||
|------|---------|-------|
|
||||
| Build project | `just build` | Builds Android debug |
|
||||
| Run tests | `just test` | Runs xUnit tests |
|
||||
| Generate card database | `just gen-db` | Downloads from Scryfall, computes hashes |
|
||||
| Publish app | `just publish` | Creates release APK |
|
||||
| Hot reload dev | `just dev` | Uses `dotnet watch` |
|
||||
| Start emulator | `just emu` | Virtual camera with Serra Angel |
|
||||
| Install to device | `just install` | Installs release APK |
|
||||
|
||||
### Database Generator Options
|
||||
|
||||
```bash
|
||||
just gen-db # Default: 500 cards with test images
|
||||
dotnet run --project tools/DbGenerator -- -c 1000 # More cards
|
||||
dotnet run --project tools/DbGenerator -- --force # Rebuild from scratch
|
||||
dotnet run --project tools/DbGenerator -- --no-test-cards # Skip priority test cards
|
||||
```
|
||||
|
||||
## Project Structure
|
||||
|
||||
- `src/Scry.App` - MAUI mobile app
|
||||
- `src/Scry.Core` - Core library (recognition, hashing, database)
|
||||
- `test/Scry.Tests` - Unit tests
|
||||
- `tools/DbGenerator` - Card hash database generator
|
||||
```
|
||||
src/
|
||||
├── Scry.App/ # MAUI mobile app (Android target)
|
||||
│ ├── Views/ # XAML pages (ScanPage, CollectionPage, etc.)
|
||||
│ ├── ViewModels/ # MVVM ViewModels using CommunityToolkit.Mvvm
|
||||
│ ├── Services/ # App-layer services (ICardRecognitionService, ICardRepository)
|
||||
│ ├── Converters/ # XAML value converters
|
||||
│ ├── Models/ # App-specific models (CollectionEntry)
|
||||
│ └── Resources/Raw/ # Bundled card_hashes.db
|
||||
│
|
||||
└── Scry.Core/ # Platform-independent core library
|
||||
├── Recognition/ # CardRecognitionService, RecognitionOptions
|
||||
├── Imaging/ # PerceptualHash, ImagePreprocessor, CardDetector
|
||||
├── Data/ # CardDatabase (SQLite)
|
||||
├── Models/ # Card, Oracle, Set, ScanResult
|
||||
└── Scryfall/ # ScryfallClient for API/bulk data
|
||||
|
||||
## Database Generator
|
||||
test/
|
||||
└── Scry.Tests/ # xUnit tests
|
||||
|
||||
The `just gen-db` command:
|
||||
- Builds the DbGenerator tool
|
||||
- Runs it against `src/Scry.App/Resources/Raw/card_hashes.db`
|
||||
- Supports incremental updates (only downloads missing cards)
|
||||
- Prefers LEA/LEB (Alpha/Beta) sets for reference_alpha test cards
|
||||
tools/
|
||||
└── DbGenerator/ # CLI tool to generate card_hashes.db
|
||||
|
||||
Use `--force` flag to rebuild from scratch if needed.
|
||||
TestImages/ # Test images organized by category
|
||||
├── reference_alpha/ # Alpha/Beta cards for testing
|
||||
├── single_cards/ # Individual card photos
|
||||
├── varying_quality/ # Different lighting/quality
|
||||
├── hands/ # Cards held in hand
|
||||
├── foil/ # Foil cards with glare
|
||||
└── ... # More categories
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
### Recognition Pipeline
|
||||
|
||||
```
|
||||
Camera Image
|
||||
│
|
||||
▼
|
||||
┌─────────────────────┐
|
||||
│ CardDetector │ ← Edge detection, find card quad
|
||||
│ (optional) │
|
||||
└─────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────┐
|
||||
│ PerspectiveCorrection│ ← Warp to rectangle
|
||||
└─────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────┐
|
||||
│ ImagePreprocessor │ ← CLAHE for lighting normalization
|
||||
│ (ApplyClahe) │
|
||||
└─────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────┐
|
||||
│ PerceptualHash │ ← Compute 192-bit color hash (24 bytes)
|
||||
│ (ComputeColorHash) │
|
||||
└─────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────┐
|
||||
│ CardRecognitionService│ ← Hamming distance match against DB
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
### Data Model
|
||||
|
||||
Three-table schema mirroring Scryfall:
|
||||
|
||||
- **oracles** - Abstract game cards (one per unique card name)
|
||||
- **sets** - MTG sets with metadata
|
||||
- **cards** - Printings with perceptual hashes (one per unique artwork)
|
||||
|
||||
The `Card` model includes denormalized Oracle fields for convenience.
|
||||
|
||||
### Key Classes
|
||||
|
||||
| Class | Purpose |
|
||||
|-------|---------|
|
||||
| `CardRecognitionService` | Main recognition logic, caches DB, handles rotation matching |
|
||||
| `PerceptualHash` | DCT-based color hash (192-bit = 8 bytes × 3 RGB channels) |
|
||||
| `ImagePreprocessor` | CLAHE, resize, grayscale conversions |
|
||||
| `CardDetector` | Edge detection + contour analysis to find card boundaries |
|
||||
| `PerspectiveCorrection` | Warp detected quad to rectangle |
|
||||
| `CardDatabase` | SQLite wrapper with batch insert, queries |
|
||||
| `ScryfallClient` | Bulk data streaming, image downloads |
|
||||
|
||||
## Code Conventions
|
||||
|
||||
### General
|
||||
|
||||
- **Target**: .NET 10.0 (net10.0-android for app, net10.0 for Core/tools)
|
||||
- **Nullable**: Enabled everywhere (`<Nullable>enable</Nullable>`)
|
||||
- **Warnings as errors**: `<TreatWarningsAsErrors>true</TreatWarningsAsErrors>`
|
||||
- **Central package management**: Versions in `Directory.Packages.props`
|
||||
|
||||
### C# Style
|
||||
|
||||
- Records for data models (`record Card`, `record ScanResult`)
|
||||
- `required` properties for non-nullable required fields
|
||||
- Extension methods for conversions (`ScryfallCard.ToCard()`)
|
||||
- Static classes for pure functions (`PerceptualHash`, `ImagePreprocessor`)
|
||||
- `using` declarations (not `using` blocks) for disposables
|
||||
- File-scoped namespaces
|
||||
- Primary constructors where appropriate
|
||||
- `CancellationToken` parameter on all async methods
|
||||
|
||||
### MVVM (App layer)
|
||||
|
||||
- `CommunityToolkit.Mvvm` for source generators
|
||||
- `[ObservableProperty]` attributes for bindable properties
|
||||
- `[RelayCommand]` for commands
|
||||
- ViewModels in `Scry.ViewModels` namespace
|
||||
|
||||
### Naming
|
||||
|
||||
- Services: `ICardRecognitionService`, `CardRecognitionService`
|
||||
- Database methods: `GetCardsWithHashAsync`, `InsertCardBatchAsync`
|
||||
- Hash methods: `ComputeColorHash`, `HammingDistance`
|
||||
- Test methods: `RecognizeAsync_ExactMatch_ReturnsSuccess`
|
||||
|
||||
## Testing
|
||||
|
||||
Tests are in `test/Scry.Tests` using xUnit.
|
||||
|
||||
```bash
|
||||
just test # Run all tests
|
||||
dotnet test --filter "FullyQualifiedName~PerceptualHash" # Filter by name
|
||||
```
|
||||
|
||||
### Test Categories
|
||||
|
||||
| Test Class | Tests |
|
||||
|------------|-------|
|
||||
| `PerceptualHashTests` | Hash computation, Hamming distance |
|
||||
| `CardRecognitionTests` | End-to-end recognition |
|
||||
| `CardDatabaseTests` | SQLite CRUD operations |
|
||||
| `ImagePreprocessorTests` | CLAHE, resize |
|
||||
| `RobustnessAnalysisTests` | Multiple image variations |
|
||||
|
||||
### Test Images
|
||||
|
||||
TestImages directory contains categorized reference images:
|
||||
- `reference_alpha/` - Alpha/Beta cards (matching DbGenerator priority cards)
|
||||
- `single_cards/` - Clean single card photos
|
||||
- `varying_quality/` - Different lighting/blur conditions
|
||||
|
||||
## Key Algorithms
|
||||
|
||||
### Perceptual Hash (pHash)
|
||||
|
||||
Color-aware 192-bit hash:
|
||||
1. Resize to 32×32
|
||||
2. For each RGB channel:
|
||||
- Compute 2D DCT
|
||||
- Extract 8×8 low-frequency coefficients (skip DC)
|
||||
- Compare each to median → 63 bits per channel
|
||||
3. Concatenate R, G, B hashes → 24 bytes (192 bits)
|
||||
|
||||
Matching uses Hamming distance with threshold ≤25 bits and minimum confidence 85%.
|
||||
|
||||
### CLAHE (Contrast Limited Adaptive Histogram Equalization)
|
||||
|
||||
Applied in LAB color space to L channel only:
|
||||
- Tile-based histogram equalization (8×8 tiles)
|
||||
- Clip limit prevents over-amplification of noise
|
||||
- Bilinear interpolation between tiles for smooth output
|
||||
|
||||
### Card Detection
|
||||
|
||||
Pure SkiaSharp implementation:
|
||||
1. Grayscale → Gaussian blur → Canny edge detection
|
||||
2. Contour tracing via flood fill
|
||||
3. Douglas-Peucker simplification → Convex hull
|
||||
4. Find best quadrilateral matching MTG aspect ratio (88/63 ≈ 1.397)
|
||||
5. Order corners: top-left, top-right, bottom-right, bottom-left
|
||||
|
||||
## Debug Mode
|
||||
|
||||
Set `RecognitionOptions.DebugOutputDirectory` to save pipeline stages:
|
||||
- `01_input.png` - Original image
|
||||
- `02_detection.png` - Card detection visualization
|
||||
- `03_perspective_corrected.png` - Warped card
|
||||
- `05_clahe_*.png` - After CLAHE preprocessing
|
||||
|
||||
On Android: `/sdcard/Download/scry-debug` (pull with `adb pull`)
|
||||
|
||||
## Dependencies
|
||||
|
||||
### Core Library (Scry.Core)
|
||||
|
||||
- **SkiaSharp** - Image processing, DCT, edge detection
|
||||
- **Microsoft.Data.Sqlite** - SQLite database
|
||||
- **Microsoft.Extensions.Options** - Options pattern
|
||||
|
||||
### App (Scry.App)
|
||||
|
||||
- **CommunityToolkit.Maui** - MAUI extensions
|
||||
- **CommunityToolkit.Maui.Camera** - Camera integration
|
||||
- **CommunityToolkit.Mvvm** - MVVM source generators
|
||||
|
||||
### DbGenerator Tool
|
||||
|
||||
- **Spectre.Console** / **Spectre.Console.Cli** - Rich terminal UI
|
||||
|
||||
## Common Tasks
|
||||
|
||||
### Adding a New Card to Priority Test Set
|
||||
|
||||
1. Add image to `TestImages/reference_alpha/` or appropriate folder
|
||||
2. Add entry to `GenerateCommand.PriorityCardsWithSets` dictionary
|
||||
3. Run `just gen-db` to regenerate database
|
||||
|
||||
### Debugging Recognition Failures
|
||||
|
||||
1. Enable debug output in `MauiProgram.cs`:
|
||||
```csharp
|
||||
options.DebugOutputDirectory = "/sdcard/Download/scry-debug";
|
||||
```
|
||||
2. Run recognition
|
||||
3. Pull debug images: `adb pull /sdcard/Download/scry-debug`
|
||||
4. Compare `05_clahe_*.png` with reference images in database
|
||||
|
||||
### Modifying Hash Algorithm
|
||||
|
||||
1. Update `PerceptualHash.ComputeColorHash()`
|
||||
2. Update `CardRecognitionService.ColorHashBits` constant
|
||||
3. Regenerate database: `just gen-db --force`
|
||||
4. Run tests: `just test`
|
||||
|
||||
## Gotchas
|
||||
|
||||
1. **Hash size is 24 bytes (192 bits)** - 3 RGB channels × 8 bytes each
|
||||
2. **Confidence threshold is 85%** - Configurable in `CardRecognitionService.MinConfidence`
|
||||
3. **Card detection is optional** - Controlled by `RecognitionOptions.EnableCardDetection`
|
||||
4. **Rotation matching tries 4 orientations** - Controlled by `RecognitionOptions.EnableRotationMatching`
|
||||
5. **Database is bundled in APK** - Copied on first run to app data directory
|
||||
6. **Multi-face cards** - Only front face image is used for hashing
|
||||
7. **Rate limiting** - DbGenerator uses 50ms delay between Scryfall image downloads
|
||||
|
||||
## CI/CD
|
||||
|
||||
Forgejo Actions workflow (`.forgejo/workflows/release.yml`):
|
||||
- Builds for win-x64, linux-x64, osx-x64
|
||||
- Creates "standard" and "embedded" (with APK) variants
|
||||
- Publishes to Forgejo releases
|
||||
|
||||
## External Resources
|
||||
|
||||
- [Scryfall API](https://scryfall.com/docs/api) - Card data source
|
||||
- [CARD_RECOGNITION.md](docs/CARD_RECOGNITION.md) - Detailed architecture doc
|
||||
|
|
|
|||
|
|
@ -17,6 +17,9 @@
|
|||
/>
|
||||
<PackageVersion Include="SkiaSharp" Version="3.119.0" />
|
||||
<PackageVersion Include="Microsoft.Data.Sqlite" Version="9.0.1" />
|
||||
<PackageVersion Include="Microsoft.Extensions.Options" Version="10.0.0" />
|
||||
<PackageVersion Include="Spectre.Console" Version="0.50.0" />
|
||||
<PackageVersion Include="Spectre.Console.Cli" Version="0.50.0" />
|
||||
|
||||
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
|
||||
<PackageVersion Include="xunit" Version="2.9.2" />
|
||||
|
|
|
|||
BIN
screen.png
BIN
screen.png
Binary file not shown.
|
Before Width: | Height: | Size: 66 KiB |
|
|
@ -30,6 +30,17 @@ public static class MauiProgram
|
|||
EnsureDatabaseCopied(dbPath);
|
||||
return new CardDatabase(dbPath);
|
||||
});
|
||||
|
||||
// Recognition options - configure debug output in DEBUG builds
|
||||
builder.Services.Configure<RecognitionOptions>(options =>
|
||||
{
|
||||
#if DEBUG && ANDROID
|
||||
// Use Download folder for easy adb pull access
|
||||
options.DebugOutputDirectory = "/sdcard/Download/scry-debug";
|
||||
#elif DEBUG
|
||||
options.DebugOutputDirectory = "./debug";
|
||||
#endif
|
||||
});
|
||||
builder.Services.AddSingleton<CardRecognitionService>();
|
||||
|
||||
// App Services
|
||||
|
|
@ -57,14 +68,25 @@ public static class MauiProgram
|
|||
|
||||
private static void EnsureDatabaseCopied(string targetPath)
|
||||
{
|
||||
if (File.Exists(targetPath))
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
using var stream = FileSystem.OpenAppPackageFileAsync("card_hashes.db").GetAwaiter().GetResult();
|
||||
using var bundledStream = FileSystem.OpenAppPackageFileAsync("card_hashes.db").GetAwaiter().GetResult();
|
||||
|
||||
if (File.Exists(targetPath))
|
||||
{
|
||||
// Compare sizes - if bundled is larger, replace
|
||||
var existingSize = new FileInfo(targetPath).Length;
|
||||
var bundledSize = bundledStream.Length;
|
||||
|
||||
if (bundledSize <= existingSize)
|
||||
return;
|
||||
|
||||
// Bundled db is larger, delete old and copy new
|
||||
File.Delete(targetPath);
|
||||
}
|
||||
|
||||
using var fileStream = File.Create(targetPath);
|
||||
stream.CopyTo(fileStream);
|
||||
bundledStream.CopyTo(fileStream);
|
||||
}
|
||||
catch
|
||||
{
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -29,6 +29,11 @@
|
|||
<UseInterpreter>true</UseInterpreter>
|
||||
</PropertyGroup>
|
||||
|
||||
<!-- Watch additional files for dotnet watch -->
|
||||
<ItemGroup>
|
||||
<Watch Include="Resources\Raw\card_hashes.db" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<!-- Display name -->
|
||||
<ApplicationTitle>Scry</ApplicationTitle>
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Scry.Core.Data;
|
||||
using Scry.Core.Imaging;
|
||||
using Scry.Core.Models;
|
||||
|
|
@ -9,6 +10,7 @@ namespace Scry.Core.Recognition;
|
|||
public class CardRecognitionService : IDisposable
|
||||
{
|
||||
private readonly CardDatabase _database;
|
||||
private readonly RecognitionOptions _options;
|
||||
private List<Card>? _cardCache;
|
||||
private readonly SemaphoreSlim _cacheLock = new(1, 1);
|
||||
|
||||
|
|
@ -16,21 +18,14 @@ public class CardRecognitionService : IDisposable
|
|||
private const int MatchThreshold = 25;
|
||||
private const float MinConfidence = 0.85f;
|
||||
|
||||
/// <summary>
|
||||
/// Enable card detection and perspective correction.
|
||||
/// When disabled, assumes the input image is already a cropped card.
|
||||
/// </summary>
|
||||
public bool EnableCardDetection { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Try multiple rotations (0°, 90°, 180°, 270°) when matching.
|
||||
/// Useful when card orientation is unknown.
|
||||
/// </summary>
|
||||
public bool EnableRotationMatching { get; set; } = true;
|
||||
|
||||
public CardRecognitionService(CardDatabase database)
|
||||
public CardRecognitionService(CardDatabase database, IOptions<RecognitionOptions> options)
|
||||
{
|
||||
_database = database;
|
||||
_options = options.Value;
|
||||
}
|
||||
|
||||
public CardRecognitionService(CardDatabase database) : this(database, Options.Create(new RecognitionOptions()))
|
||||
{
|
||||
}
|
||||
|
||||
public async Task<ScanResult> RecognizeAsync(Stream imageStream, CancellationToken ct = default)
|
||||
|
|
@ -60,6 +55,14 @@ public class CardRecognitionService : IDisposable
|
|||
public async Task<ScanResult> RecognizeAsync(SKBitmap bitmap, CancellationToken ct = default)
|
||||
{
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
var debugDir = _options.DebugOutputDirectory;
|
||||
var debugEnabled = !string.IsNullOrEmpty(debugDir);
|
||||
|
||||
if (debugEnabled)
|
||||
{
|
||||
Directory.CreateDirectory(debugDir!);
|
||||
SaveDebugImage(bitmap, debugDir!, "01_input");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
|
|
@ -75,14 +78,26 @@ public class CardRecognitionService : IDisposable
|
|||
SKBitmap cardImage;
|
||||
bool cardDetected = false;
|
||||
|
||||
if (EnableCardDetection)
|
||||
if (_options.EnableCardDetection)
|
||||
{
|
||||
var detection = CardDetector.DetectCard(bitmap);
|
||||
|
||||
if (debugEnabled)
|
||||
{
|
||||
// Save detection visualization
|
||||
SaveDetectionDebugImage(bitmap, detection, debugDir!);
|
||||
}
|
||||
|
||||
if (detection.Found)
|
||||
{
|
||||
cardImage = PerspectiveCorrection.WarpPerspective(bitmap, detection.Corners);
|
||||
cardDetected = true;
|
||||
Console.WriteLine($"[Scry] Card detected with confidence {detection.Confidence:P0}");
|
||||
|
||||
if (debugEnabled)
|
||||
{
|
||||
SaveDebugImage(cardImage, debugDir!, "03_perspective_corrected");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -99,9 +114,9 @@ public class CardRecognitionService : IDisposable
|
|||
try
|
||||
{
|
||||
// Step 2: Try matching with rotation variants (if enabled)
|
||||
var bestMatch = EnableRotationMatching
|
||||
? await FindBestMatchWithRotationsAsync(cardImage, cards, ct)
|
||||
: FindBestMatchSingle(cardImage, cards);
|
||||
var bestMatch = _options.EnableRotationMatching
|
||||
? await FindBestMatchWithRotationsAsync(cardImage, cards, debugDir, ct)
|
||||
: FindBestMatchSingle(cardImage, cards, debugDir);
|
||||
|
||||
stopwatch.Stop();
|
||||
|
||||
|
|
@ -198,6 +213,7 @@ public class CardRecognitionService : IDisposable
|
|||
private Task<(Card Card, int Distance, int Rotation)?> FindBestMatchWithRotationsAsync(
|
||||
SKBitmap cardImage,
|
||||
List<Card> candidates,
|
||||
string? debugDir,
|
||||
CancellationToken ct)
|
||||
{
|
||||
return Task.Run(() =>
|
||||
|
|
@ -205,6 +221,7 @@ public class CardRecognitionService : IDisposable
|
|||
Card? bestMatch = null;
|
||||
var bestDistance = int.MaxValue;
|
||||
var bestRotation = 0;
|
||||
var debugEnabled = !string.IsNullOrEmpty(debugDir);
|
||||
|
||||
var rotations = new[] { 0, 90, 180, 270 };
|
||||
|
||||
|
|
@ -215,8 +232,19 @@ public class CardRecognitionService : IDisposable
|
|||
using var rotated = rotation == 0 ? null : RotateImage(cardImage, rotation);
|
||||
var imageToHash = rotated ?? cardImage;
|
||||
|
||||
if (debugEnabled && rotation != 0)
|
||||
{
|
||||
SaveDebugImage(imageToHash, debugDir!, $"04_rotated_{rotation}");
|
||||
}
|
||||
|
||||
// Apply CLAHE and compute hash
|
||||
using var preprocessed = ImagePreprocessor.ApplyClahe(imageToHash);
|
||||
|
||||
if (debugEnabled)
|
||||
{
|
||||
SaveDebugImage(preprocessed, debugDir!, $"05_clahe_{rotation}");
|
||||
}
|
||||
|
||||
var queryHash = PerceptualHash.ComputeColorHash(preprocessed);
|
||||
|
||||
// Find best match for this rotation
|
||||
|
|
@ -252,10 +280,19 @@ public class CardRecognitionService : IDisposable
|
|||
/// </summary>
|
||||
private (Card Card, int Distance, int Rotation)? FindBestMatchSingle(
|
||||
SKBitmap cardImage,
|
||||
List<Card> candidates)
|
||||
List<Card> candidates,
|
||||
string? debugDir)
|
||||
{
|
||||
var debugEnabled = !string.IsNullOrEmpty(debugDir);
|
||||
|
||||
// Apply CLAHE and compute hash
|
||||
using var preprocessed = ImagePreprocessor.ApplyClahe(cardImage);
|
||||
|
||||
if (debugEnabled)
|
||||
{
|
||||
SaveDebugImage(preprocessed, debugDir!, "05_clahe_0");
|
||||
}
|
||||
|
||||
var queryHash = PerceptualHash.ComputeColorHash(preprocessed);
|
||||
|
||||
Card? bestMatch = null;
|
||||
|
|
@ -308,6 +345,83 @@ public class CardRecognitionService : IDisposable
|
|||
return rotated;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Save a debug image to disk.
|
||||
/// </summary>
|
||||
private static void SaveDebugImage(SKBitmap bitmap, string directory, string name)
|
||||
{
|
||||
var path = Path.Combine(directory, $"{name}.png");
|
||||
using var image = SKImage.FromBitmap(bitmap);
|
||||
using var data = image.Encode(SKEncodedImageFormat.Png, 100);
|
||||
using var stream = File.OpenWrite(path);
|
||||
data.SaveTo(stream);
|
||||
Console.WriteLine($"[Scry Debug] Saved: {path}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Save a debug image showing the card detection result.
|
||||
/// </summary>
|
||||
private static void SaveDetectionDebugImage(SKBitmap original, CardDetector.CardDetectionResult detection, string directory)
|
||||
{
|
||||
using var annotated = new SKBitmap(original.Width, original.Height, original.ColorType, original.AlphaType);
|
||||
using var canvas = new SKCanvas(annotated);
|
||||
|
||||
canvas.DrawBitmap(original, 0, 0);
|
||||
|
||||
if (detection.Found && detection.Corners.Length == 4)
|
||||
{
|
||||
// Draw detected corners and edges
|
||||
using var cornerPaint = new SKPaint
|
||||
{
|
||||
Color = SKColors.Red,
|
||||
Style = SKPaintStyle.Fill,
|
||||
IsAntialias = true
|
||||
};
|
||||
|
||||
using var edgePaint = new SKPaint
|
||||
{
|
||||
Color = SKColors.Lime,
|
||||
Style = SKPaintStyle.Stroke,
|
||||
StrokeWidth = 3,
|
||||
IsAntialias = true
|
||||
};
|
||||
|
||||
var corners = detection.Corners;
|
||||
|
||||
// Draw edges
|
||||
for (int i = 0; i < 4; i++)
|
||||
{
|
||||
var p1 = corners[i];
|
||||
var p2 = corners[(i + 1) % 4];
|
||||
canvas.DrawLine(p1.X, p1.Y, p2.X, p2.Y, edgePaint);
|
||||
}
|
||||
|
||||
// Draw corners
|
||||
foreach (var corner in corners)
|
||||
{
|
||||
canvas.DrawCircle(corner.X, corner.Y, 8, cornerPaint);
|
||||
}
|
||||
}
|
||||
|
||||
// Add debug text
|
||||
using var textPaint = new SKPaint
|
||||
{
|
||||
Color = detection.Found ? SKColors.Lime : SKColors.Red,
|
||||
IsAntialias = true
|
||||
};
|
||||
using var font = new SKFont
|
||||
{
|
||||
Size = 24
|
||||
};
|
||||
|
||||
var message = detection.Found
|
||||
? $"Detected: {detection.Confidence:P0}"
|
||||
: $"Not found: {detection.DebugMessage}";
|
||||
canvas.DrawText(message, 10, 30, SKTextAlign.Left, font, textPaint);
|
||||
|
||||
SaveDebugImage(annotated, directory, "02_detection");
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_cacheLock.Dispose();
|
||||
|
|
|
|||
24
src/Scry.Core/Recognition/RecognitionOptions.cs
Normal file
24
src/Scry.Core/Recognition/RecognitionOptions.cs
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
namespace Scry.Core.Recognition;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for card recognition.
|
||||
/// </summary>
|
||||
public class RecognitionOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// When set, saves debug images of each pipeline step to this directory.
|
||||
/// </summary>
|
||||
public string? DebugOutputDirectory { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Enable card detection and perspective correction.
|
||||
/// When disabled, assumes the input image is already a cropped card.
|
||||
/// </summary>
|
||||
public bool EnableCardDetection { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Try multiple rotations (0°, 90°, 180°, 270°) when matching.
|
||||
/// Useful when card orientation is unknown.
|
||||
/// </summary>
|
||||
public bool EnableRotationMatching { get; set; } = true;
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@
|
|||
<ItemGroup>
|
||||
<PackageReference Include="SkiaSharp" />
|
||||
<PackageReference Include="Microsoft.Data.Sqlite" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@
|
|||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="SkiaSharp" />
|
||||
<PackageReference Include="Spectre.Console" />
|
||||
<PackageReference Include="Spectre.Console.Cli" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
|||
495
tools/DbGenerator/GenerateCommand.cs
Normal file
495
tools/DbGenerator/GenerateCommand.cs
Normal file
|
|
@ -0,0 +1,495 @@
|
|||
using System.ComponentModel;
|
||||
using Scry.Core.Data;
|
||||
using Scry.Core.Imaging;
|
||||
using Scry.Core.Models;
|
||||
using Scry.Core.Scryfall;
|
||||
using SkiaSharp;
|
||||
using Spectre.Console;
|
||||
using Spectre.Console.Cli;
|
||||
|
||||
namespace DbGenerator;
|
||||
|
||||
public sealed class GenerateSettings : CommandSettings
|
||||
{
|
||||
[CommandArgument(0, "[output]")]
|
||||
[Description("Output database file path")]
|
||||
[DefaultValue("card_hashes.db")]
|
||||
public string Output { get; set; } = "card_hashes.db";
|
||||
|
||||
[CommandOption("-c|--count")]
|
||||
[Description("Maximum number of cards to include")]
|
||||
[DefaultValue(500)]
|
||||
public int Count { get; set; } = 500;
|
||||
|
||||
[CommandOption("--include-test-cards")]
|
||||
[Description("Include priority test cards (default: true)")]
|
||||
[DefaultValue(true)]
|
||||
public bool IncludeTestCards { get; set; } = true;
|
||||
|
||||
[CommandOption("--no-test-cards")]
|
||||
[Description("Exclude priority test cards")]
|
||||
public bool NoTestCards { get; set; }
|
||||
|
||||
[CommandOption("-f|--force")]
|
||||
[Description("Force rebuild from scratch")]
|
||||
public bool Force { get; set; }
|
||||
}
|
||||
|
||||
public sealed class GenerateCommand : AsyncCommand<GenerateSettings>
|
||||
{
|
||||
// Cards that should be included for testing with preferred sets
|
||||
private static readonly Dictionary<string, string[]> PriorityCardsWithSets = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
// From reference_alpha/ - prefer LEA (Alpha) or LEB (Beta) for classic look
|
||||
["Ancestral Recall"] = ["lea", "leb"],
|
||||
["Badlands"] = ["lea", "leb"],
|
||||
["Balance"] = ["lea", "leb"],
|
||||
["Bayou"] = ["lea", "leb"],
|
||||
["Birds of Paradise"] = ["lea", "leb"],
|
||||
["Black Lotus"] = ["lea", "leb"],
|
||||
["Channel"] = ["lea", "leb"],
|
||||
["Chaos Orb"] = ["lea", "leb"],
|
||||
["Clone"] = ["lea", "leb"],
|
||||
["Control Magic"] = ["lea", "leb"],
|
||||
["Counterspell"] = ["lea", "leb"],
|
||||
["Dark Ritual"] = ["lea", "leb"],
|
||||
["Demonic Tutor"] = ["lea", "leb"],
|
||||
["Disenchant"] = ["lea", "leb"],
|
||||
["Fireball"] = ["lea", "leb"],
|
||||
["Force of Nature"] = ["lea", "leb"],
|
||||
["Fork"] = ["lea", "leb"],
|
||||
["Giant Growth"] = ["lea", "leb"],
|
||||
["Hypnotic Specter"] = ["lea", "leb"],
|
||||
["Lightning Bolt"] = ["lea", "leb"],
|
||||
["Llanowar Elves"] = ["lea", "leb"],
|
||||
["Mahamoti Djinn"] = ["lea", "leb"],
|
||||
["Mind Twist"] = ["lea", "leb"],
|
||||
["Mox Emerald"] = ["lea", "leb"],
|
||||
["Mox Jet"] = ["lea", "leb"],
|
||||
["Mox Pearl"] = ["lea", "leb"],
|
||||
["Mox Ruby"] = ["lea", "leb"],
|
||||
["Mox Sapphire"] = ["lea", "leb"],
|
||||
["Nightmare"] = ["lea", "leb"],
|
||||
["Plateau"] = ["lea", "leb"],
|
||||
["Regrowth"] = ["lea", "leb"],
|
||||
["Rock Hydra"] = ["lea", "leb"],
|
||||
["Royal Assassin"] = ["lea", "leb"],
|
||||
["Savannah"] = ["lea", "leb"],
|
||||
["Scrubland"] = ["lea", "leb"],
|
||||
["Serra Angel"] = ["lea", "leb"],
|
||||
["Shivan Dragon"] = ["lea", "leb"],
|
||||
["Sol Ring"] = ["lea", "leb"],
|
||||
["Swords to Plowshares"] = ["lea", "leb"],
|
||||
["Taiga"] = ["lea", "leb"],
|
||||
["Time Walk"] = ["lea", "leb"],
|
||||
["Timetwister"] = ["lea", "leb"],
|
||||
["Tropical Island"] = ["lea", "leb"],
|
||||
["Tundra"] = ["lea", "leb"],
|
||||
["Underground Sea"] = ["lea", "leb"],
|
||||
["Wheel of Fortune"] = ["lea", "leb"],
|
||||
["Wrath of God"] = ["lea", "leb"],
|
||||
|
||||
// From reference/ - any set is fine
|
||||
["Brainstorm"] = [],
|
||||
["Force of Will"] = [],
|
||||
["Griselbrand"] = [],
|
||||
["Lotus Petal"] = [],
|
||||
["Ponder"] = [],
|
||||
["Show and Tell"] = [],
|
||||
["Volcanic Island"] = [],
|
||||
["Wasteland"] = [],
|
||||
|
||||
// From single_cards/ - any set is fine
|
||||
["Adanto Vanguard"] = [],
|
||||
["Angel of Sanctions"] = [],
|
||||
["Attunement"] = [],
|
||||
["Avaricious Dragon"] = [],
|
||||
["Burgeoning"] = [],
|
||||
["Jarad, Golgari Lich Lord"] = [],
|
||||
["Meletis Charlatan"] = [],
|
||||
["Mindstab Thrull"] = [],
|
||||
["Pacifism"] = [],
|
||||
["Platinum Angel"] = [],
|
||||
["Queen Marchesa"] = [],
|
||||
["Spellseeker"] = [],
|
||||
["Tarmogoyf"] = [],
|
||||
["Thought Reflection"] = [],
|
||||
["Unsummon"] = [],
|
||||
|
||||
// From varying_quality - prefer older sets
|
||||
["Dragon Whelp"] = ["lea", "leb"],
|
||||
["Evil Eye of Orms-by-Gore"] = [],
|
||||
["Instill Energy"] = ["lea", "leb"],
|
||||
|
||||
// Popular cards for general testing
|
||||
["Lightning Helix"] = [],
|
||||
["Path to Exile"] = [],
|
||||
["Thoughtseize"] = [],
|
||||
["Fatal Push"] = [],
|
||||
["Snapcaster Mage"] = [],
|
||||
["Jace, the Mind Sculptor"] = [],
|
||||
["Liliana of the Veil"] = [],
|
||||
["Noble Hierarch"] = [],
|
||||
["Goblin Guide"] = [],
|
||||
["Eidolon of the Great Revel"] = [],
|
||||
};
|
||||
|
||||
public override async Task<int> ExecuteAsync(CommandContext context, GenerateSettings settings)
|
||||
{
|
||||
var outputDb = settings.Output;
|
||||
var maxCards = settings.Count;
|
||||
var includeTestCards = settings.IncludeTestCards && !settings.NoTestCards;
|
||||
var forceRebuild = settings.Force;
|
||||
|
||||
// Header
|
||||
AnsiConsole.Write(new FigletText("Scry DB Gen").Color(Color.Blue));
|
||||
|
||||
var configTable = new Table()
|
||||
.Border(TableBorder.Rounded)
|
||||
.AddColumn("Setting")
|
||||
.AddColumn("Value");
|
||||
|
||||
configTable.AddRow("Output", outputDb);
|
||||
configTable.AddRow("Max Cards", maxCards.ToString());
|
||||
configTable.AddRow("Test Cards", includeTestCards ? "[green]Yes[/]" : "[grey]No[/]");
|
||||
configTable.AddRow("Force Rebuild", forceRebuild ? "[yellow]Yes[/]" : "[grey]No[/]");
|
||||
|
||||
AnsiConsole.Write(configTable);
|
||||
AnsiConsole.WriteLine();
|
||||
|
||||
var priorityCards = new HashSet<string>(PriorityCardsWithSets.Keys, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Force rebuild if requested
|
||||
if (forceRebuild && File.Exists(outputDb))
|
||||
{
|
||||
AnsiConsole.MarkupLine("[yellow]Force rebuild requested, removing existing database...[/]");
|
||||
File.Delete(outputDb);
|
||||
}
|
||||
|
||||
using var httpClient = new HttpClient();
|
||||
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("Scry/1.0 (MTG Card Scanner - Database Generator)");
|
||||
|
||||
using var scryfallClient = new ScryfallClient(httpClient);
|
||||
using var db = new CardDatabase(outputDb);
|
||||
|
||||
// Check existing database state
|
||||
var existingCardIds = await db.GetExistingCardIdsAsync();
|
||||
var existingCardNames = await db.GetExistingCardNamesAsync();
|
||||
var existingOracleIds = await db.GetExistingOracleIdsAsync();
|
||||
var existingSetIds = await db.GetExistingSetIdsAsync();
|
||||
var existingCount = await db.GetCardCountAsync();
|
||||
var storedScryfallDate = await db.GetMetadataAsync("scryfall_updated_at");
|
||||
|
||||
var dbStateTable = new Table()
|
||||
.Border(TableBorder.Rounded)
|
||||
.Title("[blue]Current Database State[/]")
|
||||
.AddColumn("Metric")
|
||||
.AddColumn("Count", c => c.RightAligned());
|
||||
|
||||
dbStateTable.AddRow("Cards", existingCount.ToString());
|
||||
dbStateTable.AddRow("Oracles", existingOracleIds.Count.ToString());
|
||||
dbStateTable.AddRow("Sets", existingSetIds.Count.ToString());
|
||||
|
||||
AnsiConsole.Write(dbStateTable);
|
||||
AnsiConsole.WriteLine();
|
||||
|
||||
// Fetch all sets
|
||||
List<ScryfallSet> scryfallSets = [];
|
||||
await AnsiConsole.Status()
|
||||
.Spinner(Spinner.Known.Dots)
|
||||
.StartAsync("Fetching sets from Scryfall...", async ctx =>
|
||||
{
|
||||
scryfallSets = await scryfallClient.GetAllSetsAsync();
|
||||
});
|
||||
|
||||
AnsiConsole.MarkupLine($"[green]✓[/] Found [blue]{scryfallSets.Count}[/] sets");
|
||||
|
||||
var setsById = scryfallSets.ToDictionary(s => s.Id ?? "", s => s);
|
||||
var setsByCode = scryfallSets.ToDictionary(s => s.Code ?? "", s => s, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Insert any new sets
|
||||
var newSets = scryfallSets
|
||||
.Where(s => s.Id != null && !existingSetIds.Contains(s.Id))
|
||||
.Select(s => s.ToSet())
|
||||
.ToList();
|
||||
|
||||
if (newSets.Count > 0)
|
||||
{
|
||||
AnsiConsole.MarkupLine($"[green]✓[/] Inserting [blue]{newSets.Count}[/] new sets");
|
||||
await db.InsertSetBatchAsync(newSets);
|
||||
}
|
||||
|
||||
// Fetch bulk data info
|
||||
BulkDataInfo? bulkInfo = null;
|
||||
await AnsiConsole.Status()
|
||||
.Spinner(Spinner.Known.Dots)
|
||||
.StartAsync("Fetching bulk data info...", async ctx =>
|
||||
{
|
||||
bulkInfo = await scryfallClient.GetBulkDataInfoAsync("unique_artwork");
|
||||
});
|
||||
|
||||
if (bulkInfo?.DownloadUri == null)
|
||||
{
|
||||
AnsiConsole.MarkupLine("[red]✗ Failed to get bulk data info from Scryfall[/]");
|
||||
return 1;
|
||||
}
|
||||
|
||||
AnsiConsole.MarkupLine($"[green]✓[/] Scryfall data last updated: [blue]{bulkInfo.UpdatedAt:yyyy-MM-dd HH:mm}[/]");
|
||||
|
||||
// Check if we need to update at all
|
||||
var scryfallDateStr = bulkInfo.UpdatedAt?.ToString("O") ?? "";
|
||||
var needsUpdate = existingCount == 0 ||
|
||||
storedScryfallDate != scryfallDateStr ||
|
||||
existingCount < maxCards;
|
||||
|
||||
// Also check if all priority cards exist
|
||||
var missingPriorityCards = includeTestCards
|
||||
? priorityCards.Where(c => !existingCardNames.Contains(c)).ToList()
|
||||
: [];
|
||||
|
||||
if (missingPriorityCards is not [])
|
||||
{
|
||||
AnsiConsole.MarkupLine($"[yellow]![/] Missing [blue]{missingPriorityCards.Count}[/] priority cards");
|
||||
needsUpdate = true;
|
||||
}
|
||||
|
||||
if (!needsUpdate)
|
||||
{
|
||||
AnsiConsole.MarkupLine("[green]✓ Database is up-to-date, no changes needed[/]");
|
||||
return 0;
|
||||
}
|
||||
|
||||
AnsiConsole.WriteLine();
|
||||
|
||||
var newCards = new List<Card>();
|
||||
var newOracles = new Dictionary<string, Oracle>();
|
||||
var processed = 0;
|
||||
var errors = 0;
|
||||
var skipped = 0;
|
||||
var priorityFound = 0;
|
||||
var priorityNeeded = includeTestCards ? priorityCards.Count : 0;
|
||||
|
||||
// Track which priority cards we've already found with their set
|
||||
var foundPriorityWithSet = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Helper to check if a set is preferred for a priority card
|
||||
static bool IsPreferredSet(string cardName, string setCode)
|
||||
{
|
||||
if (!PriorityCardsWithSets.TryGetValue(cardName, out var preferredSets))
|
||||
return false;
|
||||
|
||||
return preferredSets.Length == 0 || preferredSets.Contains(setCode, StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
await AnsiConsole.Progress()
|
||||
.AutoClear(false)
|
||||
.HideCompleted(false)
|
||||
.Columns(
|
||||
new RemainingTimeColumn(),
|
||||
new SpinnerColumn(),
|
||||
new ProgressBarColumn(),
|
||||
new PercentageColumn(),
|
||||
new TaskDescriptionColumn()
|
||||
{
|
||||
Alignment = Justify.Left,
|
||||
}
|
||||
)
|
||||
.StartAsync(async ctx =>
|
||||
{
|
||||
var downloadTask = ctx.AddTask("[blue]Downloading & processing cards[/]", maxValue: maxCards);
|
||||
var priorityTask = ctx.AddTask("[green]Priority cards[/]", maxValue: priorityNeeded);
|
||||
|
||||
await foreach (var scryfallCard in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadUri))
|
||||
{
|
||||
// Skip non-English cards
|
||||
if (scryfallCard.Lang != "en")
|
||||
continue;
|
||||
|
||||
var imageUri = scryfallCard.GetImageUri("normal");
|
||||
if (string.IsNullOrEmpty(imageUri))
|
||||
continue;
|
||||
|
||||
var cardId = scryfallCard.Id ?? Guid.NewGuid().ToString();
|
||||
var cardName = scryfallCard.Name ?? "Unknown";
|
||||
var setCode = scryfallCard.Set ?? "???";
|
||||
var oracleId = scryfallCard.OracleId ?? cardId;
|
||||
var setId = scryfallCard.SetId ?? "";
|
||||
|
||||
// Check if this card already exists in the database
|
||||
if (existingCardIds.Contains(cardId))
|
||||
{
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if this is a priority card we might need
|
||||
var isPriorityCard = includeTestCards && priorityCards.Contains(cardName);
|
||||
var isPreferred = isPriorityCard && IsPreferredSet(cardName, setCode);
|
||||
|
||||
// If this priority card already found with preferred set, skip
|
||||
if (isPriorityCard && foundPriorityWithSet.TryGetValue(cardName, out var existingSet))
|
||||
{
|
||||
if (IsPreferredSet(cardName, existingSet))
|
||||
continue;
|
||||
if (!isPreferred)
|
||||
continue;
|
||||
}
|
||||
|
||||
// Calculate how many slots we have left
|
||||
var totalCards = existingCount + newCards.Count;
|
||||
var priorityRemaining = priorityNeeded - foundPriorityWithSet.Count;
|
||||
var slotsForNonPriority = maxCards - priorityRemaining;
|
||||
|
||||
// Skip if we have enough non-priority cards and this isn't priority
|
||||
if (!isPriorityCard && totalCards >= slotsForNonPriority)
|
||||
continue;
|
||||
|
||||
// Download and process image
|
||||
try
|
||||
{
|
||||
downloadTask.Description = $"[blue]{Markup.Escape(cardName.Length > 30 ? cardName[..27] + "..." : cardName)}[/]";
|
||||
|
||||
var imageBytes = await httpClient.GetByteArrayAsync(imageUri);
|
||||
using var bitmap = SKBitmap.Decode(imageBytes);
|
||||
|
||||
if (bitmap == null)
|
||||
{
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Apply CLAHE preprocessing and compute hash
|
||||
using var preprocessed = ImagePreprocessor.ApplyClahe(bitmap);
|
||||
var hash = PerceptualHash.ComputeColorHash(preprocessed);
|
||||
|
||||
// Create Card (printing) with hash
|
||||
var card = scryfallCard.ToCard() with { Hash = hash };
|
||||
newCards.Add(card);
|
||||
|
||||
// Track Oracle if we haven't seen it
|
||||
if (!existingOracleIds.Contains(oracleId) && !newOracles.ContainsKey(oracleId))
|
||||
{
|
||||
newOracles[oracleId] = scryfallCard.ToOracle();
|
||||
}
|
||||
|
||||
if (isPriorityCard)
|
||||
{
|
||||
foundPriorityWithSet[cardName] = setCode;
|
||||
priorityFound++;
|
||||
priorityTask.Increment(1);
|
||||
}
|
||||
|
||||
processed++;
|
||||
downloadTask.Increment(1);
|
||||
|
||||
// Check if we have enough cards
|
||||
var foundAllPriority = foundPriorityWithSet.Count >= priorityNeeded;
|
||||
if (existingCount + newCards.Count >= maxCards && foundAllPriority)
|
||||
break;
|
||||
|
||||
// Rate limit to be nice to Scryfall
|
||||
await Task.Delay(50);
|
||||
}
|
||||
catch
|
||||
{
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
downloadTask.Value = downloadTask.MaxValue;
|
||||
priorityTask.Value = priorityTask.MaxValue;
|
||||
});
|
||||
|
||||
AnsiConsole.WriteLine();
|
||||
|
||||
// Summary table
|
||||
var summaryTable = new Table()
|
||||
.Border(TableBorder.Rounded)
|
||||
.Title("[blue]Processing Summary[/]")
|
||||
.AddColumn("Metric")
|
||||
.AddColumn("Count", c => c.RightAligned());
|
||||
|
||||
summaryTable.AddRow("Skipped (already in DB)", skipped.ToString());
|
||||
summaryTable.AddRow("Newly processed", $"[green]{processed}[/]");
|
||||
summaryTable.AddRow("New oracles", newOracles.Count.ToString());
|
||||
summaryTable.AddRow("Priority cards found", $"{priorityFound}/{priorityNeeded}");
|
||||
summaryTable.AddRow("Errors", errors > 0 ? $"[red]{errors}[/]" : "0");
|
||||
|
||||
AnsiConsole.Write(summaryTable);
|
||||
AnsiConsole.WriteLine();
|
||||
|
||||
// Insert oracles first (cards reference them)
|
||||
if (newOracles.Count > 0)
|
||||
{
|
||||
await AnsiConsole.Status()
|
||||
.Spinner(Spinner.Known.Dots)
|
||||
.StartAsync($"Inserting {newOracles.Count} new oracles...", async ctx =>
|
||||
{
|
||||
await db.InsertOracleBatchAsync(newOracles.Values);
|
||||
});
|
||||
AnsiConsole.MarkupLine($"[green]✓[/] Inserted [blue]{newOracles.Count}[/] oracles");
|
||||
}
|
||||
|
||||
if (newCards.Count > 0)
|
||||
{
|
||||
await AnsiConsole.Status()
|
||||
.Spinner(Spinner.Known.Dots)
|
||||
.StartAsync($"Inserting {newCards.Count} new cards...", async ctx =>
|
||||
{
|
||||
await db.InsertCardBatchAsync(newCards);
|
||||
});
|
||||
AnsiConsole.MarkupLine($"[green]✓[/] Inserted [blue]{newCards.Count}[/] cards");
|
||||
}
|
||||
|
||||
await db.SetMetadataAsync("generated_at", DateTime.UtcNow.ToString("O"));
|
||||
await db.SetMetadataAsync("scryfall_updated_at", scryfallDateStr);
|
||||
|
||||
var finalCardCount = await db.GetCardCountAsync();
|
||||
var finalOracleCount = await db.GetOracleCountAsync();
|
||||
var finalSetCount = await db.GetSetCountAsync();
|
||||
|
||||
await db.SetMetadataAsync("card_count", finalCardCount.ToString());
|
||||
await db.SetMetadataAsync("oracle_count", finalOracleCount.ToString());
|
||||
await db.SetMetadataAsync("set_count", finalSetCount.ToString());
|
||||
|
||||
AnsiConsole.WriteLine();
|
||||
|
||||
var finalTable = new Table()
|
||||
.Border(TableBorder.Double)
|
||||
.Title("[green]Final Database State[/]")
|
||||
.AddColumn("Metric")
|
||||
.AddColumn("Count", c => c.RightAligned());
|
||||
|
||||
finalTable.AddRow("Cards", $"[green]{finalCardCount}[/]");
|
||||
finalTable.AddRow("Oracles", $"[green]{finalOracleCount}[/]");
|
||||
finalTable.AddRow("Sets", $"[green]{finalSetCount}[/]");
|
||||
finalTable.AddRow("Output", $"[blue]{outputDb}[/]");
|
||||
|
||||
AnsiConsole.Write(finalTable);
|
||||
|
||||
// Report missing priority cards
|
||||
if (includeTestCards)
|
||||
{
|
||||
var missing = priorityCards.Where(c => !foundPriorityWithSet.ContainsKey(c)).ToList();
|
||||
|
||||
if (missing.Count > 0)
|
||||
{
|
||||
AnsiConsole.WriteLine();
|
||||
AnsiConsole.MarkupLine($"[yellow]Missing priority cards ({missing.Count}):[/]");
|
||||
|
||||
var tree = new Tree("[yellow]Missing Cards[/]");
|
||||
foreach (var name in missing.Take(20))
|
||||
{
|
||||
tree.AddNode($"[grey]{Markup.Escape(name)}[/]");
|
||||
}
|
||||
if (missing.Count > 20)
|
||||
{
|
||||
tree.AddNode($"[grey]... and {missing.Count - 20} more[/]");
|
||||
}
|
||||
AnsiConsole.Write(tree);
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,408 +1,12 @@
|
|||
using Scry.Core.Data;
|
||||
using Scry.Core.Imaging;
|
||||
using Scry.Core.Models;
|
||||
using Scry.Core.Scryfall;
|
||||
using SkiaSharp;
|
||||
using DbGenerator;
|
||||
using Spectre.Console.Cli;
|
||||
|
||||
// Generate a card hash database from Scryfall images
|
||||
// Usage: dotnet run -- <output-db> [--count N] [--include-test-cards] [--force]
|
||||
var app = new CommandApp<GenerateCommand>();
|
||||
|
||||
var outputDb = args.Length > 0 ? args[0] : "card_hashes.db";
|
||||
var maxCards = 500;
|
||||
var includeTestCards = true;
|
||||
var forceRebuild = false;
|
||||
|
||||
// Parse arguments
|
||||
for (var i = 0; i < args.Length; i++)
|
||||
app.Configure(config =>
|
||||
{
|
||||
if (args[i] == "--count" && i + 1 < args.Length && int.TryParse(args[i + 1], out var parsedCount))
|
||||
{
|
||||
maxCards = parsedCount;
|
||||
i++;
|
||||
}
|
||||
else if (args[i] == "--include-test-cards")
|
||||
{
|
||||
includeTestCards = true;
|
||||
}
|
||||
else if (args[i] == "--no-test-cards")
|
||||
{
|
||||
includeTestCards = false;
|
||||
}
|
||||
else if (args[i] == "--force")
|
||||
{
|
||||
forceRebuild = true;
|
||||
}
|
||||
}
|
||||
config.SetApplicationName("dbgen");
|
||||
config.SetApplicationVersion("1.0.0");
|
||||
});
|
||||
|
||||
Console.WriteLine($"Generating hash database with up to {maxCards} cards");
|
||||
Console.WriteLine($"Output: {outputDb}");
|
||||
Console.WriteLine($"Include test cards: {includeTestCards}");
|
||||
Console.WriteLine($"Force rebuild: {forceRebuild}");
|
||||
Console.WriteLine();
|
||||
|
||||
// Cards that should be included for testing with preferred sets
|
||||
// Key: card name, Value: preferred set codes (first match wins) or empty for any
|
||||
var priorityCardsWithSets = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
// From reference_alpha/ - prefer LEA (Alpha) or LEB (Beta) for classic look
|
||||
["Ancestral Recall"] = ["lea", "leb"],
|
||||
["Badlands"] = ["lea", "leb"],
|
||||
["Balance"] = ["lea", "leb"],
|
||||
["Bayou"] = ["lea", "leb"],
|
||||
["Birds of Paradise"] = ["lea", "leb"],
|
||||
["Black Lotus"] = ["lea", "leb"],
|
||||
["Channel"] = ["lea", "leb"],
|
||||
["Chaos Orb"] = ["lea", "leb"],
|
||||
["Clone"] = ["lea", "leb"],
|
||||
["Control Magic"] = ["lea", "leb"],
|
||||
["Counterspell"] = ["lea", "leb"],
|
||||
["Dark Ritual"] = ["lea", "leb"],
|
||||
["Demonic Tutor"] = ["lea", "leb"],
|
||||
["Disenchant"] = ["lea", "leb"],
|
||||
["Fireball"] = ["lea", "leb"],
|
||||
["Force of Nature"] = ["lea", "leb"],
|
||||
["Fork"] = ["lea", "leb"],
|
||||
["Giant Growth"] = ["lea", "leb"],
|
||||
["Hypnotic Specter"] = ["lea", "leb"],
|
||||
["Lightning Bolt"] = ["lea", "leb"],
|
||||
["Llanowar Elves"] = ["lea", "leb"],
|
||||
["Mahamoti Djinn"] = ["lea", "leb"],
|
||||
["Mind Twist"] = ["lea", "leb"],
|
||||
["Mox Emerald"] = ["lea", "leb"],
|
||||
["Mox Jet"] = ["lea", "leb"],
|
||||
["Mox Pearl"] = ["lea", "leb"],
|
||||
["Mox Ruby"] = ["lea", "leb"],
|
||||
["Mox Sapphire"] = ["lea", "leb"],
|
||||
["Nightmare"] = ["lea", "leb"],
|
||||
["Plateau"] = ["lea", "leb"],
|
||||
["Regrowth"] = ["lea", "leb"],
|
||||
["Rock Hydra"] = ["lea", "leb"],
|
||||
["Royal Assassin"] = ["lea", "leb"],
|
||||
["Savannah"] = ["lea", "leb"],
|
||||
["Scrubland"] = ["lea", "leb"],
|
||||
["Serra Angel"] = ["lea", "leb"],
|
||||
["Shivan Dragon"] = ["lea", "leb"],
|
||||
["Sol Ring"] = ["lea", "leb"],
|
||||
["Swords to Plowshares"] = ["lea", "leb"],
|
||||
["Taiga"] = ["lea", "leb"],
|
||||
["Time Walk"] = ["lea", "leb"],
|
||||
["Timetwister"] = ["lea", "leb"],
|
||||
["Tropical Island"] = ["lea", "leb"],
|
||||
["Tundra"] = ["lea", "leb"],
|
||||
["Underground Sea"] = ["lea", "leb"],
|
||||
["Wheel of Fortune"] = ["lea", "leb"],
|
||||
["Wrath of God"] = ["lea", "leb"],
|
||||
|
||||
// From reference/ - any set is fine
|
||||
["Brainstorm"] = [],
|
||||
["Force of Will"] = [],
|
||||
["Griselbrand"] = [],
|
||||
["Lotus Petal"] = [],
|
||||
["Ponder"] = [],
|
||||
["Show and Tell"] = [],
|
||||
["Volcanic Island"] = [],
|
||||
["Wasteland"] = [],
|
||||
|
||||
// From single_cards/ - any set is fine
|
||||
["Adanto Vanguard"] = [],
|
||||
["Angel of Sanctions"] = [],
|
||||
["Attunement"] = [],
|
||||
["Avaricious Dragon"] = [],
|
||||
["Burgeoning"] = [],
|
||||
["Jarad, Golgari Lich Lord"] = [],
|
||||
["Meletis Charlatan"] = [],
|
||||
["Mindstab Thrull"] = [],
|
||||
["Pacifism"] = [],
|
||||
["Platinum Angel"] = [],
|
||||
["Queen Marchesa"] = [],
|
||||
["Spellseeker"] = [],
|
||||
["Tarmogoyf"] = [],
|
||||
["Thought Reflection"] = [],
|
||||
["Unsummon"] = [],
|
||||
|
||||
// From varying_quality - prefer older sets
|
||||
["Dragon Whelp"] = ["lea", "leb"],
|
||||
["Evil Eye of Orms-by-Gore"] = [],
|
||||
["Instill Energy"] = ["lea", "leb"],
|
||||
|
||||
// Popular cards for general testing
|
||||
["Lightning Helix"] = [],
|
||||
["Path to Exile"] = [],
|
||||
["Thoughtseize"] = [],
|
||||
["Fatal Push"] = [],
|
||||
["Snapcaster Mage"] = [],
|
||||
["Jace, the Mind Sculptor"] = [],
|
||||
["Liliana of the Veil"] = [],
|
||||
["Noble Hierarch"] = [],
|
||||
["Goblin Guide"] = [],
|
||||
["Eidolon of the Great Revel"] = [],
|
||||
};
|
||||
|
||||
var priorityCards = new HashSet<string>(priorityCardsWithSets.Keys, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Force rebuild if requested
|
||||
if (forceRebuild && File.Exists(outputDb))
|
||||
{
|
||||
Console.WriteLine("Force rebuild requested, removing existing database...");
|
||||
File.Delete(outputDb);
|
||||
}
|
||||
|
||||
using var httpClient = new HttpClient();
|
||||
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("Scry/1.0 (MTG Card Scanner - Database Generator)");
|
||||
|
||||
using var scryfallClient = new ScryfallClient(httpClient);
|
||||
using var db = new CardDatabase(outputDb);
|
||||
|
||||
// Check existing database state
|
||||
var existingCardIds = await db.GetExistingCardIdsAsync();
|
||||
var existingCardNames = await db.GetExistingCardNamesAsync();
|
||||
var existingOracleIds = await db.GetExistingOracleIdsAsync();
|
||||
var existingSetIds = await db.GetExistingSetIdsAsync();
|
||||
var existingCount = await db.GetCardCountAsync();
|
||||
var storedScryfallDate = await db.GetMetadataAsync("scryfall_updated_at");
|
||||
|
||||
Console.WriteLine($"Existing database has {existingCount} cards, {existingOracleIds.Count} oracles, {existingSetIds.Count} sets");
|
||||
|
||||
// Fetch all sets first
|
||||
Console.WriteLine("Fetching sets from Scryfall...");
|
||||
var scryfallSets = await scryfallClient.GetAllSetsAsync();
|
||||
var setsById = scryfallSets.ToDictionary(s => s.Id ?? "", s => s);
|
||||
var setsByCode = scryfallSets.ToDictionary(s => s.Code ?? "", s => s, StringComparer.OrdinalIgnoreCase);
|
||||
Console.WriteLine($"Found {scryfallSets.Count} sets");
|
||||
|
||||
// Insert any new sets
|
||||
var newSets = scryfallSets
|
||||
.Where(s => s.Id != null && !existingSetIds.Contains(s.Id))
|
||||
.Select(s => s.ToSet())
|
||||
.ToList();
|
||||
|
||||
if (newSets.Count > 0)
|
||||
{
|
||||
Console.WriteLine($"Inserting {newSets.Count} new sets...");
|
||||
await db.InsertSetBatchAsync(newSets);
|
||||
}
|
||||
|
||||
Console.WriteLine("Fetching bulk data info from Scryfall...");
|
||||
var bulkInfo = await scryfallClient.GetBulkDataInfoAsync("unique_artwork");
|
||||
|
||||
if (bulkInfo?.DownloadUri == null)
|
||||
{
|
||||
Console.WriteLine("Failed to get bulk data info from Scryfall");
|
||||
return 1;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Scryfall data last updated: {bulkInfo.UpdatedAt}");
|
||||
|
||||
// Check if we need to update at all
|
||||
var scryfallDateStr = bulkInfo.UpdatedAt?.ToString("O") ?? "";
|
||||
var needsUpdate = existingCount == 0 ||
|
||||
storedScryfallDate != scryfallDateStr ||
|
||||
existingCount < maxCards;
|
||||
|
||||
// Also check if all priority cards exist
|
||||
var missingPriorityCards = includeTestCards
|
||||
? priorityCards.Where(c => !existingCardNames.Contains(c)).ToList()
|
||||
: new List<string>();
|
||||
|
||||
if (missingPriorityCards.Count > 0)
|
||||
{
|
||||
Console.WriteLine($"Missing {missingPriorityCards.Count} priority cards");
|
||||
needsUpdate = true;
|
||||
}
|
||||
|
||||
if (!needsUpdate)
|
||||
{
|
||||
Console.WriteLine("Database is up-to-date, no changes needed");
|
||||
return 0;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Downloading card data from: {bulkInfo.DownloadUri}");
|
||||
Console.WriteLine();
|
||||
|
||||
var newCards = new List<Card>();
|
||||
var newOracles = new Dictionary<string, Oracle>();
|
||||
var processed = 0;
|
||||
var errors = 0;
|
||||
var skipped = 0;
|
||||
var priorityFound = 0;
|
||||
var priorityNeeded = includeTestCards ? priorityCards.Count : 0;
|
||||
|
||||
// Track which priority cards we've already found with their set
|
||||
// Key: card name, Value: set code
|
||||
var foundPriorityWithSet = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Helper to check if a set is preferred for a priority card
|
||||
bool IsPreferredSet(string cardName, string setCode)
|
||||
{
|
||||
if (!priorityCardsWithSets.TryGetValue(cardName, out var preferredSets))
|
||||
return false;
|
||||
return preferredSets.Length == 0 || preferredSets.Contains(setCode, StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
await foreach (var scryfallCard in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadUri))
|
||||
{
|
||||
// Skip non-English cards
|
||||
if (scryfallCard.Lang != "en")
|
||||
continue;
|
||||
|
||||
var imageUri = scryfallCard.GetImageUri("normal");
|
||||
if (string.IsNullOrEmpty(imageUri))
|
||||
continue;
|
||||
|
||||
var cardId = scryfallCard.Id ?? Guid.NewGuid().ToString();
|
||||
var cardName = scryfallCard.Name ?? "Unknown";
|
||||
var setCode = scryfallCard.Set ?? "???";
|
||||
var oracleId = scryfallCard.OracleId ?? cardId;
|
||||
var setId = scryfallCard.SetId ?? "";
|
||||
|
||||
// Check if this card already exists in the database
|
||||
if (existingCardIds.Contains(cardId))
|
||||
{
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if this is a priority card we might need
|
||||
var isPriorityCard = includeTestCards && priorityCards.Contains(cardName);
|
||||
var isPreferred = isPriorityCard && IsPreferredSet(cardName, setCode);
|
||||
|
||||
// If this priority card already found with preferred set, skip
|
||||
if (isPriorityCard && foundPriorityWithSet.TryGetValue(cardName, out var existingSet))
|
||||
{
|
||||
if (IsPreferredSet(cardName, existingSet))
|
||||
{
|
||||
// Already have preferred version
|
||||
continue;
|
||||
}
|
||||
// We have a non-preferred version; if this is preferred, we'll replace
|
||||
if (!isPreferred)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate how many slots we have left
|
||||
var totalCards = existingCount + newCards.Count;
|
||||
var priorityRemaining = priorityNeeded - foundPriorityWithSet.Count;
|
||||
var slotsForNonPriority = maxCards - priorityRemaining;
|
||||
|
||||
// Skip if we have enough non-priority cards and this isn't priority
|
||||
if (!isPriorityCard && totalCards >= slotsForNonPriority)
|
||||
continue;
|
||||
|
||||
// Download and process image
|
||||
try
|
||||
{
|
||||
Console.Write($"[{processed + 1}] {cardName}... ");
|
||||
|
||||
var imageBytes = await httpClient.GetByteArrayAsync(imageUri);
|
||||
using var bitmap = SKBitmap.Decode(imageBytes);
|
||||
|
||||
if (bitmap == null)
|
||||
{
|
||||
Console.WriteLine("decode failed");
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Apply CLAHE preprocessing and compute hash
|
||||
using var preprocessed = ImagePreprocessor.ApplyClahe(bitmap);
|
||||
var hash = PerceptualHash.ComputeColorHash(preprocessed);
|
||||
|
||||
// Create Card (printing) with hash
|
||||
var card = scryfallCard.ToCard() with { Hash = hash };
|
||||
newCards.Add(card);
|
||||
|
||||
// Track Oracle if we haven't seen it
|
||||
if (!existingOracleIds.Contains(oracleId) && !newOracles.ContainsKey(oracleId))
|
||||
{
|
||||
newOracles[oracleId] = scryfallCard.ToOracle();
|
||||
}
|
||||
|
||||
if (isPriorityCard)
|
||||
{
|
||||
foundPriorityWithSet[cardName] = setCode;
|
||||
priorityFound++;
|
||||
Console.WriteLine($"OK (priority, {setCode})");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"OK ({setCode})");
|
||||
}
|
||||
|
||||
processed++;
|
||||
|
||||
// Check if we have enough cards
|
||||
var foundAllPriority = foundPriorityWithSet.Count >= priorityNeeded;
|
||||
if (existingCount + newCards.Count >= maxCards && foundAllPriority)
|
||||
{
|
||||
Console.WriteLine($"\nReached {maxCards} cards limit with all priority cards");
|
||||
break;
|
||||
}
|
||||
|
||||
// Rate limit to be nice to Scryfall
|
||||
await Task.Delay(50);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"error: {ex.Message}");
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($"Skipped (already in DB): {skipped}");
|
||||
Console.WriteLine($"Newly processed: {processed} cards");
|
||||
Console.WriteLine($"New oracles: {newOracles.Count}");
|
||||
Console.WriteLine($"New priority cards found: {priorityFound}");
|
||||
Console.WriteLine($"Total priority cards: {foundPriorityWithSet.Count}/{priorityNeeded}");
|
||||
Console.WriteLine($"Errors: {errors}");
|
||||
Console.WriteLine();
|
||||
|
||||
// Insert oracles first (cards reference them)
|
||||
if (newOracles.Count > 0)
|
||||
{
|
||||
Console.WriteLine($"Inserting {newOracles.Count} new oracles...");
|
||||
await db.InsertOracleBatchAsync(newOracles.Values);
|
||||
}
|
||||
|
||||
if (newCards.Count > 0)
|
||||
{
|
||||
Console.WriteLine($"Inserting {newCards.Count} new cards...");
|
||||
await db.InsertCardBatchAsync(newCards);
|
||||
}
|
||||
|
||||
await db.SetMetadataAsync("generated_at", DateTime.UtcNow.ToString("O"));
|
||||
await db.SetMetadataAsync("scryfall_updated_at", scryfallDateStr);
|
||||
|
||||
var finalCardCount = await db.GetCardCountAsync();
|
||||
var finalOracleCount = await db.GetOracleCountAsync();
|
||||
var finalSetCount = await db.GetSetCountAsync();
|
||||
|
||||
await db.SetMetadataAsync("card_count", finalCardCount.ToString());
|
||||
await db.SetMetadataAsync("oracle_count", finalOracleCount.ToString());
|
||||
await db.SetMetadataAsync("set_count", finalSetCount.ToString());
|
||||
|
||||
Console.WriteLine($"Database now has {finalCardCount} cards, {finalOracleCount} oracles, {finalSetCount} sets: {outputDb}");
|
||||
|
||||
// Report missing priority cards
|
||||
if (includeTestCards)
|
||||
{
|
||||
var missing = priorityCards.Where(c => !foundPriorityWithSet.ContainsKey(c)).ToList();
|
||||
|
||||
if (missing.Count > 0)
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($"Missing priority cards ({missing.Count}):");
|
||||
foreach (var name in missing.Take(20))
|
||||
{
|
||||
Console.WriteLine($" - {name}");
|
||||
}
|
||||
if (missing.Count > 20)
|
||||
{
|
||||
Console.WriteLine($" ... and {missing.Count - 20} more");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
return await app.RunAsync(args);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue