From 54ba7496c693697072813e0fe536dc90ed71cc0b Mon Sep 17 00:00:00 2001 From: Chris Kruining Date: Thu, 5 Feb 2026 11:34:57 +0100 Subject: [PATCH] . --- docs/CARD_RECOGNITION.md | 65 +- src/Scry.App/MauiProgram.cs | 4 +- src/Scry.App/Resources/Raw/card_hashes.db | Bin 188416 -> 241664 bytes .../Services/MockCardRecognitionService.cs | 42 +- src/Scry.App/ViewModels/SettingsViewModel.cs | 16 +- src/Scry.Core/Data/CardDatabase.cs | 739 ++++++++++++++++++ src/Scry.Core/Data/CardHashDatabase.cs | 234 ------ src/Scry.Core/Models/Card.cs | 149 +++- src/Scry.Core/Models/CardHash.cs | 11 - src/Scry.Core/Models/Oracle.cs | 73 ++ src/Scry.Core/Models/Set.cs | 57 ++ .../Recognition/CardRecognitionService.cs | 63 +- .../Recognition/HashDatabaseSyncService.cs | 80 +- src/Scry.Core/Scryfall/ScryfallClient.cs | 153 +++- test/Scry.Tests/CardDatabaseTests.cs | 304 +++++++ test/Scry.Tests/CardHashDatabaseTests.cs | 146 ---- test/Scry.Tests/CardRecognitionTests.cs | 62 +- test/Scry.Tests/RobustnessAnalysisTests.cs | 63 +- tools/DbGenerator/Program.cs | 95 ++- 19 files changed, 1765 insertions(+), 591 deletions(-) create mode 100644 src/Scry.Core/Data/CardDatabase.cs delete mode 100644 src/Scry.Core/Data/CardHashDatabase.cs delete mode 100644 src/Scry.Core/Models/CardHash.cs create mode 100644 src/Scry.Core/Models/Oracle.cs create mode 100644 src/Scry.Core/Models/Set.cs create mode 100644 test/Scry.Tests/CardDatabaseTests.cs delete mode 100644 test/Scry.Tests/CardHashDatabaseTests.cs diff --git a/docs/CARD_RECOGNITION.md b/docs/CARD_RECOGNITION.md index 591293f..b788b81 100644 --- a/docs/CARD_RECOGNITION.md +++ b/docs/CARD_RECOGNITION.md @@ -313,22 +313,63 @@ public MatchResult Match(byte[] queryHash, CardDatabase db) ### Database Schema +The schema mirrors Scryfall's data model with three main tables: + ```sql -CREATE TABLE cards ( - id TEXT PRIMARY KEY, -- Scryfall ID - oracle_id TEXT NOT NULL, +-- Abstract game cards (oracle) +CREATE TABLE oracles ( + id TEXT PRIMARY KEY, -- Scryfall oracle_id name TEXT NOT NULL, - set_code TEXT NOT NULL, - collector_number TEXT, - illustration_id TEXT, - image_url TEXT, - art_hash BLOB, -- 256-bit hash of art region - full_hash BLOB, -- 256-bit hash of full card - color_hash BLOB -- 768-bit color-aware hash + mana_cost TEXT, + cmc REAL, + type_line TEXT, + oracle_text TEXT, + colors TEXT, -- JSON array + color_identity TEXT, -- JSON array + keywords TEXT, -- JSON array + reserved INTEGER DEFAULT 0, + legalities TEXT, -- JSON object + power TEXT, + toughness TEXT ); -CREATE INDEX idx_cards_oracle ON cards(oracle_id); -CREATE INDEX idx_cards_illustration ON cards(illustration_id); +-- MTG sets +CREATE TABLE sets ( + id TEXT PRIMARY KEY, -- Scryfall set id + code TEXT NOT NULL UNIQUE, -- e.g., "lea", "mh2" + name TEXT NOT NULL, -- e.g., "Limited Edition Alpha" + set_type TEXT, -- e.g., "expansion", "core" + released_at TEXT, + card_count INTEGER, + icon_svg_uri TEXT, + digital INTEGER DEFAULT 0, + parent_set_code TEXT, + block TEXT +); + +-- Card printings with perceptual hashes +CREATE TABLE cards ( + id TEXT PRIMARY KEY, -- Scryfall card ID (printing) + oracle_id TEXT NOT NULL, -- FK to oracles + set_id TEXT NOT NULL, -- FK to sets + set_code TEXT, + name TEXT NOT NULL, + collector_number TEXT, + rarity TEXT, + artist TEXT, + illustration_id TEXT, -- Same across printings with identical art + image_uri TEXT, + hash BLOB, -- Perceptual hash for matching + lang TEXT DEFAULT 'en', + prices_usd REAL, + prices_usd_foil REAL, + FOREIGN KEY (oracle_id) REFERENCES oracles(id), + FOREIGN KEY (set_id) REFERENCES sets(id) +); + +CREATE INDEX idx_cards_oracle_id ON cards(oracle_id); +CREATE INDEX idx_cards_set_id ON cards(set_id); +CREATE INDEX idx_cards_name ON cards(name); ``` ### Phase 2: Enhanced (Add OCR Fallback) diff --git a/src/Scry.App/MauiProgram.cs b/src/Scry.App/MauiProgram.cs index 0eb375e..7b6252c 100644 --- a/src/Scry.App/MauiProgram.cs +++ b/src/Scry.App/MauiProgram.cs @@ -24,11 +24,11 @@ public static class MauiProgram }); // Core Services (from Scry.Core) - builder.Services.AddSingleton(sp => + builder.Services.AddSingleton(sp => { var dbPath = Path.Combine(FileSystem.AppDataDirectory, "card_hashes.db"); EnsureDatabaseCopied(dbPath); - return new CardHashDatabase(dbPath); + return new CardDatabase(dbPath); }); builder.Services.AddSingleton(); diff --git a/src/Scry.App/Resources/Raw/card_hashes.db b/src/Scry.App/Resources/Raw/card_hashes.db index 27cc00324fa6f359ed118a837efdc40f0a0b8e74..793bbc45b8b2f5f0e1669fa0ca4967709c44ca5a 100644 GIT binary patch delta 1900 zcmZoTz}@hGZ-TU-AOiz~H3I_hPSi0r7G%(?D&*z=!63}~nt|^$|3SVCK4#t+-b>tT zd3Cv?IQMfKa&vH2uv>8YvfX3rWpiSEy|FQpRW($YP25nNu`n|)CAA_mr6N8#u_&cD zzBsicJ~=-n70k1qtj{NEhY(co_fr5%Xuy zp&i82^cQ3k*A-{XMAHf~Ix_`qjU`C4J;eUZ6oht&63qw!HgRKd#!@r`@{1Cab5i4> zW>|pCh$xPSiz1AH3u(3p@v)1Gi!*jGmn0_TKwJ)XfthoVt7C|(LWrZ2k1NFQ3L0Dr zP>`82c^Jy@%o|>nQP*6~mnVec2Us{}^5ajCUgH0knEk82{P1emn$ko%`Pr=(YQb7amX-$P7 zS2x!nS3hUhU?6eQdy7+mW6}{C@_jrb5awFQ&Zv-(Gvj3Pw~n5rFkU^o_-;&?yf-yU75-GdGW<% z=~z-gN@jXyNn#F6kAjOUB+dvo4BL!8Bf(bkLCr)r;3Y7~0 DVh|(a delta 73 zcmZp8z}IkqdxEqe2Ll6x9s>ffOw=)!pA1OOyh5l#RA diff --git a/src/Scry.App/Services/MockCardRecognitionService.cs b/src/Scry.App/Services/MockCardRecognitionService.cs index 16102ca..a2db310 100644 --- a/src/Scry.App/Services/MockCardRecognitionService.cs +++ b/src/Scry.App/Services/MockCardRecognitionService.cs @@ -12,93 +12,99 @@ public class MockCardRecognitionService : ICardRecognitionService [ new Card { - Id = "sol-ring-c21", + Id = "4cbc6901-6a4a-4d0a-83ea-7eefa3b35021", + OracleId = "orb-sol-ring", + SetId = "set-c21", Name = "Sol Ring", SetCode = "C21", SetName = "Commander 2021", CollectorNumber = "263", - ScryfallId = "4cbc6901-6a4a-4d0a-83ea-7eefa3b35021", ImageUri = "https://cards.scryfall.io/normal/front/4/c/4cbc6901-6a4a-4d0a-83ea-7eefa3b35021.jpg", ManaCost = "{1}", TypeLine = "Artifact", OracleText = "{T}: Add {C}{C}.", Rarity = "uncommon", - PriceUsd = 1.50m + PricesUsd = 1.50m }, new Card { - Id = "lightning-bolt-2xm", + Id = "e3285e6b-3e79-4d7c-bf96-d920f973b122", + OracleId = "orb-lightning-bolt", + SetId = "set-2xm", Name = "Lightning Bolt", SetCode = "2XM", SetName = "Double Masters", CollectorNumber = "129", - ScryfallId = "e3285e6b-3e79-4d7c-bf96-d920f973b122", ImageUri = "https://cards.scryfall.io/normal/front/e/3/e3285e6b-3e79-4d7c-bf96-d920f973b122.jpg", ManaCost = "{R}", TypeLine = "Instant", OracleText = "Lightning Bolt deals 3 damage to any target.", Rarity = "uncommon", - PriceUsd = 2.00m + PricesUsd = 2.00m }, new Card { - Id = "counterspell-cmr", + Id = "ce30f926-bc06-46ee-9f35-0c32659a1b1c", + OracleId = "orb-counterspell", + SetId = "set-cmr", Name = "Counterspell", SetCode = "CMR", SetName = "Commander Legends", CollectorNumber = "395", - ScryfallId = "ce30f926-bc06-46ee-9f35-0c32659a1b1c", ImageUri = "https://cards.scryfall.io/normal/front/c/e/ce30f926-bc06-46ee-9f35-0c32659a1b1c.jpg", ManaCost = "{U}{U}", TypeLine = "Instant", OracleText = "Counter target spell.", Rarity = "uncommon", - PriceUsd = 1.25m + PricesUsd = 1.25m }, new Card { - Id = "llanowar-elves-m19", + Id = "73542c66-eb3a-46e8-a8f6-5f02087b28cf", + OracleId = "orb-llanowar-elves", + SetId = "set-m19", Name = "Llanowar Elves", SetCode = "M19", SetName = "Core Set 2019", CollectorNumber = "314", - ScryfallId = "73542c66-eb3a-46e8-a8f6-5f02087b28cf", ImageUri = "https://cards.scryfall.io/normal/front/7/3/73542c66-eb3a-46e8-a8f6-5f02087b28cf.jpg", ManaCost = "{G}", TypeLine = "Creature — Elf Druid", OracleText = "{T}: Add {G}.", Rarity = "common", - PriceUsd = 0.25m + PricesUsd = 0.25m }, new Card { - Id = "swords-to-plowshares-cmr", + Id = "b8dcd4a6-5c0b-4e1e-8e1d-2e2e5c1d6a1e", + OracleId = "orb-swords-to-plowshares", + SetId = "set-cmr", Name = "Swords to Plowshares", SetCode = "CMR", SetName = "Commander Legends", CollectorNumber = "387", - ScryfallId = "b8dcd4a6-5c0b-4e1e-8e1d-2e2e5c1d6a1e", ImageUri = "https://cards.scryfall.io/normal/front/b/8/b8dcd4a6-5c0b-4e1e-8e1d-2e2e5c1d6a1e.jpg", ManaCost = "{W}", TypeLine = "Instant", OracleText = "Exile target creature. Its controller gains life equal to its power.", Rarity = "uncommon", - PriceUsd = 3.50m + PricesUsd = 3.50m }, new Card { - Id = "black-lotus-lea", + Id = "bd8fa327-dd41-4737-8f19-2cf5eb1f7c2e", + OracleId = "orb-black-lotus", + SetId = "set-lea", Name = "Black Lotus", SetCode = "LEA", SetName = "Limited Edition Alpha", CollectorNumber = "232", - ScryfallId = "bd8fa327-dd41-4737-8f19-2cf5eb1f7c2e", ImageUri = "https://cards.scryfall.io/normal/front/b/d/bd8fa327-dd41-4737-8f19-2cf5eb1f7c2e.jpg", ManaCost = "{0}", TypeLine = "Artifact", OracleText = "{T}, Sacrifice Black Lotus: Add three mana of any one color.", Rarity = "rare", - PriceUsd = 500000.00m + PricesUsd = 500000.00m } ]; diff --git a/src/Scry.App/ViewModels/SettingsViewModel.cs b/src/Scry.App/ViewModels/SettingsViewModel.cs index c84bc3c..407a913 100644 --- a/src/Scry.App/ViewModels/SettingsViewModel.cs +++ b/src/Scry.App/ViewModels/SettingsViewModel.cs @@ -6,15 +6,21 @@ namespace Scry.ViewModels; public partial class SettingsViewModel : ObservableObject { - private readonly CardHashDatabase _database; + private readonly CardDatabase _database; [ObservableProperty] private int _cardCount; + [ObservableProperty] + private int _oracleCount; + + [ObservableProperty] + private int _setCount; + [ObservableProperty] private string? _statusMessage; - public SettingsViewModel(CardHashDatabase database) + public SettingsViewModel(CardDatabase database) { _database = database; } @@ -22,7 +28,9 @@ public partial class SettingsViewModel : ObservableObject [RelayCommand] private async Task LoadAsync() { - CardCount = await _database.GetHashCountAsync(); - StatusMessage = $"Database ready with {CardCount:N0} cards"; + CardCount = await _database.GetCardCountAsync(); + OracleCount = await _database.GetOracleCountAsync(); + SetCount = await _database.GetSetCountAsync(); + StatusMessage = $"Database ready: {CardCount:N0} cards, {OracleCount:N0} oracles, {SetCount:N0} sets"; } } diff --git a/src/Scry.Core/Data/CardDatabase.cs b/src/Scry.Core/Data/CardDatabase.cs new file mode 100644 index 0000000..23a3dac --- /dev/null +++ b/src/Scry.Core/Data/CardDatabase.cs @@ -0,0 +1,739 @@ +using Microsoft.Data.Sqlite; +using Scry.Core.Models; + +namespace Scry.Core.Data; + +/// +/// SQLite database for storing card data and perceptual hashes. +/// Schema mirrors Scryfall's data model: oracles (game cards), sets, and cards (printings). +/// +public class CardDatabase : IDisposable +{ + private readonly SqliteConnection _connection; + private readonly string _dbPath; + + public CardDatabase(string dbPath) + { + _dbPath = dbPath; + _connection = new SqliteConnection($"Data Source={dbPath}"); + _connection.Open(); + Initialize(); + } + + private void Initialize() + { + using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + -- Abstract game cards (oracle) + CREATE TABLE IF NOT EXISTS oracles ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + mana_cost TEXT, + cmc REAL, + type_line TEXT, + oracle_text TEXT, + colors TEXT, + color_identity TEXT, + keywords TEXT, + reserved INTEGER DEFAULT 0, + legalities TEXT, + power TEXT, + toughness TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_oracles_name ON oracles(name); + + -- MTG sets + CREATE TABLE IF NOT EXISTS sets ( + id TEXT PRIMARY KEY, + code TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + set_type TEXT, + released_at TEXT, + card_count INTEGER, + icon_svg_uri TEXT, + digital INTEGER DEFAULT 0, + parent_set_code TEXT, + block TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_sets_code ON sets(code); + + -- Card printings with hashes + CREATE TABLE IF NOT EXISTS cards ( + id TEXT PRIMARY KEY, + oracle_id TEXT NOT NULL, + set_id TEXT NOT NULL, + set_code TEXT, + name TEXT NOT NULL, + collector_number TEXT, + rarity TEXT, + artist TEXT, + illustration_id TEXT, + image_uri TEXT, + hash BLOB, + lang TEXT DEFAULT 'en', + prices_usd REAL, + prices_usd_foil REAL, + FOREIGN KEY (oracle_id) REFERENCES oracles(id), + FOREIGN KEY (set_id) REFERENCES sets(id) + ); + + CREATE INDEX IF NOT EXISTS idx_cards_oracle_id ON cards(oracle_id); + CREATE INDEX IF NOT EXISTS idx_cards_set_id ON cards(set_id); + CREATE INDEX IF NOT EXISTS idx_cards_name ON cards(name); + CREATE INDEX IF NOT EXISTS idx_cards_set_code ON cards(set_code); + + -- Metadata for tracking sync state + CREATE TABLE IF NOT EXISTS metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + """; + cmd.ExecuteNonQuery(); + } + + #region Metadata + + public async Task GetMetadataAsync(string key, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT value FROM metadata WHERE key = $key"; + cmd.Parameters.AddWithValue("$key", key); + + var result = await cmd.ExecuteScalarAsync(ct); + return result as string; + } + + public async Task SetMetadataAsync(string key, string value, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + INSERT OR REPLACE INTO metadata (key, value) VALUES ($key, $value) + """; + cmd.Parameters.AddWithValue("$key", key); + cmd.Parameters.AddWithValue("$value", value); + await cmd.ExecuteNonQueryAsync(ct); + } + + #endregion + + #region Oracles + + public async Task InsertOracleAsync(Oracle oracle, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + INSERT OR REPLACE INTO oracles + (id, name, mana_cost, cmc, type_line, oracle_text, colors, color_identity, keywords, reserved, legalities, power, toughness) + VALUES ($id, $name, $mana_cost, $cmc, $type_line, $oracle_text, $colors, $color_identity, $keywords, $reserved, $legalities, $power, $toughness) + """; + cmd.Parameters.AddWithValue("$id", oracle.Id); + cmd.Parameters.AddWithValue("$name", oracle.Name); + cmd.Parameters.AddWithValue("$mana_cost", oracle.ManaCost ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$cmc", oracle.Cmc ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$type_line", oracle.TypeLine ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$oracle_text", oracle.OracleText ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$colors", oracle.Colors ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$color_identity", oracle.ColorIdentity ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$keywords", oracle.Keywords ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$reserved", oracle.Reserved ? 1 : 0); + cmd.Parameters.AddWithValue("$legalities", oracle.Legalities ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$power", oracle.Power ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$toughness", oracle.Toughness ?? (object)DBNull.Value); + + await cmd.ExecuteNonQueryAsync(ct); + } + + public async Task InsertOracleBatchAsync(IEnumerable oracles, CancellationToken ct = default) + { + await using var transaction = await _connection.BeginTransactionAsync(ct); + + try + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + INSERT OR REPLACE INTO oracles + (id, name, mana_cost, cmc, type_line, oracle_text, colors, color_identity, keywords, reserved, legalities, power, toughness) + VALUES ($id, $name, $mana_cost, $cmc, $type_line, $oracle_text, $colors, $color_identity, $keywords, $reserved, $legalities, $power, $toughness) + """; + + var idParam = cmd.Parameters.Add("$id", SqliteType.Text); + var nameParam = cmd.Parameters.Add("$name", SqliteType.Text); + var manaCostParam = cmd.Parameters.Add("$mana_cost", SqliteType.Text); + var cmcParam = cmd.Parameters.Add("$cmc", SqliteType.Real); + var typeLineParam = cmd.Parameters.Add("$type_line", SqliteType.Text); + var oracleTextParam = cmd.Parameters.Add("$oracle_text", SqliteType.Text); + var colorsParam = cmd.Parameters.Add("$colors", SqliteType.Text); + var colorIdentityParam = cmd.Parameters.Add("$color_identity", SqliteType.Text); + var keywordsParam = cmd.Parameters.Add("$keywords", SqliteType.Text); + var reservedParam = cmd.Parameters.Add("$reserved", SqliteType.Integer); + var legalitiesParam = cmd.Parameters.Add("$legalities", SqliteType.Text); + var powerParam = cmd.Parameters.Add("$power", SqliteType.Text); + var toughnessParam = cmd.Parameters.Add("$toughness", SqliteType.Text); + + foreach (var oracle in oracles) + { + ct.ThrowIfCancellationRequested(); + + idParam.Value = oracle.Id; + nameParam.Value = oracle.Name; + manaCostParam.Value = oracle.ManaCost ?? (object)DBNull.Value; + cmcParam.Value = oracle.Cmc ?? (object)DBNull.Value; + typeLineParam.Value = oracle.TypeLine ?? (object)DBNull.Value; + oracleTextParam.Value = oracle.OracleText ?? (object)DBNull.Value; + colorsParam.Value = oracle.Colors ?? (object)DBNull.Value; + colorIdentityParam.Value = oracle.ColorIdentity ?? (object)DBNull.Value; + keywordsParam.Value = oracle.Keywords ?? (object)DBNull.Value; + reservedParam.Value = oracle.Reserved ? 1 : 0; + legalitiesParam.Value = oracle.Legalities ?? (object)DBNull.Value; + powerParam.Value = oracle.Power ?? (object)DBNull.Value; + toughnessParam.Value = oracle.Toughness ?? (object)DBNull.Value; + + await cmd.ExecuteNonQueryAsync(ct); + } + + await transaction.CommitAsync(ct); + } + catch + { + await transaction.RollbackAsync(ct); + throw; + } + } + + public async Task GetOracleByIdAsync(string id, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + SELECT id, name, mana_cost, cmc, type_line, oracle_text, colors, color_identity, keywords, reserved, legalities, power, toughness + FROM oracles WHERE id = $id + """; + cmd.Parameters.AddWithValue("$id", id); + + await using var reader = await cmd.ExecuteReaderAsync(ct); + if (await reader.ReadAsync(ct)) + { + return ReadOracle(reader); + } + return null; + } + + public async Task GetOracleByNameAsync(string name, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + SELECT id, name, mana_cost, cmc, type_line, oracle_text, colors, color_identity, keywords, reserved, legalities, power, toughness + FROM oracles WHERE name = $name COLLATE NOCASE + """; + cmd.Parameters.AddWithValue("$name", name); + + await using var reader = await cmd.ExecuteReaderAsync(ct); + if (await reader.ReadAsync(ct)) + { + return ReadOracle(reader); + } + return null; + } + + public async Task> GetExistingOracleIdsAsync(CancellationToken ct = default) + { + var ids = new HashSet(); + + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT id FROM oracles"; + + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + ids.Add(reader.GetString(0)); + } + + return ids; + } + + public async Task GetOracleCountAsync(CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM oracles"; + var result = await cmd.ExecuteScalarAsync(ct); + return Convert.ToInt32(result); + } + + private static Oracle ReadOracle(SqliteDataReader reader) => new() + { + Id = reader.GetString(0), + Name = reader.GetString(1), + ManaCost = reader.IsDBNull(2) ? null : reader.GetString(2), + Cmc = reader.IsDBNull(3) ? null : reader.GetDouble(3), + TypeLine = reader.IsDBNull(4) ? null : reader.GetString(4), + OracleText = reader.IsDBNull(5) ? null : reader.GetString(5), + Colors = reader.IsDBNull(6) ? null : reader.GetString(6), + ColorIdentity = reader.IsDBNull(7) ? null : reader.GetString(7), + Keywords = reader.IsDBNull(8) ? null : reader.GetString(8), + Reserved = reader.GetInt32(9) != 0, + Legalities = reader.IsDBNull(10) ? null : reader.GetString(10), + Power = reader.IsDBNull(11) ? null : reader.GetString(11), + Toughness = reader.IsDBNull(12) ? null : reader.GetString(12), + }; + + #endregion + + #region Sets + + public async Task InsertSetAsync(Set set, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + INSERT OR REPLACE INTO sets + (id, code, name, set_type, released_at, card_count, icon_svg_uri, digital, parent_set_code, block) + VALUES ($id, $code, $name, $set_type, $released_at, $card_count, $icon_svg_uri, $digital, $parent_set_code, $block) + """; + cmd.Parameters.AddWithValue("$id", set.Id); + cmd.Parameters.AddWithValue("$code", set.Code); + cmd.Parameters.AddWithValue("$name", set.Name); + cmd.Parameters.AddWithValue("$set_type", set.SetType ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$released_at", set.ReleasedAt ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$card_count", set.CardCount ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$icon_svg_uri", set.IconSvgUri ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$digital", set.Digital ? 1 : 0); + cmd.Parameters.AddWithValue("$parent_set_code", set.ParentSetCode ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$block", set.Block ?? (object)DBNull.Value); + + await cmd.ExecuteNonQueryAsync(ct); + } + + public async Task InsertSetBatchAsync(IEnumerable sets, CancellationToken ct = default) + { + await using var transaction = await _connection.BeginTransactionAsync(ct); + + try + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + INSERT OR REPLACE INTO sets + (id, code, name, set_type, released_at, card_count, icon_svg_uri, digital, parent_set_code, block) + VALUES ($id, $code, $name, $set_type, $released_at, $card_count, $icon_svg_uri, $digital, $parent_set_code, $block) + """; + + var idParam = cmd.Parameters.Add("$id", SqliteType.Text); + var codeParam = cmd.Parameters.Add("$code", SqliteType.Text); + var nameParam = cmd.Parameters.Add("$name", SqliteType.Text); + var setTypeParam = cmd.Parameters.Add("$set_type", SqliteType.Text); + var releasedAtParam = cmd.Parameters.Add("$released_at", SqliteType.Text); + var cardCountParam = cmd.Parameters.Add("$card_count", SqliteType.Integer); + var iconSvgUriParam = cmd.Parameters.Add("$icon_svg_uri", SqliteType.Text); + var digitalParam = cmd.Parameters.Add("$digital", SqliteType.Integer); + var parentSetCodeParam = cmd.Parameters.Add("$parent_set_code", SqliteType.Text); + var blockParam = cmd.Parameters.Add("$block", SqliteType.Text); + + foreach (var set in sets) + { + ct.ThrowIfCancellationRequested(); + + idParam.Value = set.Id; + codeParam.Value = set.Code; + nameParam.Value = set.Name; + setTypeParam.Value = set.SetType ?? (object)DBNull.Value; + releasedAtParam.Value = set.ReleasedAt ?? (object)DBNull.Value; + cardCountParam.Value = set.CardCount ?? (object)DBNull.Value; + iconSvgUriParam.Value = set.IconSvgUri ?? (object)DBNull.Value; + digitalParam.Value = set.Digital ? 1 : 0; + parentSetCodeParam.Value = set.ParentSetCode ?? (object)DBNull.Value; + blockParam.Value = set.Block ?? (object)DBNull.Value; + + await cmd.ExecuteNonQueryAsync(ct); + } + + await transaction.CommitAsync(ct); + } + catch + { + await transaction.RollbackAsync(ct); + throw; + } + } + + public async Task GetSetByIdAsync(string id, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + SELECT id, code, name, set_type, released_at, card_count, icon_svg_uri, digital, parent_set_code, block + FROM sets WHERE id = $id + """; + cmd.Parameters.AddWithValue("$id", id); + + await using var reader = await cmd.ExecuteReaderAsync(ct); + if (await reader.ReadAsync(ct)) + { + return ReadSet(reader); + } + return null; + } + + public async Task GetSetByCodeAsync(string code, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + SELECT id, code, name, set_type, released_at, card_count, icon_svg_uri, digital, parent_set_code, block + FROM sets WHERE code = $code COLLATE NOCASE + """; + cmd.Parameters.AddWithValue("$code", code); + + await using var reader = await cmd.ExecuteReaderAsync(ct); + if (await reader.ReadAsync(ct)) + { + return ReadSet(reader); + } + return null; + } + + public async Task> GetExistingSetIdsAsync(CancellationToken ct = default) + { + var ids = new HashSet(); + + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT id FROM sets"; + + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + ids.Add(reader.GetString(0)); + } + + return ids; + } + + public async Task GetSetCountAsync(CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM sets"; + var result = await cmd.ExecuteScalarAsync(ct); + return Convert.ToInt32(result); + } + + private static Set ReadSet(SqliteDataReader reader) => new() + { + Id = reader.GetString(0), + Code = reader.GetString(1), + Name = reader.GetString(2), + SetType = reader.IsDBNull(3) ? null : reader.GetString(3), + ReleasedAt = reader.IsDBNull(4) ? null : reader.GetString(4), + CardCount = reader.IsDBNull(5) ? null : reader.GetInt32(5), + IconSvgUri = reader.IsDBNull(6) ? null : reader.GetString(6), + Digital = reader.GetInt32(7) != 0, + ParentSetCode = reader.IsDBNull(8) ? null : reader.GetString(8), + Block = reader.IsDBNull(9) ? null : reader.GetString(9), + }; + + #endregion + + #region Cards (Printings) + + public async Task InsertCardAsync(Card card, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + INSERT OR REPLACE INTO cards + (id, oracle_id, set_id, set_code, name, collector_number, rarity, artist, illustration_id, image_uri, hash, lang, prices_usd, prices_usd_foil) + VALUES ($id, $oracle_id, $set_id, $set_code, $name, $collector_number, $rarity, $artist, $illustration_id, $image_uri, $hash, $lang, $prices_usd, $prices_usd_foil) + """; + cmd.Parameters.AddWithValue("$id", card.Id); + cmd.Parameters.AddWithValue("$oracle_id", card.OracleId); + cmd.Parameters.AddWithValue("$set_id", card.SetId); + cmd.Parameters.AddWithValue("$set_code", card.SetCode ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$name", card.Name); + cmd.Parameters.AddWithValue("$collector_number", card.CollectorNumber ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$rarity", card.Rarity ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$artist", card.Artist ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$illustration_id", card.IllustrationId ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$image_uri", card.ImageUri ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$hash", card.Hash ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$lang", card.Lang ?? "en"); + cmd.Parameters.AddWithValue("$prices_usd", card.PricesUsd ?? (object)DBNull.Value); + cmd.Parameters.AddWithValue("$prices_usd_foil", card.PricesUsdFoil ?? (object)DBNull.Value); + + await cmd.ExecuteNonQueryAsync(ct); + } + + public async Task InsertCardBatchAsync(IEnumerable cards, CancellationToken ct = default) + { + await using var transaction = await _connection.BeginTransactionAsync(ct); + + try + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + INSERT OR REPLACE INTO cards + (id, oracle_id, set_id, set_code, name, collector_number, rarity, artist, illustration_id, image_uri, hash, lang, prices_usd, prices_usd_foil) + VALUES ($id, $oracle_id, $set_id, $set_code, $name, $collector_number, $rarity, $artist, $illustration_id, $image_uri, $hash, $lang, $prices_usd, $prices_usd_foil) + """; + + var idParam = cmd.Parameters.Add("$id", SqliteType.Text); + var oracleIdParam = cmd.Parameters.Add("$oracle_id", SqliteType.Text); + var setIdParam = cmd.Parameters.Add("$set_id", SqliteType.Text); + var setCodeParam = cmd.Parameters.Add("$set_code", SqliteType.Text); + var nameParam = cmd.Parameters.Add("$name", SqliteType.Text); + var collectorNumberParam = cmd.Parameters.Add("$collector_number", SqliteType.Text); + var rarityParam = cmd.Parameters.Add("$rarity", SqliteType.Text); + var artistParam = cmd.Parameters.Add("$artist", SqliteType.Text); + var illustrationIdParam = cmd.Parameters.Add("$illustration_id", SqliteType.Text); + var imageUriParam = cmd.Parameters.Add("$image_uri", SqliteType.Text); + var hashParam = cmd.Parameters.Add("$hash", SqliteType.Blob); + var langParam = cmd.Parameters.Add("$lang", SqliteType.Text); + var pricesUsdParam = cmd.Parameters.Add("$prices_usd", SqliteType.Real); + var pricesUsdFoilParam = cmd.Parameters.Add("$prices_usd_foil", SqliteType.Real); + + foreach (var card in cards) + { + ct.ThrowIfCancellationRequested(); + + idParam.Value = card.Id; + oracleIdParam.Value = card.OracleId; + setIdParam.Value = card.SetId; + setCodeParam.Value = card.SetCode ?? (object)DBNull.Value; + nameParam.Value = card.Name; + collectorNumberParam.Value = card.CollectorNumber ?? (object)DBNull.Value; + rarityParam.Value = card.Rarity ?? (object)DBNull.Value; + artistParam.Value = card.Artist ?? (object)DBNull.Value; + illustrationIdParam.Value = card.IllustrationId ?? (object)DBNull.Value; + imageUriParam.Value = card.ImageUri ?? (object)DBNull.Value; + hashParam.Value = card.Hash ?? (object)DBNull.Value; + langParam.Value = card.Lang ?? "en"; + pricesUsdParam.Value = card.PricesUsd ?? (object)DBNull.Value; + pricesUsdFoilParam.Value = card.PricesUsdFoil ?? (object)DBNull.Value; + + await cmd.ExecuteNonQueryAsync(ct); + } + + await transaction.CommitAsync(ct); + } + catch + { + await transaction.RollbackAsync(ct); + throw; + } + } + + public async Task GetCardByIdAsync(string id, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + SELECT id, oracle_id, set_id, set_code, name, collector_number, rarity, artist, illustration_id, image_uri, hash, lang, prices_usd, prices_usd_foil + FROM cards WHERE id = $id + """; + cmd.Parameters.AddWithValue("$id", id); + + await using var reader = await cmd.ExecuteReaderAsync(ct); + if (await reader.ReadAsync(ct)) + { + return ReadCard(reader); + } + return null; + } + + public async Task> GetCardsByOracleIdAsync(string oracleId, CancellationToken ct = default) + { + var cards = new List(); + + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + SELECT id, oracle_id, set_id, set_code, name, collector_number, rarity, artist, illustration_id, image_uri, hash, lang, prices_usd, prices_usd_foil + FROM cards WHERE oracle_id = $oracle_id + """; + cmd.Parameters.AddWithValue("$oracle_id", oracleId); + + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + cards.Add(ReadCard(reader)); + } + + return cards; + } + + public async Task> GetCardsByNameAsync(string name, CancellationToken ct = default) + { + var cards = new List(); + + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + SELECT id, oracle_id, set_id, set_code, name, collector_number, rarity, artist, illustration_id, image_uri, hash, lang, prices_usd, prices_usd_foil + FROM cards WHERE name = $name COLLATE NOCASE + """; + cmd.Parameters.AddWithValue("$name", name); + + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + cards.Add(ReadCard(reader)); + } + + return cards; + } + + public async Task> GetAllCardsAsync(CancellationToken ct = default) + { + var cards = new List(); + + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + SELECT id, oracle_id, set_id, set_code, name, collector_number, rarity, artist, illustration_id, image_uri, hash, lang, prices_usd, prices_usd_foil + FROM cards + """; + + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + cards.Add(ReadCard(reader)); + } + + return cards; + } + + public async Task> GetCardsWithHashAsync(CancellationToken ct = default) + { + var cards = new List(); + + await using var cmd = _connection.CreateCommand(); + // Join with oracles and sets to get denormalized data + cmd.CommandText = """ + SELECT c.id, c.oracle_id, c.set_id, c.set_code, c.name, c.collector_number, c.rarity, c.artist, + c.illustration_id, c.image_uri, c.hash, c.lang, c.prices_usd, c.prices_usd_foil, + o.mana_cost, o.type_line, o.oracle_text, o.power, o.toughness, + s.name as set_name + FROM cards c + LEFT JOIN oracles o ON c.oracle_id = o.id + LEFT JOIN sets s ON c.set_id = s.id + WHERE c.hash IS NOT NULL + """; + + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + cards.Add(ReadCardWithOracle(reader)); + } + + return cards; + } + + public async Task> GetExistingCardIdsAsync(CancellationToken ct = default) + { + var ids = new HashSet(); + + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT id FROM cards"; + + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + ids.Add(reader.GetString(0)); + } + + return ids; + } + + public async Task> GetExistingCardNamesAsync(CancellationToken ct = default) + { + var names = new HashSet(StringComparer.OrdinalIgnoreCase); + + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT DISTINCT name FROM cards"; + + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + names.Add(reader.GetString(0)); + } + + return names; + } + + public async Task GetCardCountAsync(CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM cards"; + var result = await cmd.ExecuteScalarAsync(ct); + return Convert.ToInt32(result); + } + + public async Task DeleteCardByIdAsync(string id, CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "DELETE FROM cards WHERE id = $id"; + cmd.Parameters.AddWithValue("$id", id); + await cmd.ExecuteNonQueryAsync(ct); + } + + public async Task ClearCardsAsync(CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = "DELETE FROM cards"; + await cmd.ExecuteNonQueryAsync(ct); + } + + public async Task ClearAllAsync(CancellationToken ct = default) + { + await using var cmd = _connection.CreateCommand(); + cmd.CommandText = """ + DELETE FROM cards; + DELETE FROM oracles; + DELETE FROM sets; + DELETE FROM metadata; + """; + await cmd.ExecuteNonQueryAsync(ct); + } + + private static Card ReadCard(SqliteDataReader reader) => new() + { + Id = reader.GetString(0), + OracleId = reader.GetString(1), + SetId = reader.GetString(2), + SetCode = reader.IsDBNull(3) ? null : reader.GetString(3), + Name = reader.GetString(4), + CollectorNumber = reader.IsDBNull(5) ? null : reader.GetString(5), + Rarity = reader.IsDBNull(6) ? null : reader.GetString(6), + Artist = reader.IsDBNull(7) ? null : reader.GetString(7), + IllustrationId = reader.IsDBNull(8) ? null : reader.GetString(8), + ImageUri = reader.IsDBNull(9) ? null : reader.GetString(9), + Hash = reader.IsDBNull(10) ? null : (byte[])reader.GetValue(10), + Lang = reader.IsDBNull(11) ? null : reader.GetString(11), + PricesUsd = reader.IsDBNull(12) ? null : (decimal)reader.GetDouble(12), + PricesUsdFoil = reader.IsDBNull(13) ? null : (decimal)reader.GetDouble(13), + }; + + /// + /// Reads a card with joined Oracle and Set data (columns 14-19). + /// + private static Card ReadCardWithOracle(SqliteDataReader reader) => new() + { + Id = reader.GetString(0), + OracleId = reader.GetString(1), + SetId = reader.GetString(2), + SetCode = reader.IsDBNull(3) ? null : reader.GetString(3), + Name = reader.GetString(4), + CollectorNumber = reader.IsDBNull(5) ? null : reader.GetString(5), + Rarity = reader.IsDBNull(6) ? null : reader.GetString(6), + Artist = reader.IsDBNull(7) ? null : reader.GetString(7), + IllustrationId = reader.IsDBNull(8) ? null : reader.GetString(8), + ImageUri = reader.IsDBNull(9) ? null : reader.GetString(9), + Hash = reader.IsDBNull(10) ? null : (byte[])reader.GetValue(10), + Lang = reader.IsDBNull(11) ? null : reader.GetString(11), + PricesUsd = reader.IsDBNull(12) ? null : (decimal)reader.GetDouble(12), + PricesUsdFoil = reader.IsDBNull(13) ? null : (decimal)reader.GetDouble(13), + // Denormalized Oracle fields (from JOIN) + ManaCost = reader.IsDBNull(14) ? null : reader.GetString(14), + TypeLine = reader.IsDBNull(15) ? null : reader.GetString(15), + OracleText = reader.IsDBNull(16) ? null : reader.GetString(16), + Power = reader.IsDBNull(17) ? null : reader.GetString(17), + Toughness = reader.IsDBNull(18) ? null : reader.GetString(18), + SetName = reader.IsDBNull(19) ? null : reader.GetString(19), + }; + + #endregion + + public void Dispose() + { + _connection.Dispose(); + } +} diff --git a/src/Scry.Core/Data/CardHashDatabase.cs b/src/Scry.Core/Data/CardHashDatabase.cs deleted file mode 100644 index cc2629a..0000000 --- a/src/Scry.Core/Data/CardHashDatabase.cs +++ /dev/null @@ -1,234 +0,0 @@ -using Microsoft.Data.Sqlite; -using Scry.Core.Models; - -namespace Scry.Core.Data; - -public class CardHashDatabase : IDisposable -{ - private readonly SqliteConnection _connection; - private readonly string _dbPath; - - public CardHashDatabase(string dbPath) - { - _dbPath = dbPath; - _connection = new SqliteConnection($"Data Source={dbPath}"); - _connection.Open(); - Initialize(); - } - - private void Initialize() - { - using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - CREATE TABLE IF NOT EXISTS card_hashes ( - card_id TEXT PRIMARY KEY, - name TEXT NOT NULL, - set_code TEXT NOT NULL, - collector_number TEXT, - hash BLOB NOT NULL, - image_uri TEXT - ); - - CREATE INDEX IF NOT EXISTS idx_card_hashes_name ON card_hashes(name); - CREATE INDEX IF NOT EXISTS idx_card_hashes_set ON card_hashes(set_code); - - CREATE TABLE IF NOT EXISTS metadata ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - """; - cmd.ExecuteNonQuery(); - } - - public async Task GetMetadataAsync(string key, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "SELECT value FROM metadata WHERE key = $key"; - cmd.Parameters.AddWithValue("$key", key); - - var result = await cmd.ExecuteScalarAsync(ct); - return result as string; - } - - public async Task SetMetadataAsync(string key, string value, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - INSERT OR REPLACE INTO metadata (key, value) VALUES ($key, $value) - """; - cmd.Parameters.AddWithValue("$key", key); - cmd.Parameters.AddWithValue("$value", value); - await cmd.ExecuteNonQueryAsync(ct); - } - - public async Task InsertHashAsync(CardHash hash, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - INSERT OR REPLACE INTO card_hashes - (card_id, name, set_code, collector_number, hash, image_uri) - VALUES ($card_id, $name, $set_code, $collector_number, $hash, $image_uri) - """; - cmd.Parameters.AddWithValue("$card_id", hash.CardId); - cmd.Parameters.AddWithValue("$name", hash.Name); - cmd.Parameters.AddWithValue("$set_code", hash.SetCode); - cmd.Parameters.AddWithValue("$collector_number", hash.CollectorNumber ?? (object)DBNull.Value); - cmd.Parameters.AddWithValue("$hash", hash.Hash); - cmd.Parameters.AddWithValue("$image_uri", hash.ImageUri ?? (object)DBNull.Value); - - await cmd.ExecuteNonQueryAsync(ct); - } - - public async Task InsertHashBatchAsync(IEnumerable hashes, CancellationToken ct = default) - { - await using var transaction = await _connection.BeginTransactionAsync(ct); - - try - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - INSERT OR REPLACE INTO card_hashes - (card_id, name, set_code, collector_number, hash, image_uri) - VALUES ($card_id, $name, $set_code, $collector_number, $hash, $image_uri) - """; - - var cardIdParam = cmd.Parameters.Add("$card_id", SqliteType.Text); - var nameParam = cmd.Parameters.Add("$name", SqliteType.Text); - var setCodeParam = cmd.Parameters.Add("$set_code", SqliteType.Text); - var collectorNumberParam = cmd.Parameters.Add("$collector_number", SqliteType.Text); - var hashParam = cmd.Parameters.Add("$hash", SqliteType.Blob); - var imageUriParam = cmd.Parameters.Add("$image_uri", SqliteType.Text); - - foreach (var hash in hashes) - { - ct.ThrowIfCancellationRequested(); - - cardIdParam.Value = hash.CardId; - nameParam.Value = hash.Name; - setCodeParam.Value = hash.SetCode; - collectorNumberParam.Value = hash.CollectorNumber ?? (object)DBNull.Value; - hashParam.Value = hash.Hash; - imageUriParam.Value = hash.ImageUri ?? (object)DBNull.Value; - - await cmd.ExecuteNonQueryAsync(ct); - } - - await transaction.CommitAsync(ct); - } - catch - { - await transaction.RollbackAsync(ct); - throw; - } - } - - public async Task> GetAllHashesAsync(CancellationToken ct = default) - { - var hashes = new List(); - - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "SELECT card_id, name, set_code, collector_number, hash, image_uri FROM card_hashes"; - - await using var reader = await cmd.ExecuteReaderAsync(ct); - while (await reader.ReadAsync(ct)) - { - hashes.Add(new CardHash - { - CardId = reader.GetString(0), - Name = reader.GetString(1), - SetCode = reader.GetString(2), - CollectorNumber = reader.IsDBNull(3) ? null : reader.GetString(3), - Hash = (byte[])reader.GetValue(4), - ImageUri = reader.IsDBNull(5) ? null : reader.GetString(5) - }); - } - - return hashes; - } - - public async Task GetHashCountAsync(CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "SELECT COUNT(*) FROM card_hashes"; - var result = await cmd.ExecuteScalarAsync(ct); - return Convert.ToInt32(result); - } - - public async Task GetHashByIdAsync(string cardId, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = """ - SELECT card_id, name, set_code, collector_number, hash, image_uri - FROM card_hashes WHERE card_id = $card_id - """; - cmd.Parameters.AddWithValue("$card_id", cardId); - - await using var reader = await cmd.ExecuteReaderAsync(ct); - if (await reader.ReadAsync(ct)) - { - return new CardHash - { - CardId = reader.GetString(0), - Name = reader.GetString(1), - SetCode = reader.GetString(2), - CollectorNumber = reader.IsDBNull(3) ? null : reader.GetString(3), - Hash = (byte[])reader.GetValue(4), - ImageUri = reader.IsDBNull(5) ? null : reader.GetString(5) - }; - } - - return null; - } - - public async Task> GetExistingCardIdsAsync(CancellationToken ct = default) - { - var ids = new HashSet(); - - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "SELECT card_id FROM card_hashes"; - - await using var reader = await cmd.ExecuteReaderAsync(ct); - while (await reader.ReadAsync(ct)) - { - ids.Add(reader.GetString(0)); - } - - return ids; - } - - public async Task> GetExistingCardNamesAsync(CancellationToken ct = default) - { - var names = new HashSet(StringComparer.OrdinalIgnoreCase); - - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "SELECT DISTINCT name FROM card_hashes"; - - await using var reader = await cmd.ExecuteReaderAsync(ct); - while (await reader.ReadAsync(ct)) - { - names.Add(reader.GetString(0)); - } - - return names; - } - - public async Task DeleteByCardIdAsync(string cardId, CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "DELETE FROM card_hashes WHERE card_id = $card_id"; - cmd.Parameters.AddWithValue("$card_id", cardId); - await cmd.ExecuteNonQueryAsync(ct); - } - - public async Task ClearAsync(CancellationToken ct = default) - { - await using var cmd = _connection.CreateCommand(); - cmd.CommandText = "DELETE FROM card_hashes"; - await cmd.ExecuteNonQueryAsync(ct); - } - - public void Dispose() - { - _connection.Dispose(); - } -} diff --git a/src/Scry.Core/Models/Card.cs b/src/Scry.Core/Models/Card.cs index 42944c6..50d3bbd 100644 --- a/src/Scry.Core/Models/Card.cs +++ b/src/Scry.Core/Models/Card.cs @@ -1,32 +1,143 @@ namespace Scry.Core.Models; +/// +/// Represents a specific printing of a card in a set. +/// Maps to Scryfall's Card object (which is really a printing). +/// Contains the perceptual hash for image matching. +/// Includes denormalized Oracle data for convenience. +/// public record Card { + /// + /// Scryfall's unique card ID for this specific printing. + /// public required string Id { get; init; } - public required string Name { get; init; } - public string? SetCode { get; init; } - public string? SetName { get; init; } - public string? CollectorNumber { get; init; } - public string? ScryfallId { get; init; } - public string? Rarity { get; init; } - public string? ManaCost { get; init; } - public string? TypeLine { get; init; } - public string? OracleText { get; init; } - public string? ImageUri { get; init; } - public string? ImageUriSmall { get; init; } - public string? ImageUriLarge { get; init; } - public string? Artist { get; init; } - public string? Lang { get; init; } - public decimal? PriceUsd { get; init; } - public decimal? PriceUsdFoil { get; init; } /// - /// Alias for ImageUri for compatibility with App layer + /// Oracle ID linking to the abstract game card. + /// + public required string OracleId { get; init; } + + /// + /// Set ID this printing belongs to. + /// + public required string SetId { get; init; } + + /// + /// Set code (e.g., "lea", "mh2") - denormalized for convenience. + /// + public string? SetCode { get; init; } + + /// + /// Set name - denormalized for convenience. + /// + public string? SetName { get; init; } + + /// + /// Card name - denormalized from Oracle for convenience. + /// + public required string Name { get; init; } + + /// + /// Collector number within the set. + /// + public string? CollectorNumber { get; init; } + + /// + /// Rarity (common, uncommon, rare, mythic). + /// + public string? Rarity { get; init; } + + /// + /// Artist name. + /// + public string? Artist { get; init; } + + /// + /// Illustration ID - same across printings with identical art. + /// + public string? IllustrationId { get; init; } + + /// + /// URI to the card image (normal size). + /// + public string? ImageUri { get; init; } + + /// + /// Perceptual hash for image matching. + /// + public byte[]? Hash { get; init; } + + /// + /// Language code (e.g., "en", "ja"). + /// + public string? Lang { get; init; } + + /// + /// USD price for non-foil. + /// + public decimal? PricesUsd { get; init; } + + /// + /// USD price for foil. + /// + public decimal? PricesUsdFoil { get; init; } + + #region Denormalized Oracle Fields (for App layer convenience) + + /// + /// Mana cost in Scryfall notation (e.g., "{2}{U}{U}"). + /// Denormalized from Oracle. + /// + public string? ManaCost { get; init; } + + /// + /// Full type line (e.g., "Legendary Creature — Human Wizard"). + /// Denormalized from Oracle. + /// + public string? TypeLine { get; init; } + + /// + /// Official Oracle rules text. + /// Denormalized from Oracle. + /// + public string? OracleText { get; init; } + + /// + /// Power for creatures (may contain non-numeric values like "*"). + /// Denormalized from Oracle. + /// + public string? Power { get; init; } + + /// + /// Toughness for creatures (may contain non-numeric values like "*"). + /// Denormalized from Oracle. + /// + public string? Toughness { get; init; } + + #endregion + + #region Compatibility Aliases + + /// + /// Alias for ImageUri for compatibility. /// public string? ImageUrl => ImageUri; /// - /// Alias for PriceUsd for compatibility with App layer + /// Alias for PricesUsd for compatibility. /// - public decimal? Price => PriceUsd; + public decimal? Price => PricesUsd; + + /// + /// Alias for Id (Scryfall ID) for compatibility. + /// + public string ScryfallId => Id; + + /// + /// Alias for PricesUsd for compatibility. + /// + public decimal? PriceUsd => PricesUsd; + + #endregion } diff --git a/src/Scry.Core/Models/CardHash.cs b/src/Scry.Core/Models/CardHash.cs deleted file mode 100644 index e0724a5..0000000 --- a/src/Scry.Core/Models/CardHash.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Scry.Core.Models; - -public record CardHash -{ - public required string CardId { get; init; } - public required string Name { get; init; } - public required string SetCode { get; init; } - public string? CollectorNumber { get; init; } - public required byte[] Hash { get; init; } - public string? ImageUri { get; init; } -} diff --git a/src/Scry.Core/Models/Oracle.cs b/src/Scry.Core/Models/Oracle.cs new file mode 100644 index 0000000..9b76120 --- /dev/null +++ b/src/Scry.Core/Models/Oracle.cs @@ -0,0 +1,73 @@ +namespace Scry.Core.Models; + +/// +/// Represents an abstract game card - the rules object shared across all printings. +/// Maps to Scryfall's oracle_id concept. +/// +public record Oracle +{ + /// + /// Scryfall's oracle_id - unique identifier for this game card across all printings. + /// + public required string Id { get; init; } + + /// + /// The card name (e.g., "Lightning Bolt"). + /// + public required string Name { get; init; } + + /// + /// Mana cost in Scryfall notation (e.g., "{2}{U}{U}"). + /// + public string? ManaCost { get; init; } + + /// + /// Mana value (converted mana cost). + /// + public double? Cmc { get; init; } + + /// + /// Full type line (e.g., "Legendary Creature — Human Wizard"). + /// + public string? TypeLine { get; init; } + + /// + /// Official Oracle rules text. + /// + public string? OracleText { get; init; } + + /// + /// Card colors as JSON array (e.g., ["U", "R"]). + /// + public string? Colors { get; init; } + + /// + /// Color identity for Commander as JSON array. + /// + public string? ColorIdentity { get; init; } + + /// + /// Keywords as JSON array (e.g., ["Flying", "Trample"]). + /// + public string? Keywords { get; init; } + + /// + /// Whether this card is on the Reserved List. + /// + public bool Reserved { get; init; } + + /// + /// Format legalities as JSON object. + /// + public string? Legalities { get; init; } + + /// + /// Power for creatures (may contain non-numeric values like "*"). + /// + public string? Power { get; init; } + + /// + /// Toughness for creatures (may contain non-numeric values like "*"). + /// + public string? Toughness { get; init; } +} diff --git a/src/Scry.Core/Models/Set.cs b/src/Scry.Core/Models/Set.cs new file mode 100644 index 0000000..c2f1b19 --- /dev/null +++ b/src/Scry.Core/Models/Set.cs @@ -0,0 +1,57 @@ +namespace Scry.Core.Models; + +/// +/// Represents an MTG set. Maps to Scryfall's Set object. +/// +public record Set +{ + /// + /// Scryfall's unique set ID. + /// + public required string Id { get; init; } + + /// + /// Unique 3-6 letter set code (e.g., "lea", "mh2"). + /// + public required string Code { get; init; } + + /// + /// English name of the set (e.g., "Limited Edition Alpha"). + /// + public required string Name { get; init; } + + /// + /// Set classification (e.g., "expansion", "core", "masters", "commander"). + /// + public string? SetType { get; init; } + + /// + /// Release date in ISO 8601 format. + /// + public string? ReleasedAt { get; init; } + + /// + /// Number of cards in the set. + /// + public int? CardCount { get; init; } + + /// + /// URI to the set's icon SVG. + /// + public string? IconSvgUri { get; init; } + + /// + /// Whether this is a digital-only set. + /// + public bool Digital { get; init; } + + /// + /// Parent set code for promo/token sets. + /// + public string? ParentSetCode { get; init; } + + /// + /// Block name, if applicable. + /// + public string? Block { get; init; } +} diff --git a/src/Scry.Core/Recognition/CardRecognitionService.cs b/src/Scry.Core/Recognition/CardRecognitionService.cs index a417aad..fd00568 100644 --- a/src/Scry.Core/Recognition/CardRecognitionService.cs +++ b/src/Scry.Core/Recognition/CardRecognitionService.cs @@ -8,8 +8,8 @@ namespace Scry.Core.Recognition; public class CardRecognitionService : IDisposable { - private readonly CardHashDatabase _database; - private List? _hashCache; + private readonly CardDatabase _database; + private List? _cardCache; private readonly SemaphoreSlim _cacheLock = new(1, 1); private const int ColorHashBits = 192; @@ -28,7 +28,7 @@ public class CardRecognitionService : IDisposable /// public bool EnableRotationMatching { get; set; } = true; - public CardRecognitionService(CardHashDatabase database) + public CardRecognitionService(CardDatabase database) { _database = database; } @@ -63,12 +63,12 @@ public class CardRecognitionService : IDisposable try { - var hashes = await GetHashCacheAsync(ct); - Console.WriteLine($"[Scry] Database has {hashes.Count} hashes"); + var cards = await GetCardCacheAsync(ct); + Console.WriteLine($"[Scry] Database has {cards.Count} cards with hashes"); - if (hashes.Count == 0) + if (cards.Count == 0) { - return ScanResult.Failed("No card hashes in database. Run sync first."); + return ScanResult.Failed("No cards in database. Run sync first."); } // Step 1: Detect and extract card from image (if enabled) @@ -100,8 +100,8 @@ public class CardRecognitionService : IDisposable { // Step 2: Try matching with rotation variants (if enabled) var bestMatch = EnableRotationMatching - ? await FindBestMatchWithRotationsAsync(cardImage, hashes, ct) - : FindBestMatchSingle(cardImage, hashes); + ? await FindBestMatchWithRotationsAsync(cardImage, cards, ct) + : FindBestMatchSingle(cardImage, cards); stopwatch.Stop(); @@ -110,25 +110,16 @@ public class CardRecognitionService : IDisposable return ScanResult.Failed($"No match found (detection={cardDetected})"); } - var (cardHash, distance, rotation) = bestMatch.Value; + var (matchedCard, distance, rotation) = bestMatch.Value; var confidence = PerceptualHash.CalculateConfidence(distance, ColorHashBits); - Console.WriteLine($"[Scry] Best match: {cardHash.Name}, distance={distance}, confidence={confidence:P0}, rotation={rotation}°"); + Console.WriteLine($"[Scry] Best match: {matchedCard.Name}, distance={distance}, confidence={confidence:P0}, rotation={rotation}°"); if (confidence < MinConfidence) { return ScanResult.Failed($"Match confidence too low: {confidence:P0}"); } - var card = new Card - { - Id = cardHash.CardId, - Name = cardHash.Name, - SetCode = cardHash.SetCode, - CollectorNumber = cardHash.CollectorNumber, - ImageUri = cardHash.ImageUri - }; - - return ScanResult.Matched(card, confidence, distance, stopwatch.Elapsed); + return ScanResult.Matched(matchedCard, confidence, distance, stopwatch.Elapsed); } finally { @@ -176,7 +167,7 @@ public class CardRecognitionService : IDisposable await _cacheLock.WaitAsync(); try { - _hashCache = null; + _cardCache = null; } finally { @@ -184,16 +175,16 @@ public class CardRecognitionService : IDisposable } } - private async Task> GetHashCacheAsync(CancellationToken ct) + private async Task> GetCardCacheAsync(CancellationToken ct) { - if (_hashCache != null) - return _hashCache; + if (_cardCache != null) + return _cardCache; await _cacheLock.WaitAsync(ct); try { - _hashCache ??= await _database.GetAllHashesAsync(ct); - return _hashCache; + _cardCache ??= await _database.GetCardsWithHashAsync(ct); + return _cardCache; } finally { @@ -204,14 +195,14 @@ public class CardRecognitionService : IDisposable /// /// Find best match trying all 4 rotations (0°, 90°, 180°, 270°). /// - private Task<(CardHash Hash, int Distance, int Rotation)?> FindBestMatchWithRotationsAsync( + private Task<(Card Card, int Distance, int Rotation)?> FindBestMatchWithRotationsAsync( SKBitmap cardImage, - List candidates, + List candidates, CancellationToken ct) { return Task.Run(() => { - CardHash? bestMatch = null; + Card? bestMatch = null; var bestDistance = int.MaxValue; var bestRotation = 0; @@ -231,7 +222,7 @@ public class CardRecognitionService : IDisposable // Find best match for this rotation foreach (var candidate in candidates) { - if (candidate.Hash.Length != queryHash.Length) + if (candidate.Hash == null || candidate.Hash.Length != queryHash.Length) continue; var distance = PerceptualHash.HammingDistance(queryHash, candidate.Hash); @@ -252,27 +243,27 @@ public class CardRecognitionService : IDisposable if (bestMatch == null || bestDistance > MatchThreshold) return null; - return ((CardHash Hash, int Distance, int Rotation)?)(bestMatch, bestDistance, bestRotation); + return ((Card Card, int Distance, int Rotation)?)(bestMatch, bestDistance, bestRotation); }, ct); } /// /// Find best match without rotation (single orientation). /// - private (CardHash Hash, int Distance, int Rotation)? FindBestMatchSingle( + private (Card Card, int Distance, int Rotation)? FindBestMatchSingle( SKBitmap cardImage, - List candidates) + List candidates) { // Apply CLAHE and compute hash using var preprocessed = ImagePreprocessor.ApplyClahe(cardImage); var queryHash = PerceptualHash.ComputeColorHash(preprocessed); - CardHash? bestMatch = null; + Card? bestMatch = null; var bestDistance = int.MaxValue; foreach (var candidate in candidates) { - if (candidate.Hash.Length != queryHash.Length) + if (candidate.Hash == null || candidate.Hash.Length != queryHash.Length) continue; var distance = PerceptualHash.HammingDistance(queryHash, candidate.Hash); diff --git a/src/Scry.Core/Recognition/HashDatabaseSyncService.cs b/src/Scry.Core/Recognition/HashDatabaseSyncService.cs index ee7a5c1..80402ae 100644 --- a/src/Scry.Core/Recognition/HashDatabaseSyncService.cs +++ b/src/Scry.Core/Recognition/HashDatabaseSyncService.cs @@ -9,12 +9,12 @@ namespace Scry.Core.Recognition; public class HashDatabaseSyncService { private readonly ScryfallClient _scryfallClient; - private readonly CardHashDatabase _database; + private readonly CardDatabase _database; private readonly HttpClient _imageClient; public event Action? OnProgress; - public HashDatabaseSyncService(ScryfallClient scryfallClient, CardHashDatabase database, HttpClient? imageClient = null) + public HashDatabaseSyncService(ScryfallClient scryfallClient, CardDatabase database, HttpClient? imageClient = null) { _scryfallClient = scryfallClient; _database = database; @@ -29,6 +29,21 @@ public class HashDatabaseSyncService try { + // Fetch all sets first + ReportProgress(new SyncProgress { Stage = SyncStage.Initializing, Message = "Fetching sets..." }); + var scryfallSets = await _scryfallClient.GetAllSetsAsync(ct); + var existingSetIds = await _database.GetExistingSetIdsAsync(ct); + + var newSets = scryfallSets + .Where(s => s.Id != null && !existingSetIds.Contains(s.Id)) + .Select(s => s.ToSet()) + .ToList(); + + if (newSets.Count > 0) + { + await _database.InsertSetBatchAsync(newSets, ct); + } + var bulkInfo = await _scryfallClient.GetBulkDataInfoAsync(options.BulkDataType, ct); if (bulkInfo?.DownloadUri == null) { @@ -50,21 +65,25 @@ public class HashDatabaseSyncService ReportProgress(new SyncProgress { Stage = SyncStage.Downloading, Message = "Downloading card data..." }); - var batch = new List(); + var existingOracleIds = await _database.GetExistingOracleIdsAsync(ct); + var cardBatch = new List(); + var oracleBatch = new Dictionary(); var processed = 0; var errors = 0; - await foreach (var card in _scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadUri, ct)) + await foreach (var scryfallCard in _scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadUri, ct)) { ct.ThrowIfCancellationRequested(); - if (card.Lang != "en" && !options.IncludeNonEnglish) + if (scryfallCard.Lang != "en" && !options.IncludeNonEnglish) continue; - var imageUri = card.GetImageUri(options.ImageSize); + var imageUri = scryfallCard.GetImageUri(options.ImageSize); if (string.IsNullOrEmpty(imageUri)) continue; + var oracleId = scryfallCard.OracleId ?? scryfallCard.Id ?? ""; + try { var imageBytes = await DownloadWithRetryAsync(imageUri, options.MaxRetries, ct); @@ -84,23 +103,34 @@ public class HashDatabaseSyncService using var preprocessed = ImagePreprocessor.ApplyClahe(bitmap); var hash = PerceptualHash.ComputeColorHash(preprocessed); - batch.Add(new CardHash + // Track oracle if new + if (!existingOracleIds.Contains(oracleId) && !oracleBatch.ContainsKey(oracleId)) { - CardId = card.Id ?? Guid.NewGuid().ToString(), - Name = card.Name ?? "Unknown", - SetCode = card.Set ?? "???", - CollectorNumber = card.CollectorNumber, - Hash = hash, - ImageUri = imageUri - }); + oracleBatch[oracleId] = scryfallCard.ToOracle(); + } + + // Create card with hash + var card = scryfallCard.ToCard() with { Hash = hash }; + cardBatch.Add(card); processed++; - if (batch.Count >= options.BatchSize) + if (cardBatch.Count >= options.BatchSize) { - await _database.InsertHashBatchAsync(batch, ct); - result.ProcessedCards += batch.Count; - batch.Clear(); + // Insert oracles first + if (oracleBatch.Count > 0) + { + await _database.InsertOracleBatchAsync(oracleBatch.Values, ct); + foreach (var id in oracleBatch.Keys) + { + existingOracleIds.Add(id); + } + oracleBatch.Clear(); + } + + await _database.InsertCardBatchAsync(cardBatch, ct); + result.ProcessedCards += cardBatch.Count; + cardBatch.Clear(); ReportProgress(new SyncProgress { @@ -121,14 +151,20 @@ public class HashDatabaseSyncService if (options.StopOnError) throw; - result.Errors.Add($"{card.Name}: {ex.Message}"); + result.Errors.Add($"{scryfallCard.Name}: {ex.Message}"); } } - if (batch.Count > 0) + // Insert remaining batches + if (oracleBatch.Count > 0) { - await _database.InsertHashBatchAsync(batch, ct); - result.ProcessedCards += batch.Count; + await _database.InsertOracleBatchAsync(oracleBatch.Values, ct); + } + + if (cardBatch.Count > 0) + { + await _database.InsertCardBatchAsync(cardBatch, ct); + result.ProcessedCards += cardBatch.Count; } await _database.SetMetadataAsync("last_sync_date", DateTime.UtcNow.ToString("O"), ct); diff --git a/src/Scry.Core/Scryfall/ScryfallClient.cs b/src/Scry.Core/Scryfall/ScryfallClient.cs index 62e8551..58cf64c 100644 --- a/src/Scry.Core/Scryfall/ScryfallClient.cs +++ b/src/Scry.Core/Scryfall/ScryfallClient.cs @@ -1,5 +1,6 @@ using System.IO.Compression; using System.Text.Json; +using System.Text.Json.Serialization; using Scry.Core.Models; namespace Scry.Core.Scryfall; @@ -8,6 +9,7 @@ public class ScryfallClient : IDisposable { private readonly HttpClient _httpClient; private const string BulkDataUrl = "https://api.scryfall.com/bulk-data"; + private const string SetsUrl = "https://api.scryfall.com/sets"; public ScryfallClient(HttpClient? httpClient = null) { @@ -24,6 +26,27 @@ public class ScryfallClient : IDisposable d.Type?.Equals(type, StringComparison.OrdinalIgnoreCase) == true); } + public async Task> GetAllSetsAsync(CancellationToken ct = default) + { + var sets = new List(); + var url = SetsUrl; + + while (!string.IsNullOrEmpty(url)) + { + var response = await _httpClient.GetStringAsync(url, ct); + var setsResponse = JsonSerializer.Deserialize(response, JsonOptions); + + if (setsResponse?.Data != null) + { + sets.AddRange(setsResponse.Data); + } + + url = setsResponse?.NextPage; + } + + return sets; + } + public async IAsyncEnumerable StreamBulkDataAsync( string downloadUri, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct = default) @@ -72,6 +95,8 @@ public class ScryfallClient : IDisposable }; } +#region API Response Models + public record BulkDataResponse { public List? Data { get; init; } @@ -87,21 +112,62 @@ public record BulkDataInfo public long? Size { get; init; } } -public record ScryfallCard +public record SetsResponse +{ + public List? Data { get; init; } + public bool HasMore { get; init; } + public string? NextPage { get; init; } +} + +public record ScryfallSet { public string? Id { get; init; } + public string? Code { get; init; } public string? Name { get; init; } + public string? SetType { get; init; } + public string? ReleasedAt { get; init; } + public int? CardCount { get; init; } + public string? IconSvgUri { get; init; } + public bool Digital { get; init; } + public string? ParentSetCode { get; init; } + public string? Block { get; init; } +} + +public record ScryfallCard +{ + // Core identifiers + public string? Id { get; init; } + public string? OracleId { get; init; } + + // Oracle/game card fields + public string? Name { get; init; } + public string? ManaCost { get; init; } + public double? Cmc { get; init; } + public string? TypeLine { get; init; } + public string? OracleText { get; init; } + public List? Colors { get; init; } + public List? ColorIdentity { get; init; } + public List? Keywords { get; init; } + public bool Reserved { get; init; } + public Dictionary? Legalities { get; init; } + public string? Power { get; init; } + public string? Toughness { get; init; } + + // Printing-specific fields public string? Set { get; init; } + public string? SetId { get; init; } public string? SetName { get; init; } public string? CollectorNumber { get; init; } public string? Rarity { get; init; } - public string? ManaCost { get; init; } - public string? TypeLine { get; init; } - public string? OracleText { get; init; } - public string? Lang { get; init; } public string? Artist { get; init; } + public string? IllustrationId { get; init; } + public string? Lang { get; init; } + + // Images and prices public ImageUris? ImageUris { get; init; } public Prices? Prices { get; init; } + + // Multi-face cards public List? CardFaces { get; init; } } @@ -128,11 +194,45 @@ public record CardFace public string? ManaCost { get; init; } public string? TypeLine { get; init; } public string? OracleText { get; init; } + public List? Colors { get; init; } + public string? Power { get; init; } + public string? Toughness { get; init; } public ImageUris? ImageUris { get; init; } } +#endregion + +#region Extension Methods + public static class ScryfallCardExtensions { + /// + /// Extracts the Oracle (abstract game card) from a Scryfall card. + /// + public static Oracle ToOracle(this ScryfallCard scryfall) + { + return new Oracle + { + Id = scryfall.OracleId ?? scryfall.Id ?? Guid.NewGuid().ToString(), + Name = scryfall.Name ?? "Unknown", + ManaCost = scryfall.ManaCost, + Cmc = scryfall.Cmc, + TypeLine = scryfall.TypeLine, + OracleText = scryfall.OracleText, + Colors = scryfall.Colors != null ? JsonSerializer.Serialize(scryfall.Colors) : null, + ColorIdentity = scryfall.ColorIdentity != null ? JsonSerializer.Serialize(scryfall.ColorIdentity) : null, + Keywords = scryfall.Keywords != null ? JsonSerializer.Serialize(scryfall.Keywords) : null, + Reserved = scryfall.Reserved, + Legalities = scryfall.Legalities != null ? JsonSerializer.Serialize(scryfall.Legalities) : null, + Power = scryfall.Power, + Toughness = scryfall.Toughness, + }; + } + + /// + /// Converts a Scryfall card to a Card (printing) model. + /// Note: Hash must be computed separately and set on the returned Card. + /// public static Card ToCard(this ScryfallCard scryfall) { var imageUris = scryfall.ImageUris ?? scryfall.CardFaces?.FirstOrDefault()?.ImageUris; @@ -140,21 +240,46 @@ public static class ScryfallCardExtensions return new Card { Id = scryfall.Id ?? Guid.NewGuid().ToString(), - Name = scryfall.Name ?? "Unknown", + OracleId = scryfall.OracleId ?? scryfall.Id ?? Guid.NewGuid().ToString(), + SetId = scryfall.SetId ?? "", SetCode = scryfall.Set, SetName = scryfall.SetName, + Name = scryfall.Name ?? "Unknown", CollectorNumber = scryfall.CollectorNumber, Rarity = scryfall.Rarity, + Artist = scryfall.Artist, + IllustrationId = scryfall.IllustrationId, + ImageUri = imageUris?.Normal, + Lang = scryfall.Lang, + PricesUsd = decimal.TryParse(scryfall.Prices?.Usd, out var usd) ? usd : null, + PricesUsdFoil = decimal.TryParse(scryfall.Prices?.UsdFoil, out var foil) ? foil : null, + Hash = null, // Must be computed separately + // Denormalized Oracle fields ManaCost = scryfall.ManaCost, TypeLine = scryfall.TypeLine, OracleText = scryfall.OracleText, - ImageUri = imageUris?.Normal, - ImageUriSmall = imageUris?.Small, - ImageUriLarge = imageUris?.Large ?? imageUris?.Png, - Artist = scryfall.Artist, - Lang = scryfall.Lang, - PriceUsd = decimal.TryParse(scryfall.Prices?.Usd, out var usd) ? usd : null, - PriceUsdFoil = decimal.TryParse(scryfall.Prices?.UsdFoil, out var foil) ? foil : null + Power = scryfall.Power, + Toughness = scryfall.Toughness, + }; + } + + /// + /// Converts a Scryfall set to a Set model. + /// + public static Set ToSet(this ScryfallSet scryfall) + { + return new Set + { + Id = scryfall.Id ?? Guid.NewGuid().ToString(), + Code = scryfall.Code ?? "???", + Name = scryfall.Name ?? "Unknown", + SetType = scryfall.SetType, + ReleasedAt = scryfall.ReleasedAt, + CardCount = scryfall.CardCount, + IconSvgUri = scryfall.IconSvgUri, + Digital = scryfall.Digital, + ParentSetCode = scryfall.ParentSetCode, + Block = scryfall.Block, }; } @@ -173,3 +298,5 @@ public static class ScryfallCardExtensions }; } } + +#endregion diff --git a/test/Scry.Tests/CardDatabaseTests.cs b/test/Scry.Tests/CardDatabaseTests.cs new file mode 100644 index 0000000..090998e --- /dev/null +++ b/test/Scry.Tests/CardDatabaseTests.cs @@ -0,0 +1,304 @@ +using Microsoft.Data.Sqlite; +using Scry.Core.Data; +using Scry.Core.Models; +using Xunit; + +namespace Scry.Tests; + +public class CardDatabaseTests : IDisposable +{ + private readonly string _dbPath; + private readonly CardDatabase _database; + + public CardDatabaseTests() + { + _dbPath = Path.Combine(Path.GetTempPath(), $"scry_test_{Guid.NewGuid()}.db"); + _database = new CardDatabase(_dbPath); + } + + [Fact] + public async Task InsertCard_ThenRetrieve_ReturnsMatch() + { + // First insert oracle and set (foreign keys) + var oracle = new Oracle + { + Id = "oracle-1", + Name = "Test Card", + ManaCost = "{1}{U}", + TypeLine = "Creature" + }; + await _database.InsertOracleAsync(oracle); + + var set = new Set + { + Id = "set-1", + Code = "TST", + Name = "Test Set" + }; + await _database.InsertSetAsync(set); + + var card = new Card + { + Id = "test-id", + OracleId = "oracle-1", + SetId = "set-1", + SetCode = "TST", + Name = "Test Card", + CollectorNumber = "1", + Hash = new byte[] { 0x01, 0x02, 0x03 }, + ImageUri = "https://example.com/image.jpg" + }; + + await _database.InsertCardAsync(card); + var retrieved = await _database.GetCardByIdAsync("test-id"); + + Assert.NotNull(retrieved); + Assert.Equal("Test Card", retrieved.Name); + Assert.Equal("TST", retrieved.SetCode); + Assert.Equal(card.Hash, retrieved.Hash); + } + + [Fact] + public async Task InsertCardBatch_InsertsAllCards() + { + // Insert oracle first + var oracle = new Oracle { Id = "oracle-batch", Name = "Batch Card" }; + await _database.InsertOracleAsync(oracle); + + var set = new Set { Id = "set-batch", Code = "TST", Name = "Test Set" }; + await _database.InsertSetAsync(set); + + var cards = Enumerable.Range(0, 100).Select(i => new Card + { + Id = $"card-{i}", + OracleId = "oracle-batch", + SetId = "set-batch", + SetCode = "TST", + Name = $"Card {i}", + Hash = new byte[] { (byte)i } + }).ToList(); + + await _database.InsertCardBatchAsync(cards); + var count = await _database.GetCardCountAsync(); + + Assert.Equal(100, count); + } + + [Fact] + public async Task GetAllCards_ReturnsAllCards() + { + var oracle = new Oracle { Id = "oracle-all", Name = "All Card" }; + await _database.InsertOracleAsync(oracle); + + var set = new Set { Id = "set-all", Code = "TST", Name = "Test Set" }; + await _database.InsertSetAsync(set); + + var cards = Enumerable.Range(0, 10).Select(i => new Card + { + Id = $"card-{i}", + OracleId = "oracle-all", + SetId = "set-all", + SetCode = "TST", + Name = $"Card {i}", + Hash = new byte[] { (byte)i } + }).ToList(); + + await _database.InsertCardBatchAsync(cards); + var all = await _database.GetAllCardsAsync(); + + Assert.Equal(10, all.Count); + } + + [Fact] + public async Task GetCardsByOracleId_ReturnsAllPrintings() + { + var oracle = new Oracle { Id = "oracle-multi", Name = "Multi Print Card" }; + await _database.InsertOracleAsync(oracle); + + var set1 = new Set { Id = "set-1", Code = "S1", Name = "Set 1" }; + var set2 = new Set { Id = "set-2", Code = "S2", Name = "Set 2" }; + await _database.InsertSetAsync(set1); + await _database.InsertSetAsync(set2); + + var cards = new[] + { + new Card { Id = "print-1", OracleId = "oracle-multi", SetId = "set-1", SetCode = "S1", Name = "Multi Print Card", Hash = new byte[] { 0x01 } }, + new Card { Id = "print-2", OracleId = "oracle-multi", SetId = "set-2", SetCode = "S2", Name = "Multi Print Card", Hash = new byte[] { 0x02 } }, + }; + + await _database.InsertCardBatchAsync(cards); + var printings = await _database.GetCardsByOracleIdAsync("oracle-multi"); + + Assert.Equal(2, printings.Count); + } + + [Fact] + public async Task Metadata_SetAndGet() + { + await _database.SetMetadataAsync("test_key", "test_value"); + var value = await _database.GetMetadataAsync("test_key"); + + Assert.Equal("test_value", value); + } + + [Fact] + public async Task ClearCards_RemovesAllCards() + { + var oracle = new Oracle { Id = "oracle-clear", Name = "Clear Card" }; + await _database.InsertOracleAsync(oracle); + + var set = new Set { Id = "set-clear", Code = "TST", Name = "Test Set" }; + await _database.InsertSetAsync(set); + + var cards = Enumerable.Range(0, 10).Select(i => new Card + { + Id = $"card-{i}", + OracleId = "oracle-clear", + SetId = "set-clear", + SetCode = "TST", + Name = $"Card {i}", + Hash = new byte[] { (byte)i } + }).ToList(); + + await _database.InsertCardBatchAsync(cards); + await _database.ClearCardsAsync(); + var count = await _database.GetCardCountAsync(); + + Assert.Equal(0, count); + } + + [Fact] + public async Task InsertCard_DuplicateId_Updates() + { + var oracle = new Oracle { Id = "oracle-dup", Name = "Dup Card" }; + await _database.InsertOracleAsync(oracle); + + var set = new Set { Id = "set-dup", Code = "TST", Name = "Test Set" }; + await _database.InsertSetAsync(set); + + var card1 = new Card + { + Id = "duplicate-id", + OracleId = "oracle-dup", + SetId = "set-dup", + SetCode = "TST", + Name = "Original Name", + Hash = new byte[] { 0x01 } + }; + + var card2 = new Card + { + Id = "duplicate-id", + OracleId = "oracle-dup", + SetId = "set-dup", + SetCode = "TST", + Name = "Updated Name", + Hash = new byte[] { 0x02 } + }; + + await _database.InsertCardAsync(card1); + await _database.InsertCardAsync(card2); + + var retrieved = await _database.GetCardByIdAsync("duplicate-id"); + + Assert.NotNull(retrieved); + Assert.Equal("Updated Name", retrieved.Name); + Assert.Equal(new byte[] { 0x02 }, retrieved.Hash); + } + + [Fact] + public async Task InsertOracle_ThenRetrieveByName() + { + var oracle = new Oracle + { + Id = "oracle-name", + Name = "Lightning Bolt", + ManaCost = "{R}", + Cmc = 1, + TypeLine = "Instant", + OracleText = "Lightning Bolt deals 3 damage to any target." + }; + + await _database.InsertOracleAsync(oracle); + var retrieved = await _database.GetOracleByNameAsync("Lightning Bolt"); + + Assert.NotNull(retrieved); + Assert.Equal("{R}", retrieved.ManaCost); + Assert.Equal(1, retrieved.Cmc); + } + + [Fact] + public async Task InsertSet_ThenRetrieveByCode() + { + var set = new Set + { + Id = "set-lea", + Code = "lea", + Name = "Limited Edition Alpha", + SetType = "expansion", + ReleasedAt = "1993-08-05", + CardCount = 295 + }; + + await _database.InsertSetAsync(set); + var retrieved = await _database.GetSetByCodeAsync("lea"); + + Assert.NotNull(retrieved); + Assert.Equal("Limited Edition Alpha", retrieved.Name); + Assert.Equal(295, retrieved.CardCount); + } + + [Fact] + public async Task GetCardsWithHash_OnlyReturnsCardsWithHash() + { + var oracle = new Oracle { Id = "oracle-hash", Name = "Hash Card" }; + await _database.InsertOracleAsync(oracle); + + var set = new Set { Id = "set-hash", Code = "TST", Name = "Test Set" }; + await _database.InsertSetAsync(set); + + var cardWithHash = new Card + { + Id = "card-with-hash", + OracleId = "oracle-hash", + SetId = "set-hash", + SetCode = "TST", + Name = "Has Hash", + Hash = new byte[] { 0x01 } + }; + + var cardWithoutHash = new Card + { + Id = "card-no-hash", + OracleId = "oracle-hash", + SetId = "set-hash", + SetCode = "TST", + Name = "No Hash", + Hash = null + }; + + await _database.InsertCardAsync(cardWithHash); + await _database.InsertCardAsync(cardWithoutHash); + + var cardsWithHash = await _database.GetCardsWithHashAsync(); + + Assert.Single(cardsWithHash); + Assert.Equal("card-with-hash", cardsWithHash[0].Id); + } + + public void Dispose() + { + _database.Dispose(); + SqliteConnection.ClearAllPools(); + try + { + if (File.Exists(_dbPath)) + { + File.Delete(_dbPath); + } + } + catch (IOException) + { + } + } +} diff --git a/test/Scry.Tests/CardHashDatabaseTests.cs b/test/Scry.Tests/CardHashDatabaseTests.cs deleted file mode 100644 index e6d8944..0000000 --- a/test/Scry.Tests/CardHashDatabaseTests.cs +++ /dev/null @@ -1,146 +0,0 @@ -using Microsoft.Data.Sqlite; -using Scry.Core.Data; -using Scry.Core.Models; -using Xunit; - -namespace Scry.Tests; - -public class CardHashDatabaseTests : IDisposable -{ - private readonly string _dbPath; - private readonly CardHashDatabase _database; - - public CardHashDatabaseTests() - { - _dbPath = Path.Combine(Path.GetTempPath(), $"scry_test_{Guid.NewGuid()}.db"); - _database = new CardHashDatabase(_dbPath); - } - - [Fact] - public async Task InsertHash_ThenRetrieve_ReturnsMatch() - { - var hash = new CardHash - { - CardId = "test-id", - Name = "Test Card", - SetCode = "TST", - CollectorNumber = "1", - Hash = new byte[] { 0x01, 0x02, 0x03 }, - ImageUri = "https://example.com/image.jpg" - }; - - await _database.InsertHashAsync(hash); - var retrieved = await _database.GetHashByIdAsync("test-id"); - - Assert.NotNull(retrieved); - Assert.Equal("Test Card", retrieved.Name); - Assert.Equal("TST", retrieved.SetCode); - Assert.Equal(hash.Hash, retrieved.Hash); - } - - [Fact] - public async Task InsertHashBatch_InsertsAllHashes() - { - var hashes = Enumerable.Range(0, 100).Select(i => new CardHash - { - CardId = $"card-{i}", - Name = $"Card {i}", - SetCode = "TST", - Hash = new byte[] { (byte)i } - }).ToList(); - - await _database.InsertHashBatchAsync(hashes); - var count = await _database.GetHashCountAsync(); - - Assert.Equal(100, count); - } - - [Fact] - public async Task GetAllHashes_ReturnsAllHashes() - { - var hashes = Enumerable.Range(0, 10).Select(i => new CardHash - { - CardId = $"card-{i}", - Name = $"Card {i}", - SetCode = "TST", - Hash = new byte[] { (byte)i } - }).ToList(); - - await _database.InsertHashBatchAsync(hashes); - var all = await _database.GetAllHashesAsync(); - - Assert.Equal(10, all.Count); - } - - [Fact] - public async Task Metadata_SetAndGet() - { - await _database.SetMetadataAsync("test_key", "test_value"); - var value = await _database.GetMetadataAsync("test_key"); - - Assert.Equal("test_value", value); - } - - [Fact] - public async Task Clear_RemovesAllHashes() - { - var hashes = Enumerable.Range(0, 10).Select(i => new CardHash - { - CardId = $"card-{i}", - Name = $"Card {i}", - SetCode = "TST", - Hash = new byte[] { (byte)i } - }).ToList(); - - await _database.InsertHashBatchAsync(hashes); - await _database.ClearAsync(); - var count = await _database.GetHashCountAsync(); - - Assert.Equal(0, count); - } - - [Fact] - public async Task InsertHash_DuplicateId_Updates() - { - var hash1 = new CardHash - { - CardId = "duplicate-id", - Name = "Original Name", - SetCode = "TST", - Hash = new byte[] { 0x01 } - }; - - var hash2 = new CardHash - { - CardId = "duplicate-id", - Name = "Updated Name", - SetCode = "TST", - Hash = new byte[] { 0x02 } - }; - - await _database.InsertHashAsync(hash1); - await _database.InsertHashAsync(hash2); - - var retrieved = await _database.GetHashByIdAsync("duplicate-id"); - - Assert.NotNull(retrieved); - Assert.Equal("Updated Name", retrieved.Name); - Assert.Equal(new byte[] { 0x02 }, retrieved.Hash); - } - - public void Dispose() - { - _database.Dispose(); - SqliteConnection.ClearAllPools(); - try - { - if (File.Exists(_dbPath)) - { - File.Delete(_dbPath); - } - } - catch (IOException) - { - } - } -} diff --git a/test/Scry.Tests/CardRecognitionTests.cs b/test/Scry.Tests/CardRecognitionTests.cs index 35130fb..005abbe 100644 --- a/test/Scry.Tests/CardRecognitionTests.cs +++ b/test/Scry.Tests/CardRecognitionTests.cs @@ -13,14 +13,14 @@ public class CardRecognitionTests : IDisposable { private readonly ITestOutputHelper _output; private readonly string _dbPath; - private readonly CardHashDatabase _database; + private readonly CardDatabase _database; private readonly CardRecognitionService _recognitionService; public CardRecognitionTests(ITestOutputHelper output) { _output = output; _dbPath = Path.Combine(Path.GetTempPath(), $"scry_recognition_test_{Guid.NewGuid()}.db"); - _database = new CardHashDatabase(_dbPath); + _database = new CardDatabase(_dbPath); _recognitionService = new CardRecognitionService(_database); } @@ -32,7 +32,7 @@ public class CardRecognitionTests : IDisposable var result = await _recognitionService.RecognizeAsync(bitmap); Assert.False(result.Success); - Assert.Contains("No card hashes", result.ErrorMessage); + Assert.Contains("No cards", result.ErrorMessage); } [Fact] @@ -41,11 +41,17 @@ public class CardRecognitionTests : IDisposable using var bitmap = CreateTestBitmap(100, 100); var hash = _recognitionService.ComputeHash(bitmap); - await _database.InsertHashAsync(new CardHash + // Insert oracle and set first + await _database.InsertOracleAsync(new Oracle { Id = "oracle-test", Name = "Test Card" }); + await _database.InsertSetAsync(new Set { Id = "set-test", Code = "TST", Name = "Test Set" }); + + await _database.InsertCardAsync(new Card { - CardId = "test-card", - Name = "Test Card", + Id = "test-card", + OracleId = "oracle-test", + SetId = "set-test", SetCode = "TST", + Name = "Test Card", Hash = hash, ImageUri = "https://example.com/test.jpg" }); @@ -78,11 +84,16 @@ public class CardRecognitionTests : IDisposable var hash = _recognitionService.ComputeHash(bitmap); var cardName = Path.GetFileNameWithoutExtension(imagePath); - await _database.InsertHashAsync(new CardHash + await _database.InsertOracleAsync(new Oracle { Id = $"oracle-{cardName}", Name = cardName }); + await _database.InsertSetAsync(new Set { Id = "set-ref", Code = "REF", Name = "Reference Set" }); + + await _database.InsertCardAsync(new Card { - CardId = cardName, - Name = cardName, + Id = cardName, + OracleId = $"oracle-{cardName}", + SetId = "set-ref", SetCode = "REF", + Name = cardName, Hash = hash }); await _recognitionService.InvalidateCacheAsync(); @@ -121,7 +132,7 @@ public class CardRecognitionTests : IDisposable return; } - using var testDb = new CardHashDatabase(dbPath); + using var testDb = new CardDatabase(dbPath); using var testRecognition = new CardRecognitionService(testDb); using var bitmap = SKBitmap.Decode(imagePath); @@ -129,31 +140,31 @@ public class CardRecognitionTests : IDisposable // First, just compute hash and check distance manually var queryHash = testRecognition.ComputeHash(bitmap); - var allHashes = await testDb.GetAllHashesAsync(); + var allCards = await testDb.GetCardsWithHashAsync(); _output.WriteLine($"Query hash length: {queryHash.Length} bytes"); - _output.WriteLine($"Database has {allHashes.Count} cards"); + _output.WriteLine($"Database has {allCards.Count} cards with hashes"); // Find Serra Angel and compute distance - var serraHash = allHashes.FirstOrDefault(h => h.Name == "Serra Angel"); - if (serraHash != null) + var serraCard = allCards.FirstOrDefault(c => c.Name == "Serra Angel"); + if (serraCard?.Hash != null) { - var distance = PerceptualHash.HammingDistance(queryHash, serraHash.Hash); - _output.WriteLine($"Serra Angel hash length: {serraHash.Hash.Length} bytes"); + var distance = PerceptualHash.HammingDistance(queryHash, serraCard.Hash); + _output.WriteLine($"Serra Angel hash length: {serraCard.Hash.Length} bytes"); _output.WriteLine($"Distance to Serra Angel: {distance}"); } // Find the actual best match int bestDistance = int.MaxValue; string? bestName = null; - foreach (var hash in allHashes) + foreach (var card in allCards) { - if (hash.Hash.Length != queryHash.Length) continue; - var dist = PerceptualHash.HammingDistance(queryHash, hash.Hash); + if (card.Hash == null || card.Hash.Length != queryHash.Length) continue; + var dist = PerceptualHash.HammingDistance(queryHash, card.Hash); if (dist < bestDistance) { bestDistance = dist; - bestName = hash.Name; + bestName = card.Name; } } _output.WriteLine($"Best match: {bestName}, distance: {bestDistance}"); @@ -180,11 +191,16 @@ public class CardRecognitionTests : IDisposable using var bitmap = CreateTestBitmap(200, 300); var hash = _recognitionService.ComputeHash(bitmap); - await _database.InsertHashAsync(new CardHash + await _database.InsertOracleAsync(new Oracle { Id = "oracle-timing", Name = "Timing Test Card" }); + await _database.InsertSetAsync(new Set { Id = "set-timing", Code = "TST", Name = "Test Set" }); + + await _database.InsertCardAsync(new Card { - CardId = "timing-test", - Name = "Timing Test Card", + Id = "timing-test", + OracleId = "oracle-timing", + SetId = "set-timing", SetCode = "TST", + Name = "Timing Test Card", Hash = hash }); await _recognitionService.InvalidateCacheAsync(); diff --git a/test/Scry.Tests/RobustnessAnalysisTests.cs b/test/Scry.Tests/RobustnessAnalysisTests.cs index faa6810..c335347 100644 --- a/test/Scry.Tests/RobustnessAnalysisTests.cs +++ b/test/Scry.Tests/RobustnessAnalysisTests.cs @@ -15,14 +15,14 @@ public class RobustnessAnalysisTests : IDisposable { private readonly ITestOutputHelper _output; private readonly string _dbPath; - private readonly CardHashDatabase _database; + private readonly CardDatabase _database; private readonly CardRecognitionService _recognitionService; public RobustnessAnalysisTests(ITestOutputHelper output) { _output = output; _dbPath = Path.Combine(Path.GetTempPath(), $"scry_robustness_test_{Guid.NewGuid()}.db"); - _database = new CardHashDatabase(_dbPath); + _database = new CardDatabase(_dbPath); _recognitionService = new CardRecognitionService(_database); } @@ -52,11 +52,16 @@ public class RobustnessAnalysisTests : IDisposable var originalHash = _recognitionService.ComputeHash(original); // Register original in database - await _database.InsertHashAsync(new CardHash + await _database.InsertOracleAsync(new Oracle { Id = "oracle-serra", Name = "Serra Angel" }); + await _database.InsertSetAsync(new Set { Id = "set-lea", Code = "LEA", Name = "Alpha" }); + + await _database.InsertCardAsync(new Card { - CardId = "serra-angel", - Name = "Serra Angel", + Id = "serra-angel", + OracleId = "oracle-serra", + SetId = "set-lea", SetCode = "LEA", + Name = "Serra Angel", Hash = originalHash }); await _recognitionService.InvalidateCacheAsync(); @@ -113,11 +118,16 @@ public class RobustnessAnalysisTests : IDisposable var originalHash = _recognitionService.ComputeHash(original); - await _database.InsertHashAsync(new CardHash + await _database.InsertOracleAsync(new Oracle { Id = "oracle-serra", Name = "Serra Angel" }); + await _database.InsertSetAsync(new Set { Id = "set-lea", Code = "LEA", Name = "Alpha" }); + + await _database.InsertCardAsync(new Card { - CardId = "serra-angel", - Name = "Serra Angel", + Id = "serra-angel", + OracleId = "oracle-serra", + SetId = "set-lea", SetCode = "LEA", + Name = "Serra Angel", Hash = originalHash }); await _recognitionService.InvalidateCacheAsync(); @@ -167,11 +177,16 @@ public class RobustnessAnalysisTests : IDisposable var originalHash = _recognitionService.ComputeHash(original); - await _database.InsertHashAsync(new CardHash + await _database.InsertOracleAsync(new Oracle { Id = "oracle-serra", Name = "Serra Angel" }); + await _database.InsertSetAsync(new Set { Id = "set-lea", Code = "LEA", Name = "Alpha" }); + + await _database.InsertCardAsync(new Card { - CardId = "serra-angel", - Name = "Serra Angel", + Id = "serra-angel", + OracleId = "oracle-serra", + SetId = "set-lea", SetCode = "LEA", + Name = "Serra Angel", Hash = originalHash }); await _recognitionService.InvalidateCacheAsync(); @@ -218,11 +233,16 @@ public class RobustnessAnalysisTests : IDisposable var originalHash = _recognitionService.ComputeHash(original); - await _database.InsertHashAsync(new CardHash + await _database.InsertOracleAsync(new Oracle { Id = "oracle-serra", Name = "Serra Angel" }); + await _database.InsertSetAsync(new Set { Id = "set-lea", Code = "LEA", Name = "Alpha" }); + + await _database.InsertCardAsync(new Card { - CardId = "serra-angel", - Name = "Serra Angel", + Id = "serra-angel", + OracleId = "oracle-serra", + SetId = "set-lea", SetCode = "LEA", + Name = "Serra Angel", Hash = originalHash }); await _recognitionService.InvalidateCacheAsync(); @@ -265,11 +285,16 @@ public class RobustnessAnalysisTests : IDisposable var originalHash = _recognitionService.ComputeHash(original); - await _database.InsertHashAsync(new CardHash + await _database.InsertOracleAsync(new Oracle { Id = "oracle-serra", Name = "Serra Angel" }); + await _database.InsertSetAsync(new Set { Id = "set-lea", Code = "LEA", Name = "Alpha" }); + + await _database.InsertCardAsync(new Card { - CardId = "serra-angel", - Name = "Serra Angel", + Id = "serra-angel", + OracleId = "oracle-serra", + SetId = "set-lea", SetCode = "LEA", + Name = "Serra Angel", Hash = originalHash }); await _recognitionService.InvalidateCacheAsync(); @@ -312,14 +337,14 @@ public class RobustnessAnalysisTests : IDisposable return; } - using var prodDb = new CardHashDatabase(dbPath); + using var prodDb = new CardDatabase(dbPath); using var prodRecognition = new CardRecognitionService(prodDb); var testImagesDir = Path.Combine(rootDir, "TestImages"); var categoriesToTest = new[] { "real_photos", "varying_quality", "angled", "low_light" }; _output.WriteLine("=== Real-World Recognition Test ==="); - _output.WriteLine($"Database cards: {(await prodDb.GetAllHashesAsync()).Count}"); + _output.WriteLine($"Database cards: {(await prodDb.GetCardsWithHashAsync()).Count}"); _output.WriteLine(""); foreach (var category in categoriesToTest) diff --git a/tools/DbGenerator/Program.cs b/tools/DbGenerator/Program.cs index 7857845..5717926 100644 --- a/tools/DbGenerator/Program.cs +++ b/tools/DbGenerator/Program.cs @@ -151,15 +151,36 @@ using var httpClient = new HttpClient(); httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("Scry/1.0 (MTG Card Scanner - Database Generator)"); using var scryfallClient = new ScryfallClient(httpClient); -using var db = new CardHashDatabase(outputDb); +using var db = new CardDatabase(outputDb); // Check existing database state var existingCardIds = await db.GetExistingCardIdsAsync(); var existingCardNames = await db.GetExistingCardNamesAsync(); -var existingCount = await db.GetHashCountAsync(); +var existingOracleIds = await db.GetExistingOracleIdsAsync(); +var existingSetIds = await db.GetExistingSetIdsAsync(); +var existingCount = await db.GetCardCountAsync(); var storedScryfallDate = await db.GetMetadataAsync("scryfall_updated_at"); -Console.WriteLine($"Existing database has {existingCount} cards"); +Console.WriteLine($"Existing database has {existingCount} cards, {existingOracleIds.Count} oracles, {existingSetIds.Count} sets"); + +// Fetch all sets first +Console.WriteLine("Fetching sets from Scryfall..."); +var scryfallSets = await scryfallClient.GetAllSetsAsync(); +var setsById = scryfallSets.ToDictionary(s => s.Id ?? "", s => s); +var setsByCode = scryfallSets.ToDictionary(s => s.Code ?? "", s => s, StringComparer.OrdinalIgnoreCase); +Console.WriteLine($"Found {scryfallSets.Count} sets"); + +// Insert any new sets +var newSets = scryfallSets + .Where(s => s.Id != null && !existingSetIds.Contains(s.Id)) + .Select(s => s.ToSet()) + .ToList(); + +if (newSets.Count > 0) +{ + Console.WriteLine($"Inserting {newSets.Count} new sets..."); + await db.InsertSetBatchAsync(newSets); +} Console.WriteLine("Fetching bulk data info from Scryfall..."); var bulkInfo = await scryfallClient.GetBulkDataInfoAsync("unique_artwork"); @@ -198,7 +219,8 @@ if (!needsUpdate) Console.WriteLine($"Downloading card data from: {bulkInfo.DownloadUri}"); Console.WriteLine(); -var newHashes = new List(); +var newCards = new List(); +var newOracles = new Dictionary(); var processed = 0; var errors = 0; var skipped = 0; @@ -209,10 +231,6 @@ var priorityNeeded = includeTestCards ? priorityCards.Count : 0; // Key: card name, Value: set code var foundPriorityWithSet = new Dictionary(StringComparer.OrdinalIgnoreCase); -// Track deferred priority cards that might get a better set later -// Key: card name, Value: list of (cardHash, setCode) candidates -var deferredPriority = new Dictionary>(StringComparer.OrdinalIgnoreCase); - // Helper to check if a set is preferred for a priority card bool IsPreferredSet(string cardName, string setCode) { @@ -221,19 +239,21 @@ bool IsPreferredSet(string cardName, string setCode) return preferredSets.Length == 0 || preferredSets.Contains(setCode, StringComparer.OrdinalIgnoreCase); } -await foreach (var card in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadUri)) +await foreach (var scryfallCard in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadUri)) { // Skip non-English cards - if (card.Lang != "en") + if (scryfallCard.Lang != "en") continue; - var imageUri = card.GetImageUri("normal"); + var imageUri = scryfallCard.GetImageUri("normal"); if (string.IsNullOrEmpty(imageUri)) continue; - var cardId = card.Id ?? Guid.NewGuid().ToString(); - var cardName = card.Name ?? "Unknown"; - var setCode = card.Set ?? "???"; + var cardId = scryfallCard.Id ?? Guid.NewGuid().ToString(); + var cardName = scryfallCard.Name ?? "Unknown"; + var setCode = scryfallCard.Set ?? "???"; + var oracleId = scryfallCard.OracleId ?? cardId; + var setId = scryfallCard.SetId ?? ""; // Check if this card already exists in the database if (existingCardIds.Contains(cardId)) @@ -262,7 +282,7 @@ await foreach (var card in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadU } // Calculate how many slots we have left - var totalCards = existingCount + newHashes.Count; + var totalCards = existingCount + newCards.Count; var priorityRemaining = priorityNeeded - foundPriorityWithSet.Count; var slotsForNonPriority = maxCards - priorityRemaining; @@ -289,17 +309,15 @@ await foreach (var card in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadU using var preprocessed = ImagePreprocessor.ApplyClahe(bitmap); var hash = PerceptualHash.ComputeColorHash(preprocessed); - var cardHash = new CardHash - { - CardId = cardId, - Name = cardName, - SetCode = setCode, - CollectorNumber = card.CollectorNumber, - Hash = hash, - ImageUri = imageUri - }; + // Create Card (printing) with hash + var card = scryfallCard.ToCard() with { Hash = hash }; + newCards.Add(card); - newHashes.Add(cardHash); + // Track Oracle if we haven't seen it + if (!existingOracleIds.Contains(oracleId) && !newOracles.ContainsKey(oracleId)) + { + newOracles[oracleId] = scryfallCard.ToOracle(); + } if (isPriorityCard) { @@ -316,7 +334,7 @@ await foreach (var card in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadU // Check if we have enough cards var foundAllPriority = foundPriorityWithSet.Count >= priorityNeeded; - if (existingCount + newHashes.Count >= maxCards && foundAllPriority) + if (existingCount + newCards.Count >= maxCards && foundAllPriority) { Console.WriteLine($"\nReached {maxCards} cards limit with all priority cards"); break; @@ -335,24 +353,37 @@ await foreach (var card in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadU Console.WriteLine(); Console.WriteLine($"Skipped (already in DB): {skipped}"); Console.WriteLine($"Newly processed: {processed} cards"); +Console.WriteLine($"New oracles: {newOracles.Count}"); Console.WriteLine($"New priority cards found: {priorityFound}"); Console.WriteLine($"Total priority cards: {foundPriorityWithSet.Count}/{priorityNeeded}"); Console.WriteLine($"Errors: {errors}"); Console.WriteLine(); -if (newHashes.Count > 0) +// Insert oracles first (cards reference them) +if (newOracles.Count > 0) { - Console.WriteLine($"Inserting {newHashes.Count} new hashes into database..."); - await db.InsertHashBatchAsync(newHashes); + Console.WriteLine($"Inserting {newOracles.Count} new oracles..."); + await db.InsertOracleBatchAsync(newOracles.Values); +} + +if (newCards.Count > 0) +{ + Console.WriteLine($"Inserting {newCards.Count} new cards..."); + await db.InsertCardBatchAsync(newCards); } await db.SetMetadataAsync("generated_at", DateTime.UtcNow.ToString("O")); await db.SetMetadataAsync("scryfall_updated_at", scryfallDateStr); -var finalCount = await db.GetHashCountAsync(); -await db.SetMetadataAsync("card_count", finalCount.ToString()); +var finalCardCount = await db.GetCardCountAsync(); +var finalOracleCount = await db.GetOracleCountAsync(); +var finalSetCount = await db.GetSetCountAsync(); -Console.WriteLine($"Database now has {finalCount} cards: {outputDb}"); +await db.SetMetadataAsync("card_count", finalCardCount.ToString()); +await db.SetMetadataAsync("oracle_count", finalOracleCount.ToString()); +await db.SetMetadataAsync("set_count", finalSetCount.ToString()); + +Console.WriteLine($"Database now has {finalCardCount} cards, {finalOracleCount} oracles, {finalSetCount} sets: {outputDb}"); // Report missing priority cards if (includeTestCards)