.
This commit is contained in:
Chris Kruining
parent
86aa0f856c
commit
0801ceee6a
310 changed files with 6712 additions and 418 deletions
377
tools/DbGenerator/Program.cs
Normal file
377
tools/DbGenerator/Program.cs
Normal file
|
|
@ -0,0 +1,377 @@
|
|||
using Scry.Core.Data;
|
||||
using Scry.Core.Imaging;
|
||||
using Scry.Core.Models;
|
||||
using Scry.Core.Scryfall;
|
||||
using SkiaSharp;
|
||||
|
||||
// Generate a card hash database from Scryfall images
|
||||
// Usage: dotnet run -- <output-db> [--count N] [--include-test-cards] [--force]
|
||||
|
||||
var outputDb = args.Length > 0 ? args[0] : "card_hashes.db";
|
||||
var maxCards = 500;
|
||||
var includeTestCards = true;
|
||||
var forceRebuild = false;
|
||||
|
||||
// Parse arguments
|
||||
for (var i = 0; i < args.Length; i++)
|
||||
{
|
||||
if (args[i] == "--count" && i + 1 < args.Length && int.TryParse(args[i + 1], out var parsedCount))
|
||||
{
|
||||
maxCards = parsedCount;
|
||||
i++;
|
||||
}
|
||||
else if (args[i] == "--include-test-cards")
|
||||
{
|
||||
includeTestCards = true;
|
||||
}
|
||||
else if (args[i] == "--no-test-cards")
|
||||
{
|
||||
includeTestCards = false;
|
||||
}
|
||||
else if (args[i] == "--force")
|
||||
{
|
||||
forceRebuild = true;
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine($"Generating hash database with up to {maxCards} cards");
|
||||
Console.WriteLine($"Output: {outputDb}");
|
||||
Console.WriteLine($"Include test cards: {includeTestCards}");
|
||||
Console.WriteLine($"Force rebuild: {forceRebuild}");
|
||||
Console.WriteLine();
|
||||
|
||||
// Cards that should be included for testing with preferred sets
|
||||
// Key: card name, Value: preferred set codes (first match wins) or empty for any
|
||||
var priorityCardsWithSets = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
// From reference_alpha/ - prefer LEA (Alpha) or LEB (Beta) for classic look
|
||||
["Ancestral Recall"] = ["lea", "leb"],
|
||||
["Badlands"] = ["lea", "leb"],
|
||||
["Balance"] = ["lea", "leb"],
|
||||
["Bayou"] = ["lea", "leb"],
|
||||
["Birds of Paradise"] = ["lea", "leb"],
|
||||
["Black Lotus"] = ["lea", "leb"],
|
||||
["Channel"] = ["lea", "leb"],
|
||||
["Chaos Orb"] = ["lea", "leb"],
|
||||
["Clone"] = ["lea", "leb"],
|
||||
["Control Magic"] = ["lea", "leb"],
|
||||
["Counterspell"] = ["lea", "leb"],
|
||||
["Dark Ritual"] = ["lea", "leb"],
|
||||
["Demonic Tutor"] = ["lea", "leb"],
|
||||
["Disenchant"] = ["lea", "leb"],
|
||||
["Fireball"] = ["lea", "leb"],
|
||||
["Force of Nature"] = ["lea", "leb"],
|
||||
["Fork"] = ["lea", "leb"],
|
||||
["Giant Growth"] = ["lea", "leb"],
|
||||
["Hypnotic Specter"] = ["lea", "leb"],
|
||||
["Lightning Bolt"] = ["lea", "leb"],
|
||||
["Llanowar Elves"] = ["lea", "leb"],
|
||||
["Mahamoti Djinn"] = ["lea", "leb"],
|
||||
["Mind Twist"] = ["lea", "leb"],
|
||||
["Mox Emerald"] = ["lea", "leb"],
|
||||
["Mox Jet"] = ["lea", "leb"],
|
||||
["Mox Pearl"] = ["lea", "leb"],
|
||||
["Mox Ruby"] = ["lea", "leb"],
|
||||
["Mox Sapphire"] = ["lea", "leb"],
|
||||
["Nightmare"] = ["lea", "leb"],
|
||||
["Plateau"] = ["lea", "leb"],
|
||||
["Regrowth"] = ["lea", "leb"],
|
||||
["Rock Hydra"] = ["lea", "leb"],
|
||||
["Royal Assassin"] = ["lea", "leb"],
|
||||
["Savannah"] = ["lea", "leb"],
|
||||
["Scrubland"] = ["lea", "leb"],
|
||||
["Serra Angel"] = ["lea", "leb"],
|
||||
["Shivan Dragon"] = ["lea", "leb"],
|
||||
["Sol Ring"] = ["lea", "leb"],
|
||||
["Swords to Plowshares"] = ["lea", "leb"],
|
||||
["Taiga"] = ["lea", "leb"],
|
||||
["Time Walk"] = ["lea", "leb"],
|
||||
["Timetwister"] = ["lea", "leb"],
|
||||
["Tropical Island"] = ["lea", "leb"],
|
||||
["Tundra"] = ["lea", "leb"],
|
||||
["Underground Sea"] = ["lea", "leb"],
|
||||
["Wheel of Fortune"] = ["lea", "leb"],
|
||||
["Wrath of God"] = ["lea", "leb"],
|
||||
|
||||
// From reference/ - any set is fine
|
||||
["Brainstorm"] = [],
|
||||
["Force of Will"] = [],
|
||||
["Griselbrand"] = [],
|
||||
["Lotus Petal"] = [],
|
||||
["Ponder"] = [],
|
||||
["Show and Tell"] = [],
|
||||
["Volcanic Island"] = [],
|
||||
["Wasteland"] = [],
|
||||
|
||||
// From single_cards/ - any set is fine
|
||||
["Adanto Vanguard"] = [],
|
||||
["Angel of Sanctions"] = [],
|
||||
["Attunement"] = [],
|
||||
["Avaricious Dragon"] = [],
|
||||
["Burgeoning"] = [],
|
||||
["Jarad, Golgari Lich Lord"] = [],
|
||||
["Meletis Charlatan"] = [],
|
||||
["Mindstab Thrull"] = [],
|
||||
["Pacifism"] = [],
|
||||
["Platinum Angel"] = [],
|
||||
["Queen Marchesa"] = [],
|
||||
["Spellseeker"] = [],
|
||||
["Tarmogoyf"] = [],
|
||||
["Thought Reflection"] = [],
|
||||
["Unsummon"] = [],
|
||||
|
||||
// From varying_quality - prefer older sets
|
||||
["Dragon Whelp"] = ["lea", "leb"],
|
||||
["Evil Eye of Orms-by-Gore"] = [],
|
||||
["Instill Energy"] = ["lea", "leb"],
|
||||
|
||||
// Popular cards for general testing
|
||||
["Lightning Helix"] = [],
|
||||
["Path to Exile"] = [],
|
||||
["Thoughtseize"] = [],
|
||||
["Fatal Push"] = [],
|
||||
["Snapcaster Mage"] = [],
|
||||
["Jace, the Mind Sculptor"] = [],
|
||||
["Liliana of the Veil"] = [],
|
||||
["Noble Hierarch"] = [],
|
||||
["Goblin Guide"] = [],
|
||||
["Eidolon of the Great Revel"] = [],
|
||||
};
|
||||
|
||||
var priorityCards = new HashSet<string>(priorityCardsWithSets.Keys, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Force rebuild if requested
|
||||
if (forceRebuild && File.Exists(outputDb))
|
||||
{
|
||||
Console.WriteLine("Force rebuild requested, removing existing database...");
|
||||
File.Delete(outputDb);
|
||||
}
|
||||
|
||||
using var httpClient = new HttpClient();
|
||||
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("Scry/1.0 (MTG Card Scanner - Database Generator)");
|
||||
|
||||
using var scryfallClient = new ScryfallClient(httpClient);
|
||||
using var db = new CardHashDatabase(outputDb);
|
||||
|
||||
// Check existing database state
|
||||
var existingCardIds = await db.GetExistingCardIdsAsync();
|
||||
var existingCardNames = await db.GetExistingCardNamesAsync();
|
||||
var existingCount = await db.GetHashCountAsync();
|
||||
var storedScryfallDate = await db.GetMetadataAsync("scryfall_updated_at");
|
||||
|
||||
Console.WriteLine($"Existing database has {existingCount} cards");
|
||||
|
||||
Console.WriteLine("Fetching bulk data info from Scryfall...");
|
||||
var bulkInfo = await scryfallClient.GetBulkDataInfoAsync("unique_artwork");
|
||||
|
||||
if (bulkInfo?.DownloadUri == null)
|
||||
{
|
||||
Console.WriteLine("Failed to get bulk data info from Scryfall");
|
||||
return 1;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Scryfall data last updated: {bulkInfo.UpdatedAt}");
|
||||
|
||||
// Check if we need to update at all
|
||||
var scryfallDateStr = bulkInfo.UpdatedAt?.ToString("O") ?? "";
|
||||
var needsUpdate = existingCount == 0 ||
|
||||
storedScryfallDate != scryfallDateStr ||
|
||||
existingCount < maxCards;
|
||||
|
||||
// Also check if all priority cards exist
|
||||
var missingPriorityCards = includeTestCards
|
||||
? priorityCards.Where(c => !existingCardNames.Contains(c)).ToList()
|
||||
: new List<string>();
|
||||
|
||||
if (missingPriorityCards.Count > 0)
|
||||
{
|
||||
Console.WriteLine($"Missing {missingPriorityCards.Count} priority cards");
|
||||
needsUpdate = true;
|
||||
}
|
||||
|
||||
if (!needsUpdate)
|
||||
{
|
||||
Console.WriteLine("Database is up-to-date, no changes needed");
|
||||
return 0;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Downloading card data from: {bulkInfo.DownloadUri}");
|
||||
Console.WriteLine();
|
||||
|
||||
var newHashes = new List<CardHash>();
|
||||
var processed = 0;
|
||||
var errors = 0;
|
||||
var skipped = 0;
|
||||
var priorityFound = 0;
|
||||
var priorityNeeded = includeTestCards ? priorityCards.Count : 0;
|
||||
|
||||
// Track which priority cards we've already found with their set
|
||||
// Key: card name, Value: set code
|
||||
var foundPriorityWithSet = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Track deferred priority cards that might get a better set later
|
||||
// Key: card name, Value: list of (cardHash, setCode) candidates
|
||||
var deferredPriority = new Dictionary<string, List<(CardHash hash, string set)>>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Helper to check if a set is preferred for a priority card
|
||||
bool IsPreferredSet(string cardName, string setCode)
|
||||
{
|
||||
if (!priorityCardsWithSets.TryGetValue(cardName, out var preferredSets))
|
||||
return false;
|
||||
return preferredSets.Length == 0 || preferredSets.Contains(setCode, StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
await foreach (var card in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadUri))
|
||||
{
|
||||
// Skip non-English cards
|
||||
if (card.Lang != "en")
|
||||
continue;
|
||||
|
||||
var imageUri = card.GetImageUri("normal");
|
||||
if (string.IsNullOrEmpty(imageUri))
|
||||
continue;
|
||||
|
||||
var cardId = card.Id ?? Guid.NewGuid().ToString();
|
||||
var cardName = card.Name ?? "Unknown";
|
||||
var setCode = card.Set ?? "???";
|
||||
|
||||
// Check if this card already exists in the database
|
||||
if (existingCardIds.Contains(cardId))
|
||||
{
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if this is a priority card we might need
|
||||
var isPriorityCard = includeTestCards && priorityCards.Contains(cardName);
|
||||
var isPreferred = isPriorityCard && IsPreferredSet(cardName, setCode);
|
||||
|
||||
// If this priority card already found with preferred set, skip
|
||||
if (isPriorityCard && foundPriorityWithSet.TryGetValue(cardName, out var existingSet))
|
||||
{
|
||||
if (IsPreferredSet(cardName, existingSet))
|
||||
{
|
||||
// Already have preferred version
|
||||
continue;
|
||||
}
|
||||
// We have a non-preferred version; if this is preferred, we'll replace
|
||||
if (!isPreferred)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate how many slots we have left
|
||||
var totalCards = existingCount + newHashes.Count;
|
||||
var priorityRemaining = priorityNeeded - foundPriorityWithSet.Count;
|
||||
var slotsForNonPriority = maxCards - priorityRemaining;
|
||||
|
||||
// Skip if we have enough non-priority cards and this isn't priority
|
||||
if (!isPriorityCard && totalCards >= slotsForNonPriority)
|
||||
continue;
|
||||
|
||||
// Download and process image
|
||||
try
|
||||
{
|
||||
Console.Write($"[{processed + 1}] {cardName}... ");
|
||||
|
||||
var imageBytes = await httpClient.GetByteArrayAsync(imageUri);
|
||||
using var bitmap = SKBitmap.Decode(imageBytes);
|
||||
|
||||
if (bitmap == null)
|
||||
{
|
||||
Console.WriteLine("decode failed");
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Apply CLAHE preprocessing and compute hash
|
||||
using var preprocessed = ImagePreprocessor.ApplyClahe(bitmap);
|
||||
var hash = PerceptualHash.ComputeColorHash(preprocessed);
|
||||
|
||||
var cardHash = new CardHash
|
||||
{
|
||||
CardId = cardId,
|
||||
Name = cardName,
|
||||
SetCode = setCode,
|
||||
CollectorNumber = card.CollectorNumber,
|
||||
Hash = hash,
|
||||
ImageUri = imageUri
|
||||
};
|
||||
|
||||
newHashes.Add(cardHash);
|
||||
|
||||
if (isPriorityCard)
|
||||
{
|
||||
foundPriorityWithSet[cardName] = setCode;
|
||||
priorityFound++;
|
||||
Console.WriteLine($"OK (priority, {setCode})");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"OK ({setCode})");
|
||||
}
|
||||
|
||||
processed++;
|
||||
|
||||
// Check if we have enough cards
|
||||
var foundAllPriority = foundPriorityWithSet.Count >= priorityNeeded;
|
||||
if (existingCount + newHashes.Count >= maxCards && foundAllPriority)
|
||||
{
|
||||
Console.WriteLine($"\nReached {maxCards} cards limit with all priority cards");
|
||||
break;
|
||||
}
|
||||
|
||||
// Rate limit to be nice to Scryfall
|
||||
await Task.Delay(50);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"error: {ex.Message}");
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($"Skipped (already in DB): {skipped}");
|
||||
Console.WriteLine($"Newly processed: {processed} cards");
|
||||
Console.WriteLine($"New priority cards found: {priorityFound}");
|
||||
Console.WriteLine($"Total priority cards: {foundPriorityWithSet.Count}/{priorityNeeded}");
|
||||
Console.WriteLine($"Errors: {errors}");
|
||||
Console.WriteLine();
|
||||
|
||||
if (newHashes.Count > 0)
|
||||
{
|
||||
Console.WriteLine($"Inserting {newHashes.Count} new hashes into database...");
|
||||
await db.InsertHashBatchAsync(newHashes);
|
||||
}
|
||||
|
||||
await db.SetMetadataAsync("generated_at", DateTime.UtcNow.ToString("O"));
|
||||
await db.SetMetadataAsync("scryfall_updated_at", scryfallDateStr);
|
||||
|
||||
var finalCount = await db.GetHashCountAsync();
|
||||
await db.SetMetadataAsync("card_count", finalCount.ToString());
|
||||
|
||||
Console.WriteLine($"Database now has {finalCount} cards: {outputDb}");
|
||||
|
||||
// Report missing priority cards
|
||||
if (includeTestCards)
|
||||
{
|
||||
var missing = priorityCards.Where(c => !foundPriorityWithSet.ContainsKey(c)).ToList();
|
||||
|
||||
if (missing.Count > 0)
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($"Missing priority cards ({missing.Count}):");
|
||||
foreach (var name in missing.Take(20))
|
||||
{
|
||||
Console.WriteLine($" - {name}");
|
||||
}
|
||||
if (missing.Count > 20)
|
||||
{
|
||||
Console.WriteLine($" ... and {missing.Count - 20} more");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
Loading…
Add table
Add a link
Reference in a new issue