chris/scry
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

.

Browse source
This commit is contained in:
Chris Kruining 2026-02-05 11:34:57 +01:00
parent 0801ceee6a
commit 54ba7496c6
No known key found for this signature in database
GPG key ID: EB894A3560CCCAD2
19 changed files with 1765 additions and 591 deletions
Download patch file Download diff file Expand all files Collapse all files

95
tools/DbGenerator/Program.cs
View file

@ -151,15 +151,36 @@ using var httpClient = new HttpClient();
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("Scry/1.0 (MTG Card Scanner - Database Generator)");
using var scryfallClient = new ScryfallClient(httpClient);
using var db = new CardHashDatabase(outputDb);
using var db = new CardDatabase(outputDb);
// Check existing database state
var existingCardIds = await db.GetExistingCardIdsAsync();
var existingCardNames = await db.GetExistingCardNamesAsync();
var existingCount = await db.GetHashCountAsync();
var existingOracleIds = await db.GetExistingOracleIdsAsync();
var existingSetIds = await db.GetExistingSetIdsAsync();
var existingCount = await db.GetCardCountAsync();
var storedScryfallDate = await db.GetMetadataAsync("scryfall_updated_at");
Console.WriteLine($"Existing database has {existingCount} cards");
Console.WriteLine($"Existing database has {existingCount} cards, {existingOracleIds.Count} oracles, {existingSetIds.Count} sets");
// Fetch all sets first
Console.WriteLine("Fetching sets from Scryfall...");
var scryfallSets = await scryfallClient.GetAllSetsAsync();
var setsById = scryfallSets.ToDictionary(s => s.Id ?? "", s => s);
var setsByCode = scryfallSets.ToDictionary(s => s.Code ?? "", s => s, StringComparer.OrdinalIgnoreCase);
Console.WriteLine($"Found {scryfallSets.Count} sets");
// Insert any new sets
var newSets = scryfallSets
.Where(s => s.Id != null && !existingSetIds.Contains(s.Id))
.Select(s => s.ToSet())
.ToList();
if (newSets.Count > 0)
{
Console.WriteLine($"Inserting {newSets.Count} new sets...");
await db.InsertSetBatchAsync(newSets);
}
Console.WriteLine("Fetching bulk data info from Scryfall...");
var bulkInfo = await scryfallClient.GetBulkDataInfoAsync("unique_artwork");
@ -198,7 +219,8 @@ if (!needsUpdate)
Console.WriteLine($"Downloading card data from: {bulkInfo.DownloadUri}");
Console.WriteLine();
var newHashes = new List<CardHash>();
var newCards = new List<Card>();
var newOracles = new Dictionary<string, Oracle>();
var processed = 0;
var errors = 0;
var skipped = 0;
@ -209,10 +231,6 @@ var priorityNeeded = includeTestCards ? priorityCards.Count : 0;
// Key: card name, Value: set code
var foundPriorityWithSet = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
// Track deferred priority cards that might get a better set later
// Key: card name, Value: list of (cardHash, setCode) candidates
var deferredPriority = new Dictionary<string, List<(CardHash hash, string set)>>(StringComparer.OrdinalIgnoreCase);
// Helper to check if a set is preferred for a priority card
bool IsPreferredSet(string cardName, string setCode)
{
@ -221,19 +239,21 @@ bool IsPreferredSet(string cardName, string setCode)
return preferredSets.Length == 0 || preferredSets.Contains(setCode, StringComparer.OrdinalIgnoreCase);
}
await foreach (var card in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadUri))
await foreach (var scryfallCard in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadUri))
{
// Skip non-English cards
if (card.Lang != "en")
if (scryfallCard.Lang != "en")
continue;
var imageUri = card.GetImageUri("normal");
var imageUri = scryfallCard.GetImageUri("normal");
if (string.IsNullOrEmpty(imageUri))
continue;
var cardId = card.Id ?? Guid.NewGuid().ToString();
var cardName = card.Name ?? "Unknown";
var setCode = card.Set ?? "???";
var cardId = scryfallCard.Id ?? Guid.NewGuid().ToString();
var cardName = scryfallCard.Name ?? "Unknown";
var setCode = scryfallCard.Set ?? "???";
var oracleId = scryfallCard.OracleId ?? cardId;
var setId = scryfallCard.SetId ?? "";
// Check if this card already exists in the database
if (existingCardIds.Contains(cardId))
@ -262,7 +282,7 @@ await foreach (var card in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadU
}
// Calculate how many slots we have left
var totalCards = existingCount + newHashes.Count;
var totalCards = existingCount + newCards.Count;
var priorityRemaining = priorityNeeded - foundPriorityWithSet.Count;
var slotsForNonPriority = maxCards - priorityRemaining;
@ -289,17 +309,15 @@ await foreach (var card in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadU
using var preprocessed = ImagePreprocessor.ApplyClahe(bitmap);
var hash = PerceptualHash.ComputeColorHash(preprocessed);
var cardHash = new CardHash
{
CardId = cardId,
Name = cardName,
SetCode = setCode,
CollectorNumber = card.CollectorNumber,
Hash = hash,
ImageUri = imageUri
};
// Create Card (printing) with hash
var card = scryfallCard.ToCard() with { Hash = hash };
newCards.Add(card);
newHashes.Add(cardHash);
// Track Oracle if we haven't seen it
if (!existingOracleIds.Contains(oracleId) && !newOracles.ContainsKey(oracleId))
{
newOracles[oracleId] = scryfallCard.ToOracle();
}
if (isPriorityCard)
{
@ -316,7 +334,7 @@ await foreach (var card in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadU
// Check if we have enough cards
var foundAllPriority = foundPriorityWithSet.Count >= priorityNeeded;
if (existingCount + newHashes.Count >= maxCards && foundAllPriority)
if (existingCount + newCards.Count >= maxCards && foundAllPriority)
{
Console.WriteLine($"\nReached {maxCards} cards limit with all priority cards");
break;
@ -335,24 +353,37 @@ await foreach (var card in scryfallClient.StreamBulkDataAsync(bulkInfo.DownloadU
Console.WriteLine();
Console.WriteLine($"Skipped (already in DB): {skipped}");
Console.WriteLine($"Newly processed: {processed} cards");
Console.WriteLine($"New oracles: {newOracles.Count}");
Console.WriteLine($"New priority cards found: {priorityFound}");
Console.WriteLine($"Total priority cards: {foundPriorityWithSet.Count}/{priorityNeeded}");
Console.WriteLine($"Errors: {errors}");
Console.WriteLine();
if (newHashes.Count > 0)
// Insert oracles first (cards reference them)
if (newOracles.Count > 0)
{
Console.WriteLine($"Inserting {newHashes.Count} new hashes into database...");
await db.InsertHashBatchAsync(newHashes);
Console.WriteLine($"Inserting {newOracles.Count} new oracles...");
await db.InsertOracleBatchAsync(newOracles.Values);
}
if (newCards.Count > 0)
{
Console.WriteLine($"Inserting {newCards.Count} new cards...");
await db.InsertCardBatchAsync(newCards);
}
await db.SetMetadataAsync("generated_at", DateTime.UtcNow.ToString("O"));
await db.SetMetadataAsync("scryfall_updated_at", scryfallDateStr);
var finalCount = await db.GetHashCountAsync();
await db.SetMetadataAsync("card_count", finalCount.ToString());
var finalCardCount = await db.GetCardCountAsync();
var finalOracleCount = await db.GetOracleCountAsync();
var finalSetCount = await db.GetSetCountAsync();
Console.WriteLine($"Database now has {finalCount} cards: {outputDb}");
await db.SetMetadataAsync("card_count", finalCardCount.ToString());
await db.SetMetadataAsync("oracle_count", finalOracleCount.ToString());
await db.SetMetadataAsync("set_count", finalSetCount.ToString());
Console.WriteLine($"Database now has {finalCardCount} cards, {finalOracleCount} oracles, {finalSetCount} sets: {outputDb}");
// Report missing priority cards
if (includeTestCards)