From ddb9caec4819605eba073d1aaad98ef0d8966086 Mon Sep 17 00:00:00 2001 From: Sol Date: Sat, 21 Nov 2015 03:44:02 +0000 Subject: [PATCH] - Adding/updating some more tools that use mtgjson data --- .gitattributes | 1 + forge-gui/tools/EditionTracking.py | 65 ++++++++++-- forge-gui/tools/jsonOraclizer.py | 155 +++++++++++++++++++++++++++++ 3 files changed, 215 insertions(+), 6 deletions(-) create mode 100644 forge-gui/tools/jsonOraclizer.py diff --git a/.gitattributes b/.gitattributes index e3bfba83dcd..2c8ce02f337 100644 --- a/.gitattributes +++ b/.gitattributes @@ -18391,6 +18391,7 @@ forge-gui/tools/extractSetInfo.py -text forge-gui/tools/formatCards -text forge-gui/tools/formats.txt -text forge-gui/tools/guilds.xlsx -text +forge-gui/tools/jsonOraclizer.py -text forge-gui/tools/missingPreconFiles.py -text forge-gui/tools/mtg-data.txt svneol=native#text/plain forge-gui/tools/mtgdata-sets-to-forge.txt svneol=native#text/plain diff --git a/forge-gui/tools/EditionTracking.py b/forge-gui/tools/EditionTracking.py index 401cd749d85..e755b0fe936 100644 --- a/forge-gui/tools/EditionTracking.py +++ b/forge-gui/tools/EditionTracking.py @@ -1,11 +1,15 @@ #!/usr/bin/env python ############IMPLEMENTATION FOLLOWS############ +import json import os,sys,fnmatch,re +import requests toolsDir = os.path.abspath(os.path.dirname( __file__ )) resDir = os.path.abspath(os.path.join(toolsDir, '..', 'res')) editionsDir = os.path.join(resDir, 'editions') +allJson = os.path.join(toolsDir, 'AllCards.json') +allJsonUrl = 'http://mtgjson.com/json/AllCards.json' def initializeEditions(): ignoredTypes = [ "From_the_Vault", "Duel_Decks", "Online", "Premium_Deck_Series" ] @@ -50,7 +54,26 @@ def initializeEditions(): print "These sets will be ignored in some output files", ignoredSet def initializeOracleText(): - pass + print "Initializing Oracle text" + oracleDict = None + if not os.path.exists(allJson): + print "Need to download Json file..." + r = requests.get(allJsonUrl) + with open(allJson, 'w') as f: + f.write(r.text) + + oracleDict = r.json() + + else: + with open(allJson) as f: + oracleDict = json.loads(f.read()) + + print "Found Oracle text for ", len(oracleDict) + return oracleDict + +def normalizeOracle(oracle): + return oracle.replace(u'\u2014', '-').replace(u'\u2018', "'").replace(u'\u201c', '"').replace(u'\u201d', '"').replace(u'\u2022', '-') + def initializeForgeCards(): #Parse Forge @@ -185,8 +208,12 @@ def printDistinctOracle(missingSet, fileName): with open(filePath, "w") as outfile: for s in missing: if s: - oracle = mtgOracleCards.get(s, "") - outfile.write("%s\n%s" % (s, oracle)) + try: + oracle = normalizeOracle(mtgOracleCards.get(s).get('text')) + outfile.write("%s\n%s\n\n" % (s, oracle)) + except: + outfile.write("%s\n\n" % (s)) + print "Failed to grab oracle for ", s outfile.write("\n") @@ -198,7 +225,6 @@ if __name__ == '__main__': forgeFolderFiles = [] forgeCards = [] mtgDataCards = {} - mtgOracleCards = {} setCodes = [] setCodeToName = {} forgeCardCount = 0 @@ -215,6 +241,7 @@ if __name__ == '__main__': initializeEditions() initializeForgeCards() + mtgOracleCards = initializeOracleText() #Compare datasets and output results print("Comparing datasets and outputting results.") @@ -263,10 +290,36 @@ if __name__ == '__main__': for everyImplemented in currentImplemented : output.write(everyImplemented + '\n') output.write("\n") - output.write("Missing (" + str(len(currentMissing)) + "):\n") + output.write("Missing (%s):\n" % len(currentMissing)) for everyMissing in currentMissing : output.write(everyMissing + '\n') - output.write(mtgOracleCards.get(everyMissing, "")) + orc = mtgOracleCards.get(everyMissing) + try: + if 'manaCost' in orc: + output.write(orc.get('manaCost') + '\n') + + #output.write(' '.join(orc.get('supertypes', []))) + #output.write(' '.join(orc.get('types', []))) + #output.write(' '.join(orc.get('subtypes', []))) + output.write(normalizeOracle(orc.get('type'))) + output.write('\n') + + if 'power' in orc: + output.write( "PT:" + orc.get('power') + '/' + orc.get('toughness') + '\n') + if 'loyalty' in orc: + output.write('Loyalty:' + orc.get('loyalty')) + except: + print "some issue?" + + # Blah + + + try: + text = normalizeOracle(orc.get('text')) + except: + text = '' + + output.write(text + '\n\n') output.write("\n") output.write("Total: " + str(total) + "\n") output.write("Percentage implemented: " + str(round(percentage,2)) + "%\n") diff --git a/forge-gui/tools/jsonOraclizer.py b/forge-gui/tools/jsonOraclizer.py new file mode 100644 index 00000000000..b282738da1d --- /dev/null +++ b/forge-gui/tools/jsonOraclizer.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python + +# This python script is designed to handle the following: individual cards located in /res/cardsfolder/* +# Insert of Oracle data into data files from mtg-data.txt +# Future possibilities. Using mtg-data to add SetInfo data and other Outside Game Data (Type, PT, etc) +# Hopefully the oracleScript can replace both SetInfo Scripts by current SetInfo scripts by expanding their current functionality + +# python oracleScript.py +# If you run oracleScript without parameters it will run for all sets on the local mtgdata.txt + + +import os, fnmatch, re, sys +import json +from urllib import urlopen + +pathToMtgData = os.path.join(sys.path[0], "AllCards.json") + +singleSet = False +onlineOptions = [ 'false', 'f', 'no', 'n' ] +offlineSource = True +setAbbr = None + +if len(sys.argv) > 1: + offlineSource = (sys.argv[1].lower() not in onlineOptions) + print "Using AllCards.txt: " + str(offlineSource) + +if len(sys.argv) > 2: + if offlineSource: + print "Running for all sets when in Offline mode" + else: + setAbbr = sys.argv[2] + print "Using Set: " + setAbbr + +elif not offlineSource: + print "Please provide a set abbreviation when in Online Mode. eg: python oracleScript.py False GTC" + + +mtgData = None +if offlineSource: + parseFrom = open(pathToMtgData, 'r') + load = json.loads(parseFrom.read()) + parseFrom.close() + if singleSet: + mtgData = {} + for c in load['cards']: + mtgData[c['name']] = c + + else: + mtgData = load + + print "Number of cards loaded.. %s" % len(mtgData) +else: + # Load Spoiler view of the set + parseFrom = urlopen("http://magiccards.info/query?q=e:%s&v=spoiler&s=cname" % (setAbbr)) + mtgData = parseFrom.read() + parseFrom.close() + print "Size of parse data: %s" % len(mtgData) + +folder = os.path.join(sys.path[0], '..', 'res', 'cardsfolder') +err = open(os.path.join(sys.path[0], 'jsonOraclizerLog.log'), 'w') + +oracleStr = 'Oracle:' + +def writeOutCard(root, fileName, lines, oracle): + cardfile = open(os.path.join(root, fileName), 'w') + cardfile.write(lines) + + cardfile.write('Oracle:%s\n' % oracle) + cardfile.close() + + +def getOracleFromMtgData(name): + data = mtgData.get(name, None) + + if data is None: + err.write(name + '... NOT FOUND\n') + return None + + txt = data.get('text', '').replace(u'\u2014', '-').replace(u'\u2022', '-').replace(u'\u2212', '-') + return txt + +def getOracleFromMagicCardsInfo(name): + # Requires set to grab Oracle text from magiccards.info for simplicity meetings + # http://magiccards.info/query?q=e%3Agtc&v=spoiler&s=cname + search = '">%s' % name + found = mtgData.find(search) + + if found == -1: + err.write(name + '... NOT FOUND\n') + return None + + endFound = mtgData.find('

', found) + block = mtgData[found:endFound] + startOracle = '

' + + oracleStart = block.find(startOracle) + oracleBlock = block[oracleStart:] + oracle = oracleBlock[len(startOracle):].replace('

', '\\n') + return oracle + + +def hasOracleLine(cardFile, lines, offlineSource=True): + # Start parsing the rest of the data file + hasOracle = False + + for line in cardFile.readlines(): + line = line.strip() + # Skip empty lines + if line == '': + continue + + if line.find(oracleStr) != -1: + hasOracle = True + break + + else: + lines += line + '\n' + + cardFile.close() + return hasOracle, lines + +# parse cardsfolder for Card Lines and Rarity/Picture SVars. Filling in any gaps +for root, dirnames, filenames in os.walk(folder): + for fileName in fnmatch.filter(filenames, '*.txt'): + if fileName.startswith('.'): + continue + + file = open(os.path.join(root, fileName), 'r') + cleanName = fileName.replace('.txt', '') + + line = file.readline().strip() + # Handle name and creation + name = line.replace('Name:', '') + + hasOracle, lines = hasOracleLine(file, line + '\n', offlineSource) + + if hasOracle: + #print name + " already has Oracle" + continue + + if offlineSource: + oracle = getOracleFromMtgData(name) + else: + oracle = getOracleFromMagicCardsInfo(name) + + if oracle is None: + continue + + print name + print " => %s \n" % (oracle) + writeOutCard(root, fileName, lines, oracle) + + err.write(name + '... Updated\n') + +err.close()