#!/usr/bin/env python3
"""Little script to scrap data from play-in.com for french translations."""
__author__ = 'add-le'
import sys
import requests
import re
file = open('../res/languages/cardnames-fr-FR.txt', 'r')
lines = file.readlines()
file.close()
counter = 0
#Scrap from play-in.com
url = 'https://www.play-in.com/recherche/result.php?s='
#Constants
SUCCESS_STATUS = 200
def convertMana(cardInfo: str) -> str:
"""Convert HTML tag to Forge MTG compatible tags."""
cardInfo = cardInfo.replace('
', '\\n')
#Replace all correspondances of [0-9A-Z]
symbol = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '100', '(', ')']
for sym in symbol:
cardInfo = cardInfo.replace("
", '{' + sym + '}')
return cardInfo
#Manuel scrap
if len(sys.argv) >= 2:
if sys.argv[1] == '-u' or sys.argv[1] == '-url':
if len(sys.argv) <= 2:
print('Missing second arg : url to scrap')
exit(1)
output = open('cardnames-fr-FR-missing.txt', 'a')
response = requests.get(sys.argv[2])
#Founded
if response.status_code == SUCCESS_STATUS:
#Get brut chaos the data from web page
try:
match = re.search(re.compile('
.+?<\/div>
<\/div>
.+?<\/div>
<\/div>
.+?<\/div>(
|<\/div>)'), response.content.decode('utf-8').replace('\n', '')).group()
#Split in different tags, interesting data
translatedName: str = re.findall(r'
.+?<\/title>', response.content.decode('utf-8').replace('\n', ''))[0].split('')[1].split(' -')[0]
orignalName: str = re.findall(r'.+?<\/title>', response.content.decode('utf-8').replace('\n', ''))[0].split('')[1].split(' -')[1].split(' -')[0]
translatedType: str = match.split('')[1].split('
')[0]
translatedInfo: str = convertMana(match.split('
')[1].split('
')[0])
#print(orignalName + '|' + translatedName + '|' + translatedType.replace(' ', '') + '|' + translatedInfo)
output.writelines(orignalName + '|' + translatedName + '|' + translatedType.replace(' ', '') + '|' + translatedInfo + '\n')
except IndexError:
#Request probably gives multiple value
print(response.url)
output.close()
exit(0)
output = open('cardnames-fr-FR-missing.txt', 'a')
#Show only missing lines
for line in lines:
#None complete line
if line.endswith('||\n'):
engName = line.split('|')[0]
#Check already done
with open('cardnames-fr-FR-missing.txt') as f:
if engName in f.read():
continue
response = requests.get(url + engName)
#Founded
if response.url.endswith(engName) == False and response.status_code == SUCCESS_STATUS:
#print(counter, end='\r', flush=True)
#Get brut chaos the data from web page
try:
match = re.search(re.compile('
.+?<\/div>
<\/div>
.+?<\/div>
<\/div>
.+?<\/div>(
|<\/div>)'), response.content.decode('utf-8').replace('\n', '')).group()
#Split in different tags, interesting data
translatedName: str = re.findall(r'
.+?<\/title>', response.content.decode('utf-8').replace('\n', ''))[0].split('')[1].split(' -')[0]
translatedType: str = match.split('')[1].split('
')[0]
translatedInfo: str = convertMana(match.split('
')[1].split('
')[0])
#print(engName + '|' + translatedName + '|' + translatedType.replace(' ', '') + '|' + translatedInfo)
output.writelines(engName + '|' + translatedName + '|' + translatedType.replace(' ', '') + '|' + translatedInfo + '\n')
except IndexError:
#Request probably gives multiple value
print(response.url)
except AttributeError:
#Request probably gives multiple value
print(response.url)
except UnicodeDecodeError:
#Request probably gives multiple value
print(response.url)
counter += 1
output.close()
print('end: ', counter)