From c2915d909dde18904870589b0871a22a73c6ab10 Mon Sep 17 00:00:00 2001 From: Chris H Date: Sun, 7 Jan 2018 21:45:21 -0500 Subject: [PATCH 1/6] Update auto-fetcher to use Set LQ Pictures --- .../src/main/java/forge/ImageFetcher.java | 13 ++++++------- .../main/java/forge/properties/ForgeConstants.java | 3 ++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/forge-gui-desktop/src/main/java/forge/ImageFetcher.java b/forge-gui-desktop/src/main/java/forge/ImageFetcher.java index ac179136018..4634dee32fa 100644 --- a/forge-gui-desktop/src/main/java/forge/ImageFetcher.java +++ b/forge-gui-desktop/src/main/java/forge/ImageFetcher.java @@ -51,7 +51,11 @@ public class ImageFetcher { final String filename = ImageUtil.getImageKey(paperCard, backFace, true); destFile = new File(ForgeConstants.CACHE_CARD_PICS_DIR + "/" + filename + ".jpg"); - // First, try to fetch from magiccards.info, if we have the collector's number to generate a URL. + // First try to download the LQ Set URL, then fetch from scryfall/magiccards.info + StringBuilder setDownload = new StringBuilder(ForgeConstants.URL_PIC_DOWNLOAD); + setDownload.append(ImageUtil.getDownloadUrl(paperCard, backFace)); + downloadUrls.add(setDownload.toString()); + final StaticData data = StaticData.instance(); final int cardNum = data.getCommonCards().getCardCollectorNumber(paperCard.getName(), paperCard.getEdition()); if (cardNum != -1) { @@ -60,14 +64,9 @@ public class ImageFetcher { suffix = (backFace ? "b" : "a"); } final String editionMciCode = data.getEditions().getMciCodeByCode(paperCard.getEdition()); + downloadUrls.add(String.format("https://img.scryfall.com/cards/normal/en/%s/%d%s.jpg", editionMciCode, cardNum, suffix)); downloadUrls.add(String.format("http://magiccards.info/scans/en/%s/%d%s.jpg", editionMciCode, cardNum, suffix)); } - - // Otherwise, try the LQ image URL. - final String[] fallbackUrlParts = ImageUtil.getDownloadUrlAndDestination(ForgeConstants.CACHE_CARD_PICS_DIR, paperCard, backFace); - if (fallbackUrlParts != null) { - downloadUrls.add(fallbackUrlParts[0]); - } } else if (prefix.equals(ImageKeys.TOKEN_PREFIX)) { if (tokenImages == null) { tokenImages = new HashMap<>(); diff --git a/forge-gui/src/main/java/forge/properties/ForgeConstants.java b/forge-gui/src/main/java/forge/properties/ForgeConstants.java index c69c982a92b..eb58b3e76ec 100644 --- a/forge-gui/src/main/java/forge/properties/ForgeConstants.java +++ b/forge-gui/src/main/java/forge/properties/ForgeConstants.java @@ -276,7 +276,8 @@ public final class ForgeConstants { // URLs private static final String URL_CARDFORGE = "https://downloads.cardforge.org"; public static final String URL_PIC_DOWNLOAD = URL_CARDFORGE + "/images/cards/"; - public static final String URL_PRICE_DOWNLOAD = "https://downloads.cardforge.org/all-prices.txt"; + public static final String URL_TOKEN_DOWNLOAD = URL_CARDFORGE + "/images/tokens/"; + public static final String URL_PRICE_DOWNLOAD = URL_CARDFORGE + "/all-prices.txt"; // Constants for Display Card Identity game setting public static final String DISP_CURRENT_COLORS_ALWAYS = "Always"; From d60ee9307d7d280d5a9f6e2da8a18f6db242d828 Mon Sep 17 00:00:00 2001 From: Chris H Date: Mon, 8 Jan 2018 21:16:26 -0500 Subject: [PATCH 2/6] Grab LQ picture from primary server --- .../src/main/java/forge/card/CardRules.java | 4 +++ .../src/main/java/forge/util/ImageUtil.java | 31 ------------------- .../forge/download/GuiDownloadPicturesLQ.java | 16 +++++----- 3 files changed, 13 insertions(+), 38 deletions(-) diff --git a/forge-core/src/main/java/forge/card/CardRules.java b/forge-core/src/main/java/forge/card/CardRules.java index 0837697d1eb..61bb1a29dc2 100644 --- a/forge-core/src/main/java/forge/card/CardRules.java +++ b/forge-core/src/main/java/forge/card/CardRules.java @@ -231,7 +231,11 @@ public final class CardRules implements ICardCharacteristics { // Downloadable image private String dlUrl; private String dlUrlOtherSide; + + @Deprecated public String getPictureUrl(boolean backface ) { return backface ? dlUrlOtherSide : dlUrl; } + + @Deprecated public void setDlUrls(String[] dlUrls) { this.dlUrl = dlUrls[0]; this.dlUrlOtherSide = dlUrls[1]; } public ColorSet getColorIdentity() { diff --git a/forge-core/src/main/java/forge/util/ImageUtil.java b/forge-core/src/main/java/forge/util/ImageUtil.java index 29ddd5a9c06..8dbbe9c6186 100644 --- a/forge-core/src/main/java/forge/util/ImageUtil.java +++ b/forge-core/src/main/java/forge/util/ImageUtil.java @@ -118,37 +118,6 @@ public class ImageUtil { public static String getDownloadUrl(PaperCard cp, boolean backFace) { return getImageRelativePath(cp, backFace, true, true); } - - public static String[] getDownloadUrlAndDestination(String cacheCardPicsDir, PaperCard c, boolean backFace) { - final CardRules cardRules = c.getRules(); - final String urls = cardRules.getPictureUrl(backFace); - if (StringUtils.isEmpty(urls)) { - return null; - } - - String filename = ImageUtil.getImageKey(c, backFace, false); - final File destFile = new File(cacheCardPicsDir, filename + ".jpg"); - if (destFile.exists()) { - return null; - } - - filename = destFile.getAbsolutePath(); - - final String urlToDownload; - int urlIndex = 0; - int allUrlsLen = 1; - if (!urls.contains("\\")) { - urlToDownload = urls; - } else { - final String[] allUrls = urls.split("\\\\"); - allUrlsLen = allUrls.length; - urlIndex = (c.getArtIndex()-1) % allUrlsLen; - urlToDownload = allUrls[urlIndex]; - } - // System.out.println(c.getName() + "|" + c.getEdition() + " - " + c.getArtIndex() + " -> " + urlIndex + "/" + allUrlsLen + " === " + filename + " <<< " + urlToDownload); - - return new String[] { urlToDownload, filename }; - } public static String toMWSFilename(String in) { final StringBuilder out = new StringBuilder(); diff --git a/forge-gui/src/main/java/forge/download/GuiDownloadPicturesLQ.java b/forge-gui/src/main/java/forge/download/GuiDownloadPicturesLQ.java index 6707f6251c4..6861b10e402 100644 --- a/forge-gui/src/main/java/forge/download/GuiDownloadPicturesLQ.java +++ b/forge-gui/src/main/java/forge/download/GuiDownloadPicturesLQ.java @@ -22,6 +22,7 @@ import forge.model.FModel; import forge.properties.ForgeConstants; import forge.util.ImageUtil; +import java.io.File; import java.util.Map; import java.util.TreeMap; @@ -53,15 +54,16 @@ public class GuiDownloadPicturesLQ extends GuiDownloadService { } private static void addDLObject(final PaperCard c, final Map downloads, final boolean backFace) { - String[] result = ImageUtil.getDownloadUrlAndDestination(ForgeConstants.CACHE_CARD_PICS_DIR, c, backFace); - if (result == null) { - return; - } - final String urlToDownload = result[0]; - final String destPath = result[1]; + final String destPath = ForgeConstants.CACHE_CARD_PICS_DIR + ImageUtil.getImageKey(c, backFace, false); if (downloads.containsKey(destPath)) { return; } - downloads.put(destPath, urlToDownload); + + File destFile = new File(destPath); + if (destFile.exists()) { + return; + } + + downloads.put(destPath, ForgeConstants.URL_PIC_DOWNLOAD + ImageUtil.getDownloadUrl(c, backFace)); } } From d646240c6346cb517754fbbc74271f5998772e18 Mon Sep 17 00:00:00 2001 From: Chris H Date: Sat, 13 Jan 2018 20:14:06 -0500 Subject: [PATCH 3/6] Update magiccards.info protocol --- forge-gui-desktop/src/main/java/forge/ImageFetcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/forge-gui-desktop/src/main/java/forge/ImageFetcher.java b/forge-gui-desktop/src/main/java/forge/ImageFetcher.java index 4634dee32fa..09b6b2141cb 100644 --- a/forge-gui-desktop/src/main/java/forge/ImageFetcher.java +++ b/forge-gui-desktop/src/main/java/forge/ImageFetcher.java @@ -65,7 +65,7 @@ public class ImageFetcher { } final String editionMciCode = data.getEditions().getMciCodeByCode(paperCard.getEdition()); downloadUrls.add(String.format("https://img.scryfall.com/cards/normal/en/%s/%d%s.jpg", editionMciCode, cardNum, suffix)); - downloadUrls.add(String.format("http://magiccards.info/scans/en/%s/%d%s.jpg", editionMciCode, cardNum, suffix)); + downloadUrls.add(String.format("https://magiccards.info/scans/en/%s/%d%s.jpg", editionMciCode, cardNum, suffix)); } } else if (prefix.equals(ImageKeys.TOKEN_PREFIX)) { if (tokenImages == null) { From 43455723f633032f0540efec433158a2598ab43a Mon Sep 17 00:00:00 2001 From: Chris H Date: Sun, 14 Jan 2018 17:49:46 -0500 Subject: [PATCH 4/6] Check for existing images in a single IO request. --- .../forge/download/GuiDownloadPicturesLQ.java | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/forge-gui/src/main/java/forge/download/GuiDownloadPicturesLQ.java b/forge-gui/src/main/java/forge/download/GuiDownloadPicturesLQ.java index 6861b10e402..e50e519849f 100644 --- a/forge-gui/src/main/java/forge/download/GuiDownloadPicturesLQ.java +++ b/forge-gui/src/main/java/forge/download/GuiDownloadPicturesLQ.java @@ -23,6 +23,8 @@ import forge.properties.ForgeConstants; import forge.util.ImageUtil; import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Map; import java.util.TreeMap; @@ -36,15 +38,18 @@ public class GuiDownloadPicturesLQ extends GuiDownloadService { protected final Map getNeededFiles() { final Map downloads = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + File f = new File(ForgeConstants.CACHE_CARD_PICS_DIR); + ArrayList existingImages = new ArrayList(Arrays.asList(f.list())); + for (final PaperCard c : FModel.getMagicDb().getCommonCards().getAllCards()) { - addDLObject(c, downloads, false); + addDLObject(c, downloads, false, existingImages); if (ImageUtil.hasBackFacePicture(c)) { - addDLObject(c, downloads, true); + addDLObject(c, downloads, true, existingImages); } } for (final PaperCard c : FModel.getMagicDb().getVariantCards().getAllCards()) { - addDLObject(c, downloads, false); + addDLObject(c, downloads, false, existingImages); } // Add missing tokens to the list of things to download. @@ -53,14 +58,15 @@ public class GuiDownloadPicturesLQ extends GuiDownloadService { return downloads; } - private static void addDLObject(final PaperCard c, final Map downloads, final boolean backFace) { - final String destPath = ForgeConstants.CACHE_CARD_PICS_DIR + ImageUtil.getImageKey(c, backFace, false); - if (downloads.containsKey(destPath)) { + private static void addDLObject(final PaperCard c, final Map downloads, final boolean backFace, ArrayList existingImages) { + final String imageKey = ImageUtil.getImageKey(c, backFace, false); + final String destPath = ForgeConstants.CACHE_CARD_PICS_DIR + imageKey; + + if (existingImages.contains(imageKey)) { return; } - File destFile = new File(destPath); - if (destFile.exists()) { + if (downloads.containsKey(destPath)) { return; } From 23618de189663e7f1d61f9555ac977a955c2150b Mon Sep 17 00:00:00 2001 From: Chris H Date: Sun, 14 Jan 2018 23:19:25 -0500 Subject: [PATCH 5/6] Improve LQ Pictures Downloader --- .../forge/download/GuiDownloadPicturesLQ.java | 35 ++++++++++++------- .../forge/download/GuiDownloadService.java | 26 ++++++++++++++ .../download/GuiDownloadSetPicturesLQ.java | 24 ------------- 3 files changed, 48 insertions(+), 37 deletions(-) diff --git a/forge-gui/src/main/java/forge/download/GuiDownloadPicturesLQ.java b/forge-gui/src/main/java/forge/download/GuiDownloadPicturesLQ.java index e50e519849f..6bcbee8ce0a 100644 --- a/forge-gui/src/main/java/forge/download/GuiDownloadPicturesLQ.java +++ b/forge-gui/src/main/java/forge/download/GuiDownloadPicturesLQ.java @@ -17,18 +17,20 @@ */ package forge.download; +import forge.StaticData; import forge.item.PaperCard; import forge.model.FModel; import forge.properties.ForgeConstants; import forge.util.ImageUtil; import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Map; -import java.util.TreeMap; +import java.util.*; public class GuiDownloadPicturesLQ extends GuiDownloadService { + final Map downloads = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); + Set existingSets; + ArrayList existingImages; + @Override public String getTitle() { return "Download LQ Card Pictures"; @@ -36,20 +38,19 @@ public class GuiDownloadPicturesLQ extends GuiDownloadService { @Override protected final Map getNeededFiles() { - final Map downloads = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - File f = new File(ForgeConstants.CACHE_CARD_PICS_DIR); - ArrayList existingImages = new ArrayList(Arrays.asList(f.list())); + existingImages = new ArrayList(Arrays.asList(f.list())); + existingSets = retrieveManifestDirectory(); for (final PaperCard c : FModel.getMagicDb().getCommonCards().getAllCards()) { - addDLObject(c, downloads, false, existingImages); + addDLObject(c, false); if (ImageUtil.hasBackFacePicture(c)) { - addDLObject(c, downloads, true, existingImages); + addDLObject(c, true); } } for (final PaperCard c : FModel.getMagicDb().getVariantCards().getAllCards()) { - addDLObject(c, downloads, false, existingImages); + addDLObject(c, false); } // Add missing tokens to the list of things to download. @@ -58,11 +59,11 @@ public class GuiDownloadPicturesLQ extends GuiDownloadService { return downloads; } - private static void addDLObject(final PaperCard c, final Map downloads, final boolean backFace, ArrayList existingImages) { + private void addDLObject(final PaperCard c, final boolean backFace) { final String imageKey = ImageUtil.getImageKey(c, backFace, false); - final String destPath = ForgeConstants.CACHE_CARD_PICS_DIR + imageKey; + final String destPath = ForgeConstants.CACHE_CARD_PICS_DIR + imageKey + ".jpg"; - if (existingImages.contains(imageKey)) { + if (existingImages.contains(imageKey + ".jpg")) { return; } @@ -70,6 +71,14 @@ public class GuiDownloadPicturesLQ extends GuiDownloadService { return; } + final String setCode3 = c.getEdition(); + final String setCode2 = StaticData.instance().getEditions().getCode2ByCode(setCode3); + + if (!(existingSets.contains(setCode3) || existingSets.contains(setCode2))) { + // If set doesn't exist on server, don't try to download cards for it + return; + } + downloads.put(destPath, ForgeConstants.URL_PIC_DOWNLOAD + ImageUtil.getDownloadUrl(c, backFace)); } } diff --git a/forge-gui/src/main/java/forge/download/GuiDownloadService.java b/forge-gui/src/main/java/forge/download/GuiDownloadService.java index c09427dc659..eadd9cc4e8d 100644 --- a/forge-gui/src/main/java/forge/download/GuiDownloadService.java +++ b/forge-gui/src/main/java/forge/download/GuiDownloadService.java @@ -25,11 +25,14 @@ import forge.error.BugReporter; import forge.interfaces.IButton; import forge.interfaces.IProgressBar; import forge.interfaces.ITextField; +import forge.properties.ForgeConstants; import forge.util.FileUtil; +import forge.util.HttpUtil; import org.apache.commons.lang3.tuple.Pair; import java.io.*; import java.net.*; +import java.util.HashSet; import java.util.Map; import java.util.Map.Entry; @@ -340,4 +343,27 @@ public abstract class GuiDownloadService implements Runnable { } } } + + protected static HashSet retrieveManifestDirectory() { + String manifestUrl = ForgeConstants.URL_PIC_DOWNLOAD; + HashSet existingSets = new HashSet<>(); + + String response = HttpUtil.getURL(manifestUrl); + + if (response == null) return null; + + String[] strings = response.split(" 4 || idx == -1) { + continue; + } + + String set = s.substring(0, idx); + existingSets.add(set); + } + + return existingSets; + } } diff --git a/forge-gui/src/main/java/forge/download/GuiDownloadSetPicturesLQ.java b/forge-gui/src/main/java/forge/download/GuiDownloadSetPicturesLQ.java index bad57de0b12..b5ce44df1cb 100644 --- a/forge-gui/src/main/java/forge/download/GuiDownloadSetPicturesLQ.java +++ b/forge-gui/src/main/java/forge/download/GuiDownloadSetPicturesLQ.java @@ -23,7 +23,6 @@ import forge.card.CardEdition; import forge.item.PaperCard; import forge.model.FModel; import forge.properties.ForgeConstants; -import forge.util.HttpUtil; import forge.util.ImageUtil; import org.apache.commons.lang3.StringUtils; @@ -87,27 +86,4 @@ public class GuiDownloadSetPicturesLQ extends GuiDownloadService { downloads.put(destFile.getAbsolutePath(), ForgeConstants.URL_PIC_DOWNLOAD + urlPath); } } - - private HashSet retrieveManifestDirectory() { - String manifestUrl = ForgeConstants.URL_PIC_DOWNLOAD; - HashSet existingSets = new HashSet<>(); - - String response = HttpUtil.getURL(manifestUrl); - - if (response == null) return null; - - String[] strings = response.split(" 4 || idx == -1) { - continue; - } - - String set = s.substring(0, idx); - existingSets.add(set); - } - - return existingSets; - } } From 25ca71a97a23368bea98f63f017677b9d5577de9 Mon Sep 17 00:00:00 2001 From: Chris H Date: Mon, 15 Jan 2018 17:10:53 -0500 Subject: [PATCH 6/6] Add Picture work to Changes file --- forge-gui/release-files/CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/forge-gui/release-files/CHANGES.txt b/forge-gui/release-files/CHANGES.txt index 5944badb670..d82813ed8be 100644 --- a/forge-gui/release-files/CHANGES.txt +++ b/forge-gui/release-files/CHANGES.txt @@ -1,3 +1,6 @@ +- Download Pictures - +Improvements to Download Pictures and Auto fetcher + - Bug fixes - As always, this release of Forge features an assortment of bug fixes and improvements based on user feedback during the previous release run.