Merge branch 'automatic_image_downloader' into 'master'

Automatic image downloader

See merge request core-developers/forge!107

forge-core/src/main/java/forge/card/CardRules.java

@@ -231,7 +231,11 @@ public final class CardRules implements ICardCharacteristics {
     // Downloadable image
     private String dlUrl;
     private String dlUrlOtherSide;
+
+    @Deprecated
     public String getPictureUrl(boolean backface ) { return backface ? dlUrlOtherSide : dlUrl; }
+
+    @Deprecated
     public void setDlUrls(String[] dlUrls) { this.dlUrl = dlUrls[0]; this.dlUrlOtherSide = dlUrls[1]; }
 
     public ColorSet getColorIdentity() {

forge-core/src/main/java/forge/util/ImageUtil.java

@@ -118,37 +118,6 @@ public class ImageUtil {
     public static String getDownloadUrl(PaperCard cp, boolean backFace) {
         return getImageRelativePath(cp, backFace, true, true);
     }
-    
-    public static String[] getDownloadUrlAndDestination(String cacheCardPicsDir, PaperCard c, boolean backFace) {
-        final CardRules cardRules = c.getRules();
-        final String urls = cardRules.getPictureUrl(backFace);
-        if (StringUtils.isEmpty(urls)) {
-            return null;
-        }
-
-        String filename = ImageUtil.getImageKey(c, backFace, false);
-        final File destFile = new File(cacheCardPicsDir, filename + ".jpg");
-        if (destFile.exists()) {
-            return null;
-        }
-
-        filename = destFile.getAbsolutePath();
-
-        final String urlToDownload;
-        int urlIndex = 0;
-        int allUrlsLen = 1;
-        if (!urls.contains("\\")) {
-            urlToDownload = urls;
-        } else {
-            final String[] allUrls = urls.split("\\\\");
-            allUrlsLen = allUrls.length;
-            urlIndex = (c.getArtIndex()-1) % allUrlsLen;
-            urlToDownload = allUrls[urlIndex];
-        }
-        // System.out.println(c.getName() + "|" + c.getEdition() + " - " + c.getArtIndex() + " -> " + urlIndex + "/" + allUrlsLen + " === " + filename + " <<< " + urlToDownload);
-
-        return new String[] { urlToDownload, filename };
-    }
 
     public static String toMWSFilename(String in) {
         final StringBuilder out = new StringBuilder();

forge-gui-desktop/src/main/java/forge/ImageFetcher.java

@@ -51,7 +51,11 @@ public class ImageFetcher {
             final String filename = ImageUtil.getImageKey(paperCard, backFace, true);
             destFile = new File(ForgeConstants.CACHE_CARD_PICS_DIR + "/" + filename + ".jpg");
 
-            // First, try to fetch from magiccards.info, if we have the collector's number to generate a URL.
+            // First try to download the LQ Set URL, then fetch from scryfall/magiccards.info
+            StringBuilder setDownload = new StringBuilder(ForgeConstants.URL_PIC_DOWNLOAD);
+            setDownload.append(ImageUtil.getDownloadUrl(paperCard, backFace));
+            downloadUrls.add(setDownload.toString());
+
             final StaticData data = StaticData.instance();
             final int cardNum = data.getCommonCards().getCardCollectorNumber(paperCard.getName(), paperCard.getEdition());
             if (cardNum != -1)  {
@@ -60,13 +64,8 @@ public class ImageFetcher {
                     suffix = (backFace ? "b" : "a");
                 }
                 final String editionMciCode = data.getEditions().getMciCodeByCode(paperCard.getEdition());
-                downloadUrls.add(String.format("http://magiccards.info/scans/en/%s/%d%s.jpg", editionMciCode, cardNum, suffix));
+                downloadUrls.add(String.format("https://img.scryfall.com/cards/normal/en/%s/%d%s.jpg", editionMciCode, cardNum, suffix));
-            }
+                downloadUrls.add(String.format("https://magiccards.info/scans/en/%s/%d%s.jpg", editionMciCode, cardNum, suffix));
-
-            // Otherwise, try the LQ image URL.
-            final String[] fallbackUrlParts = ImageUtil.getDownloadUrlAndDestination(ForgeConstants.CACHE_CARD_PICS_DIR, paperCard, backFace);
-            if (fallbackUrlParts != null) {
-                downloadUrls.add(fallbackUrlParts[0]);
             }
         } else if (prefix.equals(ImageKeys.TOKEN_PREFIX)) {
             if (tokenImages == null) {

forge-gui/release-files/CHANGES.txt

@@ -1,3 +1,6 @@
+- Download Pictures -
+Improvements to Download Pictures and Auto fetcher
+
 - Bug fixes -
 As always, this release of Forge features an assortment of bug fixes and improvements based on user feedback during the previous release run.
 

forge-gui/src/main/java/forge/download/GuiDownloadPicturesLQ.java

@@ -17,15 +17,20 @@
  */
 package forge.download;
 
+import forge.StaticData;
 import forge.item.PaperCard;
 import forge.model.FModel;
 import forge.properties.ForgeConstants;
 import forge.util.ImageUtil;
 
-import java.util.Map;
+import java.io.File;
-import java.util.TreeMap;
+import java.util.*;
 
 public class GuiDownloadPicturesLQ extends GuiDownloadService {
+    final Map<String, String> downloads = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+    Set<String> existingSets;
+    ArrayList<String> existingImages;
+
     @Override
     public String getTitle() {
         return "Download LQ Card Pictures";
@@ -33,17 +38,19 @@ public class GuiDownloadPicturesLQ extends GuiDownloadService {
 
     @Override
     protected final Map<String, String> getNeededFiles() {
-        final Map<String, String> downloads = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+        File f = new File(ForgeConstants.CACHE_CARD_PICS_DIR);
+        existingImages = new ArrayList<String>(Arrays.asList(f.list()));
+        existingSets = retrieveManifestDirectory();
 
         for (final PaperCard c : FModel.getMagicDb().getCommonCards().getAllCards()) {
-            addDLObject(c, downloads, false);
+            addDLObject(c, false);
             if (ImageUtil.hasBackFacePicture(c)) {
-                addDLObject(c, downloads, true);
+                addDLObject(c, true);
             }
         }
 
         for (final PaperCard c : FModel.getMagicDb().getVariantCards().getAllCards()) {
-            addDLObject(c, downloads, false);
+            addDLObject(c, false);
         }
 
         // Add missing tokens to the list of things to download.
@@ -52,16 +59,26 @@ public class GuiDownloadPicturesLQ extends GuiDownloadService {
         return downloads;
     }
 
-    private static void addDLObject(final PaperCard c, final Map<String, String> downloads, final boolean backFace) {
+    private void addDLObject(final PaperCard c, final boolean backFace) {
-        String[] result = ImageUtil.getDownloadUrlAndDestination(ForgeConstants.CACHE_CARD_PICS_DIR, c, backFace);
+        final String imageKey = ImageUtil.getImageKey(c, backFace, false);
-        if (result == null) {
+        final String destPath = ForgeConstants.CACHE_CARD_PICS_DIR + imageKey  + ".jpg";
+
+        if (existingImages.contains(imageKey + ".jpg")) {
             return;
         }
-        final String urlToDownload = result[0];
+
-        final String destPath = result[1];
         if (downloads.containsKey(destPath)) {
             return;
         }
-        downloads.put(destPath, urlToDownload);
+
+        final String setCode3 = c.getEdition();
+        final String setCode2 = StaticData.instance().getEditions().getCode2ByCode(setCode3);
+
+        if (!(existingSets.contains(setCode3) || existingSets.contains(setCode2))) {
+            // If set doesn't exist on server, don't try to download cards for it
+            return;
+        }
+
+        downloads.put(destPath, ForgeConstants.URL_PIC_DOWNLOAD + ImageUtil.getDownloadUrl(c, backFace));
     }
 }

forge-gui/src/main/java/forge/download/GuiDownloadService.java

@@ -25,11 +25,14 @@ import forge.error.BugReporter;
 import forge.interfaces.IButton;
 import forge.interfaces.IProgressBar;
 import forge.interfaces.ITextField;
+import forge.properties.ForgeConstants;
 import forge.util.FileUtil;
+import forge.util.HttpUtil;
 import org.apache.commons.lang3.tuple.Pair;
 
 import java.io.*;
 import java.net.*;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Map.Entry;
 
@@ -340,4 +343,27 @@ public abstract class GuiDownloadService implements Runnable {
             }
         }
     }
+
+    protected static HashSet<String> retrieveManifestDirectory() {
+        String manifestUrl = ForgeConstants.URL_PIC_DOWNLOAD;
+        HashSet<String> existingSets = new HashSet<>();
+
+        String response = HttpUtil.getURL(manifestUrl);
+
+        if (response == null)  return null;
+
+        String[] strings = response.split("<a href=\"");
+
+        for (String s : strings) {
+            int idx = s.indexOf('/');
+            if (!Character.isLetterOrDigit(s.charAt(0)) || idx > 4 || idx == -1) {
+                continue;
+            }
+
+            String set = s.substring(0, idx);
+            existingSets.add(set);
+        }
+
+        return existingSets;
+    }
 }

forge-gui/src/main/java/forge/download/GuiDownloadSetPicturesLQ.java

@@ -23,7 +23,6 @@ import forge.card.CardEdition;
 import forge.item.PaperCard;
 import forge.model.FModel;
 import forge.properties.ForgeConstants;
-import forge.util.HttpUtil;
 import forge.util.ImageUtil;
 import org.apache.commons.lang3.StringUtils;
 
@@ -87,27 +86,4 @@ public class GuiDownloadSetPicturesLQ extends GuiDownloadService {
             downloads.put(destFile.getAbsolutePath(), ForgeConstants.URL_PIC_DOWNLOAD + urlPath);
         }
     }
-
-    private HashSet<String> retrieveManifestDirectory() {
-        String manifestUrl = ForgeConstants.URL_PIC_DOWNLOAD;
-        HashSet<String> existingSets = new HashSet<>();
-
-        String response = HttpUtil.getURL(manifestUrl);
-
-        if (response == null)  return null;
-
-        String[] strings = response.split("<a href=\"");
-
-        for (String s : strings) {
-            int idx = s.indexOf('/');
-            if (!Character.isLetterOrDigit(s.charAt(0)) || idx > 4 || idx == -1) {
-                continue;
-            }
-
-            String set = s.substring(0, idx);
-            existingSets.add(set);
-        }
-
-        return existingSets;
-    }
 }

forge-gui/src/main/java/forge/properties/ForgeConstants.java

@@ -276,7 +276,8 @@ public final class ForgeConstants {
     // URLs
     private static final String URL_CARDFORGE = "https://downloads.cardforge.org";
     public static final String URL_PIC_DOWNLOAD = URL_CARDFORGE + "/images/cards/";
-    public static final String URL_PRICE_DOWNLOAD = "https://downloads.cardforge.org/all-prices.txt";
+    public static final String URL_TOKEN_DOWNLOAD = URL_CARDFORGE + "/images/tokens/";
+    public static final String URL_PRICE_DOWNLOAD = URL_CARDFORGE + "/all-prices.txt";
 
     // Constants for Display Card Identity game setting
     public static final String DISP_CURRENT_COLORS_ALWAYS = "Always";