From 4280852e06bad8abb7f23a5fa633c265ccbcc1d6 Mon Sep 17 00:00:00 2001 From: Maxmtg Date: Sun, 17 Mar 2013 18:20:16 +0000 Subject: [PATCH] GuiDownload uses map of (local_path -> url) as core object Download code uses NIO --- .../gui/download/GuiDownloadPicturesLQ.java | 58 +++--- .../forge/gui/download/GuiDownloadPrices.java | 13 +- .../gui/download/GuiDownloadQuestImages.java | 7 +- .../download/GuiDownloadSetPicturesLQ.java | 12 +- .../forge/gui/download/GuiDownloader.java | 168 +++++++----------- 5 files changed, 117 insertions(+), 141 deletions(-) diff --git a/src/main/java/forge/gui/download/GuiDownloadPicturesLQ.java b/src/main/java/forge/gui/download/GuiDownloadPicturesLQ.java index a90496c2b0f..805e73e6a8c 100644 --- a/src/main/java/forge/gui/download/GuiDownloadPicturesLQ.java +++ b/src/main/java/forge/gui/download/GuiDownloadPicturesLQ.java @@ -18,14 +18,14 @@ package forge.gui.download; import java.io.File; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Set; +import java.util.Map; +import java.util.TreeMap; import org.apache.commons.lang3.StringUtils; import forge.ImageCache; import forge.card.CardRules; +import forge.card.CardSplitType; import forge.item.CardDb; import forge.item.CardPrinted; import forge.properties.NewConstants; @@ -37,18 +37,17 @@ public class GuiDownloadPicturesLQ extends GuiDownloader { } @Override - protected final ArrayList getNeededImages() { - ArrayList downloads = new ArrayList(); - Set filenames = new HashSet(); + protected final Map getNeededImages() { + Map downloads = new TreeMap(String.CASE_INSENSITIVE_ORDER); for (CardPrinted c : CardDb.instance().getAllCards()) { - addDLObject(c, false, downloads, filenames); - addDLObject(c, true, downloads, filenames); + addDLObject(c, downloads, false); + if ( c.getRules().getSplitType() == CardSplitType.Transform) + addDLObject(c, downloads, true); } for (CardPrinted c : CardDb.variants().getAllCards()) { - addDLObject(c, false, downloads, filenames); - addDLObject(c, true, downloads, filenames); + addDLObject(c, downloads, false); } // Add missing tokens to the list of things to download. @@ -57,26 +56,37 @@ public class GuiDownloadPicturesLQ extends GuiDownloader { return downloads; } - private void addDLObject(CardPrinted c, boolean backFace, ArrayList downloads, Set filenames) { + private void addDLObject(CardPrinted c, Map downloads, boolean backFace) { CardRules cardRules = c.getRules(); String urls = backFace ? cardRules.getPictureOtherSideUrl() : cardRules.getPictureUrl(); if (StringUtils.isEmpty(urls)) { return; } - - for (String url : urls.split("\\\\")) { - - String filename = ImageCache.getImageKey(c, backFace, false); - if (filenames.contains(filename)) { - continue; - } - filenames.add(filename); - - File destFile = new File(NewConstants.CACHE_CARD_PICS_DIR, filename + ".jpg"); - if (!destFile.exists()) { - downloads.add(new DownloadObject(url, destFile)); - } + String filename = ImageCache.getImageKey(c, backFace, false); + File destFile = new File(NewConstants.CACHE_CARD_PICS_DIR, filename + ".jpg"); + if (destFile.exists()) + return; + + filename = destFile.getAbsolutePath(); + + if (downloads.containsKey(filename)) { + return; } + + final String urlToDownload; + int urlIndex = 0; + int allUrlsLen = 1; + if (urls.indexOf("\\\\") < 0) + urlToDownload = urls; + else { + String[] allUrls = urls.split("\\\\"); + allUrlsLen = allUrls.length; + urlIndex = c.getArtIndex() % allUrlsLen; + urlToDownload = allUrls[urlIndex]; + } + + //System.out.println(c.getName() + "|" + c.getEdition() + " - " + c.getArtIndex() + " -> " + urlIndex + "/" + allUrlsLen + " === " + filename + " <<< " + urlToDownload); + downloads.put(destFile.getAbsolutePath(), urlToDownload); } } diff --git a/src/main/java/forge/gui/download/GuiDownloadPrices.java b/src/main/java/forge/gui/download/GuiDownloadPrices.java index b9e5653b0c7..7d76f116cf1 100644 --- a/src/main/java/forge/gui/download/GuiDownloadPrices.java +++ b/src/main/java/forge/gui/download/GuiDownloadPrices.java @@ -17,10 +17,8 @@ */ package forge.gui.download; -import java.io.File; -import java.util.ArrayList; - -import com.google.common.collect.Lists; +import java.util.HashMap; +import java.util.Map; import forge.properties.NewConstants; @@ -31,8 +29,9 @@ public class GuiDownloadPrices extends GuiDownloader { } @Override - protected ArrayList getNeededImages() { - final File f = new File(NewConstants.QUEST_CARD_PRICE_FILE); - return Lists.newArrayList(new DownloadObject(NewConstants.URL_PRICE_DOWNLOAD, f)); + protected Map getNeededImages() { + Map result = new HashMap(); + result.put(NewConstants.QUEST_CARD_PRICE_FILE, NewConstants.URL_PRICE_DOWNLOAD); + return result; } } diff --git a/src/main/java/forge/gui/download/GuiDownloadQuestImages.java b/src/main/java/forge/gui/download/GuiDownloadQuestImages.java index b09c8605de6..b898c0d3438 100644 --- a/src/main/java/forge/gui/download/GuiDownloadQuestImages.java +++ b/src/main/java/forge/gui/download/GuiDownloadQuestImages.java @@ -17,7 +17,8 @@ */ package forge.gui.download; -import java.util.ArrayList; +import java.util.Map; +import java.util.TreeMap; import forge.properties.NewConstants; @@ -41,9 +42,9 @@ public class GuiDownloadQuestImages extends GuiDownloader { * @return an array of {@link forge.gui.download.GuiDownloadSetPicturesLQ} objects. */ @Override - protected final ArrayList getNeededImages() { + protected final Map getNeededImages() { // read all card names and urls - final ArrayList urls = new ArrayList(); + final Map urls = new TreeMap(String.CASE_INSENSITIVE_ORDER); addMissingItems(urls, NewConstants.IMAGE_LIST_QUEST_OPPONENT_ICONS_FILE, NewConstants.CACHE_ICON_PICS_DIR); addMissingItems(urls, NewConstants.IMAGE_LIST_QUEST_PET_SHOP_ICONS_FILE, NewConstants.CACHE_ICON_PICS_DIR); diff --git a/src/main/java/forge/gui/download/GuiDownloadSetPicturesLQ.java b/src/main/java/forge/gui/download/GuiDownloadSetPicturesLQ.java index efa3d3b60f8..0dee681a295 100644 --- a/src/main/java/forge/gui/download/GuiDownloadSetPicturesLQ.java +++ b/src/main/java/forge/gui/download/GuiDownloadSetPicturesLQ.java @@ -18,7 +18,8 @@ package forge.gui.download; import java.io.File; -import java.util.ArrayList; +import java.util.Map; +import java.util.TreeMap; import org.apache.commons.lang3.StringUtils; @@ -38,8 +39,8 @@ public class GuiDownloadSetPicturesLQ extends GuiDownloader { } @Override - protected final ArrayList getNeededImages() { - ArrayList downloads = new ArrayList(); + protected final Map getNeededImages() { + Map downloads = new TreeMap(String.CASE_INSENSITIVE_ORDER); for (final CardPrinted c : Iterables.concat(CardDb.instance().getAllCards(), CardDb.variants().getAllCards())) { final String setCode3 = c.getEdition(); @@ -62,10 +63,11 @@ public class GuiDownloadSetPicturesLQ extends GuiDownloader { return downloads; } - private void addDLObject(String urlPath, String filename, ArrayList downloads) { + private void addDLObject(String urlPath, String filename, Map downloads) { File destFile = new File(NewConstants.CACHE_CARD_PICS_DIR, filename + ".jpg"); + // System.out.println(filename); if (!destFile.exists()) { - downloads.add(new DownloadObject(NewConstants.URL_PIC_DOWNLOAD + urlPath, destFile)); + downloads.put(destFile.getAbsolutePath(), NewConstants.URL_PIC_DOWNLOAD + urlPath); } } } diff --git a/src/main/java/forge/gui/download/GuiDownloader.java b/src/main/java/forge/gui/download/GuiDownloader.java index 9fe43964a01..6fd86a22f8c 100644 --- a/src/main/java/forge/gui/download/GuiDownloader.java +++ b/src/main/java/forge/gui/download/GuiDownloader.java @@ -20,8 +20,6 @@ package forge.gui.download; import java.awt.EventQueue; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -31,7 +29,10 @@ import java.net.InetSocketAddress; import java.net.MalformedURLException; import java.net.Proxy; import java.net.URL; -import java.util.ArrayList; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.util.Map; +import java.util.Map.Entry; import java.util.Random; import javax.swing.AbstractButton; @@ -107,8 +108,7 @@ public abstract class GuiDownloader extends DefaultBoundedRangeModel implements private int type; // Progress variables - private ArrayList cards; - private int card; + private Map cards; // local path -> url private boolean cancel; private final long[] times = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; private int tptr = 0; @@ -226,7 +226,6 @@ public abstract class GuiDownloader extends DefaultBoundedRangeModel implements } private void update(final int card) { - this.card = card; final class Worker implements Runnable { private final int card; @@ -248,22 +247,19 @@ public abstract class GuiDownloader extends DefaultBoundedRangeModel implements long t2Go = (GuiDownloader.this.cards.size() - this.card) * a; - boolean secOnly = true; if (t2Go > 3600000) { sb.append(String.format("%02d:", t2Go / 3600000)); t2Go = t2Go % 3600000; - secOnly = false; } if (t2Go > 60000) { sb.append(String.format("%02d:", t2Go / 60000)); t2Go = t2Go % 60000; - secOnly = false; - } - if (!secOnly) { - sb.append(String.format("%02d remaining.", t2Go / 1000)); } else { - sb.append(String.format("0:%02d remaining.", t2Go / 1000)); + sb.append("00:"); } + + sb.append(String.format("%02d remaining.", t2Go / 1000)); + } else { sb.append(String.format("%d of %d items finished! Please close!", this.card, GuiDownloader.this.cards.size())); @@ -281,11 +277,8 @@ public abstract class GuiDownloader extends DefaultBoundedRangeModel implements } public final void run() { - BufferedInputStream in; - BufferedOutputStream out; - final Random r = MyRandom.getRandom(); - + Proxy p = null; if (this.type == 0) { p = Proxy.NO_PROXY; @@ -301,81 +294,72 @@ public abstract class GuiDownloader extends DefaultBoundedRangeModel implements } } - if (p != null) { - final byte[] buf = new byte[1024]; - int len; - for (this.update(0); (this.card < this.cards.size()) && !this.cancel; this.update(this.card + 1)) { - final String url = this.cards.get(this.card).getSource(); - final File fileDest = this.cards.get(this.card).getDestination(); - final File base = fileDest.getParentFile(); + int iCard = 0; + for(Entry kv : cards.entrySet()) { + if( cancel ) + break; + update(iCard++); + + String url = kv.getValue(); + final File fileDest = new File(kv.getKey()); + final File base = fileDest.getParentFile(); - try { - // test for folder existence - if (!base.exists() && !base.mkdir()) { // create folder if not found - System.out.println("Can't create folder" + base.getAbsolutePath()); - } - - URL imageUrl = new URL(url); - HttpURLConnection conn = (HttpURLConnection) imageUrl.openConnection(); - // don't allow redirections here -- they indicate 'file not found' on the server - conn.setInstanceFollowRedirects(false); - conn.connect(); - - if (conn.getResponseCode() != 200) { - conn.disconnect(); - System.out.println("Skipped Download for: " + fileDest.getPath()); - continue; - } - - in = new BufferedInputStream(conn.getInputStream()); - out = new BufferedOutputStream(new FileOutputStream(fileDest)); - - while ((len = in.read(buf)) != -1) { - // user cancelled - if (this.cancel) { - in.close(); - out.flush(); - out.close(); - - // delete what was written so far - fileDest.delete(); - this.close(); - return; - } // if - cancel - - out.write(buf, 0, len); - } // while - read and write file - in.close(); - out.flush(); - out.close(); - } catch (final ConnectException ce) { - System.out.println("Connection refused for url: " + url); - } catch (final MalformedURLException mURLe) { - System.out.println("Error - possibly missing URL for: " + fileDest.getName()); - } catch (final FileNotFoundException fnfe) { - String formatStr = "Error - the LQ picture %s could not be found on the server. [%s] - %s"; - System.out.println(String.format(formatStr, fileDest.getName(), url, fnfe.getMessage())); - } catch (final Exception ex) { - Log.error("LQ Pictures", "Error downloading pictures", ex); + try { + // test for folder existence + if (!base.exists() && !base.mkdir()) { // create folder if not found + System.out.println("Can't create folder" + base.getAbsolutePath()); } - // throttle to reduce load on the server - try { - Thread.sleep(r.nextInt(250) + 250); - } catch (final InterruptedException e) { - Log.error("GuiDownloader", "Sleep Error", e); + URL imageUrl = new URL(url); + HttpURLConnection conn = (HttpURLConnection) imageUrl.openConnection(p); + // don't allow redirections here -- they indicate 'file not found' on the server + conn.setInstanceFollowRedirects(false); + conn.connect(); + + if (conn.getResponseCode() != 200) { + conn.disconnect(); + System.out.println("Skipped Download for: " + fileDest.getPath()); + continue; } - } // for - } + + ReadableByteChannel rbc = Channels.newChannel(conn.getInputStream()); + FileOutputStream fos = new FileOutputStream(fileDest); + fos.getChannel().transferFrom(rbc, 0, 1 << 24); + + fos.flush(); + fos.close(); + rbc.close(); + } catch (final ConnectException ce) { + System.out.println("Connection refused for url: " + url); + } catch (final MalformedURLException mURLe) { + System.out.println("Error - possibly missing URL for: " + fileDest.getName()); + } catch (final FileNotFoundException fnfe) { + String formatStr = "Error - the LQ picture %s could not be found on the server. [%s] - %s"; + System.out.println(String.format(formatStr, fileDest.getName(), url, fnfe.getMessage())); + } catch (final Exception ex) { + Log.error("LQ Pictures", "Error downloading pictures", ex); + } + + // throttle to reduce load on the server + try { + Thread.sleep(r.nextInt(50) + 50); + } catch (final InterruptedException e) { + Log.error("GuiDownloader", "Sleep Error", e); + } + } // for + if ( !cancel ) + update(cards.size()); + } - protected abstract ArrayList getNeededImages(); + protected abstract Map getNeededImages(); - protected static void addMissingItems(ArrayList list, String nameUrlFile, String dir) { + protected static void addMissingItems(Map list, String nameUrlFile, String dir) { for (Pair nameUrlPair : FileUtil.readNameUrlFile(nameUrlFile)) { File f = new File(dir, nameUrlPair.getLeft()); + //System.out.println(f.getAbsolutePath()); if (!f.exists()) { - list.add(new DownloadObject(nameUrlPair.getRight(), f)); + list.put(f.getAbsolutePath(), nameUrlPair.getRight()); } } } @@ -397,24 +381,4 @@ public abstract class GuiDownloader extends DefaultBoundedRangeModel implements } } - protected static class DownloadObject { - - private final String source; - private final File destination; - - DownloadObject(final String srcUrl, final File destFile) { - source = srcUrl; - destination = destFile; - //System.out.println(String.format("downloading %s to %s", srcUrl, destFile)); - System.out.println(String.format("Preparing to download %s", destFile)); - } - - public String getSource() { - return source; - } - - public File getDestination() { - return destination; - } - } }