From 1fb5479e4f534c530535eacf40bbdd842b0416c5 Mon Sep 17 00:00:00 2001 From: Maxmtg Date: Wed, 20 Mar 2013 21:06:12 +0000 Subject: [PATCH] CardStorageReader uses a threadPool to load all cards in several threads, got 3x faster execution on i7-2600. --- src/main/java/forge/card/CardRules.java | 4 +- src/main/java/forge/card/CardRulesReader.java | 3 +- .../card/cardfactory/CardStorageReader.java | 160 +++++++++++------- src/main/java/forge/control/FControl.java | 15 ++ .../java/forge/gui/toolbox/FProgressBar.java | 32 +++- 5 files changed, 145 insertions(+), 69 deletions(-) diff --git a/src/main/java/forge/card/CardRules.java b/src/main/java/forge/card/CardRules.java index 8f5a0972c11..7c1549e469c 100644 --- a/src/main/java/forge/card/CardRules.java +++ b/src/main/java/forge/card/CardRules.java @@ -33,8 +33,6 @@ import forge.card.mana.ManaCost; * @version $Id: CardRules.java 9708 2011-08-09 19:34:12Z jendave $ */ public final class CardRules implements ICardCharacteristics { - private final static EditionCollection editions = new EditionCollection(); // create a copy here, Singletons.model... is not initialized yet. - private final CardSplitType splitType; private final ICardFace mainPart; private final ICardFace otherPart; @@ -51,7 +49,7 @@ public final class CardRules implements ICardCharacteristics { //System.out.print(faces[0].getName()); for (Entry cs : sets.entrySet()) { - if( editions.get(cs.getKey()) != null ) + if( CardRulesReader.editions.get(cs.getKey()) != null ) setsPrinted.put(cs.getKey(), cs.getValue()); } diff --git a/src/main/java/forge/card/CardRulesReader.java b/src/main/java/forge/card/CardRulesReader.java index 2731de073fa..fb1ff50e9c2 100644 --- a/src/main/java/forge/card/CardRulesReader.java +++ b/src/main/java/forge/card/CardRulesReader.java @@ -38,7 +38,8 @@ import forge.card.mana.ManaCostShard; * @version $Id$ */ public class CardRulesReader { - + public final static EditionCollection editions = new EditionCollection(); // create a copy here, Singletons.model... is not initialized yet. + // fields to build private CardFace[] faces = new CardFace[] { null, null }; private String[] pictureUrl = new String[] { null, null }; diff --git a/src/main/java/forge/card/cardfactory/CardStorageReader.java b/src/main/java/forge/card/cardfactory/CardStorageReader.java index deffb368137..25a791665cd 100644 --- a/src/main/java/forge/card/cardfactory/CardStorageReader.java +++ b/src/main/java/forge/card/cardfactory/CardStorageReader.java @@ -27,13 +27,21 @@ import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Enumeration; import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import javax.swing.SwingUtilities; +import org.apache.commons.lang.time.StopWatch; + import forge.card.CardRules; import forge.card.CardRulesReader; +import forge.control.FControl; import forge.error.BugReporter; import forge.gui.toolbox.FProgressBar; import forge.util.FileUtil; @@ -54,19 +62,12 @@ public class CardStorageReader { /** Default charset when loading from files. */ public static final String DEFAULT_CHARSET_NAME = "US-ASCII"; - /** Special value for estimatedFilesRemaining. */ - private static final int UNKNOWN_NUMBER_OF_FILES_REMAINING = -1; - private transient File cardsfolder; private transient ZipFile zip; private transient Charset charset; private transient CardRulesReader rulesReader; - private transient Enumeration zipEnum; - - private transient long estimatedFilesRemaining = CardStorageReader.UNKNOWN_NUMBER_OF_FILES_REMAINING; - // 8/18/11 10:56 PM @@ -108,25 +109,28 @@ public class CardStorageReader { try { this.zip = new ZipFile(zipFile); } catch (final Exception exn) { - System.err.println("Error reading zip file \"" - // Braids on - // 8/18/11 10:53 - // PM - + zipFile.getAbsolutePath() + "\": " + exn + ". " + "Defaulting to txt files in \"" - + theCardsFolder.getAbsolutePath() + "\"."); + System.err.printf("Error reading zip file \"%s\": %s. Defaulting to txt files in \"%s\".%n", zipFile.getAbsolutePath(), exn, theCardsFolder.getAbsolutePath()); } - - if (this.zip != null) { - this.zipEnum = this.zip.entries(); - this.estimatedFilesRemaining = this.zip.size(); - } - } + } this.charset = Charset.forName(CardStorageReader.DEFAULT_CHARSET_NAME); } // CardReader() - + private final List loadCardsInRange(final List files, int from, int to) { + + CardRulesReader rulesReader = new CardRulesReader(); + + List result = new ArrayList(); + for(int i = from; i < to; i++) { + File cardTxtFile = files.get(i); + if (cardTxtFile.getName().endsWith(CardStorageReader.CARD_FILE_DOT_EXTENSION)) + result.add(this.loadCard(rulesReader, cardTxtFile)); + } + return result; + } + + /** * Starts reading cards into memory until the given card is found. * @@ -143,41 +147,18 @@ public class CardStorageReader { // Iterate through txt files or zip archive. // Report relevant numbers to progress monitor model. if (this.zip == null) { - List allFiles = new ArrayList(); - if (this.estimatedFilesRemaining == CardStorageReader.UNKNOWN_NUMBER_OF_FILES_REMAINING) { - fillFilesArray(allFiles, this.cardsfolder); - this.estimatedFilesRemaining = allFiles.size(); - - } - - if (barProgress != null) { - barProgress.setMaximum((int) this.estimatedFilesRemaining); - SwingUtilities.invokeLater(new Runnable() { - @Override - public void run() { - barProgress.setDescription("Loading card data: "); - } - }); - } - - for (final File cardTxtFile : allFiles) { - if (!cardTxtFile.getName().endsWith(CardStorageReader.CARD_FILE_DOT_EXTENSION)) { - barProgress.increment(); - continue; - } - - //System.out.println(cardTxtFile.getName()); - result.add(this.loadCard(cardTxtFile)); - barProgress.increment(); - - } // endfor + result = loadAllCardsFromFolder(barProgress); } else { - barProgress.setMaximum((int) this.estimatedFilesRemaining); + + Enumeration zipEnum = this.zip.entries(); + int estimatedFilesRemaining = this.zip.size(); + + barProgress.setMaximum(estimatedFilesRemaining); ZipEntry entry; // zipEnum was initialized in the constructor. - while (this.zipEnum.hasMoreElements()) { - entry = this.zipEnum.nextElement(); + while (zipEnum.hasMoreElements()) { + entry = zipEnum.nextElement(); if (entry.isDirectory() || !entry.getName().endsWith(CardStorageReader.CARD_FILE_DOT_EXTENSION)) { barProgress.increment(); @@ -189,9 +170,74 @@ public class CardStorageReader { } } // endif + barProgress.setPercentMode(false); + return result; } // loadCardsUntilYouFind(String) + private List loadAllCardsFromFolder(final FProgressBar barProgress) { + List result = new ArrayList(); + + StopWatch sw = new StopWatch(); + sw.start(); + final List allFiles = new ArrayList(); + + fillFilesArray(allFiles, this.cardsfolder); + long estimatedFilesRemaining = allFiles.size(); + + + final int NUMBER_OF_PARTS = 20; + sw.split(); + if (barProgress != null) { + barProgress.setMaximum(NUMBER_OF_PARTS); + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + barProgress.setPercentMode(true); + barProgress.setDescription("Loading card data: "); + } + }); + } + + final CountDownLatch cdl = new CountDownLatch(NUMBER_OF_PARTS); + int totalFiles = allFiles.size(); + int filesPerPart = totalFiles / NUMBER_OF_PARTS; + final List>> tasks = new ArrayList>>(); + for (int iPart = 0; iPart < NUMBER_OF_PARTS; iPart++) { + final int from = iPart * filesPerPart; + final int till = iPart == NUMBER_OF_PARTS - 1 ? totalFiles : from + filesPerPart; + tasks.add(new Callable>() { + @Override + public List call() throws Exception{ + List res = loadCardsInRange(allFiles, from, till); + barProgress.increment(); + cdl.countDown(); + return res; + } + }); + } + + try { + final ExecutorService executor = FControl.getComputingPool(0.5f); + final List>> parts = executor.invokeAll(tasks); + executor.shutdown(); + cdl.await(); + for(Future> pp : parts) { + result.addAll(pp.get()); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } + + long fs = sw.getSplitTime(); + sw.stop(); + + System.out.println("Processed " + estimatedFilesRemaining + " file objects in " + sw.getTime() + " ms, (folder scan took " + fs + " ms.)"); + return result; + } + /** * TODO: Write javadoc for this method. * @param allFiles @@ -225,13 +271,13 @@ public class CardStorageReader { * * @return the card loaded from the stream */ - protected final CardRules loadCard(final InputStream inputStream) { - this.rulesReader.reset(); + protected final CardRules loadCard(CardRulesReader reader, final InputStream inputStream) { + reader.reset(); InputStreamReader isr = new InputStreamReader(inputStream, this.charset); List allLines = FileUtil.readAllLines(isr, true); - return rulesReader.readCard(allLines); + return reader.readCard(allLines); } /** @@ -242,11 +288,11 @@ public class CardStorageReader { * * @return a new Card instance */ - protected final CardRules loadCard(final File pathToTxtFile) { + protected final CardRules loadCard(final CardRulesReader reader, final File pathToTxtFile) { FileInputStream fileInputStream = null; try { fileInputStream = new FileInputStream(pathToTxtFile); - return this.loadCard(fileInputStream); + return this.loadCard(reader, fileInputStream); } catch (final FileNotFoundException ex) { BugReporter.reportException(ex, "File \"%s\" exception", pathToTxtFile.getAbsolutePath()); throw new RuntimeException("CardReader : run error -- file exception -- filename is " @@ -273,7 +319,7 @@ public class CardStorageReader { InputStream zipInputStream = null; try { zipInputStream = this.zip.getInputStream(entry); - return this.loadCard(zipInputStream); + return this.loadCard(rulesReader, zipInputStream); } catch (final IOException exn) { throw new RuntimeException(exn); // PM diff --git a/src/main/java/forge/control/FControl.java b/src/main/java/forge/control/FControl.java index fffeba57872..0c2815e2c22 100644 --- a/src/main/java/forge/control/FControl.java +++ b/src/main/java/forge/control/FControl.java @@ -25,6 +25,8 @@ import java.awt.event.WindowEvent; import java.awt.event.WindowListener; import java.io.File; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import javax.swing.ImageIcon; import javax.swing.JLayeredPane; @@ -83,6 +85,8 @@ public enum FControl { DRAFTING_PROCESS } + private final ExecutorService threadPool = Executors.newCachedThreadPool(); + private final SoundSystem soundSystem = new SoundSystem(); /** @@ -103,6 +107,8 @@ public enum FControl { System.exit(0); } }; + + // "Close" button override during match this.waConcede = new WindowAdapter() { @@ -307,4 +313,13 @@ public enum FControl { public SoundSystem getSoundSystem() { return soundSystem; } + + public ExecutorService getThreadPool() { + return threadPool; + } + + // This pool is designed to parallel CPU or IO intensive tasks like parse cards or download images, assuming a load factor of 0.5 + public final static ExecutorService getComputingPool(float loadFactor) { + return Executors.newFixedThreadPool((int)(Runtime.getRuntime().availableProcessors() / (1-loadFactor))); + } } diff --git a/src/main/java/forge/gui/toolbox/FProgressBar.java b/src/main/java/forge/gui/toolbox/FProgressBar.java index 864b075a42d..bff0f133454 100644 --- a/src/main/java/forge/gui/toolbox/FProgressBar.java +++ b/src/main/java/forge/gui/toolbox/FProgressBar.java @@ -21,6 +21,8 @@ public class FProgressBar extends JProgressBar { private String message; private boolean showETA = true; private boolean showCount = true; + + private boolean percentMode = false; /** */ public FProgressBar() { @@ -40,6 +42,14 @@ public class FProgressBar extends JProgressBar { this.setString(s0); } + private final Runnable barIncrementor = new Runnable() { + @Override + public void run() { + FProgressBar.this.setValue(tempVal); + FProgressBar.this.setString(message); + } + }; + /** Increments bar, thread safe. Calculations executed on separate thread. */ public void increment() { //GuiUtils.checkEDT("FProgressBar$increment", false); @@ -48,7 +58,11 @@ public class FProgressBar extends JProgressBar { // String.format leads to StringBuilder anyway. Direct calls will be faster StringBuilder sb = new StringBuilder(desc); if (showCount) { - sb.append(" ").append(tempVal).append(" of ").append(getMaximum()); + sb.append(" "); + if (percentMode) + sb.append(100 * tempVal / getMaximum()).append("%"); + else + sb.append(tempVal).append(" of ").append(getMaximum()); } if (showETA) { @@ -58,13 +72,7 @@ public class FProgressBar extends JProgressBar { message = sb.toString(); // When calculations finished; EDT can be used. - SwingUtilities.invokeLater(new Runnable() { - @Override - public void run() { - FProgressBar.this.setValue(tempVal); - FProgressBar.this.setString(message); - } - }); + SwingUtilities.invokeLater(barIncrementor); } /** Resets the various values required for this class. Must be called from EDT. */ @@ -96,4 +104,12 @@ public class FProgressBar extends JProgressBar { etaSecs = (int) ((this.getMaximum() - v0) * timePerUnit) / 1000; } + public boolean isPercentMode() { + return percentMode; + } + + public void setPercentMode(boolean value) { + this.percentMode = value; + } + }