CardStorageReader uses a threadPool to load all cards in several threads, got 3x faster execution on i7-2600.

src/main/java/forge/card/CardRules.java

@@ -33,8 +33,6 @@ import forge.card.mana.ManaCost;
  * @version $Id: CardRules.java 9708 2011-08-09 19:34:12Z jendave $
  */
 public final class CardRules implements ICardCharacteristics {
-    private final static EditionCollection editions = new EditionCollection(); // create a copy here, Singletons.model... is not initialized yet.
-    
     private final CardSplitType splitType;
     private final ICardFace mainPart;
     private final ICardFace otherPart;
@@ -51,7 +49,7 @@ public final class CardRules implements ICardCharacteristics {
         //System.out.print(faces[0].getName());
         
         for (Entry<String, CardInSet> cs : sets.entrySet()) {
-            if( editions.get(cs.getKey()) != null )
+            if( CardRulesReader.editions.get(cs.getKey()) != null )
                 setsPrinted.put(cs.getKey(), cs.getValue());
         }
 

src/main/java/forge/card/CardRulesReader.java

@@ -38,7 +38,8 @@ import forge.card.mana.ManaCostShard;
  * @version $Id$
  */
 public class CardRulesReader {
-
+    public final static EditionCollection editions = new EditionCollection(); // create a copy here, Singletons.model... is not initialized yet.
+    
     // fields to build 
     private CardFace[] faces = new CardFace[] { null, null };
     private String[] pictureUrl = new String[] { null, null };

src/main/java/forge/card/cardfactory/CardStorageReader.java

@@ -27,13 +27,21 @@ import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 
 import javax.swing.SwingUtilities;
 
+import org.apache.commons.lang.time.StopWatch;
+
 import forge.card.CardRules;
 import forge.card.CardRulesReader;
+import forge.control.FControl;
 import forge.error.BugReporter;
 import forge.gui.toolbox.FProgressBar;
 import forge.util.FileUtil;
@@ -54,19 +62,12 @@ public class CardStorageReader {
     /** Default charset when loading from files. */
     public static final String DEFAULT_CHARSET_NAME = "US-ASCII";
 
-    /** Special value for estimatedFilesRemaining. */
-    private static final int UNKNOWN_NUMBER_OF_FILES_REMAINING = -1;
-
     private transient File cardsfolder;
 
     private transient ZipFile zip;
     private transient Charset charset;
     private transient CardRulesReader rulesReader;
 
-    private transient Enumeration<? extends ZipEntry> zipEnum;
-
-    private transient long estimatedFilesRemaining = CardStorageReader.UNKNOWN_NUMBER_OF_FILES_REMAINING;
-
 
     // 8/18/11 10:56 PM
 
@@ -108,25 +109,28 @@ public class CardStorageReader {
             try {
                 this.zip = new ZipFile(zipFile);
             } catch (final Exception exn) {
-                System.err.println("Error reading zip file \""
+                System.err.printf("Error reading zip file \"%s\": %s. Defaulting to txt files in \"%s\".%n", zipFile.getAbsolutePath(), exn, theCardsFolder.getAbsolutePath());
-                        // Braids on
-                        // 8/18/11 10:53
-                        // PM
-                        + zipFile.getAbsolutePath() + "\": " + exn + ". " + "Defaulting to txt files in \""
-                        + theCardsFolder.getAbsolutePath() + "\".");
             }
-
+         }
-            if (this.zip != null) {
-                this.zipEnum = this.zip.entries();
-                this.estimatedFilesRemaining = this.zip.size();
-            }
-        }
 
         this.charset = Charset.forName(CardStorageReader.DEFAULT_CHARSET_NAME);
 
     } // CardReader()
 
-
+    private final List<CardRules> loadCardsInRange(final List<File> files, int from, int to) {
+        
+        CardRulesReader rulesReader = new CardRulesReader();
+        
+        List<CardRules> result = new ArrayList<CardRules>();
+        for(int i = from; i < to; i++) {
+            File cardTxtFile = files.get(i);
+            if (cardTxtFile.getName().endsWith(CardStorageReader.CARD_FILE_DOT_EXTENSION))
+                result.add(this.loadCard(rulesReader, cardTxtFile));
+        }
+        return result;
+    }
+    
+    
     /**
      * Starts reading cards into memory until the given card is found.
      * 
@@ -143,41 +147,18 @@ public class CardStorageReader {
         // Iterate through txt files or zip archive.
         // Report relevant numbers to progress monitor model.
         if (this.zip == null) {
-            List<File> allFiles = new ArrayList<File>();
+            result = loadAllCardsFromFolder(barProgress);
-            if (this.estimatedFilesRemaining == CardStorageReader.UNKNOWN_NUMBER_OF_FILES_REMAINING) {
-                fillFilesArray(allFiles, this.cardsfolder);
-                this.estimatedFilesRemaining = allFiles.size();
-
-            }
-
-            if (barProgress != null) {
-                barProgress.setMaximum((int) this.estimatedFilesRemaining);
-                SwingUtilities.invokeLater(new Runnable() {
-                    @Override
-                    public void run() {
-                        barProgress.setDescription("Loading card data: ");
-                    }
-                });
-            }
-
-            for (final File cardTxtFile : allFiles) {
-                if (!cardTxtFile.getName().endsWith(CardStorageReader.CARD_FILE_DOT_EXTENSION)) {
-                    barProgress.increment();
-                    continue;
-                }
-
-                //System.out.println(cardTxtFile.getName());
-                result.add(this.loadCard(cardTxtFile));
-                barProgress.increment();
-
-            } // endfor
         } else {
-            barProgress.setMaximum((int) this.estimatedFilesRemaining);
+            
+            Enumeration<? extends ZipEntry> zipEnum = this.zip.entries();
+            int estimatedFilesRemaining = this.zip.size();
+            
+            barProgress.setMaximum(estimatedFilesRemaining);
             ZipEntry entry;
 
             // zipEnum was initialized in the constructor.
-            while (this.zipEnum.hasMoreElements()) {
+            while (zipEnum.hasMoreElements()) {
-                entry = this.zipEnum.nextElement();
+                entry = zipEnum.nextElement();
 
                 if (entry.isDirectory() || !entry.getName().endsWith(CardStorageReader.CARD_FILE_DOT_EXTENSION)) {
                     barProgress.increment();
@@ -189,9 +170,74 @@ public class CardStorageReader {
             }
         } // endif
 
+        barProgress.setPercentMode(false);
+
         return result;
     } // loadCardsUntilYouFind(String)
 
+    private List<CardRules> loadAllCardsFromFolder(final FProgressBar barProgress) {
+        List<CardRules> result = new ArrayList<CardRules>();
+        
+        StopWatch sw = new StopWatch();
+        sw.start();
+        final List<File> allFiles = new ArrayList<File>();
+        
+        fillFilesArray(allFiles, this.cardsfolder);
+        long estimatedFilesRemaining = allFiles.size();
+        
+
+        final int NUMBER_OF_PARTS = 20;
+        sw.split();
+        if (barProgress != null) {
+            barProgress.setMaximum(NUMBER_OF_PARTS);
+            SwingUtilities.invokeLater(new Runnable() {
+                @Override
+                public void run() {
+                    barProgress.setPercentMode(true);
+                    barProgress.setDescription("Loading card data: ");
+                }
+            });
+        }
+
+        final CountDownLatch cdl = new CountDownLatch(NUMBER_OF_PARTS);
+        int totalFiles = allFiles.size();
+        int filesPerPart = totalFiles / NUMBER_OF_PARTS; 
+        final List<Callable<List<CardRules>>> tasks = new ArrayList<Callable<List<CardRules>>>();
+        for (int iPart = 0; iPart < NUMBER_OF_PARTS; iPart++) {
+            final int from = iPart * filesPerPart;
+            final int till = iPart == NUMBER_OF_PARTS - 1 ? totalFiles : from + filesPerPart;
+            tasks.add(new Callable<List<CardRules>>() {
+                @Override
+                public List<CardRules> call() throws Exception{
+                    List<CardRules> res = loadCardsInRange(allFiles, from, till);
+                    barProgress.increment();
+                    cdl.countDown();
+                    return res;
+                }
+            });
+        }
+            
+        try {
+            final ExecutorService executor = FControl.getComputingPool(0.5f);
+            final List<Future<List<CardRules>>> parts = executor.invokeAll(tasks);
+            executor.shutdown();
+            cdl.await();
+            for(Future<List<CardRules>> pp : parts) {
+                result.addAll(pp.get());
+            }
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+        } catch (ExecutionException e) {
+            e.printStackTrace();
+        }
+        
+        long fs = sw.getSplitTime();
+        sw.stop();
+        
+        System.out.println("Processed " + estimatedFilesRemaining + " file objects in " + sw.getTime() + " ms, (folder scan took " + fs + " ms.)");
+        return result;
+    }
+
     /**
      * TODO: Write javadoc for this method.
      * @param allFiles
@@ -225,13 +271,13 @@ public class CardStorageReader {
      * 
      * @return the card loaded from the stream
      */
-    protected final CardRules loadCard(final InputStream inputStream) {
+    protected final CardRules loadCard(CardRulesReader reader, final InputStream inputStream) {
-        this.rulesReader.reset();
+        reader.reset();
 
         InputStreamReader isr = new InputStreamReader(inputStream, this.charset);
         List<String> allLines = FileUtil.readAllLines(isr, true);
 
-        return rulesReader.readCard(allLines);
+        return reader.readCard(allLines);
     }
 
     /**
@@ -242,11 +288,11 @@ public class CardStorageReader {
      * 
      * @return a new Card instance
      */
-    protected final CardRules loadCard(final File pathToTxtFile) {
+    protected final CardRules loadCard(final CardRulesReader reader, final File pathToTxtFile) {
         FileInputStream fileInputStream = null;
         try {
             fileInputStream = new FileInputStream(pathToTxtFile);
-            return this.loadCard(fileInputStream);
+            return this.loadCard(reader, fileInputStream);
         } catch (final FileNotFoundException ex) {
             BugReporter.reportException(ex, "File \"%s\" exception", pathToTxtFile.getAbsolutePath());
             throw new RuntimeException("CardReader : run error -- file exception -- filename is "
@@ -273,7 +319,7 @@ public class CardStorageReader {
         InputStream zipInputStream = null;
         try {
             zipInputStream = this.zip.getInputStream(entry);
-            return this.loadCard(zipInputStream);
+            return this.loadCard(rulesReader, zipInputStream);
         } catch (final IOException exn) {
             throw new RuntimeException(exn);
             // PM

src/main/java/forge/control/FControl.java

@@ -25,6 +25,8 @@ import java.awt.event.WindowEvent;
 import java.awt.event.WindowListener;
 import java.io.File;
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 import javax.swing.ImageIcon;
 import javax.swing.JLayeredPane;
@@ -83,6 +85,8 @@ public enum FControl {
         DRAFTING_PROCESS
     }
 
+    private final ExecutorService threadPool = Executors.newCachedThreadPool(); 
+
     private final SoundSystem soundSystem = new SoundSystem();
 
     /**
@@ -103,6 +107,8 @@ public enum FControl {
                 System.exit(0);
             }
         };
+        
+        
 
         // "Close" button override during match
         this.waConcede = new WindowAdapter() {
@@ -307,4 +313,13 @@ public enum FControl {
     public SoundSystem getSoundSystem() {
         return soundSystem;
     }
+
+    public ExecutorService getThreadPool() {
+        return threadPool;
+    }
+
+    // This pool is designed to parallel CPU or IO intensive tasks like parse cards or download images, assuming a load factor of 0.5
+    public final static ExecutorService getComputingPool(float loadFactor) {
+        return Executors.newFixedThreadPool((int)(Runtime.getRuntime().availableProcessors() / (1-loadFactor)));
+    }
 }

src/main/java/forge/gui/toolbox/FProgressBar.java

@@ -21,6 +21,8 @@ public class FProgressBar extends JProgressBar {
     private String message;
     private boolean showETA = true;
     private boolean showCount = true;
+    
+    private boolean percentMode = false;
 
     /** */
     public FProgressBar() {
@@ -40,6 +42,14 @@ public class FProgressBar extends JProgressBar {
         this.setString(s0);
     }
 
+    private final Runnable barIncrementor = new Runnable() {
+        @Override
+        public void run() {
+            FProgressBar.this.setValue(tempVal);
+            FProgressBar.this.setString(message);
+        }
+    };
+    
     /** Increments bar, thread safe. Calculations executed on separate thread. */
     public void increment() {
         //GuiUtils.checkEDT("FProgressBar$increment", false);
@@ -48,7 +58,11 @@ public class FProgressBar extends JProgressBar {
         // String.format leads to StringBuilder anyway. Direct calls will be faster
         StringBuilder sb = new StringBuilder(desc);
         if (showCount) {
-            sb.append(" ").append(tempVal).append(" of ").append(getMaximum());
+            sb.append(" ");
+            if (percentMode)
+                sb.append(100 * tempVal / getMaximum()).append("%");
+            else
+                sb.append(tempVal).append(" of ").append(getMaximum());
         }
 
         if (showETA) {
@@ -58,13 +72,7 @@ public class FProgressBar extends JProgressBar {
         message = sb.toString();
 
         // When calculations finished; EDT can be used.
-        SwingUtilities.invokeLater(new Runnable() {
+        SwingUtilities.invokeLater(barIncrementor);
-            @Override
-            public void run() {
-                FProgressBar.this.setValue(tempVal);
-                FProgressBar.this.setString(message);
-            }
-        });
     }
 
     /** Resets the various values required for this class. Must be called from EDT. */
@@ -96,4 +104,12 @@ public class FProgressBar extends JProgressBar {
         etaSecs = (int) ((this.getMaximum() - v0) * timePerUnit) / 1000;
     }
 
+    public boolean isPercentMode() {
+        return percentMode;
+    }
+
+    public void setPercentMode(boolean value) {
+        this.percentMode = value;
+    }
+
 }