From 54a39a5853d69d93ced6652de791244d2aa4ee0a Mon Sep 17 00:00:00 2001 From: Maxmtg Date: Thu, 21 Mar 2013 16:01:17 +0000 Subject: [PATCH] CardStorageReader uses threadPool for zips as well, Thread pool is not used if 1 == Runtime.getRuntime().availableProcessors() FControl static ctor prints number of cores available to JVM. --- .../card/cardfactory/CardStorageReader.java | 202 ++++++++++-------- src/main/java/forge/control/FControl.java | 12 ++ src/main/java/forge/model/FModel.java | 3 +- 3 files changed, 132 insertions(+), 85 deletions(-) diff --git a/src/main/java/forge/card/cardfactory/CardStorageReader.java b/src/main/java/forge/card/cardfactory/CardStorageReader.java index 5eae22246a8..fddcd941817 100644 --- a/src/main/java/forge/card/cardfactory/CardStorageReader.java +++ b/src/main/java/forge/card/cardfactory/CardStorageReader.java @@ -62,11 +62,16 @@ public class CardStorageReader { /** Default charset when loading from files. */ public static final String DEFAULT_CHARSET_NAME = "US-ASCII"; + final private boolean useThreadTool = FControl.isMultiCoreSystem(); + final private int NUMBER_OF_PARTS = 25; + + final private CountDownLatch cdl = new CountDownLatch(NUMBER_OF_PARTS); + final private FProgressBar barProgress = FView.SINGLETON_INSTANCE.getSplash().getProgressBar(); + private transient File cardsfolder; private transient ZipFile zip; private transient Charset charset; - private transient CardRulesReader rulesReader; // 8/18/11 10:56 PM @@ -86,8 +91,6 @@ public class CardStorageReader { public CardStorageReader(String cardDataDir, final boolean useZip) { // These read data for lightweight classes. - this.rulesReader = new CardRulesReader(); - File theCardsFolder = new File(cardDataDir); if (!theCardsFolder.exists()) { @@ -104,7 +107,6 @@ public class CardStorageReader { final File zipFile = new File(theCardsFolder, "cardsfolder.zip"); - // Prepare resources to read cards lazily. if (useZip && zipFile.exists()) { try { this.zip = new ZipFile(zipFile); @@ -124,12 +126,24 @@ public class CardStorageReader { List result = new ArrayList(); for(int i = from; i < to; i++) { File cardTxtFile = files.get(i); - if (cardTxtFile.getName().endsWith(CardStorageReader.CARD_FILE_DOT_EXTENSION)) - result.add(this.loadCard(rulesReader, cardTxtFile)); + result.add(this.loadCard(rulesReader, cardTxtFile)); } return result; } + private final List loadCardsInRangeFromZip(final List files, int from, int to) { + + CardRulesReader rulesReader = new CardRulesReader(); + + List result = new ArrayList(); + for(int i = from; i < to; i++) { + ZipEntry ze = files.get(i); + // if (ze.getName().endsWith(CardStorageReader.CARD_FILE_DOT_EXTENSION)) // already filtered! + result.add(this.loadCard(rulesReader, ze)); + } + return result; + } + /** * Starts reading cards into memory until the given card is found. @@ -141,65 +155,103 @@ public class CardStorageReader { */ public final List loadCards() { - List result = new ArrayList(); - final FProgressBar barProgress = FView.SINGLETON_INSTANCE.getSplash().getProgressBar(); - - // Iterate through txt files or zip archive. - // Report relevant numbers to progress monitor model. - if (this.zip == null) { - result = loadAllCardsFromFolder(barProgress); - } else { - - Enumeration zipEnum = this.zip.entries(); - int estimatedFilesRemaining = this.zip.size(); - - barProgress.setMaximum(estimatedFilesRemaining); - ZipEntry entry; - - // zipEnum was initialized in the constructor. - while (zipEnum.hasMoreElements()) { - entry = zipEnum.nextElement(); - - if (entry.isDirectory() || !entry.getName().endsWith(CardStorageReader.CARD_FILE_DOT_EXTENSION)) { - barProgress.increment(); - continue; - } - - result.add(this.loadCard(entry)); - barProgress.increment(); - } - } // endif - - barProgress.setPercentMode(false); - - return result; - } // loadCardsUntilYouFind(String) - - private List loadAllCardsFromFolder(final FProgressBar barProgress) { - List result = new ArrayList(); - - StopWatch sw = new StopWatch(); - sw.start(); - final List allFiles = new ArrayList(); - - fillFilesArray(allFiles, this.cardsfolder); - long estimatedFilesRemaining = allFiles.size(); - final int NUMBER_OF_PARTS = 20; - sw.split(); if (barProgress != null) { barProgress.setMaximum(NUMBER_OF_PARTS); SwingUtilities.invokeLater(new Runnable() { @Override public void run() { barProgress.setPercentMode(true); - barProgress.setDescription("Loading card data: "); + barProgress.setDescription("Loading card data: 0%"); } }); } + + final List>> tasks; + long estimatedFilesRemaining; + + // Iterate through txt files or zip archive. + // Report relevant numbers to progress monitor model. + if (this.zip == null) { + final List allFiles = new ArrayList(); + fillFilesArray(allFiles, this.cardsfolder); + estimatedFilesRemaining = allFiles.size(); + tasks = makeTaskListForFiles(allFiles); + } else { + + estimatedFilesRemaining = this.zip.size(); + ZipEntry entry; + List entries = new ArrayList(); + // zipEnum was initialized in the constructor. + Enumeration zipEnum = this.zip.entries(); + while (zipEnum.hasMoreElements()) { + entry = zipEnum.nextElement(); + if (entry.isDirectory() || !entry.getName().endsWith(CardStorageReader.CARD_FILE_DOT_EXTENSION)) + continue; + entries.add(entry); + } - final CountDownLatch cdl = new CountDownLatch(NUMBER_OF_PARTS); + tasks = makeTaskListForZip(entries); + } // endif + + return executeLoadTask(tasks, estimatedFilesRemaining); + } // loadCardsUntilYouFind(String) + + private List executeLoadTask(final List>> tasks, long totalRecords) { + List result = new ArrayList(); + StopWatch sw = new StopWatch(); + sw.start(); + try { + if ( useThreadTool ) { + final ExecutorService executor = FControl.getComputingPool(0.5f); + final List>> parts = executor.invokeAll(tasks); + executor.shutdown(); + cdl.await(); + for(Future> pp : parts) { + result.addAll(pp.get()); + } + } else { + for(Callable> c : tasks) { + result.addAll(c.call()); + } + } + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } catch (Exception e) { // this clause comes from non-threaded branch + throw new RuntimeException(e); + } + + sw.stop(); + final long timeOnParse = sw.getTime(); + System.out.printf("Read %s %s in %d ms (%d parts) %s%n", totalRecords, zip == null? "files" : "archived files", timeOnParse, NUMBER_OF_PARTS, useThreadTool ? "using thread pool" : "in same thread"); + barProgress.setPercentMode(false); + return result; + } + + private List>> makeTaskListForZip(final List entries) { + int totalFiles = entries.size(); + int filesPerPart = totalFiles / NUMBER_OF_PARTS; + final List>> tasks = new ArrayList>>(); + for (int iPart = 0; iPart < NUMBER_OF_PARTS; iPart++) { + final int from = iPart * filesPerPart; + final int till = iPart == NUMBER_OF_PARTS - 1 ? totalFiles : from + filesPerPart; + tasks.add(new Callable>() { + @Override + public List call() throws Exception{ + List res = loadCardsInRangeFromZip(entries, from, till); + barProgress.increment(); + cdl.countDown(); + return res; + } + }); + } + return tasks; + } + + private List>> makeTaskListForFiles(final List allFiles) { int totalFiles = allFiles.size(); int filesPerPart = totalFiles / NUMBER_OF_PARTS; final List>> tasks = new ArrayList>>(); @@ -216,26 +268,7 @@ public class CardStorageReader { } }); } - - try { - final ExecutorService executor = FControl.getComputingPool(0.5f); - final List>> parts = executor.invokeAll(tasks); - executor.shutdown(); - cdl.await(); - for(Future> pp : parts) { - result.addAll(pp.get()); - } - } catch (InterruptedException e) { - e.printStackTrace(); - } catch (ExecutionException e) { - e.printStackTrace(); - } - - long fs = sw.getSplitTime(); - sw.stop(); - - System.out.println("Processed " + estimatedFilesRemaining + " file objects in " + (sw.getTime() - fs) + " ms, apart from that folder scan took " + fs + " ms."); - return result; + return tasks; } /** @@ -245,19 +278,20 @@ public class CardStorageReader { */ private void fillFilesArray(List allFiles, File startDir) { String[] list = startDir.list(); - for (String filename : list) { - File entry = new File(startDir, filename); + for (String filename : list) { + File entry = new File(startDir, filename); - if (!entry.isDirectory()) { + if (!entry.isDirectory()) { + if (entry.getName().endsWith(CardStorageReader.CARD_FILE_DOT_EXTENSION)) allFiles.add(entry); - continue; - } - if (filename.startsWith(".")) { - continue; - } - - fillFilesArray(allFiles, entry); + continue; } + if (filename.startsWith(".")) { + continue; + } + + fillFilesArray(allFiles, entry); + } } @@ -315,7 +349,7 @@ public class CardStorageReader { * * @return a new Card instance */ - protected final CardRules loadCard(final ZipEntry entry) { + protected final CardRules loadCard(final CardRulesReader rulesReader, final ZipEntry entry) { InputStream zipInputStream = null; try { zipInputStream = this.zip.getInputStream(entry); diff --git a/src/main/java/forge/control/FControl.java b/src/main/java/forge/control/FControl.java index 0c2815e2c22..c12b26f0b37 100644 --- a/src/main/java/forge/control/FControl.java +++ b/src/main/java/forge/control/FControl.java @@ -89,6 +89,10 @@ public enum FControl { private final SoundSystem soundSystem = new SoundSystem(); + + static { + System.out.printf("(FControl static ctor): Running on a machine with %d cpu core(s)%n", Runtime.getRuntime().availableProcessors() ); + } /** *

* FControl. @@ -322,4 +326,12 @@ public enum FControl { public final static ExecutorService getComputingPool(float loadFactor) { return Executors.newFixedThreadPool((int)(Runtime.getRuntime().availableProcessors() / (1-loadFactor))); } + + /** + * TODO: Write javadoc for this method. + * @return + */ + public static boolean isMultiCoreSystem() { + return Runtime.getRuntime().availableProcessors() > 1; + } } diff --git a/src/main/java/forge/model/FModel.java b/src/main/java/forge/model/FModel.java index 0b2d4da6a72..56a9f31bd40 100644 --- a/src/main/java/forge/model/FModel.java +++ b/src/main/java/forge/model/FModel.java @@ -29,6 +29,7 @@ import forge.Constant; import forge.Constant.Preferences; import forge.card.BoosterData; import forge.card.CardBlock; +import forge.card.CardRulesReader; import forge.card.EditionCollection; import forge.card.FatPackData; import forge.card.FormatCollection; @@ -144,7 +145,7 @@ public enum FModel { this.questPreferences = new QuestPreferences(); this.gauntletData = new GauntletData(); - this.editions = new EditionCollection(); + this.editions = CardRulesReader.editions; // CardRules ctor cannot refer to FModel, since it is not yet build by that moment this.formats = new FormatCollection("res/blockdata/formats.txt"); this.boosters = new StorageView(new BoosterData.Reader("res/blockdata/boosters.txt")); this.tournaments = new StorageView(new BoosterData.Reader("res/blockdata/starters.txt"));