Fully working LDA based deck generation for Standard and Modern

(cherry picked from commit 892ae23)

forge-gui-desktop/src/test/java/forge/deck/generate/lda/examples/Example.java

@@ -1,59 +0,0 @@
-/*
-* Copyright 2015 Kohei Yamamoto
-*
-* Licensed under the Apache License, Version 2.0 (the "License");
-* you may not use this file except in compliance with the License.
-* You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package forge.deck.generate.lda.examples;
-
-import java.util.List;
-
-import com.google.common.base.Function;
-import forge.GuiBase;
-import forge.GuiDesktop;
-import forge.deck.generate.lda.lda.LDA;
-import static forge.deck.generate.lda.lda.inference.InferenceMethod.*;
-
-import forge.model.FModel;
-import forge.properties.ForgePreferences;
-import org.apache.commons.lang3.tuple.Pair;
-
-import forge.deck.generate.lda.dataset.Dataset;
-
-public class Example {
-    public static void main(String[] args) throws Exception {
-        GuiBase.setInterface(new GuiDesktop());
-        FModel.initialize(null, new Function<ForgePreferences, Void>()  {
-            @Override
-            public Void apply(ForgePreferences preferences) {
-                preferences.setPref(ForgePreferences.FPref.LOAD_CARD_SCRIPTS_LAZILY, false);
-                return null;
-            }
-        });
-        Dataset dataset = new Dataset(FModel.getFormats().getStandard());
-        
-        final int numTopics = 50;
-        LDA lda = new LDA(0.1, 0.1, numTopics, dataset, CGS);
-        lda.run();
-        System.out.println(lda.computePerplexity(dataset));
-
-        for (int t = 0; t < numTopics; ++t) {
-            List<Pair<String, Double>> highRankVocabs = lda.getVocabsSortedByPhi(t);
-            System.out.print("t" + t + ": ");
-            for (int i = 0; i < 20; ++i) {
-                System.out.println("[" + highRankVocabs.get(i).getLeft() + "," + highRankVocabs.get(i).getRight() + "],");
-            }
-            System.out.println();
-        }
-    }
-}

forge-gui-desktop/src/test/java/forge/planarconquestgenerate/PlanarConquestCommanderGeneraterGA.java

@@ -0,0 +1,139 @@
+package forge.planarconquestgenerate;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import forge.GuiBase;
+import forge.GuiDesktop;
+import forge.LobbyPlayer;
+import forge.StaticData;
+import forge.card.CardRulesPredicates;
+import forge.deck.*;
+import forge.deck.io.DeckStorage;
+import forge.game.GameFormat;
+import forge.game.GameRules;
+import forge.game.GameType;
+import forge.game.Match;
+import forge.game.player.RegisteredPlayer;
+import forge.item.PaperCard;
+import forge.limited.CardRanker;
+import forge.model.FModel;
+import forge.player.GamePlayerUtil;
+import forge.properties.ForgeConstants;
+import forge.properties.ForgePreferences;
+import forge.tournament.system.AbstractTournament;
+import forge.tournament.system.TournamentPairing;
+import forge.tournament.system.TournamentPlayer;
+import forge.tournament.system.TournamentSwiss;
+import forge.util.AbstractGeneticAlgorithm;
+import forge.util.MyRandom;
+import forge.util.TextUtil;
+import forge.view.SimulateMatch;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class PlanarConquestCommanderGeneraterGA extends PlanarConquestGeneraterGA {
+
+
+    private int deckCount = 0;
+
+    public static void main(String[] args){
+        test();
+    }
+
+    public static void test(){
+
+        GuiBase.setInterface(new GuiDesktop());
+        FModel.initialize(null, new Function<ForgePreferences, Void>()  {
+            @Override
+            public Void apply(ForgePreferences preferences) {
+                preferences.setPref(ForgePreferences.FPref.LOAD_CARD_SCRIPTS_LAZILY, false);
+                return null;
+            }
+        });
+        List<String> sets = new ArrayList<>();
+        sets.add("XLN");
+        sets.add("RIX");
+
+        PlanarConquestCommanderGeneraterGA ga = new PlanarConquestCommanderGeneraterGA(new GameRules(GameType.Constructed),
+                new GameFormat("conquest",sets,null),
+                DeckFormat.PlanarConquest,
+                4,
+                12,
+                16);
+        ga.run();
+        List<Deck> winners = ga.listFinalPopulation();
+
+        DeckStorage storage = new DeckStorage(new File(ForgeConstants.DECK_CONSTRUCTED_DIR), ForgeConstants.DECK_BASE_DIR);
+        int i=1;
+        for(Deck deck:winners){
+            storage.save(new Deck(deck,"GA"+i+"_"+deck.getName()));
+            i++;
+        }
+    }
+
+    public PlanarConquestCommanderGeneraterGA(GameRules rules, GameFormat gameFormat, DeckFormat deckFormat, int cardsToUse, int decksPerCard, int generations){
+        super(rules,gameFormat,deckFormat,cardsToUse,decksPerCard,generations);
+    }
+
+    @Override
+    protected void initializeCards(){
+        standardMap = CardRelationMatrixGenerator.cardPools.get(format.getName());
+        List<String> cardNames = new ArrayList<>(standardMap.keySet());
+        List<PaperCard> cards = new ArrayList<>();
+        for(String cardName:cardNames){
+            cards.add(StaticData.instance().getCommonCards().getUniqueByName(cardName));
+        }
+
+        Iterable<PaperCard> filtered= Iterables.filter(cards, Predicates.and(
+                Predicates.compose(CardRulesPredicates.IS_KEPT_IN_AI_DECKS, PaperCard.FN_GET_RULES),
+                Predicates.compose(CardRulesPredicates.Presets.IS_PLANESWALKER, PaperCard.FN_GET_RULES),
+                //Predicates.compose(CardRulesPredicates.Presets.IS_LEGENDARY, PaperCard.FN_GET_RULES),
+                gameFormat.getFilterPrinted()));
+
+        List<PaperCard> filteredList = Lists.newArrayList(filtered);
+        rankedList = CardRanker.rankCardsInDeck(filteredList);
+        List<Deck> decks = new ArrayList<>();
+        for(PaperCard card: rankedList.subList(0,cardsToUse)){
+            System.out.println(card.getName());
+            for( int i=0; i<decksPerCard;i++){
+                decks.add(getDeckForCard(card));
+            }
+        }
+        initializePopulation(decks);
+    }
+
+    @Override
+    protected Deck getDeckForCard(PaperCard card){
+        Deck genDeck =  DeckgenUtil.buildPlanarConquestCommanderDeck(card, gameFormat, deckFormat);
+        Deck d = new Deck(genDeck,genDeck.getName()+"_"+deckCount+"_"+generationCount);
+        deckCount++;
+        return d;
+    }
+
+    @Override
+    protected Deck getDeckForCard(PaperCard card, PaperCard card2){
+        return getDeckForCard(card);
+    }
+
+    @Override
+    protected Deck mutateObject(Deck parent1) {
+        PaperCard allele = parent1.getCommanders().get(0);
+        if(!standardMap.keySet().contains(allele.getName())){
+            return null;
+        }
+        return getDeckForCard(allele);
+    }
+
+    @Override
+    protected Deck createChild(Deck parent1, Deck parent2) {
+        PaperCard allele = parent1.getCommanders().get(0);
+        return getDeckForCard(allele);
+    }
+
+
+}

forge-gui-desktop/src/test/java/forge/planarconquestgenerate/PlanarConquestGeneraterGA.java

@@ -0,0 +1,277 @@
+package forge.planarconquestgenerate;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import forge.GuiBase;
+import forge.GuiDesktop;
+import forge.LobbyPlayer;
+import forge.StaticData;
+import forge.card.CardRulesPredicates;
+import forge.deck.*;
+import forge.deck.io.DeckStorage;
+import forge.game.GameFormat;
+import forge.game.GameRules;
+import forge.game.GameType;
+import forge.game.Match;
+import forge.game.player.RegisteredPlayer;
+import forge.item.PaperCard;
+import forge.limited.CardRanker;
+import forge.model.FModel;
+import forge.player.GamePlayerUtil;
+import forge.properties.ForgeConstants;
+import forge.properties.ForgePreferences;
+import forge.tournament.system.AbstractTournament;
+import forge.tournament.system.TournamentPairing;
+import forge.tournament.system.TournamentPlayer;
+import forge.tournament.system.TournamentSwiss;
+import forge.util.AbstractGeneticAlgorithm;
+import forge.util.MyRandom;
+import forge.util.TextUtil;
+import forge.view.SimulateMatch;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class PlanarConquestGeneraterGA extends AbstractGeneticAlgorithm<Deck> {
+
+    private DeckGroup deckGroup;
+    private List<TournamentPlayer> players = new ArrayList<>();
+    private TournamentSwiss tourney = null;
+    protected Map<String,List<Map.Entry<PaperCard,Integer>>> standardMap;
+    private GameRules rules;
+    protected GameFormat format = FModel.getFormats().getStandard();
+    protected int generations;
+    protected GameFormat gameFormat;
+    protected DeckFormat deckFormat;
+    protected int cardsToUse;
+    protected int decksPerCard;
+    private int deckCount = 0;
+    protected List<PaperCard> rankedList;
+
+    public static void main(String[] args){
+        test();
+    }
+
+    public static void test(){
+
+        GuiBase.setInterface(new GuiDesktop());
+        FModel.initialize(null, new Function<ForgePreferences, Void>()  {
+            @Override
+            public Void apply(ForgePreferences preferences) {
+                preferences.setPref(ForgePreferences.FPref.LOAD_CARD_SCRIPTS_LAZILY, false);
+                return null;
+            }
+        });
+        List<String> sets = new ArrayList<>();
+        sets.add("XLN");
+        sets.add("RIX");
+
+        PlanarConquestGeneraterGA ga = new PlanarConquestGeneraterGA(new GameRules(GameType.Constructed),
+                new GameFormat("conquest",sets,null),
+                DeckFormat.PlanarConquest,
+                40,
+                4,
+                10);
+        ga.run();
+        List<Deck> winners = ga.listFinalPopulation();
+
+        DeckStorage storage = new DeckStorage(new File(ForgeConstants.DECK_CONSTRUCTED_DIR), ForgeConstants.DECK_BASE_DIR);
+        int i=1;
+        for(Deck deck:winners){
+            storage.save(new Deck(deck,"GA"+i+"_"+deck.getName()));
+            i++;
+        }
+    }
+
+    public PlanarConquestGeneraterGA(GameRules rules, GameFormat gameFormat, DeckFormat deckFormat, int cardsToUse, int decksPerCard, int generations){
+        this.rules = rules;
+        rules.setGamesPerMatch(7);
+        this.gameFormat = gameFormat;
+        this.deckFormat = deckFormat;
+        this.cardsToUse = cardsToUse;
+        this.decksPerCard = decksPerCard;
+        this.generations = generations;
+        initializeCards();
+    }
+
+
+    protected void initializeCards(){
+        standardMap = CardRelationMatrixGenerator.cardPools.get(format.getName());
+        List<String> cardNames = new ArrayList<>(standardMap.keySet());
+        List<PaperCard> cards = new ArrayList<>();
+        for(String cardName:cardNames){
+            cards.add(StaticData.instance().getCommonCards().getUniqueByName(cardName));
+        }
+
+        Iterable<PaperCard> filtered= Iterables.filter(cards, Predicates.and(
+                Predicates.compose(CardRulesPredicates.IS_KEPT_IN_AI_DECKS, PaperCard.FN_GET_RULES),
+                Predicates.compose(CardRulesPredicates.Presets.IS_NON_LAND, PaperCard.FN_GET_RULES),
+                gameFormat.getFilterPrinted()));
+
+        List<PaperCard> filteredList = Lists.newArrayList(filtered);
+        setRankedList(CardRanker.rankCardsInDeck(filteredList));
+        List<Deck> decks = new ArrayList<>();
+        for(PaperCard card: getRankedList().subList(0,cardsToUse)){
+            System.out.println(card.getName());
+            for( int i=0; i<decksPerCard;i++){
+                decks.add(getDeckForCard(card));
+            }
+        }
+        initializePopulation(decks);
+    }
+
+    protected List<PaperCard> getRankedList(){
+        return rankedList;
+    }
+
+    protected void setRankedList(List<PaperCard> list){
+        rankedList = list;
+    }
+
+    protected Deck getDeckForCard(PaperCard card){
+        Deck genDeck =  DeckgenUtil.buildPlanarConquestDeck(card, gameFormat, deckFormat);
+        Deck d = new Deck(genDeck,genDeck.getName()+"_"+deckCount+"_"+generationCount);
+        deckCount++;
+        return d;
+    }
+
+    protected Deck getDeckForCard(PaperCard card, PaperCard card2){
+        Deck genDeck = DeckgenUtil.buildPlanarConquestDeck(card, card2, gameFormat, deckFormat, false);
+        Deck d = new Deck(genDeck,genDeck.getName()+"_"+deckCount+"_"+generationCount);
+        deckCount++;
+        return d;
+    }
+
+    @Override
+    protected void evaluateFitness() {
+        deckGroup = new DeckGroup("SimulatedTournament");
+        players = new ArrayList<>();
+        int i=0;
+        for(Deck d:population) {
+            deckGroup.addAiDeck(d);
+            players.add(new TournamentPlayer(GamePlayerUtil.createAiPlayer(d.getName(), 0), i));
+            ++i;
+        }
+        tourney = new TournamentSwiss(players, 2);
+        tourney = runTournament(tourney, rules, players.size(), deckGroup);
+        population = new ArrayList<>();
+        for (int k = 0; k < tourney.getAllPlayers().size(); k++) {
+            String deckName = tourney.getAllPlayers().get(k).getPlayer().getName();
+            for (Deck sortedDeck : deckGroup.getAiDecks()) {
+                if (sortedDeck.getName().equals(deckName)) {
+                    population.add(sortedDeck);
+                    break;
+                }
+            }
+        }
+        deckCount=0;
+    }
+
+    @Override
+    protected Deck mutateObject(Deck parent1) {
+        PaperCard allele = parent1.getMain().get(MyRandom.getRandom().nextInt(8));
+        if(!standardMap.keySet().contains(allele.getName())){
+            return null;
+        }
+        return getDeckForCard(allele);
+    }
+
+    @Override
+    protected Deck createChild(Deck parent1, Deck parent2) {
+        PaperCard allele = parent1.getMain().get(MyRandom.getRandom().nextInt(8));
+        PaperCard allele2 = parent2.getMain().get(MyRandom.getRandom().nextInt(8));
+        if(!standardMap.keySet().contains(allele.getName())
+                ||!standardMap.keySet().contains(allele2.getName())
+                ||allele.getName().equals(allele2.getName())){
+            return null;
+        }
+        return getDeckForCard(allele,allele2);
+    }
+
+    @Override
+    protected Deck expandPool(){
+        PaperCard seed = getRankedList().get(MyRandom.getRandom().nextInt(getRankedList().size()));
+        return getDeckForCard(seed);
+    }
+
+    @Override
+    protected boolean shouldContinue() {
+        return generationCount<generations;
+    }
+
+
+    public TournamentSwiss runTournament(TournamentSwiss tourney, GameRules rules, int numPlayers, DeckGroup deckGroup){
+        tourney.initializeTournament();
+
+        String lastWinner = "";
+        int curRound = 0;
+        System.out.println(TextUtil.concatNoSpace("Starting a tournament with ",
+                String.valueOf(numPlayers), " players over ",
+                String.valueOf(tourney.getTotalRounds()), " rounds"));
+        while(!tourney.isTournamentOver()) {
+            if (tourney.getActiveRound() != curRound) {
+                if (curRound != 0) {
+                    System.out.println(TextUtil.concatNoSpace("End Round - ", String.valueOf(curRound)));
+                }
+                curRound = tourney.getActiveRound();
+                System.out.println("");
+                System.out.println(TextUtil.concatNoSpace("Round ", String.valueOf(curRound) ," Pairings:"));
+
+                for(TournamentPairing pairing : tourney.getActivePairings()) {
+                    System.out.println(pairing.outputHeader());
+                }
+                System.out.println("");
+            }
+
+            TournamentPairing pairing = tourney.getNextPairing();
+            List<RegisteredPlayer> regPlayers = AbstractTournament.registerTournamentPlayers(pairing, deckGroup);
+
+            StringBuilder sb = new StringBuilder();
+            sb.append("Round ").append(tourney.getActiveRound()).append(" - ");
+            sb.append(pairing.outputHeader());
+            //System.out.println(sb.toString());
+
+            if (!pairing.isBye()) {
+                Match mc = new Match(rules, regPlayers, "TourneyMatch");
+
+                int exceptions = 0;
+                int iGame = 0;
+                while (!mc.isMatchOver()) {
+                    // play games until the match ends
+                    try{
+                        SimulateMatch.simulateSingleMatch(mc, iGame, false);
+                        iGame++;
+                    } catch(Exception e) {
+                        exceptions++;
+                        System.out.println(e.toString());
+                        if (exceptions > 5) {
+                            System.out.println("Exceeded number of exceptions thrown. Abandoning match...");
+                            break;
+                        } else {
+                            System.out.println("Game threw exception. Abandoning game and continuing...");
+                        }
+                    }
+
+                }
+                LobbyPlayer winner = mc.getWinner().getPlayer();
+                for (TournamentPlayer tp : pairing.getPairedPlayers()) {
+                    if (winner.equals(tp.getPlayer())) {
+                        pairing.setWinner(tp);
+                        lastWinner = winner.getName();
+                        //System.out.println(TextUtil.concatNoSpace("Match Winner - ", lastWinner, "!"));
+                        //System.out.println("");
+                        break;
+                    }
+                }
+            }
+
+            tourney.reportMatchCompletion(pairing);
+        }
+        tourney.outputTournamentResults();
+        return tourney;
+    }
+}

forge-gui/src/main/java/forge/deck/CardRelationLDAGenerator.java

@@ -0,0 +1,247 @@
+package forge.deck;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import forge.StaticData;
+import forge.card.CardRules;
+import forge.card.CardRulesPredicates;
+import forge.deck.io.CardThemedLDAIO;
+import forge.deck.io.CardThemedMatrixIO;
+import forge.deck.io.DeckStorage;
+import forge.deck.lda.dataset.Dataset;
+import forge.deck.lda.lda.LDA;
+import forge.game.GameFormat;
+import forge.item.PaperCard;
+import forge.model.FModel;
+import forge.properties.ForgeConstants;
+import forge.util.storage.IStorage;
+import forge.util.storage.StorageImmediatelySerialized;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.tuple.Pair;
+
+import java.io.File;
+import java.util.*;
+
+import static forge.deck.lda.lda.inference.InferenceMethod.CGS;
+
+/**
+ * Created by maustin on 09/05/2017.
+ */
+public final class CardRelationLDAGenerator {
+
+    public static Map<String, Map<String,List<List<String>>>> ldaPools = new HashMap();
+    /**
+        To ensure that only cards with at least 14 connections (as 14*4+4=60) are included in the card based deck
+        generation pools
+    **/
+    public static final int MIN_REQUIRED_CONNECTIONS = 14;
+
+    public static boolean initialize(){
+        List<String> formatStrings = new ArrayList<>();
+        formatStrings.add(FModel.getFormats().getStandard().getName());
+        formatStrings.add(FModel.getFormats().getModern().getName());
+        formatStrings.add(DeckFormat.Commander.toString());
+
+        for (String formatString : formatStrings){
+            if(!initializeFormat(formatString)){
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    /** Try to load matrix .dat files, otherwise check for deck folders and build .dat, otherwise return false **/
+    public static boolean initializeFormat(String format){
+        Map<String,List<List<String>>> formatMap = CardThemedLDAIO.loadLDA(format);
+        if(formatMap==null) {
+            try {
+                if (CardThemedLDAIO.getMatrixFolder(format).exists()) {
+                    if (format.equals(FModel.getFormats().getStandard().getName())) {
+                        formatMap = initializeFormat(FModel.getFormats().getStandard());
+                    } else if (format.equals(FModel.getFormats().getModern().getName())) {
+                        formatMap = initializeFormat(FModel.getFormats().getModern());
+                    } else {
+                        //formatMap = initializeCommanderFormat();
+                    }
+                    CardThemedLDAIO.saveLDA(format, formatMap);
+                } else {
+                    return false;
+                }
+            }catch (Exception e){
+                e.printStackTrace();
+                return false;
+            }
+        }
+        ldaPools.put(format, formatMap);
+        return true;
+    }
+
+    public static Map<String,List<List<String>>> initializeFormat(GameFormat format) throws Exception{
+        Dataset dataset = new Dataset(format);
+
+        final int numTopics = dataset.getNumDocs()/50;
+        LDA lda = new LDA(0.1, 0.1, numTopics, dataset, CGS);
+        lda.run();
+        System.out.println(lda.computePerplexity(dataset));
+        List<List<String>> topics = new ArrayList<>();
+        Set<String> cards = new HashSet<String>();
+        for (int t = 0; t < numTopics; ++t) {
+            List<String> topic = new ArrayList<>();
+            List<Pair<String, Double>> highRankVocabs = lda.getVocabsSortedByPhi(t);
+            if (highRankVocabs.get(0).getRight()<=0.001d){
+                continue;
+            }
+            System.out.print("t" + t + ": ");
+            int i = 0;
+            while (topic.size()<=40) {
+                String cardName = highRankVocabs.get(i).getLeft();;
+                if(!StaticData.instance().getCommonCards().getUniqueByName(cardName).getRules().getType().isBasicLand()){
+                    if(highRankVocabs.get(i).getRight()>=0.0005d){
+                        cards.add(cardName);
+                    }
+                    System.out.println("[" + highRankVocabs.get(i).getLeft() + "," + highRankVocabs.get(i).getRight() + "],");
+                    topic.add(highRankVocabs.get(i).getLeft());
+                }
+                i++;
+            }
+            System.out.println();
+            topics.add(topic);
+        }
+        Map<String,List<List<String>>> cardTopicMap = new HashMap<>();
+        for (String card:cards){
+            List<List<String>>  cardTopics = new ArrayList<>();
+            for( List<String> topic:topics){
+                if(topic.contains(card)){
+                    cardTopics.add(topic);
+                }
+            }
+            cardTopicMap.put(card,cardTopics);
+        }
+        return cardTopicMap;
+    }
+
+    public static HashMap<String,List<Map.Entry<PaperCard,Integer>>> initializeCommanderFormat(){
+
+        IStorage<Deck> decks = new StorageImmediatelySerialized<Deck>("Generator",
+                new DeckStorage(new File(ForgeConstants.DECK_GEN_DIR,DeckFormat.Commander.toString()),
+                ForgeConstants.DECK_GEN_DIR, false),
+                true);
+
+        //get all cards
+        final Iterable<PaperCard> cards = Iterables.filter(FModel.getMagicDb().getCommonCards().getUniqueCards()
+                , Predicates.compose(Predicates.not(CardRulesPredicates.Presets.IS_BASIC_LAND_NOT_WASTES), PaperCard.FN_GET_RULES));
+        List<PaperCard> cardList = Lists.newArrayList(cards);
+        cardList.add(FModel.getMagicDb().getCommonCards().getCard("Wastes"));
+        Map<String, Integer> cardIntegerMap = new HashMap<>();
+        Map<Integer, PaperCard> integerCardMap = new HashMap<>();
+        Map<String, Integer> legendIntegerMap = new HashMap<>();
+        Map<Integer, PaperCard> integerLegendMap = new HashMap<>();
+        //generate lookups for cards to link card names to matrix columns
+        for (int i=0; i<cardList.size(); ++i){
+            cardIntegerMap.put(cardList.get(i).getName(), i);
+            integerCardMap.put(i, cardList.get(i));
+        }
+
+        //filter to just legal commanders
+        List<PaperCard> legends = Lists.newArrayList(Iterables.filter(cardList,Predicates.compose(
+                new Predicate<CardRules>() {
+                    @Override
+                    public boolean apply(CardRules rules) {
+                        return DeckFormat.Commander.isLegalCommander(rules);
+                    }
+                }, PaperCard.FN_GET_RULES)));
+
+        //generate lookups for legends to link commander names to matrix rows
+        for (int i=0; i<legends.size(); ++i){
+            legendIntegerMap.put(legends.get(i).getName(), i);
+            integerLegendMap.put(i, legends.get(i));
+        }
+        int[][] matrix = new int[legends.size()][cardList.size()];
+
+        //loop through commanders and decks
+        for (PaperCard legend:legends){
+            for (Deck deck:decks){
+                //if the deck has the commander
+                if (deck.getCommanders().contains(legend)){
+                    //update the matrix by incrementing the connectivity count for each card in the deck
+                    updateLegendMatrix(deck, legend, cardIntegerMap, legendIntegerMap, matrix);
+                }
+            }
+        }
+
+        //convert the matrix into a map of pools for each commander
+        HashMap<String,List<Map.Entry<PaperCard,Integer>>> cardPools = new HashMap<>();
+        for (PaperCard card:legends){
+            int col=legendIntegerMap.get(card.getName());
+            int[] distances = matrix[col];
+            int max = (Integer) Collections.max(Arrays.asList(ArrayUtils.toObject(distances)));
+            if (max>0) {
+                List<Map.Entry<PaperCard,Integer>> deckPool=new ArrayList<>();
+                for(int k=0;k<cardList.size(); k++){
+                    if(matrix[col][k]>0){
+                        deckPool.add(new AbstractMap.SimpleEntry<PaperCard, Integer>(integerCardMap.get(k),matrix[col][k]));
+                    }
+                }
+                cardPools.put(card.getName(), deckPool);
+            }
+        }
+        return cardPools;
+    }
+
+    //update the matrix by incrementing the connectivity count for each card in the deck
+    public static void updateLegendMatrix(Deck deck, PaperCard legend, Map<String, Integer> cardIntegerMap,
+                             Map<String, Integer> legendIntegerMap, int[][] matrix){
+        for (PaperCard pairCard:Iterables.filter(deck.getMain().toFlatList(),
+                Predicates.compose(Predicates.not(CardRulesPredicates.Presets.IS_BASIC_LAND_NOT_WASTES), PaperCard.FN_GET_RULES))){
+            if (!pairCard.getName().equals(legend.getName())){
+                try {
+                    int old = matrix[legendIntegerMap.get(legend.getName())][cardIntegerMap.get(pairCard.getName())];
+                    matrix[legendIntegerMap.get(legend.getName())][cardIntegerMap.get(pairCard.getName())] = old + 1;
+                }catch (NullPointerException ne){
+                    //Todo: Not sure what was failing here
+                    ne.printStackTrace();
+                }
+            }
+
+        }
+        //add partner commanders to matrix
+        if(deck.getCommanders().size()>1){
+            for(PaperCard partner:deck.getCommanders()){
+                if(!partner.equals(legend)){
+                    int old = matrix[legendIntegerMap.get(legend.getName())][cardIntegerMap.get(partner.getName())];
+                    matrix[legendIntegerMap.get(legend.getName())][cardIntegerMap.get(partner.getName())] = old + 1;
+                }
+            }
+        }
+    }
+
+    public static class ArrayIndexComparator implements Comparator<Integer>
+    {
+        private final Integer[] array;
+
+        public ArrayIndexComparator(Integer[] array)
+        {
+            this.array = array;
+        }
+
+        public Integer[] createIndexArray()
+        {
+            Integer[] indexes = new Integer[array.length];
+            for (int i = 0; i < array.length; i++)
+            {
+                indexes[i] = i; // Autoboxing
+            }
+            return indexes;
+        }
+
+        @Override
+        public int compare(Integer index1, Integer index2)
+        {
+            // Autounbox from Integer to int to use as array indexes
+            return array[index1].compareTo(array[index2]);
+        }
+    }
+}

forge-gui/src/main/java/forge/deck/CardRelationMatrixGenerator.java

@@ -6,6 +6,7 @@ import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import forge.card.CardRules;
 import forge.card.CardRulesPredicates;
+import forge.deck.io.CardThemedLDAIO;
 import forge.deck.io.CardThemedMatrixIO;
 import forge.deck.io.DeckStorage;
 import forge.game.GameFormat;
@@ -25,6 +26,8 @@ import java.util.*;
 public final class CardRelationMatrixGenerator {
 
     public static HashMap<String,HashMap<String,List<Map.Entry<PaperCard,Integer>>>> cardPools = new HashMap<>();
+
+    public static Map<String, Map<String,List<List<String>>>> ldaPools = new HashMap();
     /**
         To ensure that only cards with at least 14 connections (as 14*4+4=60) are included in the card based deck
         generation pools
@@ -34,6 +37,7 @@ public final class CardRelationMatrixGenerator {
     public static boolean initialize(){
         List<String> formatStrings = new ArrayList<>();
         formatStrings.add(FModel.getFormats().getStandard().getName());
+        ldaPools.put(FModel.getFormats().getStandard().getName(),CardThemedLDAIO.loadLDA(FModel.getFormats().getStandard().getName()));
         formatStrings.add(FModel.getFormats().getModern().getName());
         formatStrings.add(DeckFormat.Commander.toString());
 

forge-gui/src/main/java/forge/deck/CardThemedDeckGenerator.java

@@ -14,13 +14,14 @@ import java.util.List;
 public class CardThemedDeckGenerator extends DeckProxy implements Comparable<CardThemedDeckGenerator> {
     public static List<DeckProxy> getMatrixDecks(GameFormat format, boolean isForAi){
         final List<DeckProxy> decks = new ArrayList<DeckProxy>();
-        for(String card: CardRelationMatrixGenerator.cardPools.get(format.getName()).keySet()) {
+            for(String card: CardRelationLDAGenerator.ldaPools.get(format.getName()).keySet()) {
                 //exclude non AI playables as keycards for AI decks
                 if(isForAi&&FModel.getMagicDb().getCommonCards().getUniqueByName(card).getRules().getAiHints().getRemAIDecks()){
                     continue;
                 }
                 decks.add(new CardThemedDeckGenerator(card, format, isForAi));
             }
+
         return decks;
     }
     private final String name;

forge-gui/src/main/java/forge/deck/CommanderDeckGenerator.java

@@ -54,7 +54,7 @@ public class CommanderDeckGenerator extends DeckProxy implements Comparable<Comm
         if(isCardGen){
             uniqueCards = new ItemPool<PaperCard>(PaperCard.class);
             //TODO: upate to actual Brawl model from real Brawl decks
-            Iterable<String> legendNames=CardRelationMatrixGenerator.cardPools.get(FModel.getFormats().getStandard().getName()).keySet();
+            Iterable<String> legendNames=CardRelationLDAGenerator.ldaPools.get(FModel.getFormats().getStandard().getName()).keySet();
             for(String legendName:legendNames) {
                 uniqueCards.add(FModel.getMagicDb().getCommonCards().getUniqueByName(legendName));
             }

forge-gui/src/main/java/forge/deck/DeckgenUtil.java

@@ -18,6 +18,7 @@ import forge.game.GameType;
 import forge.item.PaperCard;
 import forge.itemmanager.IItemManager;
 import forge.limited.CardThemedCommanderDeckBuilder;
+import forge.limited.CardThemedConquestDeckBuilder;
 import forge.limited.CardThemedDeckBuilder;
 import forge.model.FModel;
 import forge.properties.ForgePreferences.FPref;
@@ -31,6 +32,7 @@ import forge.util.MyRandom;
 import forge.util.gui.SOptionPane;
 import forge.util.storage.IStorage;
 
+import java.awt.print.Paper;
 import java.util.*;
 
 /** 
@@ -45,7 +47,7 @@ public class DeckgenUtil {
 
     public static Deck buildCardGenDeck(GameFormat format, boolean isForAI){
         try {
-            List<String> keys      = new ArrayList<>(CardRelationMatrixGenerator.cardPools.get(format.getName()).keySet());
+            List<String> keys      = new ArrayList<>(CardRelationLDAGenerator.ldaPools.get(format.getName()).keySet());
             String       randomKey = keys.get( MyRandom.getRandom().nextInt(keys.size()) );
             Predicate<PaperCard> cardFilter = Predicates.and(format.getFilterPrinted(),PaperCard.Predicates.name(randomKey));
             PaperCard keyCard = FModel.getMagicDb().getCommonCards().getAllCards(cardFilter).get(0);
@@ -110,6 +112,98 @@ public class DeckgenUtil {
         }
     }
 
+    public static Deck buildPlanarConquestDeck(PaperCard card, GameFormat format, DeckFormat deckFormat){
+        return buildPlanarConquestDeck(card, null, format, deckFormat, false);
+    }
+
+    public static Deck buildPlanarConquestCommanderDeck(PaperCard card, GameFormat format, DeckFormat deckFormat){
+        Deck deck = buildPlanarConquestDeck(card, null, format, deckFormat, true);
+        deck.getMain().removeAll(card);
+        deck.getOrCreate(DeckSection.Commander).add(card);
+        return deck;
+    }
+
+    public static Deck buildPlanarConquestCommanderDeck(PaperCard card, PaperCard secondKeycard, GameFormat format, DeckFormat deckFormat){
+        Deck deck = buildPlanarConquestDeck(card, secondKeycard, format, deckFormat, true);
+        deck.getMain().removeAll(card);
+        deck.getOrCreate(DeckSection.Commander).add(card);
+        return deck;
+    }
+
+    public static Deck buildPlanarConquestDeck(PaperCard card, PaperCard secondKeycard, GameFormat format, DeckFormat deckFormat, boolean forCommander){
+        final boolean isForAI = true;
+        Set<String> uniqueCards = new HashSet<>();
+        List<PaperCard> selectedCards = new ArrayList<>();
+        List<List<String>> cardArchetypes = CardRelationLDAGenerator.ldaPools.get(FModel.getFormats().getStandard().getName()).get(card.getName());
+        for(List<String> archetype:cardArchetypes){
+            for(String cardName:archetype){
+                uniqueCards.add(cardName);
+            }
+        }
+        for(String cardName:uniqueCards){
+            selectedCards.add(StaticData.instance().getCommonCards().getUniqueByName(cardName));
+        }
+        /*
+        if(secondKeycard == null){
+            //get second keycard
+            for(Map.Entry<PaperCard,Integer> pair:potentialCards){
+                preSelectedCards.add(pair.getKey());
+            }
+            //filter out land cards and if for AI non-playable cards as potential second key cards and remove cards not legal in format
+            Iterable<PaperCard> preSelectedNonLandCards;
+            preSelectedNonLandCards=Iterables.filter(preSelectedCards,Predicates.and(
+                        Predicates.compose(CardRulesPredicates.IS_KEPT_IN_AI_DECKS, PaperCard.FN_GET_RULES),
+                        Predicates.compose(CardRulesPredicates.Presets.IS_NON_LAND, PaperCard.FN_GET_RULES),
+                        format.getFilterPrinted()));
+
+            preSelectedCards= Lists.newArrayList(preSelectedNonLandCards);
+
+            //choose a second card randomly from the top 8 cards if possible
+            int randMax=4;
+            if(preSelectedCards.size()<randMax){
+                randMax=preSelectedCards.size();
+            }
+            secondKeycard = preSelectedCards.get(MyRandom.getRandom().nextInt(randMax));
+        }
+
+        List<Map.Entry<PaperCard,Integer>> potentialSecondCards = CardRelationMatrixGenerator.cardPools.get(FModel.getFormats().getStandard().getName()).get(secondKeycard.getName());
+
+        //combine card distances from second key card and re-sort
+        if(potentialSecondCards !=null && potentialSecondCards.size()>0) {
+            combineDistances(potentialCards, potentialSecondCards);
+            Collections.sort(potentialCards, new CardDistanceComparator());
+            Collections.reverse(potentialCards);
+        }
+
+        List<PaperCard> selectedCards = new ArrayList<>();
+        selectedCards.add(card);
+        selectedCards.add(secondKeycard);
+        for(Map.Entry<PaperCard,Integer> pair:potentialCards){
+            PaperCard potentialCard = pair.getKey();
+            if (!potentialCard.getName().equals(card.getName())
+                    && (forCommander || !potentialCard.getName().equals(secondKeycard.getName()))
+                    && format.getFilterPrinted().apply(potentialCard)) {
+                selectedCards.add(pair.getKey());
+            }
+        }*/
+
+        //build deck from combined list
+        CardThemedDeckBuilder dBuilder;
+
+        if(forCommander){
+            dBuilder = new CardThemedConquestDeckBuilder(card, selectedCards, format ,isForAI, deckFormat);
+        }else{
+            dBuilder = new CardThemedDeckBuilder(card,secondKeycard, selectedCards,format,isForAI, deckFormat);
+        }
+
+        Deck deck = dBuilder.buildDeck();
+        return deck;
+    }
+
+    public static Deck buildCardGenDeck(PaperCard card, GameFormat format, boolean isForAI){
+        return buildCardGenDeck(card, null, format, isForAI);
+    }
+
     /**
      * Build a deck based on the chosen card.
      *
@@ -118,9 +212,12 @@ public class DeckgenUtil {
      * @param isForAI
      * @return
      */
-    public static Deck buildCardGenDeck(PaperCard card, GameFormat format, boolean isForAI){
+    public static Deck buildCardGenDeck(PaperCard card, PaperCard secondKeycard, GameFormat format, boolean isForAI){
-        List<Map.Entry<PaperCard,Integer>> potentialCards = new ArrayList<>();
+
-        potentialCards.addAll(CardRelationMatrixGenerator.cardPools.get(format.getName()).get(card.getName()));
+            return buildLDACardGenDeck(card, format, isForAI);
+
+                /*List<Map.Entry<PaperCard,Integer>> potentialCards = new ArrayList<>();
+        potentialCards.addAll(CardRelationLDAGenerator.ldaPools.get(format.getName()).get(card.getName()));
         Collections.sort(potentialCards,new CardDistanceComparator());
         Collections.reverse(potentialCards);
         //get second keycard
@@ -141,11 +238,14 @@ public class DeckgenUtil {
         preSelectedCards= Lists.newArrayList(preSelectedNonLandCards);
 
         //choose a second card randomly from the top 8 cards if possible
-        int randMax=8;
+        if(secondKeycard == null) {
-        if(preSelectedCards.size()<randMax){
+            int randMax = 8;
-            randMax=preSelectedCards.size();
+            if (preSelectedCards.size() < randMax) {
+                randMax = preSelectedCards.size();
             }
-        PaperCard secondKeycard = preSelectedCards.get(MyRandom.getRandom().nextInt(randMax));
+            secondKeycard = preSelectedCards.get(MyRandom.getRandom().nextInt(randMax));
+        }
+
         List<Map.Entry<PaperCard,Integer>> potentialSecondCards = CardRelationMatrixGenerator.cardPools.get(format.getName()).get(secondKeycard.getName());
 
         //combine card distances from second key card and re-sort
@@ -159,7 +259,19 @@ public class DeckgenUtil {
         for(Map.Entry<PaperCard,Integer> pair:potentialCards){
             selectedCards.add(pair.getKey());
         }
-        List<PaperCard> toRemove = new ArrayList<>();
+        */
+        /*List<PaperCard> toRemove = new ArrayList<>();
+        Set<String> uniqueCards = new HashSet<>();
+        List<PaperCard> selectedCards = new ArrayList<>();
+        List<List<String>> cardArchetypes = CardRelationLDAGenerator.ldaPools.get(FModel.getFormats().getStandard().getName()).get(card.getName());
+        for(List<String> archetype:cardArchetypes){
+            for(String cardName:archetype){
+                uniqueCards.add(cardName);
+            }
+        }
+        for(String cardName:uniqueCards){
+            selectedCards.add(StaticData.instance().getCommonCards().getUniqueByName(cardName));
+        }
 
         //randomly remove cards
         int removeCount=0;
@@ -223,6 +335,82 @@ public class DeckgenUtil {
         while(deck.get(DeckSection.Sideboard).countAll()>15){
             deck.get(DeckSection.Sideboard).remove(deck.get(DeckSection.Sideboard).get(0));
         }
+        return deck;*/
+    }
+
+    /**
+     * Build a deck based on the chosen card.
+     *
+     * @param card
+     * @param format
+     * @param isForAI
+     * @return
+     */
+    public static Deck buildLDACardGenDeck(PaperCard card,GameFormat format, boolean isForAI){
+
+        List<List<String>> preSelectedCardLists = CardRelationLDAGenerator.ldaPools.get(format.getName()).get(card.getName());
+        List<String> preSelectedCardNames = preSelectedCardLists.get(MyRandom.getRandom().nextInt(preSelectedCardLists.size()));
+        List<PaperCard> selectedCards = new ArrayList<>();
+        for(String name:preSelectedCardNames){
+            PaperCard cardToAdd = StaticData.instance().getCommonCards().getUniqueByName(name);
+            //for(int i=0; i<1;++i) {
+                if(!cardToAdd.getName().equals(card.getName())) {
+                    selectedCards.add(cardToAdd);
+                }
+            //}
+        }
+
+        List<PaperCard> toRemove = new ArrayList<>();
+
+        //randomly remove cards
+        int removeCount=0;
+        int i=0;
+        for(PaperCard c:selectedCards){
+            if(MyRandom.getRandom().nextInt(100)>70+(15-(i/selectedCards.size())*selectedCards.size()) && removeCount<4 //randomly remove some cards - more likely as distance increases
+                    &&!c.getName().contains("Urza")){ //avoid breaking Tron decks
+                toRemove.add(c);
+                removeCount++;
+            }
+            if(c.getName().equals(card.getName())){//may have been added in secondary list
+                toRemove.add(c);
+            }
+            ++i;
+        }
+        selectedCards.removeAll(toRemove);
+        //Add keycard
+        List<PaperCard> playsetList = new ArrayList<>();
+        int keyCardCount=4;
+        if(card.getRules().getMainPart().getManaCost().getCMC()>7){
+            keyCardCount=1+MyRandom.getRandom().nextInt(4);
+        }else if(card.getRules().getMainPart().getManaCost().getCMC()>5){
+            keyCardCount=2+MyRandom.getRandom().nextInt(3);
+        }
+        for(int j=0;j<keyCardCount;++j) {
+            playsetList.add(card);
+        }
+        for (PaperCard c:selectedCards){
+            for(int j=0;j<4;++j) {
+                if(MyRandom.getRandom().nextInt(100)<90) {
+                    playsetList.add(c);
+                }
+            }
+        }
+
+        //build deck from combined list
+        CardThemedDeckBuilder dBuilder = new CardThemedDeckBuilder(card,null, playsetList,format,isForAI);
+        Deck deck = dBuilder.buildDeck();
+        if(deck.getMain().countAll()!=60){
+            System.out.println(deck.getMain().countAll());
+            System.out.println("Wrong card count "+deck.getMain().countAll());
+            deck=buildCardGenDeck(format,isForAI);
+        }
+        if(deck.getMain().countAll(Predicates.compose(CardRulesPredicates.Presets.IS_LAND, PaperCard.FN_GET_RULES))>27){
+            System.out.println("Too many lands "+deck.getMain().countAll(Predicates.compose(CardRulesPredicates.Presets.IS_LAND, PaperCard.FN_GET_RULES)));
+            deck=buildCardGenDeck(format,isForAI);
+        }
+        while(deck.get(DeckSection.Sideboard).countAll()>15){
+            deck.get(DeckSection.Sideboard).remove(deck.get(DeckSection.Sideboard).get(0));
+        }
         return deck;
     }
 
@@ -517,20 +705,31 @@ public class DeckgenUtil {
         IDeckGenPool cardDb;
         DeckGeneratorBase gen = null;
         PaperCard selectedPartner=null;
-        if(isCardGen){
-            List<Map.Entry<PaperCard,Integer>> potentialCards = new ArrayList<>();
-            if(format.equals(DeckFormat.Brawl)){//TODO: replace with actual Brawl based data
-                potentialCards.addAll(CardRelationMatrixGenerator.cardPools.get(FModel.getFormats().getStandard().getName()).get(commander.getName()));
-            }else {
-                potentialCards.addAll(CardRelationMatrixGenerator.cardPools.get(DeckFormat.Commander.toString()).get(commander.getName()));
-            }
-            //Collections.shuffle(potentialCards, r);
         List<PaperCard> preSelectedCards = new ArrayList<>();
+        if(isCardGen){
+            if(format.equals(DeckFormat.Brawl)){//TODO: replace with actual Brawl based data
+                Set<String> uniqueCards = new HashSet<>();
+                List<List<String>> cardArchetypes = CardRelationLDAGenerator.ldaPools.get(FModel.getFormats().getStandard().getName()).get(commander.getName());
+                for(List<String> archetype:cardArchetypes){
+                    for(String cardName:archetype){
+                        uniqueCards.add(cardName);
+                    }
+                }
+                for(String card:uniqueCards){
+                    preSelectedCards.add(StaticData.instance().getCommonCards().getUniqueByName(card));
+                }
+            }else {
+                List<Map.Entry<PaperCard,Integer>> potentialCards = new ArrayList<>();
+                potentialCards.addAll(CardRelationMatrixGenerator.cardPools.get(DeckFormat.Commander.toString()).get(commander.getName()));
                 for(Map.Entry<PaperCard,Integer> pair:potentialCards){
                     if(format.isLegalCard(pair.getKey())) {
                         preSelectedCards.add(pair.getKey());
                     }
                 }
+            }
+            //Collections.shuffle(potentialCards, r);
+
+
             //check for partner commanders
             List<PaperCard> partners=new ArrayList<>();
             for(PaperCard c:preSelectedCards){

forge-gui/src/main/java/forge/deck/io/CardThemedLDAIO.java

@@ -0,0 +1,66 @@
+package forge.deck.io;
+
+import forge.game.GameFormat;
+import forge.item.PaperCard;
+import forge.properties.ForgeConstants;
+
+import java.io.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Created by maustin on 11/05/2017.
+ */
+public class CardThemedLDAIO {
+
+    /** suffix for all gauntlet data files */
+    public static final String SUFFIX_DATA = ".lda.dat";
+
+    public static void saveLDA(String format, Map<String,List<List<String>>> map){
+        File file = getLDAFile(format);
+        ObjectOutputStream s = null;
+        try {
+            FileOutputStream f = new FileOutputStream(file);
+            s = new ObjectOutputStream(f);
+            s.writeObject(map);
+            s.close();
+        } catch (IOException e) {
+            System.out.println("Error writing matrix data: " + e);
+        } finally {
+            if(s!=null) {
+                try {
+                    s.close();
+                }catch(Exception e){
+                    e.printStackTrace();
+                }
+            }
+        }
+    }
+
+    public static Map<String,List<List<String>>> loadLDA(String format){
+        try {
+            FileInputStream fin = new FileInputStream(getLDAFile(format));
+            ObjectInputStream s = new ObjectInputStream(fin);
+            Map<String, List<List<String>>> matrix = (Map<String, List<List<String>>>) s.readObject();
+            s.close();
+            return matrix;
+        }catch (Exception e){
+            System.out.println("Error reading LDA data: " + e);
+            return null;
+        }
+
+    }
+
+    public static File getLDAFile(final String name) {
+        return new File(ForgeConstants.DECK_GEN_DIR, name + SUFFIX_DATA);
+    }
+
+    public static File getMatrixFolder(final String name) {
+        return new File(ForgeConstants.DECK_GEN_DIR, name);
+    }
+
+    public static File getLDAFile(final GameFormat gf) {
+        return getLDAFile(gf.getName());
+    }
+}

forge-gui/src/main/java/forge/deck/lda/dataset/BagOfWords.java

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.dataset;
+package forge.deck.lda.dataset;
 
 import com.google.common.base.Predicates;
 import com.google.common.collect.Iterables;

forge-gui/src/main/java/forge/deck/lda/dataset/Dataset.java

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.dataset;
+package forge.deck.lda.dataset;
 
 import forge.game.GameFormat;
 import forge.model.FModel;

forge-gui/src/main/java/forge/deck/lda/dataset/Vocabularies.java

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.dataset;
+package forge.deck.lda.dataset;
 
 import java.io.IOException;
 import java.nio.file.Files;

forge-gui/src/main/java/forge/deck/lda/dataset/Vocabulary.java

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.dataset;
+package forge.deck.lda.dataset;
 
 public class Vocabulary {
     private final int id;

forge-gui/src/main/java/forge/deck/lda/examples/Example.java

@@ -0,0 +1,87 @@
+/*
+* Copyright 2015 Kohei Yamamoto
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package forge.deck.lda.examples;
+
+import java.util.*;
+
+import forge.StaticData;
+import forge.deck.lda.lda.LDA;
+import static forge.deck.lda.lda.inference.InferenceMethod.*;
+
+import forge.deck.io.CardThemedLDAIO;
+import forge.game.GameFormat;
+import forge.model.FModel;
+import forge.properties.ForgePreferences;
+import org.apache.commons.lang3.tuple.Pair;
+
+import forge.deck.lda.dataset.Dataset;
+
+public class Example {
+    public static void main(String[] args) throws Exception {
+        /*GuiBase.setInterface(new GuiDesktop());
+        FModel.initialize(null, new Function<ForgePreferences, Void>()  {
+            @Override
+            public Void apply(ForgePreferences preferences) {
+                preferences.setPref(ForgePreferences.FPref.LOAD_CARD_SCRIPTS_LAZILY, false);
+                return null;
+            }
+        });*/
+        GameFormat format = FModel.getFormats().getStandard();
+        Dataset dataset = new Dataset(format);
+        
+        final int numTopics = 50;
+        LDA lda = new LDA(0.1, 0.1, numTopics, dataset, CGS);
+        lda.run();
+        System.out.println(lda.computePerplexity(dataset));
+        List<List<String>> topics = new ArrayList<>();
+        Set<String> cards = new HashSet<String>();
+        for (int t = 0; t < numTopics; ++t) {
+            List<String> topic = new ArrayList<>();
+            List<Pair<String, Double>> highRankVocabs = lda.getVocabsSortedByPhi(t);
+            System.out.print("t" + t + ": ");
+            int i = 0;
+            while (topic.size()<=40) {
+                String cardName = highRankVocabs.get(i).getLeft();;
+                if(!StaticData.instance().getCommonCards().getUniqueByName(cardName).getRules().getType().isBasicLand()){
+                    cards.add(cardName);
+                    System.out.println("[" + highRankVocabs.get(i).getLeft() + "," + highRankVocabs.get(i).getRight() + "],");
+                    topic.add(cardName);
+                }
+                i++;
+            }
+            System.out.println();
+            topics.add(topic);
+        }
+        Map<String,List<List<String>>> cardTopicMap = new HashMap<>();
+        for (String card:cards){
+            List<List<String>>  cardTopics = new ArrayList<>();
+            for( List<String> topic:topics){
+                if(topic.contains(card)){
+                    cardTopics.add(topic);
+                }
+            }
+            cardTopicMap.put(card,cardTopics);
+        }
+        CardThemedLDAIO.saveLDA(format.getName(), cardTopicMap);
+
+
+        Map<String,List<List<String>>> cardTopicMapIn =CardThemedLDAIO.loadLDA(format.getName());
+        System.out.println(cardTopicMapIn.get(cardTopicMapIn.keySet().iterator().next()).get(0).toString());
+
+
+    }
+}

forge-gui/src/main/java/forge/deck/lda/lda/Alpha.java

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda;
+package forge.deck.lda.lda;
 
 import java.util.List;
 import java.util.stream.Collectors;

forge-gui/src/main/java/forge/deck/lda/lda/Beta.java

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda;
+package forge.deck.lda.lda;
 
 import java.util.Arrays;
 import java.util.List;

forge-gui/src/main/java/forge/deck/lda/lda/Hyperparameters.java

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda;
+package forge.deck.lda.lda;
 
 class Hyperparameters {
     private Alpha alpha;

forge-gui/src/main/java/forge/deck/lda/lda/LDA.java

@@ -14,21 +14,21 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda;
+package forge.deck.lda.lda;
 
 import java.io.IOException;
 import java.util.List;
 
-import forge.deck.generate.lda.lda.inference.Inference;
+import forge.deck.lda.lda.inference.Inference;
-import forge.deck.generate.lda.lda.inference.InferenceFactory;
+import forge.deck.lda.lda.inference.InferenceFactory;
-import forge.deck.generate.lda.lda.inference.InferenceMethod;
+import forge.deck.lda.lda.inference.InferenceMethod;
-import forge.deck.generate.lda.lda.inference.InferenceProperties;
+import forge.deck.lda.lda.inference.InferenceProperties;
 
 import org.apache.commons.lang3.tuple.Pair;
 
-import forge.deck.generate.lda.dataset.BagOfWords;
+import forge.deck.lda.dataset.BagOfWords;
-import forge.deck.generate.lda.dataset.Dataset;
+import forge.deck.lda.dataset.Dataset;
-import forge.deck.generate.lda.dataset.Vocabularies;
+import forge.deck.lda.dataset.Vocabularies;
 
 public class LDA {
     private Hyperparameters hyperparameters;

forge-gui/src/main/java/forge/deck/lda/lda/inference/Inference.java

@@ -14,11 +14,11 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda.inference;
+package forge.deck.lda.lda.inference;
 
 import java.util.List;
 
-import forge.deck.generate.lda.lda.LDA;
+import forge.deck.lda.lda.LDA;
 
 import org.apache.commons.lang3.tuple.Pair;
 

forge-gui/src/main/java/forge/deck/lda/lda/inference/InferenceFactory.java

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda.inference;
+package forge.deck.lda.lda.inference;
 
 
 public class InferenceFactory {

forge-gui/src/main/java/forge/deck/lda/lda/inference/InferenceMethod.java

@@ -14,10 +14,10 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda.inference;
+package forge.deck.lda.lda.inference;
 
 public enum InferenceMethod {
-    CGS("forge.deck.generate.lda.lda.inference.internal.CollapsedGibbsSampler"),
+    CGS("forge.deck.lda.lda.inference.internal.CollapsedGibbsSampler"),
     // more
     ;
 

forge-gui/src/main/java/forge/deck/lda/lda/inference/InferenceProperties.java

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda.inference;
+package forge.deck.lda.lda.inference;
 
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;

forge-gui/src/main/java/forge/deck/lda/lda/inference/internal/AssignmentCounter.java

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda.inference.internal;
+package forge.deck.lda.lda.inference.internal;
 
 import java.util.List;
 import java.util.stream.Collectors;

forge-gui/src/main/java/forge/deck/lda/lda/inference/internal/CollapsedGibbsSampler.java

@@ -14,21 +14,21 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda.inference.internal;
+package forge.deck.lda.lda.inference.internal;
 
 import java.util.Arrays;
 import java.util.List;
 import java.util.stream.IntStream;
 
-import forge.deck.generate.lda.lda.LDA;
+import forge.deck.lda.lda.LDA;
-import forge.deck.generate.lda.lda.inference.Inference;
+import forge.deck.lda.lda.inference.Inference;
-import forge.deck.generate.lda.lda.inference.InferenceProperties;
+import forge.deck.lda.lda.inference.InferenceProperties;
 
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.commons.math3.distribution.EnumeratedIntegerDistribution;
 import org.apache.commons.math3.distribution.IntegerDistribution;
 
-import forge.deck.generate.lda.dataset.Vocabulary;
+import forge.deck.lda.dataset.Vocabulary;
 
 public class CollapsedGibbsSampler implements Inference {
     private LDA lda;

forge-gui/src/main/java/forge/deck/lda/lda/inference/internal/Document.java

@@ -14,11 +14,11 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda.inference.internal;
+package forge.deck.lda.lda.inference.internal;
 
 import java.util.List;
 
-import forge.deck.generate.lda.dataset.Vocabulary;
+import forge.deck.lda.dataset.Vocabulary;
 
 public class Document {
     private final int id;

forge-gui/src/main/java/forge/deck/lda/lda/inference/internal/Documents.java

@@ -14,17 +14,17 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda.inference.internal;
+package forge.deck.lda.lda.inference.internal;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 
-import forge.deck.generate.lda.lda.LDA;
+import forge.deck.lda.lda.LDA;
-import forge.deck.generate.lda.dataset.BagOfWords;
+import forge.deck.lda.dataset.BagOfWords;
-import forge.deck.generate.lda.dataset.Vocabularies;
+import forge.deck.lda.dataset.Vocabularies;
-import forge.deck.generate.lda.dataset.Vocabulary;
+import forge.deck.lda.dataset.Vocabulary;
 
 class Documents {
     private List<Document> documents;

forge-gui/src/main/java/forge/deck/lda/lda/inference/internal/Topic.java

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda.inference.internal;
+package forge.deck.lda.lda.inference.internal;
 
 class Topic {
     private final int id;

forge-gui/src/main/java/forge/deck/lda/lda/inference/internal/TopicAssignment.java

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda.inference.internal;
+package forge.deck.lda.lda.inference.internal;
 
 import java.util.ArrayList;
 import java.util.List;

forge-gui/src/main/java/forge/deck/lda/lda/inference/internal/TopicCounter.java

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda.inference.internal;
+package forge.deck.lda.lda.inference.internal;
 
 class TopicCounter {
     private AssignmentCounter topicCount;

forge-gui/src/main/java/forge/deck/lda/lda/inference/internal/Topics.java

@@ -14,19 +14,19 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda.inference.internal;
+package forge.deck.lda.lda.inference.internal;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 
-import forge.deck.generate.lda.lda.LDA;
+import forge.deck.lda.lda.LDA;
 
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Pair;
 
-import forge.deck.generate.lda.dataset.Vocabularies;
+import forge.deck.lda.dataset.Vocabularies;
 
 class Topics {
     private List<Topic> topics;

forge-gui/src/main/java/forge/deck/lda/lda/inference/internal/VocabularyCounter.java

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda.inference.internal;
+package forge.deck.lda.lda.inference.internal;
 
 class VocabularyCounter {
     private AssignmentCounter vocabCount;

forge-gui/src/main/java/forge/deck/lda/lda/inference/internal/Words.java

@@ -14,12 +14,12 @@
 * limitations under the License.
 */
 
-package forge.deck.generate.lda.lda.inference.internal;
+package forge.deck.lda.lda.inference.internal;
 
 import java.util.Collections;
 import java.util.List;
 
-import forge.deck.generate.lda.dataset.Vocabulary;
+import forge.deck.lda.dataset.Vocabulary;
 
 class Words {
     private List<Vocabulary> words;

forge-gui/src/main/java/forge/limited/CardThemedDeckBuilder.java

@@ -60,7 +60,7 @@ public class CardThemedDeckBuilder extends DeckGeneratorBase {
     protected Iterable<PaperCard> onColorNonCreatures;
     protected Iterable<PaperCard> keyCards;
 
-    protected static final boolean logToConsole = false;
+    protected static final boolean logToConsole = true;
     protected static final boolean logColorsToConsole = false;
 
     protected Map<Integer,Integer> targetCMCs;
@@ -113,9 +113,11 @@ public class CardThemedDeckBuilder extends DeckGeneratorBase {
         if(!colors.hasAllColors(keyCard.getRules().getColorIdentity().getColor())){
             colors = ColorSet.fromMask(colors.getColor() | keyCard.getRules().getColorIdentity().getColor());
         }
-        if(!colors.hasAllColors(secondKeyCard.getRules().getColorIdentity().getColor())){
+        if(secondKeyCard!=null) {
+            if (!colors.hasAllColors(secondKeyCard.getRules().getColorIdentity().getColor())) {
                 colors = ColorSet.fromMask(colors.getColor() | secondKeyCard.getRules().getColorIdentity().getColor());
             }
+        }
         numSpellsNeeded = ((Double)Math.floor(targetSize*(getCreaturePercentage()+getSpellPercentage()))).intValue();
         numCreaturesToStart = ((Double)Math.ceil(targetSize*(getCreaturePercentage()))).intValue();
         landsNeeded = ((Double)Math.ceil(targetSize*(getLandPercentage()))).intValue();;
@@ -337,7 +339,7 @@ public class CardThemedDeckBuilder extends DeckGeneratorBase {
             rankedColorList.removeAll(keyCardList);
         }
         // Add the second keycard if not land
-        if(!secondKeyCard.getRules().getMainPart().getType().isLand()) {
+        if(secondKeyCard!=null && !secondKeyCard.getRules().getMainPart().getType().isLand()) {
             Iterable<PaperCard> secondKeyCards = Iterables.filter(aiPlayables,PaperCard.Predicates.name(secondKeyCard.getName()));
             final List<PaperCard> keyCardList = Lists.newArrayList(secondKeyCards);
             deckList.addAll(keyCardList);
@@ -357,7 +359,7 @@ public class CardThemedDeckBuilder extends DeckGeneratorBase {
             landsNeeded--;
         }
         // Add the deck card
-        if(secondKeyCard.getRules().getMainPart().getType().isLand()) {
+        if(secondKeyCard!=null && secondKeyCard.getRules().getMainPart().getType().isLand()) {
             Iterable<PaperCard> secondKeyCards = Iterables.filter(aiPlayables,PaperCard.Predicates.name(secondKeyCard.getName()));
             final List<PaperCard> keyCardList = Lists.newArrayList(secondKeyCards);
             deckList.addAll(keyCardList);
@@ -480,7 +482,11 @@ public class CardThemedDeckBuilder extends DeckGeneratorBase {
      * @return name
      */
     protected String generateName() {
-        return keyCard.getName() + " - " + secondKeyCard.getName() +" based deck";
+        if(secondKeyCard!=null ) {
+            return keyCard.getName() + " - " + secondKeyCard.getName() + " based deck";
+        }else{
+            return keyCard.getName() + " based deck";
+        }
     }
 
     /**

forge-gui/src/main/java/forge/model/FModel.java

@@ -26,6 +26,7 @@ import forge.achievement.*;
 import forge.ai.AiProfileUtil;
 import forge.card.CardPreferences;
 import forge.card.CardType;
+import forge.deck.CardRelationLDAGenerator;
 import forge.deck.CardRelationMatrixGenerator;
 import forge.deck.io.DeckPreferences;
 import forge.game.GameFormat;
@@ -219,6 +220,7 @@ public final class FModel {
         if(!FModel.getPreferences().getPrefBoolean(FPref.LOAD_CARD_SCRIPTS_LAZILY)
                 &&FModel.getPreferences().getPrefBoolean(FPref.DECKGEN_CARDBASED)) {
             deckGenMatrixLoaded=CardRelationMatrixGenerator.initialize();
+            deckGenMatrixLoaded=CardRelationLDAGenerator.initialize();
         }
     }