Adds an undocumented option -XmaxPermsPerPrecompile that
supports sharded precompilations. This is useful for
compiles that have so many permutations that the resources
of the compile node are overwhelmed if they are all
precompiled together.
Review by: scottb
git-svn-id: https://google-web-toolkit.googlecode.com/svn/trunk@5088 8db76d5a-ed1c-0410-87a9-c151d255dfc7
diff --git a/dev/core/src/com/google/gwt/dev/CompilePerms.java b/dev/core/src/com/google/gwt/dev/CompilePerms.java
index e97579f..aca31b3 100644
--- a/dev/core/src/com/google/gwt/dev/CompilePerms.java
+++ b/dev/core/src/com/google/gwt/dev/CompilePerms.java
@@ -21,12 +21,13 @@
import com.google.gwt.dev.jjs.JavaToJavaScriptCompiler;
import com.google.gwt.dev.jjs.UnifiedAst;
import com.google.gwt.dev.util.FileBackedObject;
-import com.google.gwt.dev.util.Util;
import com.google.gwt.dev.util.arg.ArgHandlerLocalWorkers;
import com.google.gwt.util.tools.ArgHandlerString;
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.SortedSet;
import java.util.TreeSet;
@@ -141,7 +142,7 @@
CompilePermsOptions {
private int localWorkers;
- private int[] permsToCompile = new int[0];
+ private int[] permsToCompile;
public CompilePermsOptionsImpl() {
}
@@ -161,7 +162,7 @@
}
public int[] getPermsToCompile() {
- return permsToCompile.clone();
+ return (permsToCompile == null) ? null : permsToCompile.clone();
}
public void setLocalWorkers(int localWorkers) {
@@ -169,7 +170,8 @@
}
public void setPermsToCompile(int[] permsToCompile) {
- this.permsToCompile = permsToCompile.clone();
+ this.permsToCompile = (permsToCompile == null) ? null
+ : permsToCompile.clone();
}
}
@@ -242,6 +244,44 @@
return new File(compilerWorkDir, "permutation-" + permNumber + ".js");
}
+ /**
+ * Check whether any of the listed permutations have their precompilation in
+ * the supplied precompilation file.
+ */
+ private static boolean isPrecompileForAnyOf(
+ PrecompilationFile precompilationFile, int[] permutations) {
+ if (permutations == null) {
+ // Special case: compile everything.
+ return true;
+ }
+ for (int perm : permutations) {
+ if (precompilationFile.isForPermutation(perm)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Choose the subset of requested permutations that correspond to the
+ * indicated precompilation.
+ */
+ private static Permutation[] selectPermutationsForPrecompilation(
+ int[] permsToRun, PrecompilationFile precompilationFile,
+ Precompilation precompilation) {
+ if (permsToRun == null) {
+ // Special case: compile everything.
+ return precompilation.getPermutations();
+ }
+ ArrayList<Permutation> subPermsList = new ArrayList<Permutation>();
+ for (int perm : permsToRun) {
+ if (precompilationFile.isForPermutation(perm)) {
+ subPermsList.add(precompilation.getPermutation(perm));
+ }
+ }
+ return subPermsList.toArray(new Permutation[0]);
+ }
+
private final CompilePermsOptionsImpl options;
public CompilePerms(CompilePermsOptions options) {
@@ -250,53 +290,70 @@
public boolean run(TreeLogger logger) throws UnableToCompleteException {
for (String moduleName : options.getModuleNames()) {
- File compilerWorkDir = options.getCompilerWorkDir(moduleName);
- File precompilationFile = new File(compilerWorkDir,
- Precompile.PRECOMPILATION_FILENAME);
- if (!precompilationFile.exists()) {
- logger.log(TreeLogger.ERROR, "File not found '"
- + precompilationFile.getAbsolutePath()
- + "'; please run Precompile first");
- return false;
- }
- Precompilation precompilation;
- try {
- /*
- * TODO: don't bother deserializing the generated artifacts.
- */
- precompilation = Util.readFileAsObject(precompilationFile,
- Precompilation.class);
- } catch (ClassNotFoundException e) {
- logger.log(TreeLogger.ERROR, "Unable to deserialize '"
- + precompilationFile.getAbsolutePath() + "'", e);
- return false;
- }
-
- Permutation[] perms = precompilation.getPermutations();
- Permutation[] subPerms;
+ /*
+ * NOTE: as a special case, null means "compile everything".
+ */
int[] permsToRun = options.getPermsToCompile();
- if (permsToRun.length == 0) {
- subPerms = perms;
- } else {
- int i = 0;
- subPerms = new Permutation[permsToRun.length];
- // Range check the supplied perms.
- for (int permToRun : permsToRun) {
- if (permToRun >= perms.length) {
- logger.log(TreeLogger.ERROR, "The specified perm number '"
- + permToRun + "' is too big; the maximum value is "
- + (perms.length - 1) + "'");
- return false;
- }
- subPerms[i++] = perms[permToRun];
- }
+
+ File compilerWorkDir = options.getCompilerWorkDir(moduleName);
+ Collection<PrecompilationFile> precomps;
+ try {
+ precomps = PrecompilationFile.scanJarFile(new File(compilerWorkDir,
+ Precompile.PRECOMPILE_FILENAME));
+ } catch (IOException e) {
+ logger.log(TreeLogger.ERROR, "Failed to scan "
+ + Precompile.PRECOMPILE_FILENAME + "; has Precompile been run?", e);
+ return false;
}
- List<FileBackedObject<PermutationResult>> resultFiles = makeResultFiles(
- compilerWorkDir, subPerms);
- compile(logger, precompilation, subPerms, options.getLocalWorkers(),
- resultFiles);
+ /*
+ * Check that all requested permutations actually have a Precompile
+ * available
+ */
+ if (permsToRun != null) {
+ checking_perms : for (int perm : permsToRun) {
+ for (PrecompilationFile precomp : precomps) {
+ if (precomp.isForPermutation(perm)) {
+ continue checking_perms;
+ }
+ }
+ logger.log(TreeLogger.ERROR,
+ "No precompilation file found for permutation " + perm);
+ return false;
+ }
+ } else {
+ // TODO: validate that a contiguous set of all perms exists.
+ }
+
+ /*
+ * Perform the compiles one file at a time, to minimize the number of
+ * times a Precompilation needs to be deserialized.
+ */
+ for (PrecompilationFile precompilationFile : precomps) {
+ if (!isPrecompileForAnyOf(precompilationFile, permsToRun)) {
+ continue;
+ }
+ Precompilation precompilation;
+ try {
+ /*
+ * TODO: don't bother deserializing the generated artifacts.
+ */
+ precompilation = precompilationFile.newInstance(logger);
+ } catch (UnableToCompleteException e) {
+ return false;
+ }
+
+ // Choose which permutations go with this permutation
+ Permutation[] subPerms = selectPermutationsForPrecompilation(
+ permsToRun, precompilationFile, precompilation);
+
+ List<FileBackedObject<PermutationResult>> resultFiles = makeResultFiles(
+ compilerWorkDir, subPerms);
+ compile(logger, precompilation, subPerms, options.getLocalWorkers(),
+ resultFiles);
+ }
}
+
return true;
}
}
diff --git a/dev/core/src/com/google/gwt/dev/Link.java b/dev/core/src/com/google/gwt/dev/Link.java
index 4ea728f..4d78f19 100644
--- a/dev/core/src/com/google/gwt/dev/Link.java
+++ b/dev/core/src/com/google/gwt/dev/Link.java
@@ -37,7 +37,10 @@
import com.google.gwt.dev.util.arg.OptionWarDir;
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -312,32 +315,42 @@
public boolean run(TreeLogger logger) throws UnableToCompleteException {
for (String moduleName : options.getModuleNames()) {
File compilerWorkDir = options.getCompilerWorkDir(moduleName);
+ Collection<PrecompilationFile> precomps;
+ try {
+ precomps = PrecompilationFile.scanJarFile(new File(compilerWorkDir,
+ Precompile.PRECOMPILE_FILENAME));
+ } catch (IOException e) {
+ logger.log(TreeLogger.ERROR, "Failed to scan "
+ + Precompile.PRECOMPILE_FILENAME, e);
+ return false;
+ }
+
ModuleDef module = ModuleDefLoader.loadFromClassPath(logger, moduleName);
- File precompilationFile = new File(
- options.getCompilerWorkDir(moduleName),
- Precompile.PRECOMPILATION_FILENAME);
- if (!precompilationFile.exists()) {
- logger.log(TreeLogger.ERROR, "File not found '"
- + precompilationFile.getAbsolutePath()
+ if (precomps.isEmpty()) {
+ logger.log(TreeLogger.ERROR, "No precompilation files found in '"
+ + compilerWorkDir.getAbsolutePath()
+ "'; please run Precompile first");
return false;
}
- Precompilation precompilation;
- try {
- precompilation = Util.readFileAsObject(precompilationFile,
- Precompilation.class);
- } catch (ClassNotFoundException e) {
- logger.log(TreeLogger.ERROR, "Unable to deserialize '"
- + precompilationFile.getAbsolutePath() + "'", e);
- return false;
- }
- Permutation[] perms = precompilation.getPermutations();
- ArtifactSet generatedArtifacts = precompilation.getGeneratedArtifacts();
- JJSOptions precompileOptions = precompilation.getUnifiedAst().getOptions();
+ List<Permutation> permsList = new ArrayList<Permutation>();
+ ArtifactSet generatedArtifacts = new ArtifactSet();
+ JJSOptions precompileOptions = null;
- precompilation = null; // No longer needed, and it needs a lot of memory
+ for (PrecompilationFile precompilationFile : precomps) {
+ Precompilation precompilation;
+ try {
+ precompilation = precompilationFile.newInstance(logger);
+ } catch (UnableToCompleteException e) {
+ return false;
+ }
+ permsList.addAll(Arrays.asList(precompilation.getPermutations()));
+ generatedArtifacts.addAll(precompilation.getGeneratedArtifacts());
+ precompileOptions = precompilation.getUnifiedAst().getOptions();
+ }
+
+ Permutation[] perms = permsList.toArray(new Permutation[0]);
List<FileBackedObject<PermutationResult>> resultFiles = new ArrayList<FileBackedObject<PermutationResult>>(
perms.length);
@@ -345,8 +358,7 @@
File f = CompilePerms.makePermFilename(compilerWorkDir,
perms[i].getId());
if (!f.exists()) {
- logger.log(TreeLogger.ERROR, "File not found '"
- + precompilationFile.getAbsolutePath()
+ logger.log(TreeLogger.ERROR, "File not found '" + f.getAbsolutePath()
+ "'; please compile all permutations");
return false;
}
diff --git a/dev/core/src/com/google/gwt/dev/Precompilation.java b/dev/core/src/com/google/gwt/dev/Precompilation.java
index 4c0fa32..ac6c25a 100644
--- a/dev/core/src/com/google/gwt/dev/Precompilation.java
+++ b/dev/core/src/com/google/gwt/dev/Precompilation.java
@@ -35,23 +35,30 @@
private final Permutation[] permutations;
private final UnifiedAst unifiedAst;
+ public Precompilation(UnifiedAst unifiedAst,
+ Collection<Permutation> permutations, ArtifactSet generatedArtifacts) {
+ this(unifiedAst, permutations, 0, generatedArtifacts);
+ }
+
/**
- * Constructs a new precompilation. We create new Permutations with
- * a new id so that the ids are consecutive and correspond to the index
- * in the array.
+ * Constructs a new precompilation. We create new Permutations with a new id
+ * so that the ids are consecutive and correspond to the index in the array.
*
* @param unifiedAst the unified AST used by
* {@link com.google.gwt.dev.jjs.JavaToJavaScriptCompiler}
* @param permutations the set of permutations that can be run
+ * @param permutationBase the id to use for the first permutation
* @param generatedArtifacts the set of artifacts created by generators
*/
public Precompilation(UnifiedAst unifiedAst,
- Collection<Permutation> permutations, ArtifactSet generatedArtifacts) {
+ Collection<Permutation> permutations, int permutationBase,
+ ArtifactSet generatedArtifacts) {
+
this.unifiedAst = unifiedAst;
this.permutations = new Permutation[permutations.size()];
int i = 0;
for (Permutation permutation : permutations) {
- this.permutations[i] = new Permutation(i, permutation);
+ this.permutations[i] = new Permutation(i + permutationBase, permutation);
++i;
}
this.generatedArtifacts = generatedArtifacts;
@@ -64,6 +71,15 @@
return generatedArtifacts;
}
+ public Permutation getPermutation(int id) {
+ for (Permutation perm : permutations) {
+ if (perm.getId() == id) {
+ return perm;
+ }
+ }
+ return null;
+ }
+
/**
* Returns the set of permutations to run.
*/
diff --git a/dev/core/src/com/google/gwt/dev/PrecompilationFile.java b/dev/core/src/com/google/gwt/dev/PrecompilationFile.java
new file mode 100644
index 0000000..279b67f
--- /dev/null
+++ b/dev/core/src/com/google/gwt/dev/PrecompilationFile.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright 2009 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.dev;
+
+import com.google.gwt.core.ext.TreeLogger;
+import com.google.gwt.core.ext.UnableToCompleteException;
+import com.google.gwt.dev.util.Util;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.zip.ZipEntry;
+
+/**
+ * The backing file for a {@link Precompilation} for a subset of the
+ * permutations for a compilation. The permutations this one is for are in a
+ * consecutive range described by {@link #getFirstPerm()} and
+ * {@link #getNumPerms()}.
+ */
+public class PrecompilationFile {
+ /**
+ * Suffix for precompilation files stored in the work directory.
+ */
+ private static final String FILENAME_PREFIX = "precompilation";
+
+ /**
+ * Prefix for precompilation files stored in the work directory.
+ */
+ private static final String FILENAME_SUFFIX = ".ser";
+
+ public static String fileNameForPermutations(int firstPerm, int numPerms) {
+ return FILENAME_PREFIX + firstPerm + "-" + (firstPerm + numPerms - 1)
+ + FILENAME_SUFFIX;
+ }
+
+ public static Collection<PrecompilationFile> scanJarFile(File file)
+ throws IOException {
+ Pattern pattern = Pattern.compile("precompilation([0-9]+)-([0-9]+)\\.ser");
+
+ List<PrecompilationFile> precomps = new ArrayList<PrecompilationFile>();
+
+ JarFile jarFile = new JarFile(file);
+ Enumeration<JarEntry> entries = jarFile.entries();
+ while (entries.hasMoreElements()) {
+ JarEntry entry = entries.nextElement();
+ Matcher matcher = pattern.matcher(entry.getName());
+ if (matcher.matches()) {
+ int start = Integer.parseInt(matcher.group(1));
+ int end = Integer.parseInt(matcher.group(2));
+ int numPerms = end - start + 1;
+
+ precomps.add(new PrecompilationFile(jarFile, entry, start, numPerms));
+ }
+ }
+
+ return precomps;
+ }
+
+ private final JarFile jarFile;
+ private final ZipEntry zipEntry;
+
+ private final int firstPerm;
+
+ private final int numPerms;
+
+ public PrecompilationFile(JarFile jarFile, ZipEntry zipEntry, int firstPerm,
+ int numPerms) {
+ this.firstPerm = firstPerm;
+ this.numPerms = numPerms;
+
+ this.jarFile = jarFile;
+ this.zipEntry = zipEntry;
+ }
+
+ /**
+ * Return the first permutation this {@link Precompilation} is for.
+ */
+ public int getFirstPerm() {
+ return firstPerm;
+ }
+
+ /**
+ * Get the number of the highest permutation included in this
+ * {@link Precompilation}.
+ */
+ public int getLastPerm() {
+ return getFirstPerm() + getNumPerms() - 1;
+ }
+
+ /**
+ * Return the number of permutations in this {@link Precompilation}.
+ */
+ public int getNumPerms() {
+ return numPerms;
+ }
+
+ public boolean isForPermutation(int perm) {
+ return perm >= getFirstPerm() && perm <= getLastPerm();
+ }
+
+ public Precompilation newInstance(TreeLogger logger)
+ throws UnableToCompleteException {
+ Precompilation toReturn;
+
+ try {
+ toReturn = Util.readStreamAsObject(jarFile.getInputStream(zipEntry),
+ Precompilation.class);
+ } catch (IOException e) {
+ toReturn = null;
+ } catch (ClassNotFoundException e) {
+ logger.log(TreeLogger.ERROR, "Missing class definition", e);
+ throw new UnableToCompleteException();
+ }
+
+ if (toReturn == null) {
+ logger.log(TreeLogger.ERROR, "Unable to instantiate object");
+ throw new UnableToCompleteException();
+ }
+ return toReturn;
+ }
+}
diff --git a/dev/core/src/com/google/gwt/dev/Precompile.java b/dev/core/src/com/google/gwt/dev/Precompile.java
index 9e9b143..3fa247d 100644
--- a/dev/core/src/com/google/gwt/dev/Precompile.java
+++ b/dev/core/src/com/google/gwt/dev/Precompile.java
@@ -51,20 +51,26 @@
import com.google.gwt.dev.util.arg.ArgHandlerDumpSignatures;
import com.google.gwt.dev.util.arg.ArgHandlerEnableAssertions;
import com.google.gwt.dev.util.arg.ArgHandlerGenDir;
+import com.google.gwt.dev.util.arg.ArgHandlerMaxPermsPerPrecompile;
import com.google.gwt.dev.util.arg.ArgHandlerScriptStyle;
import com.google.gwt.dev.util.arg.ArgHandlerValidateOnlyFlag;
import com.google.gwt.dev.util.arg.OptionDisableUpdateCheck;
import com.google.gwt.dev.util.arg.OptionDumpSignatures;
import com.google.gwt.dev.util.arg.OptionGenDir;
+import com.google.gwt.dev.util.arg.OptionMaxPermsPerPrecompile;
import com.google.gwt.dev.util.arg.OptionValidateOnly;
import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.concurrent.FutureTask;
+import java.util.jar.JarOutputStream;
+import java.util.zip.ZipEntry;
/**
* Performs the first phase of compilation, generating the set of permutations
@@ -77,7 +83,7 @@
*/
public interface PrecompileOptions extends JJSOptions, CompileTaskOptions,
OptionGenDir, OptionValidateOnly, OptionDisableUpdateCheck,
- OptionDumpSignatures {
+ OptionDumpSignatures, OptionMaxPermsPerPrecompile {
}
static class ArgProcessor extends CompileArgProcessor {
@@ -94,6 +100,7 @@
registerHandler(new ArgHandlerDraftCompile(options));
registerHandler(new ArgHandlerDisableUpdateCheck(options));
registerHandler(new ArgHandlerDumpSignatures(options));
+ registerHandler(new ArgHandlerMaxPermsPerPrecompile(options));
}
@Override
@@ -108,6 +115,7 @@
private File dumpFile;
private File genDir;
private final JJSOptionsImpl jjsOptions = new JJSOptionsImpl();
+ private int maxPermsPerPrecompile;
private boolean validateOnly;
public PrecompileOptionsImpl() {
@@ -125,6 +133,7 @@
setDisableUpdateCheck(other.isUpdateCheckDisabled());
setDumpSignatureFile(other.getDumpSignatureFile());
setGenDir(other.getGenDir());
+ setMaxPermsPerPrecompile(other.getMaxPermsPerPrecompile());
setValidateOnly(other.isValidateOnly());
}
@@ -136,6 +145,10 @@
return genDir;
}
+ public int getMaxPermsPerPrecompile() {
+ return maxPermsPerPrecompile;
+ }
+
public JsOutputOption getOutput() {
return jjsOptions.getOutput();
}
@@ -208,6 +221,10 @@
this.genDir = genDir;
}
+ public void setMaxPermsPerPrecompile(int maxPermsPerPrecompile) {
+ this.maxPermsPerPrecompile = maxPermsPerPrecompile;
+ }
+
public void setOutput(JsOutputOption output) {
jjsOptions.setOutput(output);
}
@@ -288,9 +305,12 @@
}
}
- static final String PERM_COUNT_FILENAME = "permCount.txt";
+ /**
+ * The file name for the result of Precompile.
+ */
+ public static final String PRECOMPILE_FILENAME = "precompilation.ser";
- static final String PRECOMPILATION_FILENAME = "precompilation.ser";
+ static final String PERM_COUNT_FILENAME = "permCount.txt";
/**
* Performs a command-line precompile.
@@ -339,59 +359,11 @@
* @return the precompilation
*/
public static Precompilation precompile(TreeLogger logger,
- JJSOptions jjsOptions, ModuleDef module, File genDir,
+ PrecompileOptionsImpl jjsOptions, ModuleDef module, File genDir,
File generatorResourcesDir, File dumpSignatureFile) {
- try {
- CompilationState compilationState = module.getCompilationState(logger);
- if (dumpSignatureFile != null) {
- // Dump early to avoid generated types.
- SignatureDumper.dumpSignatures(logger, module.getTypeOracle(logger),
- dumpSignatureFile);
- }
-
- String[] declEntryPts = module.getEntryPointTypeNames();
- if (declEntryPts.length == 0) {
- logger.log(TreeLogger.ERROR, "Module has no entry points defined", null);
- throw new UnableToCompleteException();
- }
-
- ArtifactSet generatedArtifacts = new ArtifactSet();
- DistillerRebindPermutationOracle rpo = new DistillerRebindPermutationOracle(
- module, compilationState, generatedArtifacts,
- new PropertyPermutations(module.getProperties()), genDir,
- generatorResourcesDir);
- FragmentLoaderCreator fragmentLoaderCreator = new FragmentLoaderCreator(
- compilationState, module, genDir, generatorResourcesDir,
- generatedArtifacts);
- WebModeCompilerFrontEnd frontEnd = new WebModeCompilerFrontEnd(
- compilationState, rpo, fragmentLoaderCreator);
- PerfLogger.start("Precompile");
- UnifiedAst unifiedAst = JavaToJavaScriptCompiler.precompile(logger,
- frontEnd, declEntryPts, null, jjsOptions,
- rpo.getPermuationCount() == 1);
- PerfLogger.end();
-
- // Merge all identical permutations together.
- Permutation[] permutations = rpo.getPermutations();
- // Sort the permutations by an ordered key to ensure determinism.
- SortedMap<String, Permutation> merged = new TreeMap<String, Permutation>();
- for (Permutation permutation : permutations) {
- permutation.reduceRebindAnswers(unifiedAst.getRebindRequests());
- // Arbitrarily choose as a key the stringified map of rebind answers.
- String rebindResultsString = permutation.getRebindAnswers().toString();
- if (merged.containsKey(rebindResultsString)) {
- Permutation existing = merged.get(rebindResultsString);
- existing.mergeFrom(permutation);
- } else {
- merged.put(rebindResultsString, permutation);
- }
- }
- return new Precompilation(unifiedAst, merged.values(), generatedArtifacts);
- } catch (UnableToCompleteException e) {
- // We intentionally don't pass in the exception here since the real
- // cause has been logged.
- return null;
- }
+ return precompile(logger, jjsOptions, module, 0, 0,
+ module.getProperties().numPermutations(), genDir,
+ generatorResourcesDir, dumpSignatureFile);
}
/**
@@ -446,7 +418,66 @@
}
}
- private ModuleDef module;
+ private static Precompilation precompile(TreeLogger logger,
+ PrecompileOptionsImpl jjsOptions, ModuleDef module, int permutationBase,
+ int firstPerm, int numPerms, File genDir, File generatorResourcesDir,
+ File dumpSignatureFile) {
+
+ try {
+ CompilationState compilationState = module.getCompilationState(logger);
+ if (dumpSignatureFile != null) {
+ // Dump early to avoid generated types.
+ SignatureDumper.dumpSignatures(logger, module.getTypeOracle(logger),
+ dumpSignatureFile);
+ }
+
+ String[] declEntryPts = module.getEntryPointTypeNames();
+ if (declEntryPts.length == 0) {
+ logger.log(TreeLogger.ERROR, "Module has no entry points defined", null);
+ throw new UnableToCompleteException();
+ }
+
+ ArtifactSet generatedArtifacts = new ArtifactSet();
+ DistillerRebindPermutationOracle rpo = new DistillerRebindPermutationOracle(
+ module,
+ compilationState,
+ generatedArtifacts,
+ new PropertyPermutations(module.getProperties(), firstPerm, numPerms),
+ genDir, generatorResourcesDir);
+ FragmentLoaderCreator fragmentLoaderCreator = new FragmentLoaderCreator(
+ compilationState, module, genDir, generatorResourcesDir,
+ generatedArtifacts);
+ WebModeCompilerFrontEnd frontEnd = new WebModeCompilerFrontEnd(
+ compilationState, rpo, fragmentLoaderCreator);
+ PerfLogger.start("Precompile");
+ UnifiedAst unifiedAst = JavaToJavaScriptCompiler.precompile(logger,
+ frontEnd, declEntryPts, null, jjsOptions,
+ rpo.getPermuationCount() == 1);
+ PerfLogger.end();
+
+ // Merge all identical permutations together.
+ Permutation[] permutations = rpo.getPermutations();
+ // Sort the permutations by an ordered key to ensure determinism.
+ SortedMap<String, Permutation> merged = new TreeMap<String, Permutation>();
+ for (Permutation permutation : permutations) {
+ permutation.reduceRebindAnswers(unifiedAst.getRebindRequests());
+ // Arbitrarily choose as a key the stringified map of rebind answers.
+ String rebindResultsString = permutation.getRebindAnswers().toString();
+ if (merged.containsKey(rebindResultsString)) {
+ Permutation existing = merged.get(rebindResultsString);
+ existing.mergeFrom(permutation);
+ } else {
+ merged.put(rebindResultsString, permutation);
+ }
+ }
+ return new Precompilation(unifiedAst, merged.values(), permutationBase,
+ generatedArtifacts);
+ } catch (UnableToCompleteException e) {
+ // We intentionally don't pass in the exception here since the real
+ // cause has been logged.
+ return null;
+ }
+ }
private final PrecompileOptionsImpl options;
@@ -460,7 +491,17 @@
Util.recursiveDelete(compilerWorkDir, true);
compilerWorkDir.mkdirs();
- this.module = ModuleDefLoader.loadFromClassPath(logger, moduleName);
+ JarOutputStream precompilationJar;
+ try {
+ precompilationJar = new JarOutputStream(new FileOutputStream(new File(
+ compilerWorkDir, PRECOMPILE_FILENAME)));
+ } catch (IOException e) {
+ logger.log(TreeLogger.ERROR, "Could not create " + PRECOMPILE_FILENAME,
+ e);
+ return false;
+ }
+
+ ModuleDef module = ModuleDefLoader.loadFromClassPath(logger, moduleName);
// TODO: All JDT checks now before even building TypeOracle?
module.getCompilationState(logger);
@@ -477,21 +518,70 @@
} else {
TreeLogger branch = logger.branch(TreeLogger.INFO,
"Precompiling module " + module.getName());
- Precompilation precompilation = precompile(branch, options, module,
- options.getGenDir(), compilerWorkDir,
- options.getDumpSignatureFile());
- if (precompilation == null) {
- branch.log(TreeLogger.ERROR, "Precompilation failed");
+ int potentialPermutations = module.getProperties().numPermutations();
+ int permutationsPerIteration = options.getMaxPermsPerPrecompile();
+
+ if (permutationsPerIteration <= 0) {
+ permutationsPerIteration = potentialPermutations;
+ }
+ /*
+ * The potential number of permutations to precompile >= the actual
+ * number of permutations that end up being precompiled, because some of
+ * the permutations might collapse due to identical rebind results. So
+ * we have to track these two counts and ids separately.
+ */
+ int actualPermutations = 0;
+ for (int potentialFirstPerm = 0; potentialFirstPerm < potentialPermutations; potentialFirstPerm += permutationsPerIteration) {
+ int numPermsToPrecompile = Math.min(potentialPermutations
+ - potentialFirstPerm, permutationsPerIteration);
+
+ /*
+ * After the first precompile, we need to forcibly refresh
+ * CompilationState to clear out generated units from previous
+ * precompilations.
+ */
+ if (potentialFirstPerm != 0) {
+ module.getCompilationState(branch).refresh(branch);
+ }
+
+ Precompilation precompilation = precompile(branch, options, module,
+ actualPermutations, potentialFirstPerm, numPermsToPrecompile,
+ options.getGenDir(), compilerWorkDir,
+ options.getDumpSignatureFile());
+ if (precompilation == null) {
+ branch.log(TreeLogger.ERROR, "Precompilation failed");
+ return false;
+ }
+ int actualNumPermsPrecompiled = precompilation.getPermutations().length;
+ String precompilationFilename = PrecompilationFile.fileNameForPermutations(
+ actualPermutations, actualNumPermsPrecompiled);
+ try {
+ precompilationJar.putNextEntry(new ZipEntry(precompilationFilename));
+ Util.writeObjectToStream(precompilationJar, precompilation);
+ } catch (IOException e) {
+ branch.log(TreeLogger.ERROR,
+ "Failed to write a precompilation result", e);
+ return false;
+ }
+
+ actualPermutations += actualNumPermsPrecompiled;
+ branch.log(TreeLogger.DEBUG, "Compiled " + actualNumPermsPrecompiled
+ + " permutations starting from " + potentialFirstPerm);
+ }
+
+ try {
+ precompilationJar.close();
+ } catch (IOException e) {
+ branch.log(TreeLogger.ERROR, "Failed to finalize "
+ + PRECOMPILE_FILENAME, e);
return false;
}
- Util.writeObjectAsFile(branch, new File(compilerWorkDir,
- PRECOMPILATION_FILENAME), precompilation);
+
Util.writeStringAsFile(branch, new File(compilerWorkDir,
- PERM_COUNT_FILENAME),
- String.valueOf(precompilation.getPermutations().length));
+ PERM_COUNT_FILENAME), String.valueOf(actualPermutations));
branch.log(TreeLogger.INFO,
"Precompilation succeeded, number of permutations: "
- + precompilation.getPermutations().length);
+ + actualPermutations);
}
}
return true;
diff --git a/dev/core/src/com/google/gwt/dev/cfg/BindingProperty.java b/dev/core/src/com/google/gwt/dev/cfg/BindingProperty.java
index e62a644..af8970f 100644
--- a/dev/core/src/com/google/gwt/dev/cfg/BindingProperty.java
+++ b/dev/core/src/com/google/gwt/dev/cfg/BindingProperty.java
@@ -47,10 +47,16 @@
definedValues.add(newValue);
}
+ /**
+ * Returns the set of allowed values in sorted order.
+ */
public String[] getAllowedValues() {
return allowedValues.toArray(new String[allowedValues.size()]);
}
+ /**
+ * Returns the set of defined values in sorted order.
+ */
public String[] getDefinedValues() {
return definedValues.toArray(new String[definedValues.size()]);
}
diff --git a/dev/core/src/com/google/gwt/dev/cfg/Properties.java b/dev/core/src/com/google/gwt/dev/cfg/Properties.java
index e52bfca..79af9a3 100644
--- a/dev/core/src/com/google/gwt/dev/cfg/Properties.java
+++ b/dev/core/src/com/google/gwt/dev/cfg/Properties.java
@@ -72,6 +72,23 @@
return map.values().iterator();
}
+ /**
+ * Count the total number of permutations that this property set supports.
+ */
+ public int numPermutations() {
+ BindingProperty[] bindingPropsArray = bindingProps.toArray(new BindingProperty[0]);
+
+ int count = 1;
+
+ for (BindingProperty prop : bindingPropsArray) {
+ String[] options = prop.getAllowedValues();
+ assert (options.length > 0);
+ count *= options.length;
+ }
+
+ return count;
+ }
+
private <T extends Property> T create(String name, Class<T> clazz) {
if (clazz == null) {
throw new NullPointerException("clazz");
diff --git a/dev/core/src/com/google/gwt/dev/cfg/PropertyPermutations.java b/dev/core/src/com/google/gwt/dev/cfg/PropertyPermutations.java
index 36c8139..1e77ee2 100644
--- a/dev/core/src/com/google/gwt/dev/cfg/PropertyPermutations.java
+++ b/dev/core/src/com/google/gwt/dev/cfg/PropertyPermutations.java
@@ -15,76 +15,50 @@
*/
package com.google.gwt.dev.cfg;
-import java.util.Arrays;
+import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
import java.util.SortedSet;
/**
- * Generates all possible permutations of properties in a module.
+ * Generates all possible permutations of properties in a module. Each
+ * permutation consists of the list of active property values associated with
+ * that permutation. That list of property values is represented as an array of
+ * Strings corresponding to the list of properties returned by
+ * {@link Properties#getBindingProperties()}.
*/
public class PropertyPermutations implements Iterable<String[]> {
- private int currPermIndex;
-
- private final int lastProp;
-
- private final BindingProperty[] properties;
-
- private final String[][] values;
-
- public PropertyPermutations(Properties properties) {
- SortedSet<BindingProperty> bindingProps = properties.getBindingProperties();
- this.properties = bindingProps.toArray(new BindingProperty[bindingProps.size()]);
- lastProp = this.properties.length - 1;
- int permCount = countPermutations();
- values = new String[permCount][];
- if (this.properties.length > 0) {
- permute(null, 0);
- assert (permCount == currPermIndex);
- } else {
- values[0] = new String[0];
- }
- }
-
- public BindingProperty[] getOrderedProperties() {
- return properties;
- }
-
- public String[] getOrderedPropertyValues(int permutation) {
- return values[permutation];
- }
-
/**
- * Enumerates each permutation as an array of strings such that the index of
- * each string in the array corresponds to the property at the same index in
- * the array returned from {@link #getOrderedProperties()}.
+ * Returns the list of all permutations. This method must return results in a
+ * consistently sorted order over multiple invocations.
*/
- public Iterator<String[]> iterator() {
- return Arrays.asList(values).iterator();
- }
+ private static List<String[]> allPermutationsOf(Properties properties) {
+ BindingProperty[] bindingProperties = getOrderedPropertiesOf(properties);
- public int size() {
- return values.length;
- }
+ int permCount = properties.numPermutations();
- private int countPermutations() {
- int count = 1;
- for (int i = 0; i < properties.length; i++) {
- BindingProperty prop = properties[i];
- String[] options = getPossibilities(prop);
- assert (options.length > 0);
- count *= options.length;
+ List<String[]> permutations = new ArrayList<String[]>(permCount);
+ if (bindingProperties.length > 0) {
+ permute(bindingProperties, null, 0, permutations);
+ assert (permCount == permutations.size());
+ } else {
+ permutations.add(new String[0]);
}
- return count;
+ return permutations;
}
- private String[] getPossibilities(BindingProperty prop) {
- return prop.getAllowedValues();
+ private static BindingProperty[] getOrderedPropertiesOf(Properties properties) {
+ SortedSet<BindingProperty> bindingProps = properties.getBindingProperties();
+ return bindingProps.toArray(new BindingProperty[bindingProps.size()]);
}
- private void permute(String[] soFar, int whichProp) {
+ private static void permute(BindingProperty[] properties, String[] soFar,
+ int whichProp, List<String[]> permutations) {
+ int lastProp = properties.length - 1;
+
BindingProperty prop = properties[whichProp];
- String[] options = getPossibilities(prop);
+ String[] options = prop.getAllowedValues();
for (int i = 0; i < options.length; i++) {
String knownValue = options[i];
@@ -96,13 +70,46 @@
nextStep[whichProp] = knownValue;
if (whichProp < lastProp) {
- permute(nextStep, whichProp + 1);
+ permute(properties, nextStep, whichProp + 1, permutations);
} else {
// Finished this permutation.
- //
- values[currPermIndex] = nextStep;
- ++currPermIndex;
+ permutations.add(nextStep);
}
}
}
+
+ private final Properties properties;
+ private final List<String[]> values;
+
+ public PropertyPermutations(Properties properties) {
+ this.properties = properties;
+ this.values = allPermutationsOf(properties);
+ }
+
+ public PropertyPermutations(Properties properties, int firstPerm, int numPerms) {
+ this.properties = properties;
+ values = allPermutationsOf(properties).subList(firstPerm,
+ firstPerm + numPerms);
+ }
+
+ public BindingProperty[] getOrderedProperties() {
+ return getOrderedPropertiesOf(properties);
+ }
+
+ public String[] getOrderedPropertyValues(int permutation) {
+ return values.get(permutation);
+ }
+
+ /**
+ * Enumerates each permutation as an array of strings such that the index of
+ * each string in the array corresponds to the property at the same index in
+ * the array returned from {@link #getOrderedProperties()}.
+ */
+ public Iterator<String[]> iterator() {
+ return values.iterator();
+ }
+
+ public int size() {
+ return values.size();
+ }
}
diff --git a/dev/core/src/com/google/gwt/dev/util/Util.java b/dev/core/src/com/google/gwt/dev/util/Util.java
index dcbd806..4da79d9 100644
--- a/dev/core/src/com/google/gwt/dev/util/Util.java
+++ b/dev/core/src/com/google/gwt/dev/util/Util.java
@@ -277,7 +277,7 @@
}
}
- /**
+/**
* Escapes '&', '<', '>', '"', and '\'' to their XML entity equivalents.
*/
public static String escapeXml(String unescaped) {
@@ -617,36 +617,12 @@
public static <T extends Serializable> T readFileAsObject(File file,
Class<T> type) throws ClassNotFoundException {
FileInputStream fileInputStream = null;
- ObjectInputStream objectInputStream = null;
try {
fileInputStream = new FileInputStream(file);
- objectInputStream = new ObjectInputStream(fileInputStream);
- return type.cast(objectInputStream.readObject());
+ return readStreamAsObject(fileInputStream, type);
} catch (IOException e) {
return null;
} finally {
- Utility.close(objectInputStream);
- Utility.close(fileInputStream);
- }
- }
-
- public static Serializable[] readFileAsObjects(File file,
- Class<? extends Serializable>... types) throws ClassNotFoundException {
- FileInputStream fileInputStream = null;
- ObjectInputStream objectInputStream = null;
- try {
- fileInputStream = new FileInputStream(file);
- objectInputStream = new ObjectInputStream(fileInputStream);
- Serializable[] results = new Serializable[types.length];
- for (int i = 0; i < results.length; ++i) {
- Object object = objectInputStream.readObject();
- results[i] = types[i].cast(object);
- }
- return results;
- } catch (IOException e) {
- return null;
- } finally {
- Utility.close(objectInputStream);
Utility.close(fileInputStream);
}
}
@@ -682,6 +658,19 @@
}
}
+ public static <T extends Serializable> T readStreamAsObject(
+ InputStream inputStream, Class<T> type) throws ClassNotFoundException {
+ ObjectInputStream objectInputStream = null;
+ try {
+ objectInputStream = new ObjectInputStream(inputStream);
+ return type.cast(objectInputStream.readObject());
+ } catch (IOException e) {
+ return null;
+ } finally {
+ Utility.close(objectInputStream);
+ }
+ }
+
/**
* Reads an entire input stream as String. Closes the input stream.
*/
@@ -1100,24 +1089,32 @@
public static void writeObjectAsFile(TreeLogger logger, File file,
Serializable... objects) throws UnableToCompleteException {
FileOutputStream stream = null;
- ObjectOutputStream objectStream = null;
try {
file.getParentFile().mkdirs();
stream = new FileOutputStream(file);
- objectStream = new ObjectOutputStream(stream);
- for (Serializable object : objects) {
- objectStream.writeObject(object);
- }
+ writeObjectToStream(stream, objects);
} catch (IOException e) {
logger.log(TreeLogger.ERROR, "Unable to write file: "
+ file.getAbsolutePath(), e);
throw new UnableToCompleteException();
} finally {
- Utility.close(objectStream);
Utility.close(stream);
}
}
+ /**
+ * Serializes an object and writes it to a stream.
+ */
+ public static void writeObjectToStream(OutputStream stream,
+ Serializable... objects) throws IOException {
+ ObjectOutputStream objectStream = null;
+ objectStream = new ObjectOutputStream(stream);
+ for (Serializable object : objects) {
+ objectStream.writeObject(object);
+ }
+ objectStream.flush();
+ }
+
public static boolean writeStringAsFile(File file, String string) {
FileOutputStream stream = null;
OutputStreamWriter writer = null;
diff --git a/dev/core/src/com/google/gwt/dev/util/arg/ArgHandlerMaxPermsPerPrecompile.java b/dev/core/src/com/google/gwt/dev/util/arg/ArgHandlerMaxPermsPerPrecompile.java
new file mode 100644
index 0000000..08bc229
--- /dev/null
+++ b/dev/core/src/com/google/gwt/dev/util/arg/ArgHandlerMaxPermsPerPrecompile.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.dev.util.arg;
+
+import com.google.gwt.util.tools.ArgHandlerInt;
+
+/**
+ * Handles an argument for {@link OptionMaxPermsPerPrecompile}.
+ */
+public class ArgHandlerMaxPermsPerPrecompile extends ArgHandlerInt {
+
+ private final OptionMaxPermsPerPrecompile options;
+
+ public ArgHandlerMaxPermsPerPrecompile(OptionMaxPermsPerPrecompile options) {
+ this.options = options;
+ }
+
+ @Override
+ public String[] getDefaultArgs() {
+ return new String[] {getTag(), "-1"};
+ }
+
+ @Override
+ public String getPurpose() {
+ return "maximum permutations to compile at a time";
+ }
+
+ @Override
+ public String getTag() {
+ return "-XmaxPermsPerPrecompile";
+ }
+
+ @Override
+ public String[] getTagArgs() {
+ return new String[] {"perms"};
+ }
+
+ @Override
+ public boolean isUndocumented() {
+ return true;
+ }
+
+ @Override
+ public void setInt(int maxPerms) {
+ options.setMaxPermsPerPrecompile(maxPerms);
+ }
+}
diff --git a/dev/core/src/com/google/gwt/dev/util/arg/OptionMaxPermsPerPrecompile.java b/dev/core/src/com/google/gwt/dev/util/arg/OptionMaxPermsPerPrecompile.java
new file mode 100644
index 0000000..fdfa71f
--- /dev/null
+++ b/dev/core/src/com/google/gwt/dev/util/arg/OptionMaxPermsPerPrecompile.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.dev.util.arg;
+
+/**
+ * Option to choose the maximum number of permutations to compile at once.
+ */
+public interface OptionMaxPermsPerPrecompile {
+ /**
+ * Get the maximum number of permutations per precompile. Returns a negative
+ * number if there is no limit.
+ */
+ int getMaxPermsPerPrecompile();
+
+ void setMaxPermsPerPrecompile(int maxPerms);
+}
