Modifies the GWT compiler sharded entry points to support an alternative method of sharding where multiple instances of Precompile and CompilePerms are invoked. Review at http://gwt-code-reviews.appspot.com/1074801 git-svn-id: https://google-web-toolkit.googlecode.com/svn/trunk@9206 8db76d5a-ed1c-0410-87a9-c151d255dfc7
diff --git a/dev/core/src/com/google/gwt/dev/AnalyzeModule.java b/dev/core/src/com/google/gwt/dev/AnalyzeModule.java index 6b7886c..ae1a249 100644 --- a/dev/core/src/com/google/gwt/dev/AnalyzeModule.java +++ b/dev/core/src/com/google/gwt/dev/AnalyzeModule.java
@@ -1,12 +1,12 @@ /* * Copyright 2010 Google Inc. - * + * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of * the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the @@ -18,11 +18,11 @@ import com.google.gwt.core.ext.TreeLogger; import com.google.gwt.core.ext.UnableToCompleteException; import com.google.gwt.dev.CompileTaskRunner.CompileTask; +import com.google.gwt.dev.Precompile.PrecompileOptions; import com.google.gwt.dev.Precompile.PrecompileOptionsImpl; import com.google.gwt.dev.cfg.ModuleDef; import com.google.gwt.dev.cfg.ModuleDefLoader; import com.google.gwt.dev.cfg.PropertyPermutations; -import com.google.gwt.dev.jjs.JJSOptions; import com.google.gwt.dev.util.Memory; import com.google.gwt.dev.util.Util; import com.google.gwt.dev.util.log.speedtracer.CompilerEventType; @@ -30,6 +30,7 @@ import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger.Event; import java.io.File; +import java.io.IOException; /** * Performs the first phase of compilation, generating the set of permutations @@ -37,27 +38,13 @@ */ public class AnalyzeModule { - private interface AnalyzeModuleOptions extends JJSOptions, CompileTaskOptions { - } - - static class ArgProcessor extends CompileArgProcessor { - public ArgProcessor(AnalyzeModuleOptions options) { - super(options); - } - - @Override - protected String getName() { - return AnalyzeModule.class.getName(); - } - } - /** - * Mirror all the compiler options, in case this makes some - * difference in module analysis. + * Mirror all the compiler options, in case this makes some difference in + * module analysis. */ - static class AnalyzeModuleOptionsImpl extends PrecompileOptionsImpl - implements AnalyzeModuleOptions { - private final PrecompileOptionsImpl precompileOptions = new PrecompileOptionsImpl(); + @SuppressWarnings("serial") + static class AnalyzeModuleOptionsImpl extends PrecompileOptionsImpl implements + AnalyzeModuleOptions { public AnalyzeModuleOptionsImpl() { } @@ -68,13 +55,40 @@ public void copyFrom(AnalyzeModuleOptions other) { super.copyFrom(other); - precompileOptions.copyFrom(other); } } + static class ArgProcessor extends Precompile.ArgProcessor { + public ArgProcessor(AnalyzeModuleOptions options) { + super(options); + } + + @Override + protected String getName() { + return AnalyzeModule.class.getName(); + } + } + + private interface AnalyzeModuleOptions extends Precompile.PrecompileOptions { + // This interface is here to support future options. + } + /** - * Performs a command-line analysis of the module with output to files - * for use in further sharded build steps. + * The options passed to the AnalyzeModule step are saved here and passed + * through to future steps. + */ + static final String OPTIONS_FILENAME = "compilerOptions.ser"; + + /** + * Count of the maximum number of permutations in the module configuration. + * Used to communicate the number of permutations defined in a module to an + * external build tool. + */ + static final String PERM_COUNT_FILENAME = "permCount.txt"; + + /** + * Performs a command-line analysis of the module with output to files for use + * in further sharded build steps. */ public static void main(String[] args) { Memory.initialize(); @@ -92,6 +106,29 @@ analyzeModuleEvent.end(); } + /** + * Loads the AnalyzeModule.OPTIONS_FILENAME data. + * + * Silently returns <code>null</code> if the file is not found or another problem is + * encountered reading the file. + */ + public static PrecompileOptions readAnalyzeModuleOptionsFile( + TreeLogger logger, File compilerWorkDir) { + File optionsFile = new File(compilerWorkDir, AnalyzeModule.OPTIONS_FILENAME); + PrecompileOptions precompilationOptions = null; + try { + precompilationOptions = Util.readFileAsObject(optionsFile, + PrecompileOptions.class); + } catch (IOException e) { + logger.log(TreeLogger.DEBUG, "Failed to read " + optionsFile + + "\nHas AnalyzeModule been run? Falling back.", e); + return null; + } catch (ClassNotFoundException e) { + logger.log(TreeLogger.ERROR, "Failed to read " + optionsFile, e); + return null; + } + return precompilationOptions; + } private final AnalyzeModuleOptionsImpl options; @@ -100,31 +137,31 @@ } public boolean run(TreeLogger logger) throws UnableToCompleteException { - for (String moduleName : options.getModuleNames()) { File compilerWorkDir = options.getCompilerWorkDir(moduleName); Util.recursiveDelete(compilerWorkDir, true); // No need to check mkdirs result because an IOException will occur anyway compilerWorkDir.mkdirs(); - + ModuleDef module = ModuleDefLoader.loadFromClassPath(logger, moduleName); - logger.log(TreeLogger.INFO, "Analyzing module " + module.getName()); /* - * Count the permutations to expose to external build tools performing - * a sharded compile. + * Count the permutations to expose to external build tools performing a + * sharded compile. */ - int numPermutations = - new PropertyPermutations(module.getProperties(), module.getActiveLinkerNames()) - .collapseProperties().size(); - Util.writeStringAsFile(logger, new File(compilerWorkDir, Precompile.PERM_COUNT_FILENAME), - String.valueOf(numPermutations)); - - // TODO(zundel): Serializing the ModuleDef structure would save time in subsequent steps. - - // TODO(zundel): Building the initial type oracle in this step would save cputime when - // the precompile step is sharded. + int numPermutations = new PropertyPermutations(module.getProperties(), + module.getActiveLinkerNames()).collapseProperties().size(); + Util.writeStringAsFile(logger, new File(compilerWorkDir, + AnalyzeModule.PERM_COUNT_FILENAME), String.valueOf(numPermutations)); + Util.writeObjectAsFile(logger, new File(compilerWorkDir, + AnalyzeModule.OPTIONS_FILENAME), options); + + // TODO(zundel): Serializing the ModuleDef structure would save time in + // subsequent steps. + + // TODO(zundel): Building the initial type oracle in this step would save + // cputime when the precompile step is sharded. } return true;
diff --git a/dev/core/src/com/google/gwt/dev/CompileOnePerm.java b/dev/core/src/com/google/gwt/dev/CompileOnePerm.java new file mode 100644 index 0000000..ddfbddd --- /dev/null +++ b/dev/core/src/com/google/gwt/dev/CompileOnePerm.java
@@ -0,0 +1,190 @@ +/* + * Copyright 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev; + +import com.google.gwt.core.ext.TreeLogger; +import com.google.gwt.core.ext.UnableToCompleteException; +import com.google.gwt.dev.CompileTaskRunner.CompileTask; +import com.google.gwt.dev.Precompile.PrecompileOptions; +import com.google.gwt.dev.cfg.ModuleDef; +import com.google.gwt.dev.cfg.ModuleDefLoader; +import com.google.gwt.dev.jjs.PermutationResult; +import com.google.gwt.dev.util.PerfCounter; +import com.google.gwt.dev.util.arg.ArgHandlerPerm; +import com.google.gwt.dev.util.arg.OptionPerm; + +import java.io.File; +import java.util.List; + +/** + * Performs the second phase of compilation, converting one of the Precompile's + * serialized AST files into JavaScript outputs. + */ +public class CompileOnePerm { + + /** + * Options for CompilePerm. + */ + public interface CompileOnePermOptions extends CompileTaskOptions, OptionPerm { + } + + static class ArgProcessor extends CompileArgProcessor { + public ArgProcessor(CompileOnePermOptions options) { + super(options); + registerHandler(new ArgHandlerPerm(options)); + } + + @Override + protected String getName() { + return CompileOnePerm.class.getName(); + } + } + + /** + * Concrete class to implement compiler perm options. + */ + static class CompileOnePermOptionsImpl extends CompileTaskOptionsImpl + implements CompileOnePermOptions { + + private int permToCompile = -1; + + public CompileOnePermOptionsImpl() { + } + + public CompileOnePermOptionsImpl(CompileOnePermOptions other) { + copyFrom(other); + } + + public void copyFrom(CompileOnePermOptions other) { + super.copyFrom(other); + setPermToCompile(other.getPermToCompile()); + } + + public int getPermToCompile() { + return permToCompile; + } + + public void setPermToCompile(int permToCompile) { + this.permToCompile = permToCompile; + } + } + + public static void main(String[] args) { + int exitCode = -1; + /* + * NOTE: main always exits with a call to System.exit to terminate any + * non-daemon threads that were started in Generators. Typically, this is to + * shutdown AWT related threads, since the contract for their termination is + * still implementation-dependent. + */ + final CompileOnePermOptions options = new CompileOnePermOptionsImpl(); + if (new ArgProcessor(options).processArgs(args)) { + CompileTask task = new CompileTask() { + public boolean run(TreeLogger logger) throws UnableToCompleteException { + return new CompileOnePerm(options).run(logger); + } + }; + if (CompileTaskRunner.runWithAppropriateLogger(options, task)) { + // Exit w/ success code. + exitCode = 0; + } + } + PerfCounter.print(); + System.exit(exitCode); + } + + /** + * Return the filename corresponding to the given permutation number, + * one-based. + */ + static File makePermFilename(File compilerWorkDir, int permNumber) { + return new File(compilerWorkDir, "permutation-" + permNumber + ".js"); + } + + /** + * Run Compile where precompilation occurred previously on sharded steps of + * Precompile. There will be a different serialized AST file named + * permutation-*.ser, one per permutation. + * + * @param logger tree logger + * @param moduleName Name of the GWT module with the entry point + * @param permId permutations to compile + * @param compilerWorkDir directory where work files are stored + * + * @return <code>true</code> if compilation succeeds + */ + private static boolean compileSpecificPermutation(TreeLogger logger, + String moduleName, PrecompileOptions precompilationOptions, int permId, + File compilerWorkDir) throws UnableToCompleteException { + + ModuleDef module = ModuleDefLoader.loadFromClassPath(logger, moduleName); + + logger = logger.branch(TreeLogger.INFO, "Compiling permutation " + permId); + + File precompilationFile = new File(compilerWorkDir, + PrecompileOnePerm.getPrecompileFilename(permId)); + Precompilation precompilation = (Precompilation) CompilePerms.readPrecompilationFile( + logger, precompilationFile); + + // Choose the permutation that goes with this precompilation + Permutation[] subPerms = CompilePerms.selectPermutationsForPrecompilation( + new int[]{permId}, precompilation); + assert subPerms.length == 1; + + PermutationResult permResult = precompilation.getUnifiedAst().compilePermutation( + logger, subPerms[0]); + Link.linkOnePermutationToJar(logger, module, + precompilation.getGeneratedArtifacts(), permResult, makePermFilename( + compilerWorkDir, permId), precompilationOptions); + return true; + } + + private final CompileOnePermOptionsImpl options; + + public CompileOnePerm(CompileOnePermOptions options) { + this.options = new CompileOnePermOptionsImpl(options); + } + + public boolean run(TreeLogger logger) throws UnableToCompleteException { + List<String> moduleNames = options.getModuleNames(); + if (moduleNames.size() != 1) { + logger.log(TreeLogger.ERROR, "Expected a single module name."); + return false; + } + int permToRun = options.getPermToCompile(); + if (permToRun < 0) { + logger.log(TreeLogger.ERROR, + "Expected argument -perm to specify the permutation to compile."); + return false; + } + + String moduleName = moduleNames.get(0); + File compilerWorkDir = options.getCompilerWorkDir(moduleName); + + // Look for the sentinel file that indicates that this compilation already + // has a precompilation result from a previous Precompile step. + PrecompileOptions precompilationOptions = AnalyzeModule.readAnalyzeModuleOptionsFile( + logger, compilerWorkDir); + if (precompilationOptions == null) { + logger.log(TreeLogger.ERROR, "Could not read file " + + AnalyzeModule.OPTIONS_FILENAME + " output from AnalyzeModule step."); + return false; + } + + return compileSpecificPermutation(logger, moduleName, + precompilationOptions, permToRun, compilerWorkDir); + } +}
diff --git a/dev/core/src/com/google/gwt/dev/CompilePerms.java b/dev/core/src/com/google/gwt/dev/CompilePerms.java index d1a7fa2..176bad1 100644 --- a/dev/core/src/com/google/gwt/dev/CompilePerms.java +++ b/dev/core/src/com/google/gwt/dev/CompilePerms.java
@@ -250,11 +250,27 @@ return new File(compilerWorkDir, "permutation-" + permNumber + ".js"); } + static PrecompilationResult readPrecompilationFile(TreeLogger logger, + File precompilationFile) { + PrecompilationResult precompileResults = null; + try { + precompileResults = Util.readFileAsObject(precompilationFile, + PrecompilationResult.class); + } catch (IOException e) { + logger.log(TreeLogger.ERROR, "Failed to read " + + precompilationFile + "\nHas Precompile been run?"); + } catch (ClassNotFoundException e) { + logger.log(TreeLogger.ERROR, "Failed to read " + + precompilationFile, e); + } + return precompileResults; + } + /** * Choose the subset of requested permutations that correspond to the * indicated precompilation. */ - private static Permutation[] selectPermutationsForPrecompilation( + static Permutation[] selectPermutationsForPrecompilation( int[] permsToRun, Precompilation precompilation) { if (permsToRun == null) { // Special case: compile everything. @@ -288,19 +304,8 @@ File precompilationFile = new File(compilerWorkDir, Precompile.PRECOMPILE_FILENAME); - PrecompilationResult precompileResults; - try { - precompileResults = Util.readFileAsObject(precompilationFile, - PrecompilationResult.class); - } catch (IOException e) { - logger.log(TreeLogger.ERROR, "Failed to read " - + Precompile.PRECOMPILE_FILENAME + "; has Precompile been run?"); - return false; - } catch (ClassNotFoundException e) { - logger.log(TreeLogger.ERROR, "Failed to read " - + Precompile.PRECOMPILE_FILENAME, e); - return false; - } + PrecompilationResult precompileResults = readPrecompilationFile(logger, + precompilationFile); if (precompileResults instanceof PrecompileOptions) { PrecompileOptions precompilationOptions = (PrecompileOptions) precompileResults;
diff --git a/dev/core/src/com/google/gwt/dev/DistillerRebindPermutationOracle.java b/dev/core/src/com/google/gwt/dev/DistillerRebindPermutationOracle.java new file mode 100644 index 0000000..babb988 --- /dev/null +++ b/dev/core/src/com/google/gwt/dev/DistillerRebindPermutationOracle.java
@@ -0,0 +1,124 @@ +/* + * Copyright 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev; + +import com.google.gwt.core.ext.TreeLogger; +import com.google.gwt.core.ext.UnableToCompleteException; +import com.google.gwt.core.ext.linker.ArtifactSet; +import com.google.gwt.dev.cfg.BindingProperty; +import com.google.gwt.dev.cfg.ConfigurationProperty; +import com.google.gwt.dev.cfg.ModuleDef; +import com.google.gwt.dev.cfg.PropertyPermutations; +import com.google.gwt.dev.cfg.Rules; +import com.google.gwt.dev.cfg.StaticPropertyOracle; +import com.google.gwt.dev.javac.CompilationState; +import com.google.gwt.dev.javac.StandardGeneratorContext; +import com.google.gwt.dev.jdt.RebindOracle; +import com.google.gwt.dev.jdt.RebindPermutationOracle; +import com.google.gwt.dev.shell.StandardRebindOracle; +import com.google.gwt.dev.util.Util; +import com.google.gwt.dev.util.collect.HashSet; +import com.google.gwt.dev.util.log.speedtracer.CompilerEventType; +import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger; +import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger.Event; + +import java.io.File; +import java.util.Set; +import java.util.SortedSet; + +/** + * Implementation of RebindPermutationOracle used by Precompile. + */ +class DistillerRebindPermutationOracle implements RebindPermutationOracle { + private CompilationState compilationState; + private StandardGeneratorContext generatorContext; + private final Permutation[] permutations; + private final StaticPropertyOracle[] propertyOracles; + private final RebindOracle[] rebindOracles; + + public DistillerRebindPermutationOracle(ModuleDef module, + CompilationState compilationState, ArtifactSet generatorArtifacts, + PropertyPermutations perms, File genDir) { + this.compilationState = compilationState; + permutations = new Permutation[perms.size()]; + propertyOracles = new StaticPropertyOracle[perms.size()]; + rebindOracles = new RebindOracle[perms.size()]; + generatorContext = new StandardGeneratorContext(compilationState, module, + genDir, generatorArtifacts); + BindingProperty[] orderedProps = perms.getOrderedProperties(); + SortedSet<ConfigurationProperty> configPropSet = module.getProperties().getConfigurationProperties(); + ConfigurationProperty[] configProps = configPropSet.toArray(new ConfigurationProperty[configPropSet.size()]); + Rules rules = module.getRules(); + for (int i = 0; i < rebindOracles.length; ++i) { + String[] orderedPropValues = perms.getOrderedPropertyValues(i); + propertyOracles[i] = new StaticPropertyOracle(orderedProps, + orderedPropValues, configProps); + rebindOracles[i] = new StandardRebindOracle(propertyOracles[i], rules, + generatorContext); + permutations[i] = new Permutation(i, propertyOracles[i]); + } + } + + public void clear() { + generatorContext.clear(); + compilationState = null; + generatorContext = null; + } + + public String[] getAllPossibleRebindAnswers(TreeLogger logger, + String requestTypeName) throws UnableToCompleteException { + + String msg = "Computing all possible rebind results for '" + + requestTypeName + "'"; + logger = logger.branch(TreeLogger.DEBUG, msg, null); + + Set<String> answers = new HashSet<String>(); + Event getAllRebindsEvent = SpeedTracerLogger.start(CompilerEventType.GET_ALL_REBINDS); + for (int i = 0; i < getPermuationCount(); ++i) { + String resultTypeName = rebindOracles[i].rebind(logger, requestTypeName); + answers.add(resultTypeName); + // Record the correct answer into each permutation. + permutations[i].putRebindAnswer(requestTypeName, resultTypeName); + } + String[] result = Util.toArray(String.class, answers); + getAllRebindsEvent.end(); + return result; + } + + public CompilationState getCompilationState() { + return compilationState; + } + + public StandardGeneratorContext getGeneratorContext() { + return generatorContext; + } + + public int getPermuationCount() { + return rebindOracles.length; + } + + public Permutation[] getPermutations() { + return permutations; + } + + public StaticPropertyOracle getPropertyOracle(int permNumber) { + return propertyOracles[permNumber]; + } + + public RebindOracle getRebindOracle(int permNumber) { + return rebindOracles[permNumber]; + } +}
diff --git a/dev/core/src/com/google/gwt/dev/Link.java b/dev/core/src/com/google/gwt/dev/Link.java index 9c46fc3..261f522 100644 --- a/dev/core/src/com/google/gwt/dev/Link.java +++ b/dev/core/src/com/google/gwt/dev/Link.java
@@ -67,6 +67,7 @@ import java.util.jar.JarFile; import java.util.jar.JarOutputStream; import java.util.zip.ZipEntry; +import java.util.zip.ZipException; /** * Performs the last phase of compilation, merging the compilation outputs. @@ -199,8 +200,7 @@ if (deployDir.equals(extrasDir)) { deployFileSet = extrasFileSet; } else { - deployFileSet = chooseOutputFileSet(deployDir, - module.getName() + "/"); + deployFileSet = chooseOutputFileSet(deployDir, module.getName() + "/"); } doProduceOutput(logger, artifacts, linkerContext, chooseOutputFileSet( outDir, module.getName() + "/"), deployFileSet, extrasFileSet); @@ -220,7 +220,7 @@ if (jarFile.exists()) { boolean success = jarFile.delete(); if (!success) { - logger.log(Type.ERROR, "Linker output file " + jarFile.getName() + logger.log(Type.ERROR, "Linker output file " + jarFile.getName() + " already exists and can't be deleted."); } } @@ -523,8 +523,8 @@ * @param linkerContext * @return prefixed path */ - private static String prefixArtifactPath( - EmittedArtifact art, StandardLinkerContext linkerContext) { + private static String prefixArtifactPath(EmittedArtifact art, + StandardLinkerContext linkerContext) { String pathWithLinkerName = linkerContext.getExtraPathForLinker( art.getLinker(), art.getPartialPath()); if (pathWithLinkerName.startsWith("/")) { @@ -539,7 +539,15 @@ final ArtifactSet artifacts = new ArtifactSet(); for (File resultFile : resultFiles) { - JarFile jarFile = new JarFile(resultFile); + JarFile jarFile = null; + try { + jarFile = new JarFile(resultFile); + } catch (ZipException ze) { + logger.log(TreeLogger.ERROR, "Error opening " + resultFile + + " as jar file.", ze); + throw new UnableToCompleteException(); + } + Enumeration<JarEntry> entries = jarFile.entries(); while (entries.hasMoreElements()) { JarEntry entry = entries.nextElement(); @@ -599,26 +607,32 @@ ModuleDef module = ModuleDefLoader.loadFromClassPath(logger, moduleName); File compilerWorkDir = options.getCompilerWorkDir(moduleName); - PrecompilationResult precompileResults; - try { - precompileResults = Util.readFileAsObject(new File(compilerWorkDir, - Precompile.PRECOMPILE_FILENAME), PrecompilationResult.class); - } catch (ClassNotFoundException e) { - logger.log(TreeLogger.ERROR, "Error reading " - + Precompile.PRECOMPILE_FILENAME); - return false; - } catch (IOException e) { - logger.log(TreeLogger.ERROR, "Error reading " - + Precompile.PRECOMPILE_FILENAME); - return false; + + // Look for the compilerOptions file output from running AnalyzeModule + PrecompileOptions precompileOptions = AnalyzeModule.readAnalyzeModuleOptionsFile( + logger, compilerWorkDir); + + PrecompilationResult precompileResults = null; + if (precompileOptions == null) { + // Check for the output from Precompile where precompiling has + // been delegated to shards. + File precompilationFile = new File(compilerWorkDir, + Precompile.PRECOMPILE_FILENAME); + precompileResults = CompilePerms.readPrecompilationFile(logger, + precompilationFile); + if (precompileResults == null) { + return false; + } + if (precompileResults instanceof PrecompileOptions) { + precompileOptions = (PrecompileOptions) precompileResults; + } } - if (precompileResults instanceof PrecompileOptions) { + if (precompileOptions != null) { /** * Precompiling happened on the shards. */ - if (!doLinkFinal(logger, compilerWorkDir, module, - (JJSOptions) precompileResults)) { + if (!doLinkFinal(logger, compilerWorkDir, module, precompileOptions)) { return false; } continue loop_modules;
diff --git a/dev/core/src/com/google/gwt/dev/Precompile.java b/dev/core/src/com/google/gwt/dev/Precompile.java index 52b1ea3..830e240 100644 --- a/dev/core/src/com/google/gwt/dev/Precompile.java +++ b/dev/core/src/com/google/gwt/dev/Precompile.java
@@ -24,18 +24,12 @@ import com.google.gwt.core.ext.linker.impl.StandardLinkerContext; import com.google.gwt.core.ext.typeinfo.JClassType; import com.google.gwt.dev.CompileTaskRunner.CompileTask; -import com.google.gwt.dev.cfg.BindingProperty; import com.google.gwt.dev.cfg.ConfigurationProperty; import com.google.gwt.dev.cfg.ModuleDef; import com.google.gwt.dev.cfg.ModuleDefLoader; import com.google.gwt.dev.cfg.PropertyPermutations; -import com.google.gwt.dev.cfg.Rules; -import com.google.gwt.dev.cfg.StaticPropertyOracle; import com.google.gwt.dev.javac.CompilationState; import com.google.gwt.dev.javac.CompilationUnit; -import com.google.gwt.dev.javac.StandardGeneratorContext; -import com.google.gwt.dev.jdt.RebindOracle; -import com.google.gwt.dev.jdt.RebindPermutationOracle; import com.google.gwt.dev.jjs.AbstractCompiler; import com.google.gwt.dev.jjs.JJSOptions; import com.google.gwt.dev.jjs.JJSOptionsImpl; @@ -44,7 +38,6 @@ import com.google.gwt.dev.jjs.UnifiedAst; import com.google.gwt.dev.shell.CheckForUpdates; import com.google.gwt.dev.shell.CheckForUpdates.UpdateResult; -import com.google.gwt.dev.shell.StandardRebindOracle; import com.google.gwt.dev.util.CollapsedPropertyKey; import com.google.gwt.dev.util.Memory; import com.google.gwt.dev.util.Util; @@ -72,7 +65,6 @@ import com.google.gwt.dev.util.arg.OptionGenDir; import com.google.gwt.dev.util.arg.OptionMaxPermsPerPrecompile; import com.google.gwt.dev.util.arg.OptionValidateOnly; -import com.google.gwt.dev.util.collect.HashSet; import com.google.gwt.dev.util.collect.Lists; import com.google.gwt.dev.util.log.speedtracer.CompilerEventType; import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger; @@ -88,7 +80,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; @@ -308,89 +299,6 @@ } } - private static class DistillerRebindPermutationOracle implements - RebindPermutationOracle { - - private CompilationState compilationState; - private StandardGeneratorContext generatorContext; - private final Permutation[] permutations; - private final StaticPropertyOracle[] propertyOracles; - private final RebindOracle[] rebindOracles; - - public DistillerRebindPermutationOracle(ModuleDef module, - CompilationState compilationState, ArtifactSet generatorArtifacts, - PropertyPermutations perms, File genDir) { - this.compilationState = compilationState; - permutations = new Permutation[perms.size()]; - propertyOracles = new StaticPropertyOracle[perms.size()]; - rebindOracles = new RebindOracle[perms.size()]; - generatorContext = new StandardGeneratorContext(compilationState, module, - genDir, generatorArtifacts); - BindingProperty[] orderedProps = perms.getOrderedProperties(); - SortedSet<ConfigurationProperty> configPropSet = module.getProperties().getConfigurationProperties(); - ConfigurationProperty[] configProps = configPropSet.toArray(new ConfigurationProperty[configPropSet.size()]); - Rules rules = module.getRules(); - for (int i = 0; i < rebindOracles.length; ++i) { - String[] orderedPropValues = perms.getOrderedPropertyValues(i); - propertyOracles[i] = new StaticPropertyOracle(orderedProps, - orderedPropValues, configProps); - rebindOracles[i] = new StandardRebindOracle(propertyOracles[i], rules, - generatorContext); - permutations[i] = new Permutation(i, propertyOracles[i]); - } - } - - public void clear() { - generatorContext.clear(); - compilationState = null; - generatorContext = null; - } - - public String[] getAllPossibleRebindAnswers(TreeLogger logger, - String requestTypeName) throws UnableToCompleteException { - - String msg = "Computing all possible rebind results for '" - + requestTypeName + "'"; - logger = logger.branch(TreeLogger.DEBUG, msg, null); - - Set<String> answers = new HashSet<String>(); - Event getAllRebindsEvent = SpeedTracerLogger.start(CompilerEventType.GET_ALL_REBINDS); - for (int i = 0; i < getPermuationCount(); ++i) { - String resultTypeName = rebindOracles[i].rebind(logger, requestTypeName); - answers.add(resultTypeName); - // Record the correct answer into each permutation. - permutations[i].putRebindAnswer(requestTypeName, resultTypeName); - } - String[] result = Util.toArray(String.class, answers); - getAllRebindsEvent.end(); - return result; - } - - public CompilationState getCompilationState() { - return compilationState; - } - - public StandardGeneratorContext getGeneratorContext() { - return generatorContext; - } - - public int getPermuationCount() { - return rebindOracles.length; - } - - public Permutation[] getPermutations() { - return permutations; - } - - public StaticPropertyOracle getPropertyOracle(int permNumber) { - return propertyOracles[permNumber]; - } - - public RebindOracle getRebindOracle(int permNumber) { - return rebindOracles[permNumber]; - } - } - /** * Creates a Graphics2D context in a thread in order to go ahead and get first * time initialization out of the way. Delays ranging from 200ms to 6s have @@ -413,12 +321,23 @@ }; /** - * The file name for the result of Precompile. + * The file name for the max number of permutations output as plain text. */ - public static final String PRECOMPILE_FILENAME = "precompilation.ser"; - static final String PERM_COUNT_FILENAME = "permCount.txt"; + static final String PRECOMPILE_FILENAME = Precompile.PRECOMPILE_FILENAME_PREFIX + + Precompile.PRECOMPILE_FILENAME_SUFFIX; + + /** + * The file name for the serialized AST artifact from the Precompile step. + * Sometimes this file is overloaded and only contains a PrecompileOptions + * object to indicate that precompilation should run inside the CompilePerms + * step. + */ + static final String PRECOMPILE_FILENAME_PREFIX = "precompilation"; + + static final String PRECOMPILE_FILENAME_SUFFIX = ".ser"; + /** * Performs a command-line precompile. */ @@ -486,7 +405,7 @@ * @param logger a logger to use * @param jjsOptions a set of compiler options * @param module the module to compile - * @param genDir optional directory to dump generated source, may be + * @param genDir optional directory to dump generated source, may be * <code>null</code> */ public static boolean validate(TreeLogger logger, JJSOptions jjsOptions, @@ -529,6 +448,39 @@ } } + /** + * Create a list of all possible permutations configured for this module after + * collapsing soft permutations. + */ + static List<PropertyPermutations> getCollapsedPermutations(ModuleDef module) { + PropertyPermutations allPermutations = new PropertyPermutations( + module.getProperties(), module.getActiveLinkerNames()); + List<PropertyPermutations> collapsedPermutations = allPermutations.collapseProperties(); + return collapsedPermutations; + } + + static AbstractCompiler getCompiler(ModuleDef module) { + ConfigurationProperty compilerClassProp = module.getProperties().createConfiguration( + "x.compiler.class", false); + String compilerClassName = compilerClassProp.getValue(); + if (compilerClassName == null || compilerClassName.length() == 0) { + return new JavaScriptCompiler(); + } + Throwable caught; + try { + Class<?> compilerClass = Class.forName(compilerClassName); + return (AbstractCompiler) compilerClass.newInstance(); + } catch (ClassNotFoundException e) { + caught = e; + } catch (InstantiationException e) { + caught = e; + } catch (IllegalAccessException e) { + caught = e; + } + throw new RuntimeException("Unable to instantiate compiler class '" + + compilerClassName + "'", caught); + } + static Precompilation precompile(TreeLogger logger, JJSOptions jjsOptions, ModuleDef module, int permutationBase, PropertyPermutations allPermutations, File genDir) { @@ -623,7 +575,7 @@ precompilationMetrics.setPermuationIds(ids); // TODO(zundel): Right now this double counts module load and // precompile time. It correctly counts the amount of time spent - // in this process. The elapsed time in ModuleMetricsArtifact + // in this process. The elapsed time in ModuleMetricsArtifact // represents time which could be done once for all permutations. precompilationMetrics.setElapsedMilliseconds(System.currentTimeMillis() - startTimeMilliseconds); @@ -646,29 +598,6 @@ "Aborting compile due to errors in some input files"); throw new UnableToCompleteException(); } - - private static AbstractCompiler getCompiler(ModuleDef module) { - ConfigurationProperty compilerClassProp = module.getProperties().createConfiguration( - "x.compiler.class", false); - String compilerClassName = compilerClassProp.getValue(); - if (compilerClassName == null || compilerClassName.length() == 0) { - return new JavaScriptCompiler(); - } - Throwable caught; - try { - Class<?> compilerClass = Class.forName(compilerClassName); - return (AbstractCompiler) compilerClass.newInstance(); - } catch (ClassNotFoundException e) { - caught = e; - } catch (InstantiationException e) { - caught = e; - } catch (IllegalAccessException e) { - caught = e; - } - throw new RuntimeException("Unable to instantiate compiler class '" - + compilerClassName + "'", caught); - } - /** * This merges Permutations that can be considered equivalent by considering * their collapsed properties. The list passed into this method may have @@ -716,7 +645,6 @@ permutations.set(i, new Permutation(i, permutations.get(i))); } } - private final PrecompileOptionsImpl options; public Precompile(PrecompileOptions options) { @@ -731,7 +659,8 @@ for (String moduleName : options.getModuleNames()) { File compilerWorkDir = options.getCompilerWorkDir(moduleName); Util.recursiveDelete(compilerWorkDir, true); - // No need to check mkdirs result because an IOException will occur anyway. + // No need to check mkdirs result because an IOException will occur + // anyway. compilerWorkDir.mkdirs(); File precompilationFile = new File(compilerWorkDir, PRECOMPILE_FILENAME);
diff --git a/dev/core/src/com/google/gwt/dev/PrecompileOnePerm.java b/dev/core/src/com/google/gwt/dev/PrecompileOnePerm.java new file mode 100644 index 0000000..f02a115 --- /dev/null +++ b/dev/core/src/com/google/gwt/dev/PrecompileOnePerm.java
@@ -0,0 +1,255 @@ +/* + * Copyright 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev; + +import com.google.gwt.core.ext.TreeLogger; +import com.google.gwt.core.ext.UnableToCompleteException; +import com.google.gwt.core.ext.linker.impl.StandardLinkerContext; +import com.google.gwt.dev.CompileTaskRunner.CompileTask; +import com.google.gwt.dev.cfg.ModuleDef; +import com.google.gwt.dev.cfg.ModuleDefLoader; +import com.google.gwt.dev.cfg.PropertyPermutations; +import com.google.gwt.dev.shell.CheckForUpdates; +import com.google.gwt.dev.shell.CheckForUpdates.UpdateResult; +import com.google.gwt.dev.util.Memory; +import com.google.gwt.dev.util.Util; +import com.google.gwt.dev.util.arg.ArgHandlerPerm; +import com.google.gwt.dev.util.arg.OptionPerm; +import com.google.gwt.dev.util.log.speedtracer.CompilerEventType; +import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger; +import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger.Event; + +import java.io.File; +import java.util.List; +import java.util.concurrent.FutureTask; + +/** + * Creates a ready-to-compile AST for a single permutation. + * + * Also collapses similar permutations and builds them together. + */ +public class PrecompileOnePerm { + + /** + * The set of options for the precompiler. + */ + public interface PrecompileOnePermOptions extends + Precompile.PrecompileOptions, OptionPerm { + } + + static class ArgProcessor extends Precompile.ArgProcessor { + public ArgProcessor(PrecompileOnePermOptions options) { + super(options); + registerHandler(new ArgHandlerPerm(options)); + } + + @Override + protected String getName() { + return PrecompileOnePerm.class.getName(); + } + } + + @SuppressWarnings("serial") + static class PrecompileOnePermOptionsImpl extends + Precompile.PrecompileOptionsImpl implements PrecompileOnePermOptions { + + int permToCompile = -1; + + public PrecompileOnePermOptionsImpl() { + } + + public PrecompileOnePermOptionsImpl(PrecompileOnePermOptions other) { + copyFrom(other); + } + + public void copyFrom(PrecompileOnePermOptions other) { + super.copyFrom(other); + setPermToCompile(other.getPermToCompile()); + } + + public int getPermToCompile() { + return permToCompile; + } + + public void setPermToCompile(int permToCompile) { + this.permToCompile = permToCompile; + } + } + + /** + * Performs a command-line precompile. + */ + public static void main(String[] args) { + Memory.initialize(); + SpeedTracerLogger.init(); + Event precompileEvent = SpeedTracerLogger.start(CompilerEventType.PRECOMPILE); + if (System.getProperty("gwt.jjs.dumpAst") != null) { + System.out.println("Will dump AST to: " + + System.getProperty("gwt.jjs.dumpAst")); + } + + /* + * NOTE: main always exits with a call to System.exit to terminate any + * non-daemon threads that were started in Generators. Typically, this is to + * shutdown AWT related threads, since the contract for their termination is + * still implementation-dependent. + */ + final PrecompileOnePermOptions options = new PrecompileOnePermOptionsImpl(); + boolean success = false; + if (new ArgProcessor(options).processArgs(args)) { + CompileTask task = new CompileTask() { + public boolean run(TreeLogger logger) throws UnableToCompleteException { + FutureTask<UpdateResult> updater = null; + if (!options.isUpdateCheckDisabled()) { + updater = CheckForUpdates.checkForUpdatesInBackgroundThread(logger, + CheckForUpdates.ONE_DAY); + } + boolean success = new PrecompileOnePerm(options).run(logger); + if (success) { + CheckForUpdates.logUpdateAvailable(logger, updater); + } + return success; + } + }; + if (CompileTaskRunner.runWithAppropriateLogger(options, task)) { + // Exit w/ success code. + success = true; + } + } + precompileEvent.end(); + System.exit(success ? 0 : 1); + } + + /** + * Returns a filename for the serialized AST when precompile is performed + * separately on a per-permutation basis. + */ + static String getPrecompileFilename(int permutationBase) { + return Precompile.PRECOMPILE_FILENAME_PREFIX + "-" + permutationBase + + Precompile.PRECOMPILE_FILENAME_SUFFIX; + } + + private static boolean validateOptions(TreeLogger logger, PrecompileOnePermOptions options) { + // Fatal Errors + if (options.getModuleNames().size() != 1) { + logger.log(TreeLogger.ERROR, "Expected a single module name."); + return false; + } + if (options.getPermToCompile() < 0) { + logger.log(TreeLogger.ERROR, + "Expected argument -perm to specify the permutation to compile."); + return false; + } + + // Warnings + if (!options.isEnabledGeneratingOnShards()) { + logger.log(TreeLogger.WARN, + "-XdisableGeneratingOnShards has no effect in PrecompileOnePerm"); + } + if (options.getMaxPermsPerPrecompile() != -1) { + logger.log(TreeLogger.WARN, + "-XmaxPermsPerPrecompile has no effect in PrecompileOnePerm"); + } + + return true; + } + + private final PrecompileOnePermOptionsImpl options; + + public PrecompileOnePerm(PrecompileOnePermOptions options) { + this.options = new PrecompileOnePermOptionsImpl(options); + } + + public boolean run(TreeLogger logger) throws UnableToCompleteException { + + if (!validateOptions(logger, options)) { + return false; + } + + // Avoid early optimizations since permutation compiles will run separately. + options.setOptimizePrecompile(false); + + List<String> moduleNames = options.getModuleNames(); + + int permToRun = options.getPermToCompile(); + + String moduleName = moduleNames.get(0); + File compilerWorkDir = options.getCompilerWorkDir(moduleName); + + ModuleDef module = ModuleDefLoader.loadFromClassPath(logger, moduleName); + StandardLinkerContext linkerContext = new StandardLinkerContext( + TreeLogger.NULL, module, options); + + if (!linkerContext.allLinkersAreShardable()) { + logger.log(TreeLogger.ERROR, + "This compilation mode requires all linkers to be shardable."); + return false; + } + + Precompile.PrecompileOptions optionsFileData = AnalyzeModule.readAnalyzeModuleOptionsFile( + logger, compilerWorkDir); + if (optionsFileData == null) { + logger.log(TreeLogger.ERROR, "Couldn't find " + + AnalyzeModule.OPTIONS_FILENAME + " in " + compilerWorkDir); + return false; + } + logger.log(TreeLogger.INFO, "Precompiling only specified permutations"); + + if (options.isValidateOnly()) { + TreeLogger branch = logger.branch(TreeLogger.INFO, + "Validating compilation " + module.getName()); + if (!Precompile.validate(branch, options, module, options.getGenDir())) { + branch.log(TreeLogger.ERROR, "Validation failed"); + return false; + } + branch.log(TreeLogger.INFO, "Validation succeeded"); + } else { + TreeLogger branch = logger.branch(TreeLogger.INFO, "Precompiling module " + + module.getName()); + if (!precompilePermutation(logger, compilerWorkDir, module, branch, + permToRun)) { + branch.log(TreeLogger.ERROR, "Precompile failed"); + return false; + } + } + return true; + } + + private boolean precompilePermutation(TreeLogger logger, + File compilerWorkDir, ModuleDef module, TreeLogger branch, int permId) + throws UnableToCompleteException { + + // Only precompile specified permutations + List<PropertyPermutations> collapsedPermutations = + Precompile.getCollapsedPermutations(module); + + PropertyPermutations onePerm = collapsedPermutations.get(permId); + Precompilation precompilation = Precompile.precompile(branch, options, + module, permId, onePerm, options.getGenDir()); + if (precompilation == null) { + branch.log(TreeLogger.ERROR, "Precompilation failed"); + return false; + } + File precompilationFile = new File(compilerWorkDir, + getPrecompileFilename(permId)); + Util.writeObjectAsFile(logger, precompilationFile, precompilation); + + branch.log(TreeLogger.INFO, "Precompilation succeeded for permutation " + + permId); + + return true; + } +}
diff --git a/dev/core/src/com/google/gwt/dev/util/arg/ArgHandlerPerm.java b/dev/core/src/com/google/gwt/dev/util/arg/ArgHandlerPerm.java new file mode 100644 index 0000000..bb7a6a1 --- /dev/null +++ b/dev/core/src/com/google/gwt/dev/util/arg/ArgHandlerPerm.java
@@ -0,0 +1,55 @@ +/* + * Copyright 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev.util.arg; + +import com.google.gwt.util.tools.ArgHandlerInt; + +/** + * Argument handler for specifying the which permutation to run. + */ +public class ArgHandlerPerm extends ArgHandlerInt { + private final OptionPerm option; + + public ArgHandlerPerm(OptionPerm option) { + this.option = option; + } + + @Override + public String getPurpose() { + return "Specifies (0-based) the permutation to compile"; + } + + @Override + public String getTag() { + return "-perm"; + } + + @Override + public String[] getTagArgs() { + return new String[]{"perm"}; + } + + @Override + public void setInt(int value) { + // TODO Auto-generated method stub + if (value < 0) { + System.err.println(getTag() + " error: negative value '" + value + + "' is not allowed"); + } else { + option.setPermToCompile(value); + } + } +}
diff --git a/dev/core/src/com/google/gwt/dev/util/arg/OptionPerm.java b/dev/core/src/com/google/gwt/dev/util/arg/OptionPerm.java new file mode 100644 index 0000000..9d51e98 --- /dev/null +++ b/dev/core/src/com/google/gwt/dev/util/arg/OptionPerm.java
@@ -0,0 +1,32 @@ +/* + * Copyright 2010 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev.util.arg; + +/** + * Handles options for which permutation to compile. + */ +public interface OptionPerm { + /** + * Gets the permutation to compile. Returns -1 if no permutation was + * specified. + */ + int getPermToCompile(); + + /** + * Sets a permutation to compile. + */ + void setPermToCompile(int permToCompile); +}