Implements out-of-process compiles via -localWorkers command line option. Patch by: bobv (+tweaks from spoon, scottb) Review by: spoon, scottb git-svn-id: https://google-web-toolkit.googlecode.com/svn/releases/1.6@4145 8db76d5a-ed1c-0410-87a9-c151d255dfc7
diff --git a/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardCompilationResult.java b/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardCompilationResult.java index e8cdb0d..48a10bc 100644 --- a/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardCompilationResult.java +++ b/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardCompilationResult.java
@@ -17,6 +17,7 @@ import com.google.gwt.core.ext.linker.CompilationResult; import com.google.gwt.core.ext.linker.SelectionProperty; +import com.google.gwt.dev.PermutationResult; import com.google.gwt.dev.util.Util; import java.io.File; @@ -68,17 +69,17 @@ */ public static final Comparator<SortedMap<SelectionProperty, String>> MAP_COMPARATOR = new MapComparator(); - private final File cacheFile; + private final File resultFile; private transient SoftReference<String> js; private final SortedSet<SortedMap<SelectionProperty, String>> propertyValues = new TreeSet<SortedMap<SelectionProperty, String>>( MAP_COMPARATOR); private final String strongName; - public StandardCompilationResult(String js, String strongName, File cacheFile) { + public StandardCompilationResult(String js, String strongName, File resultFile) { super(StandardLinkerContext.class); this.js = new SoftReference<String>(js); this.strongName = strongName; - this.cacheFile = cacheFile; + this.resultFile = resultFile; } /** @@ -98,11 +99,20 @@ toReturn = js.get(); } if (toReturn == null) { - toReturn = Util.readFileAsString(cacheFile); - if (toReturn == null) { - throw new RuntimeException("Unexpectedly unable to read JS file '" - + cacheFile.getAbsolutePath() + "'"); + PermutationResult result; + try { + result = Util.readFileAsObject(resultFile, PermutationResult.class); + } catch (ClassNotFoundException e) { + throw new RuntimeException( + "Unexpectedly unable to read PermutationResult '" + + resultFile.getAbsolutePath() + "'", e); } + if (toReturn == null) { + throw new RuntimeException( + "Unexpectedly unable to read PermutationResult '" + + resultFile.getAbsolutePath() + "'"); + } + toReturn = result.getJs(); js = new SoftReference<String>(toReturn); } return toReturn;
diff --git a/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardLinkerContext.java b/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardLinkerContext.java index c5e4a06..55518a5 100644 --- a/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardLinkerContext.java +++ b/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardLinkerContext.java
@@ -26,6 +26,7 @@ import com.google.gwt.core.ext.linker.PublicResource; import com.google.gwt.core.ext.linker.SelectionProperty; import com.google.gwt.core.ext.linker.LinkerOrder.Order; +import com.google.gwt.dev.PermutationResult; import com.google.gwt.dev.cfg.BindingProperty; import com.google.gwt.dev.cfg.ModuleDef; import com.google.gwt.dev.cfg.Property; @@ -254,19 +255,27 @@ /** * Gets or creates a CompilationResult for the given JavaScript program. */ - public StandardCompilationResult getCompilation(TreeLogger logger, File jsFile) - throws UnableToCompleteException { - byte[] bytes = Util.readFileAsBytes(jsFile); - if (bytes == null) { - logger.log(TreeLogger.ERROR, "Unable to read file '" - + jsFile.getAbsolutePath() + "'"); + public StandardCompilationResult getCompilation(TreeLogger logger, + File resultFile) throws UnableToCompleteException { + PermutationResult permutationResult; + try { + permutationResult = Util.readFileAsObject(resultFile, + PermutationResult.class); + } catch (ClassNotFoundException e) { + logger.log(TreeLogger.ERROR, "Unable to instantiate PermutationResult", e); throw new UnableToCompleteException(); } - String strongName = Util.computeStrongName(bytes); + + if (permutationResult == null) { + logger.log(TreeLogger.ERROR, "Unable to read PermutationResult"); + throw new UnableToCompleteException(); + } + + String strongName = Util.computeStrongName(Util.getBytes(permutationResult.getJs())); StandardCompilationResult result = resultsByStrongName.get(strongName); if (result == null) { - result = new StandardCompilationResult(Util.toString(bytes), strongName, - jsFile); + result = new StandardCompilationResult(permutationResult.getJs(), + strongName, resultFile); resultsByStrongName.put(result.getStrongName(), result); artifacts.add(result); }
diff --git a/dev/core/src/com/google/gwt/dev/CompilePerms.java b/dev/core/src/com/google/gwt/dev/CompilePerms.java index 82a08cd..94ef4c4 100644 --- a/dev/core/src/com/google/gwt/dev/CompilePerms.java +++ b/dev/core/src/com/google/gwt/dev/CompilePerms.java
@@ -18,10 +18,10 @@ import com.google.gwt.core.ext.TreeLogger; import com.google.gwt.core.ext.UnableToCompleteException; import com.google.gwt.dev.CompileTaskRunner.CompileTask; -import com.google.gwt.dev.PermutationCompiler.ResultsHandler; -import com.google.gwt.dev.jjs.UnifiedAst; import com.google.gwt.dev.jjs.JavaToJavaScriptCompiler; +import com.google.gwt.dev.jjs.UnifiedAst; import com.google.gwt.dev.util.Util; +import com.google.gwt.dev.util.arg.ArgHandlerLocalWorkers; import com.google.gwt.util.tools.ArgHandlerString; import java.io.File; @@ -29,15 +29,16 @@ import java.util.TreeSet; /** - * Performs the second phase of compilation, converting the Precompile's AST into - * JavaScript outputs. + * Performs the second phase of compilation, converting the Precompile's AST + * into JavaScript outputs. */ public class CompilePerms { /** * Options for CompilePerms. */ - public interface CompilePermsOptions extends CompileTaskOptions, OptionPerms { + public interface CompilePermsOptions extends CompileTaskOptions, + OptionLocalWorkers, OptionPerms { } /** @@ -121,6 +122,7 @@ public ArgProcessor(CompilePermsOptions options) { super(options); registerHandler(new ArgHandlerPerms(options)); + registerHandler(new ArgHandlerLocalWorkers(options)); } @Override @@ -135,6 +137,7 @@ static class CompilePermsOptionsImpl extends CompileTaskOptionsImpl implements CompilePermsOptions { + private int localWorkers; private int[] permsToCompile = new int[0]; public CompilePermsOptionsImpl() { @@ -146,24 +149,40 @@ public void copyFrom(CompilePermsOptions other) { super.copyFrom(other); - setPermsToCompile(other.getPermsToCompile()); + setLocalWorkers(other.getLocalWorkers()); + } + + public int getLocalWorkers() { + return localWorkers; } public int[] getPermsToCompile() { return permsToCompile.clone(); } + public void setLocalWorkers(int localWorkers) { + this.localWorkers = localWorkers; + } + public void setPermsToCompile(int[] permsToCompile) { this.permsToCompile = permsToCompile.clone(); } } - public static String compile(TreeLogger logger, Permutation permutation, - UnifiedAst unifiedAst) { + /** + * Compile a single permutation. + */ + public static PermutationResult compile(TreeLogger logger, + Permutation permutation, UnifiedAst unifiedAst) { try { - return JavaToJavaScriptCompiler.compilePermutation(logger, unifiedAst, - permutation.getRebindAnswers()); + final String js = JavaToJavaScriptCompiler.compilePermutation(logger, + unifiedAst, permutation.getRebindAnswers()); + return new PermutationResult() { + public String getJs() { + return js; + } + }; } catch (UnableToCompleteException e) { // We intentionally don't pass in the exception here since the real // cause has been logged. @@ -171,6 +190,20 @@ } } + /** + * Compile multiple permutations. + */ + public static boolean compile(TreeLogger logger, + Precompilation precompilation, Permutation[] perms, int localWorkers, + File[] resultFiles) throws UnableToCompleteException { + final TreeLogger branch = logger.branch(TreeLogger.INFO, "Compiling " + + perms.length + " permutations"); + PermutationWorkerFactory.compilePermutations(logger, precompilation, perms, + localWorkers, resultFiles); + branch.log(TreeLogger.INFO, "Permutation compile succeeded"); + return true; + } + public static void main(String[] args) { /* * NOTE: main always exits with a call to System.exit to terminate any @@ -194,6 +227,14 @@ System.exit(1); } + public static File[] makeResultFiles(File compilerWorkDir, Permutation[] perms) { + File[] resultFiles = new File[perms.length]; + for (int i = 0; i < perms.length; ++i) { + resultFiles[i] = makePermFilename(compilerWorkDir, perms[i].getId()); + } + return resultFiles; + } + /** * Return the filename corresponding to the given permutation number, * one-based. @@ -229,17 +270,15 @@ + precompilationFile.getAbsolutePath() + "'", e); return false; } - Permutation[] perms = precompilation.getPermutations(); - UnifiedAst unifiedAst = precompilation.getUnifiedAst(); - int[] permsToRun = options.getPermsToCompile(); + Permutation[] perms = precompilation.getPermutations(); + Permutation[] subPerms; + int[] permsToRun = options.getPermsToCompile(); if (permsToRun.length == 0) { - // Compile them all. - permsToRun = new int[perms.length]; - for (int i = 0; i < permsToRun.length; ++i) { - permsToRun[i] = i; - } + subPerms = perms; } else { + int i = 0; + subPerms = new Permutation[permsToRun.length]; // Range check the supplied perms. for (int permToRun : permsToRun) { if (permToRun >= perms.length) { @@ -248,21 +287,12 @@ + (perms.length - 1) + "'"); return false; } + subPerms[i++] = perms[permToRun]; } } - final TreeLogger branch = logger.branch(TreeLogger.INFO, "Compiling " - + permsToRun.length + " permutations"); - PermutationCompiler multiThread = new PermutationCompiler(branch, - unifiedAst, perms, permsToRun); - multiThread.go(new ResultsHandler() { - public void addResult(Permutation permutation, int permNum, String js) - throws UnableToCompleteException { - Util.writeStringAsFile(branch, makePermFilename( - options.getCompilerWorkDir(), permNum), js); - } - }); - branch.log(TreeLogger.INFO, "Permutation compile succeeded"); - return true; + File[] resultFiles = makeResultFiles(options.getCompilerWorkDir(), perms); + return compile(logger, precompilation, subPerms, options.getLocalWorkers(), + resultFiles); } }
diff --git a/dev/core/src/com/google/gwt/dev/CompilePermsServer.java b/dev/core/src/com/google/gwt/dev/CompilePermsServer.java new file mode 100644 index 0000000..ace3385 --- /dev/null +++ b/dev/core/src/com/google/gwt/dev/CompilePermsServer.java
@@ -0,0 +1,309 @@ +/* + * Copyright 2008 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev; + +import com.google.gwt.core.ext.TreeLogger; +import com.google.gwt.core.ext.TreeLogger.Type; +import com.google.gwt.dev.jjs.UnifiedAst; +import com.google.gwt.dev.util.arg.ArgHandlerLogLevel; +import com.google.gwt.dev.util.arg.OptionLogLevel; +import com.google.gwt.dev.util.log.PrintWriterTreeLogger; +import com.google.gwt.util.tools.ArgHandlerString; +import com.google.gwt.util.tools.ToolBase; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.net.Socket; +import java.net.UnknownHostException; + +/** + * An out-of-process implementation of CompilePerms that will connect back to an + * existing compiler host. This class is intended to be launched by + * {@link ExternalPermutationWorkerFactory} and not by users directly. + */ +public class CompilePermsServer { + /** + * Adds host and port information. + */ + public interface CompileServerOptions extends OptionLogLevel { + String getCompileHost(); + + int getCompilePort(); + + String getCookie(); + + void setCompileHost(String host); + + void setCompilePort(int port); + + void setCookie(String cookie); + } + + static final class ArgHandlerCompileHost extends ArgHandlerString { + private final CompileServerOptions options; + + public ArgHandlerCompileHost(CompileServerOptions options) { + this.options = options; + } + + @Override + public String getPurpose() { + return "The host to which the permutation server should connect"; + } + + @Override + public String getTag() { + return "-host"; + } + + @Override + public String[] getTagArgs() { + return new String[] {"hostname"}; + } + + @Override + public boolean isRequired() { + return true; + } + + @Override + public boolean setString(String str) { + options.setCompileHost(str); + return true; + } + } + + static final class ArgHandlerCompilePort extends ArgHandlerString { + private final CompileServerOptions options; + + public ArgHandlerCompilePort(CompileServerOptions options) { + this.options = options; + } + + @Override + public String getPurpose() { + return "The port to which the permutation server should connect"; + } + + @Override + public String getTag() { + return "-port"; + } + + @Override + public String[] getTagArgs() { + return new String[] {"1234"}; + } + + @Override + public boolean isRequired() { + return true; + } + + @Override + public boolean setString(String str) { + Integer port = Integer.parseInt(str); + if (port <= 0) { + return false; + } + options.setCompilePort(port); + return true; + } + } + + static final class ArgHandlerCookie extends ArgHandlerString { + + private final CompileServerOptions options; + + public ArgHandlerCookie(CompileServerOptions option) { + this.options = option; + } + + @Override + public String getPurpose() { + return "Specifies the security cookie the server expects"; + } + + @Override + public String getTag() { + return "-cookie"; + } + + @Override + public String[] getTagArgs() { + return new String[] {"cookie"}; + } + + @Override + public boolean isRequired() { + return true; + } + + @Override + public boolean setString(String str) { + options.setCookie(str); + return true; + } + } + + static class ArgProcessor extends ToolBase { + public ArgProcessor(CompileServerOptions options) { + registerHandler(new ArgHandlerLogLevel(options)); + registerHandler(new ArgHandlerCompileHost(options)); + registerHandler(new ArgHandlerCompilePort(options)); + registerHandler(new ArgHandlerCookie(options)); + } + + /* + * Overridden to make public. + */ + @Override + public final boolean processArgs(String[] args) { + return super.processArgs(args); + } + + @Override + protected String getName() { + return CompilePermsServer.class.getName(); + } + } + + static class CompileServerOptionsImpl implements CompileServerOptions { + + private String compileHost; + private int compilePort; + private String cookie; + private Type logLevel; + + public void copyFrom(CompileServerOptions other) { + setCompileHost(other.getCompileHost()); + setCompilePort(other.getCompilePort()); + setCookie(other.getCookie()); + setLogLevel(other.getLogLevel()); + } + + public String getCompileHost() { + return compileHost; + } + + public int getCompilePort() { + return compilePort; + } + + public String getCookie() { + return cookie; + } + + public Type getLogLevel() { + return logLevel; + } + + public void setCompileHost(String host) { + assert host != null; + compileHost = host; + } + + public void setCompilePort(int port) { + assert port > 0; + compilePort = port; + } + + public void setCookie(String cookie) { + this.cookie = cookie; + } + + public void setLogLevel(Type logLevel) { + this.logLevel = logLevel; + } + } + + public static void main(String[] args) { + final CompileServerOptions options = new CompileServerOptionsImpl(); + if (new ArgProcessor(options).processArgs(args)) { + PrintWriterTreeLogger logger = new PrintWriterTreeLogger(); + logger.setMaxDetail(options.getLogLevel()); + if (run(options, logger)) { + System.exit(0); + } + } + System.exit(-1); + } + + public static boolean run(CompileServerOptions options, TreeLogger logger) { + try { + Socket s = new Socket(options.getCompileHost(), options.getCompilePort()); + logger.log(TreeLogger.DEBUG, "Socket opened"); + + ObjectOutputStream out = new ObjectOutputStream(s.getOutputStream()); + ObjectInputStream in = new ObjectInputStream(s.getInputStream()); + + // Write my cookie + out.writeUTF(options.getCookie()); + out.flush(); + + // Read the File that contains the serialized UnifiedAst + File astFile = (File) in.readObject(); + ObjectInputStream astIn = new ObjectInputStream(new FileInputStream( + astFile)); + UnifiedAst ast = (UnifiedAst) astIn.readObject(); + ast.prepare(); + logger.log(TreeLogger.SPAM, "Created new UnifiedAst instance"); + + // Report on the amount of memory we think we're using + long estimatedMemory = Runtime.getRuntime().totalMemory() + - Runtime.getRuntime().freeMemory(); + out.writeLong(estimatedMemory); + out.flush(); + + boolean keepGoing = in.readBoolean(); + while (keepGoing) { + compilePermutation(logger, ast, in, out); + + keepGoing = in.readBoolean(); + logger.log(TreeLogger.SPAM, "keepGoing = " + keepGoing); + } + + logger.log(TreeLogger.DEBUG, "Successfully terminating"); + return true; + + } catch (UnknownHostException e) { + logger.log(TreeLogger.ERROR, "Invalid hostname", e); + } catch (IOException e) { + logger.log(TreeLogger.ERROR, "Communication error", e); + } catch (ClassNotFoundException e) { + logger.log(TreeLogger.ERROR, "Probable client/server mismatch or " + + "classpath misconfiguration", e); + } + return false; + } + + static void compilePermutation(TreeLogger logger, UnifiedAst ast, + ObjectInputStream in, ObjectOutputStream out) + throws ClassNotFoundException, IOException { + Permutation p = (Permutation) in.readObject(); + logger.log(TreeLogger.SPAM, "Permutation read"); + + PermutationResult result = CompilePerms.compile(logger.branch( + TreeLogger.DEBUG, "Compiling"), p, ast); + logger.log(TreeLogger.DEBUG, "Successfully compiled permutation"); + out.writeObject(result); + out.flush(); + logger.log(TreeLogger.SPAM, "Sent result"); + } +}
diff --git a/dev/core/src/com/google/gwt/dev/CompileTaskOptionsImpl.java b/dev/core/src/com/google/gwt/dev/CompileTaskOptionsImpl.java index e0b7ead..e92d1c8 100644 --- a/dev/core/src/com/google/gwt/dev/CompileTaskOptionsImpl.java +++ b/dev/core/src/com/google/gwt/dev/CompileTaskOptionsImpl.java
@@ -24,8 +24,6 @@ */ class CompileTaskOptionsImpl implements CompileTaskOptions { - public static final String GWT_TMP_DIR = "gwt-tmp"; - private Type logLevel; private String moduleName; private boolean useGuiLogger;
diff --git a/dev/core/src/com/google/gwt/dev/CompilerOptions.java b/dev/core/src/com/google/gwt/dev/CompilerOptions.java index 0b568e4..382eee8 100644 --- a/dev/core/src/com/google/gwt/dev/CompilerOptions.java +++ b/dev/core/src/com/google/gwt/dev/CompilerOptions.java
@@ -21,5 +21,6 @@ /** * The complete set of options for the GWT compiler. */ -public interface CompilerOptions extends PrecompileOptions, LinkOptions { +public interface CompilerOptions extends PrecompileOptions, LinkOptions, + OptionLocalWorkers { }
diff --git a/dev/core/src/com/google/gwt/dev/ExternalPermutationWorkerFactory.java b/dev/core/src/com/google/gwt/dev/ExternalPermutationWorkerFactory.java new file mode 100644 index 0000000..392afea --- /dev/null +++ b/dev/core/src/com/google/gwt/dev/ExternalPermutationWorkerFactory.java
@@ -0,0 +1,376 @@ +/* + * Copyright 2008 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev; + +import com.google.gwt.core.ext.TreeLogger; +import com.google.gwt.core.ext.UnableToCompleteException; +import com.google.gwt.dev.jjs.UnifiedAst; +import com.google.gwt.dev.util.Util; + +import java.io.BufferedReader; +import java.io.EOFException; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.lang.management.ManagementFactory; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Random; +import java.util.Set; + +/** + * A PermutationWorkerFactory designed to launch instances of + * {@link CompilePermsServer}. The system property + * {@value #JAVA_COMMAND_PROPERTY} can be used to change the command used to + * launch the JVM. The system property {@link #JVM_ARGS_PROPERTY} can be used to + * override the JVM args passed to the subprocess. + */ +public class ExternalPermutationWorkerFactory extends PermutationWorkerFactory { + + /** + * Allows accept() to be called a finite number of times on a ServerSocket + * before closing the socket. + */ + private static class CountedServerSocket { + private int accepts; + private ServerSocket sock; + + public CountedServerSocket(ServerSocket sock, int maxAccepts) { + assert sock != null; + assert maxAccepts >= 1; + + this.accepts = maxAccepts; + this.sock = sock; + } + + public synchronized Socket accept() throws IOException { + assert accepts >= 0; + + if (accepts == 0) { + throw new IllegalStateException("Too many calls to accept()"); + } + + try { + return sock.accept(); + } finally { + if (--accepts == 0) { + sock.close(); + sock = null; + } + } + } + } + + private static class ExternalPermutationWorker implements PermutationWorker { + private final File astFile; + private final Set<String> cookies; + private ObjectInputStream in; + private ObjectOutputStream out; + private final CountedServerSocket serverSocket; + private Socket workerSocket; + + public ExternalPermutationWorker(CountedServerSocket sock, File astFile, + Set<String> cookies) { + this.astFile = astFile; + this.cookies = cookies; + this.serverSocket = sock; + } + + public PermutationResult compile(TreeLogger logger, Permutation permutation) + throws TransientWorkerException, UnableToCompleteException { + + // If we've just started, we need to get a connection from a subprocess + if (workerSocket == null) { + try { + /* + * We've set SO_TIMEOUT, so this may fail if the remote process never + * connects back. + */ + workerSocket = serverSocket.accept(); + + in = new ObjectInputStream(workerSocket.getInputStream()); + out = new ObjectOutputStream(workerSocket.getOutputStream()); + + // Verify we're talking to the right worker + String c = in.readUTF(); + if (!cookies.contains(c)) { + throw new TransientWorkerException("Received unknown cookie " + c, + null); + } + + out.writeObject(astFile); + + // Get the remote worker's estimate of memory use + long memoryUse = in.readLong(); + logger.log(TreeLogger.SPAM, "Remote process indicates " + memoryUse + + " bytes of memory used"); + + } catch (SocketTimeoutException e) { + throw new TransientWorkerException( + "Remote process did not connect within timeout period", e); + } catch (IOException e) { + throw new TransientWorkerException( + "Unable to communicate with worker", e); + } + } + + try { + out.writeBoolean(true); + out.writeObject(permutation); + out.flush(); + return (PermutationResult) in.readObject(); + } catch (IOException e) { + logger.log(TreeLogger.WARN, "Lost communication with remote process", e); + throw new TransientWorkerException( + "Lost communication with remote process", e); + } catch (ClassNotFoundException e) { + logger.log(TreeLogger.ERROR, "Unable to receive response", e); + throw new UnableToCompleteException(); + } + } + + public String getName() { + return "External worker " + + (workerSocket != null ? workerSocket.getRemoteSocketAddress() + : "unconnected"); + } + + public void shutdown() { + if (out != null) { + try { + out.writeBoolean(false); + out.flush(); + out.close(); + } catch (IOException e) { + // Not much to do here + } + } + + if (in != null) { + try { + in.close(); + } catch (IOException e) { + // Not much to do here + } + } + + if (workerSocket != null) { + try { + workerSocket.close(); + } catch (IOException e) { + // Nothing to do + } + } + } + } + + /** + * A system property that can be used to override the command used to invoke a + * JVM instance. + */ + public static final String JAVA_COMMAND_PROPERTY = "gwt.jjs.javaCommand"; + + /** + * A system property that can be used to override the JVM args passed to the + * subprocess. + */ + public static final String JVM_ARGS_PROPERTY = "gwt.jjs.javaArgs"; + + /** + * Launches an external worker and returns the cookie that worker should + * return via the network connection. + */ + private static String launchExternalWorker(TreeLogger logger, int port) + throws UnableToCompleteException { + + String javaCommand = System.getProperty(JAVA_COMMAND_PROPERTY, + System.getProperty("java.home") + File.separator + "bin" + + File.separator + "java"); + logger.log(TreeLogger.TRACE, "javaCommand = " + javaCommand); + + // Construct the arguments + List<String> args = new ArrayList<String>(); + args.add(javaCommand); + + // This will include -Xmx, -D, etc... + String userJvmArgs = System.getProperty(JVM_ARGS_PROPERTY); + if (userJvmArgs == null) { + args.addAll(ManagementFactory.getRuntimeMXBean().getInputArguments()); + } else { + args.addAll(Arrays.asList(userJvmArgs.split(" "))); + } + + // Determine the logLevel for the external program + TreeLogger.Type logLevel = TreeLogger.ERROR; + for (TreeLogger.Type t : TreeLogger.Type.values()) { + if (logger.isLoggable(t)) { + logLevel = t; + } else { + break; + } + } + + byte[] cookieBytes = new byte[16]; + (new Random()).nextBytes(cookieBytes); + String cookie = Util.toHexString(cookieBytes); + + // Cook up the classpath, main class, and extra args + args.addAll(Arrays.asList("-classpath", + ManagementFactory.getRuntimeMXBean().getClassPath(), + CompilePermsServer.class.getName(), "-host", "localhost", "-port", + String.valueOf(port), "-logLevel", logLevel.toString(), "-cookie", + cookie)); + + // Filter undesirable arguments + for (Iterator<String> iter = args.iterator(); iter.hasNext();) { + String arg = iter.next(); + if (arg.startsWith("-agentlib")) { + iter.remove(); + } + } + + ProcessBuilder builder = new ProcessBuilder(); + builder.command(args); + + try { + final Process proc = builder.start(); + final BufferedReader bin = new BufferedReader(new InputStreamReader( + proc.getInputStream())); + final BufferedReader berr = new BufferedReader(new InputStreamReader( + proc.getErrorStream())); + final TreeLogger procLogger = logger.branch(TreeLogger.DEBUG, + "Process output"); + + // Threads to copy stdout, stderr to the logger + new Thread(new Runnable() { + public void run() { + while (true) { + try { + String line = bin.readLine(); + if (line == null) { + break; + } + procLogger.log(TreeLogger.INFO, line); + } catch (EOFException e) { + // Ignore + } catch (IOException e) { + procLogger.log(TreeLogger.ERROR, + "Unable to read from subprocess", e); + } + } + } + }).start(); + + new Thread(new Runnable() { + public void run() { + while (true) { + try { + String line = berr.readLine(); + if (line == null) { + break; + } + procLogger.log(TreeLogger.ERROR, line); + } catch (EOFException e) { + // Ignore + } catch (IOException e) { + procLogger.log(TreeLogger.ERROR, + "Unable to read from subprocess", e); + } + } + } + }).start(); + + // The child process should not outlive this JVM + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + public void run() { + try { + proc.exitValue(); + } catch (IllegalThreadStateException e) { + proc.destroy(); + } + } + })); + + return cookie; + } catch (IOException e) { + logger.log(TreeLogger.ERROR, "Unable to start external process", e); + throw new UnableToCompleteException(); + } + } + + private ServerSocket sock; + + @Override + public Collection<PermutationWorker> getWorkers(TreeLogger logger, + UnifiedAst unifiedAst, int numWorkers) throws UnableToCompleteException { + File astFile; + try { + astFile = File.createTempFile("externalPermutationWorkerFactory", ".ser"); + astFile.deleteOnExit(); + Util.writeObjectAsFile(logger, astFile, unifiedAst); + } catch (IOException e) { + logger.log(TreeLogger.ERROR, "Unable to create temporary file", e); + throw new UnableToCompleteException(); + } + + Set<String> cookies = Collections.synchronizedSet(new HashSet<String>( + numWorkers)); + CountedServerSocket countedSock = new CountedServerSocket(sock, numWorkers); + List<PermutationWorker> toReturn = new ArrayList<PermutationWorker>( + numWorkers); + + for (int i = 0; i < numWorkers; i++) { + String cookie = launchExternalWorker(logger, sock.getLocalPort()); + cookies.add(cookie); + toReturn.add(new ExternalPermutationWorker(countedSock, astFile, cookies)); + } + + return toReturn; + } + + @Override + public void init(TreeLogger logger) throws UnableToCompleteException { + try { + sock = new ServerSocket(); + /* + * Have accept() wait no more than one minute for a connection. This + * prevents dead-head behavior. + */ + sock.setSoTimeout(60000); + sock.bind(null); + logger.log(TreeLogger.SPAM, "Listening for external workers on port " + + sock.getLocalPort()); + } catch (IOException e) { + logger.log(TreeLogger.ERROR, "Unable to create socket", e); + throw new UnableToCompleteException(); + } + } + + @Override + public boolean isLocal() { + return true; + } +}
diff --git a/dev/core/src/com/google/gwt/dev/GWTCompiler.java b/dev/core/src/com/google/gwt/dev/GWTCompiler.java index f331040..546e5ab 100644 --- a/dev/core/src/com/google/gwt/dev/GWTCompiler.java +++ b/dev/core/src/com/google/gwt/dev/GWTCompiler.java
@@ -17,12 +17,15 @@ import com.google.gwt.core.ext.TreeLogger; import com.google.gwt.core.ext.UnableToCompleteException; -import com.google.gwt.dev.CompilePerms.CompilePermsOptionsImpl; import com.google.gwt.dev.CompileTaskRunner.CompileTask; import com.google.gwt.dev.Link.LinkOptionsImpl; import com.google.gwt.dev.Precompile.PrecompileOptionsImpl; +import com.google.gwt.dev.cfg.ModuleDef; +import com.google.gwt.dev.cfg.ModuleDefLoader; +import com.google.gwt.dev.util.PerfLogger; import com.google.gwt.dev.util.Util; import com.google.gwt.dev.util.arg.ArgHandlerExtraDir; +import com.google.gwt.dev.util.arg.ArgHandlerLocalWorkers; import com.google.gwt.dev.util.arg.ArgHandlerOutDir; import com.google.gwt.dev.util.arg.ArgHandlerWorkDirOptional; import com.google.gwt.util.tools.Utility; @@ -43,6 +46,7 @@ registerHandler(new ArgHandlerWorkDirOptional(options)); registerHandler(new ArgHandlerExtraDir(options)); + registerHandler(new ArgHandlerLocalWorkers(options)); registerHandler(new ArgHandlerOutDir(options)); } @@ -56,6 +60,7 @@ CompilerOptions { private LinkOptionsImpl linkOptions = new LinkOptionsImpl(); + private int localWorkers; public GWTCompilerOptionsImpl() { } @@ -67,12 +72,17 @@ public void copyFrom(CompilerOptions other) { super.copyFrom(other); linkOptions.copyFrom(other); + localWorkers = other.getLocalWorkers(); } public File getExtraDir() { return linkOptions.getExtraDir(); } + public int getLocalWorkers() { + return localWorkers; + } + public File getOutDir() { return linkOptions.getOutDir(); } @@ -81,6 +91,10 @@ linkOptions.setExtraDir(extraDir); } + public void setLocalWorkers(int localWorkers) { + this.localWorkers = localWorkers; + } + public void setOutDir(File outDir) { linkOptions.setOutDir(outDir); } @@ -116,9 +130,15 @@ } public boolean run(TreeLogger logger) throws UnableToCompleteException { + ModuleDef module = ModuleDefLoader.loadFromClassPath(logger, + options.getModuleName()); + if (options.isValidateOnly()) { - return new Precompile(options).run(logger); + return Precompile.validate(logger, options, module, options.getGenDir(), + options.getCompilerWorkDir()); } else { + PerfLogger.start("compile"); + long compileStart = System.currentTimeMillis(); logger = logger.branch(TreeLogger.INFO, "Compiling module " + options.getModuleName()); @@ -129,26 +149,29 @@ tempWorkDir = true; } - if (new Precompile(options).run(logger)) { - /* - * TODO: use the in-memory result of Precompile to run CompilePerms - * instead of serializing through the file system. - */ - CompilePermsOptionsImpl permsOptions = new CompilePermsOptionsImpl(); - permsOptions.copyFrom(options); - if (new CompilePerms(permsOptions).run(logger)) { - if (new Link(options).run(logger)) { - logger.log(TreeLogger.INFO, "Compilation succeeded"); - return true; - } - } - } + Precompilation precompilation = Precompile.precompile(logger, options, + module, options.getGenDir(), options.getCompilerWorkDir()); - logger.log(TreeLogger.ERROR, "Compilation failed"); + Permutation[] allPerms = precompilation.getPermutations(); + File[] resultFiles = CompilePerms.makeResultFiles( + options.getCompilerWorkDir(), allPerms); + CompilePerms.compile(logger, precompilation, allPerms, + options.getLocalWorkers(), resultFiles); + + Link.link(logger.branch(TreeLogger.INFO, "Linking into " + + options.getOutDir().getPath()), module, precompilation, + resultFiles, options.getOutDir(), options.getExtraDir()); + + long compileDone = System.currentTimeMillis(); + long delta = compileDone - compileStart; + logger.log(TreeLogger.INFO, "Compilation succeeded -- " + + String.format("%.3f", delta / 1000d) + "s"); + return true; } catch (IOException e) { logger.log(TreeLogger.ERROR, "Unable to create compiler work directory", e); } finally { + PerfLogger.end(); if (tempWorkDir) { Util.recursiveDelete(options.getWorkDir(), false); }
diff --git a/dev/core/src/com/google/gwt/dev/Link.java b/dev/core/src/com/google/gwt/dev/Link.java index d476bbc..f10d910 100644 --- a/dev/core/src/com/google/gwt/dev/Link.java +++ b/dev/core/src/com/google/gwt/dev/Link.java
@@ -100,20 +100,20 @@ } public static ArtifactSet link(TreeLogger logger, ModuleDef module, - Precompilation precompilation, File[] jsFiles) + Precompilation precompilation, File[] resultFiles) throws UnableToCompleteException { StandardLinkerContext linkerContext = new StandardLinkerContext(logger, module, precompilation.getUnifiedAst().getOptions()); - return doLink(logger, linkerContext, precompilation, jsFiles); + return doLink(logger, linkerContext, precompilation, resultFiles); } public static void link(TreeLogger logger, ModuleDef module, - Precompilation precompilation, File[] jsFiles, File outDir, File extrasDir) - throws UnableToCompleteException { + Precompilation precompilation, File[] resultFiles, File outDir, + File extrasDir) throws UnableToCompleteException { StandardLinkerContext linkerContext = new StandardLinkerContext(logger, module, precompilation.getUnifiedAst().getOptions()); ArtifactSet artifacts = doLink(logger, linkerContext, precompilation, - jsFiles); + resultFiles); doProduceOutput(logger, artifacts, linkerContext, module, outDir, extrasDir); } @@ -142,15 +142,15 @@ private static ArtifactSet doLink(TreeLogger logger, StandardLinkerContext linkerContext, Precompilation precompilation, - File[] jsFiles) throws UnableToCompleteException { + File[] resultFiles) throws UnableToCompleteException { Permutation[] perms = precompilation.getPermutations(); - if (perms.length != jsFiles.length) { + if (perms.length != resultFiles.length) { throw new IllegalArgumentException( - "Mismatched jsFiles.length and permutation count"); + "Mismatched resultFiles.length and permutation count"); } for (int i = 0; i < perms.length; ++i) { - finishPermuation(logger, perms[i], jsFiles[i], linkerContext); + finishPermuation(logger, perms[i], resultFiles[i], linkerContext); } linkerContext.addOrReplaceArtifacts(precompilation.getGeneratedArtifacts()); @@ -254,11 +254,11 @@ return false; } Permutation[] perms = precompilation.getPermutations(); - File[] jsFiles = new File[perms.length]; + File[] resultFiles = new File[perms.length]; for (int i = 0; i < perms.length; ++i) { - jsFiles[i] = CompilePerms.makePermFilename(options.getCompilerWorkDir(), - i); - if (!jsFiles[i].exists()) { + resultFiles[i] = CompilePerms.makePermFilename( + options.getCompilerWorkDir(), i); + if (!resultFiles[i].exists()) { logger.log(TreeLogger.ERROR, "File not found '" + precompilationFile.getAbsolutePath() + "'; please compile all permutations"); @@ -271,7 +271,7 @@ StandardLinkerContext linkerContext = new StandardLinkerContext(branch, module, precompilation.getUnifiedAst().getOptions()); ArtifactSet artifacts = doLink(branch, linkerContext, precompilation, - jsFiles); + resultFiles); doProduceOutput(branch, artifacts, linkerContext, module, options.getOutDir(), options.getExtraDir());
diff --git a/dev/core/src/com/google/gwt/dev/OptionLocalWorkers.java b/dev/core/src/com/google/gwt/dev/OptionLocalWorkers.java new file mode 100644 index 0000000..e853888 --- /dev/null +++ b/dev/core/src/com/google/gwt/dev/OptionLocalWorkers.java
@@ -0,0 +1,31 @@ +/* + * Copyright 2008 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev; + +/** + * Controls the number of local PermutationWorkers to be used. + */ +public interface OptionLocalWorkers { + /** + * Returns the number of workers to run on this machine. + */ + int getLocalWorkers(); + + /** + * Sets the number of workers to run on this machine. + */ + void setLocalWorkers(int localWorkers); +}
diff --git a/dev/core/src/com/google/gwt/dev/Permutation.java b/dev/core/src/com/google/gwt/dev/Permutation.java index 13ca881..3eba3f2 100644 --- a/dev/core/src/com/google/gwt/dev/Permutation.java +++ b/dev/core/src/com/google/gwt/dev/Permutation.java
@@ -28,13 +28,19 @@ * Represents the state of a single permutation for compile. */ public final class Permutation implements Serializable { + private final int id; private final List<StaticPropertyOracle> propertyOracles = new ArrayList<StaticPropertyOracle>(); private final SortedMap<String, String> rebindAnswers = new TreeMap<String, String>(); - public Permutation(StaticPropertyOracle propertyOracle) { + public Permutation(int id, StaticPropertyOracle propertyOracle) { + this.id = id; this.propertyOracles.add(propertyOracle); } + public int getId() { + return id; + } + public StaticPropertyOracle[] getPropertyOracles() { return propertyOracles.toArray(new StaticPropertyOracle[propertyOracles.size()]); }
diff --git a/dev/core/src/com/google/gwt/dev/PermutationResult.java b/dev/core/src/com/google/gwt/dev/PermutationResult.java new file mode 100644 index 0000000..d0133ea --- /dev/null +++ b/dev/core/src/com/google/gwt/dev/PermutationResult.java
@@ -0,0 +1,28 @@ +/* + * Copyright 2008 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev; + +import java.io.Serializable; + +/** + * An extensible return type for the results of compiling a single permutation. + */ +public interface PermutationResult extends Serializable { + /** + * The compiled JavaScript code. + */ + String getJs(); +}
diff --git a/dev/core/src/com/google/gwt/dev/PermutationWorker.java b/dev/core/src/com/google/gwt/dev/PermutationWorker.java new file mode 100644 index 0000000..f24d35f --- /dev/null +++ b/dev/core/src/com/google/gwt/dev/PermutationWorker.java
@@ -0,0 +1,50 @@ +/* + * Copyright 2008 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev; + +import com.google.gwt.core.ext.TreeLogger; +import com.google.gwt.core.ext.UnableToCompleteException; + +/** + * Represents a facility that can compile an individual {@link Permutation}. + * Instances of PermutationWorker are expected to be created via + * {@link PermutationWorkerFactory#getWorkers(int)}. + */ +interface PermutationWorker { + + /** + * Compile a single permutation. The {@link com.google.gwt.dev.jjs.UnifiedAst} + * will have been provided to {@link PermutationWorkerFactory#getWorkers} + * method. + * + * @throws TransientWorkerException if the Permutation should be tried again + * on another worker + * @throws UnableToCompleteException due to a fatal error + */ + PermutationResult compile(TreeLogger logger, Permutation permutation) + throws TransientWorkerException, UnableToCompleteException; + + /** + * Returns a human-readable description of the worker instance. This may be + * used for error reporting. + */ + String getName(); + + /** + * Release any resources associated with the worker. + */ + void shutdown(); +}
diff --git a/dev/core/src/com/google/gwt/dev/PermutationWorkerFactory.java b/dev/core/src/com/google/gwt/dev/PermutationWorkerFactory.java new file mode 100644 index 0000000..4448c3d --- /dev/null +++ b/dev/core/src/com/google/gwt/dev/PermutationWorkerFactory.java
@@ -0,0 +1,396 @@ +/* + * Copyright 2008 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev; + +import com.google.gwt.core.ext.TreeLogger; +import com.google.gwt.core.ext.UnableToCompleteException; +import com.google.gwt.dev.jjs.InternalCompilerException; +import com.google.gwt.dev.jjs.UnifiedAst; +import com.google.gwt.dev.util.Util; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +/** + * Represents a factory for implementations of an endpoint that will invoke + * CompilePerms. Implementations of PermutationWorkerFactory should be + * default-instantiable and will have {@link #init} called immediately after + * construction. + */ +public abstract class PermutationWorkerFactory { + + /** + * This callable is responsible for compiling one permutation. It waits for an + * available worker, uses it, and returns it to the pool when finished. + */ + private static class CompileOnePermutation implements Callable<ResultStatus> { + private final BlockingQueue<PermutationWorker> availableWorkers; + private final TreeLogger logger; + private final Permutation permutation; + private final File resultFile; + + public CompileOnePermutation(TreeLogger logger, Permutation permutation, + File resultFile, BlockingQueue<PermutationWorker> availableWorkers) { + this.logger = logger; + this.permutation = permutation; + this.resultFile = resultFile; + this.availableWorkers = availableWorkers; + } + + public ResultStatus call() { + // Find a free worker + PermutationWorker worker; + try { + worker = availableWorkers.take(); + } catch (InterruptedException e) { + logger.log(TreeLogger.DEBUG, "Worker interrupted", e); + return ResultStatus.HARD_FAILURE; + } + + if (worker == noMoreWorkersWorker) { + // Shutting down + return ResultStatus.HARD_FAILURE; + } + + // Invoke the worker + try { + PermutationResult result = worker.compile(logger, permutation); + Util.writeObjectAsFile(logger, resultFile, result); + logger.log(TreeLogger.DEBUG, "Successfully compiled permutation"); + availableWorkers.add(worker); + return ResultStatus.SUCCESS; + } catch (TransientWorkerException e) { + logger.log(TreeLogger.DEBUG, "Worker died, will retry Permutation", e); + return ResultStatus.TRANSIENT_FAILURE; + } catch (UnableToCompleteException e) { + logger.log(TreeLogger.ERROR, "Unrecoverable exception, shutting down", + e); + return ResultStatus.HARD_FAILURE; + } + } + } + + private static enum ResultStatus { + /** + * A failure bad enough to merit shutting down the compilation. + */ + HARD_FAILURE, + + /** + * A successful compile. + */ + SUCCESS, + + /** + * A worker died while processing this permutation, but it is worth trying + * to compile it with another worker. + */ + TRANSIENT_FAILURE + }; + + /** + * The name of the system property used to define the workers. + */ + public static final String FACTORY_IMPL_PROPERTY = "gwt.jjs.permutationWorkerFactory"; + + /** + * This value can be passed into {@link #setLocalWorkers(int)} to indicate + * that a heuristic should be used to determine the total number of local + * workers. + */ + public static final int WORKERS_AUTO = 0; + + private static List<PermutationWorkerFactory> lazyFactories; + + private static final PermutationWorker noMoreWorkersWorker = new PermutationWorker() { + + public PermutationResult compile(TreeLogger logger, Permutation permutation) + throws TransientWorkerException, UnableToCompleteException { + throw new UnableToCompleteException(); + } + + public String getName() { + return "Marker worker indicating no more workers"; + } + + public void shutdown() { + } + }; + + /** + * Compiles all Permutations in a Precompilation and returns an array of Files + * that can be consumed by Link using the system-default + * PermutationWorkersFactories. + */ + public static void compilePermutations(TreeLogger logger, + Precompilation precompilation, int localWorkers, File[] resultFiles) + throws UnableToCompleteException { + compilePermutations(logger, precompilation, + precompilation.getPermutations(), localWorkers, resultFiles); + } + + /** + * Compiles a subset of the Permutations in a Precompilation and returns an + * array of Files that can be consumed by Link using the system-default + * PermutationWorkersFactories. + * + * @param localWorkers Set the maximum number of workers that should be + * executed on the local system by the PermutationWorkerFactory. The + * value {@link #WORKERS_AUTO} will allow the + * PermutationWorkerFactory to apply a heuristic to determine the + * correct number of local workers. + * @param resultFiles the output files to write into; must be the same length + * as permutations + */ + public static void compilePermutations(TreeLogger logger, + Precompilation precompilation, Permutation[] permutations, + int localWorkers, File[] resultFiles) throws UnableToCompleteException { + assert permutations.length == resultFiles.length; + assert Arrays.asList(precompilation.getPermutations()).containsAll( + Arrays.asList(permutations)); + + // We may have a mixed collection of workers from different factories + List<PermutationWorker> workers = new ArrayList<PermutationWorker>(); + + /* + * We can have errors below this point, there's a finally block to handle + * cleanup of workers. + */ + try { + createWorkers(logger, precompilation.getUnifiedAst(), + permutations.length, localWorkers, workers); + ExecutorService executor = Executors.newFixedThreadPool(workers.size()); + + // List of available workers. + // The extra space is for inserting nulls at shutdown time + BlockingQueue<PermutationWorker> availableWorkers = new ArrayBlockingQueue<PermutationWorker>( + 2 * workers.size()); + availableWorkers.addAll(workers); + + try { + + // Submit all tasks to the executor + + // The permutation compiles not yet finished + Queue<CompileOnePermutation> tasksOutstanding = new LinkedList<CompileOnePermutation>(); + + // The futures for the results of those compiles + Queue<Future<ResultStatus>> resultFutures = new LinkedList<Future<ResultStatus>>(); + + for (int i = 0; i < permutations.length; ++i) { + TreeLogger permLogger = logger.branch(TreeLogger.DEBUG, + "Worker permutation " + permutations[i].getId() + " of " + + permutations.length); + CompileOnePermutation task = new CompileOnePermutation(permLogger, + permutations[i], resultFiles[i], availableWorkers); + tasksOutstanding.add(task); + resultFutures.add(executor.submit(task)); + } + + // Count the number of dead workers + int numDeadWorkers = 0; + int successCount = 0; + + while (!resultFutures.isEmpty() && numDeadWorkers < workers.size()) { + assert resultFutures.size() == tasksOutstanding.size(); + + CompileOnePermutation task = tasksOutstanding.remove(); + Future<ResultStatus> future = resultFutures.remove(); + ResultStatus result; + try { + result = future.get(); + } catch (InterruptedException e) { + logger.log(TreeLogger.ERROR, + "Exiting without results due to interruption", e); + throw new UnableToCompleteException(); + } catch (ExecutionException e) { + logger.log(TreeLogger.ERROR, "A compilation failed", e); + throw new UnableToCompleteException(); + } + + if (result == ResultStatus.SUCCESS) { + ++successCount; + } else if (result == ResultStatus.TRANSIENT_FAILURE) { + // A worker died. Resubmit for the remaining workers. + ++numDeadWorkers; + tasksOutstanding.add(task); + resultFutures.add(executor.submit(task)); + } else if (result == ResultStatus.HARD_FAILURE) { + // Shut down. + break; + } else { + throw new InternalCompilerException("Unknown result type"); + } + } + + // Too many permutations is a coding error + assert successCount <= permutations.length; + + if (successCount < permutations.length) { + // Likely as not, all of the workers died + logger.log(TreeLogger.ERROR, "Not all permutation were compiled " + + successCount + " of " + permutations.length); + throw new UnableToCompleteException(); + } + } finally { + // Shut down the executor + executor.shutdown(); + + // Inform any residual CompileOnePermutation's that there aren't any + // more workers + for (int i = 0; i < workers.size(); i++) { + availableWorkers.add(noMoreWorkersWorker); + } + } + } finally { + // Shut down all workers + for (PermutationWorker worker : workers) { + worker.shutdown(); + } + } + } + + /** + * Creates one or more implementations of worker factories. This will treat + * the value of the {@value #FACTORY_IMPL_PROPERTY} system property as a + * comma-separated list of type names. + */ + private static synchronized List<PermutationWorkerFactory> createAll( + TreeLogger logger) throws UnableToCompleteException { + // NB: This is the much-derided FactoryFactory pattern + + logger = logger.branch(TreeLogger.TRACE, + "Creating PermutationWorkerFactory instances"); + + if (lazyFactories != null) { + logger.log(TreeLogger.SPAM, "Using lazy instances"); + return lazyFactories; + } + + List<PermutationWorkerFactory> mutableFactories = new ArrayList<PermutationWorkerFactory>(); + String classes = System.getProperty(FACTORY_IMPL_PROPERTY, + ThreadedPermutationWorkerFactory.class.getName() + "," + + ExternalPermutationWorkerFactory.class.getName()); + logger.log(TreeLogger.SPAM, "Factory impl property is " + classes); + + String[] classParts = classes.split(","); + for (String className : classParts) { + try { + Class<? extends PermutationWorkerFactory> clazz = Class.forName( + className).asSubclass(PermutationWorkerFactory.class); + PermutationWorkerFactory factory = clazz.newInstance(); + factory.init(logger); + mutableFactories.add(factory); + logger.log(TreeLogger.SPAM, "Added PermutationWorkerFactory " + + clazz.getName()); + } catch (ClassCastException e) { + logger.log(TreeLogger.ERROR, className + " is not a " + + PermutationWorkerFactory.class.getName()); + } catch (ClassNotFoundException e) { + logger.log(TreeLogger.ERROR, + "Unable to find PermutationWorkerFactory named " + className); + } catch (InstantiationException e) { + logger.log(TreeLogger.ERROR, + "Unable to instantiate PermutationWorkerFactory " + className, e); + } catch (IllegalAccessException e) { + logger.log(TreeLogger.ERROR, + "Unable to instantiate PermutationWorkerFactory " + className, e); + } + } + + if (mutableFactories.size() == 0) { + logger.log(TreeLogger.ERROR, + "No usable PermutationWorkerFactories available"); + throw new UnableToCompleteException(); + } + + return lazyFactories = Collections.unmodifiableList(mutableFactories); + } + + /** + * Create as many workers as possible to service the Permutations. + */ + private static void createWorkers(TreeLogger logger, UnifiedAst unifiedAst, + int workersNeeded, int localWorkers, List<PermutationWorker> workers) + throws UnableToCompleteException { + if (localWorkers <= WORKERS_AUTO) { + // TODO: something smarter? + localWorkers = 1; + } + + for (PermutationWorkerFactory factory : PermutationWorkerFactory.createAll(logger)) { + if (workersNeeded <= 0) { + break; + } + + int wanted = factory.isLocal() ? Math.min(workersNeeded, localWorkers) + : workersNeeded; + if (wanted <= 0) { + continue; + } + + Collection<PermutationWorker> newWorkers = factory.getWorkers(logger, + unifiedAst, wanted); + + workers.addAll(newWorkers); + workersNeeded -= newWorkers.size(); + if (factory.isLocal()) { + localWorkers -= newWorkers.size(); + } + } + + if (workers.size() == 0) { + logger.log(TreeLogger.ERROR, "No PermutationWorkers created"); + throw new UnableToCompleteException(); + } + } + + /** + * Return some number of PermutationWorkers. + * + * @param unifiedAst a UnifiedAst + * @param numWorkers the desired number of workers + * @return a collection of PermutationWorkers, the size of which may be less + * than <code>numWorkers</code> + */ + public abstract Collection<PermutationWorker> getWorkers(TreeLogger logger, + UnifiedAst unifiedAst, int numWorkers) throws UnableToCompleteException; + + /** + * Initialize the PermutationWorkerFactory. + */ + public abstract void init(TreeLogger logger) throws UnableToCompleteException; + + /** + * Indicates if the PermutationWorkers created by the factory consume + * computational or memory resources on the local system, as opposed to the + * per-permutation work being performed on a remote system. + */ + public abstract boolean isLocal(); +}
diff --git a/dev/core/src/com/google/gwt/dev/Precompile.java b/dev/core/src/com/google/gwt/dev/Precompile.java index a5aa4ed..bb96be3 100644 --- a/dev/core/src/com/google/gwt/dev/Precompile.java +++ b/dev/core/src/com/google/gwt/dev/Precompile.java
@@ -170,7 +170,7 @@ rebindOracles[i] = new StandardRebindOracle(compilationState, propertyOracles[i], module, rules, genDir, generatorResourcesDir, generatorArtifacts); - permutations[i] = new Permutation(propertyOracles[i]); + permutations[i] = new Permutation(i, propertyOracles[i]); } }
diff --git a/dev/core/src/com/google/gwt/dev/ThreadedPermutationWorkerFactory.java b/dev/core/src/com/google/gwt/dev/ThreadedPermutationWorkerFactory.java new file mode 100644 index 0000000..89eebf0 --- /dev/null +++ b/dev/core/src/com/google/gwt/dev/ThreadedPermutationWorkerFactory.java
@@ -0,0 +1,103 @@ +/* + * Copyright 2008 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev; + +import com.google.gwt.core.ext.TreeLogger; +import com.google.gwt.core.ext.UnableToCompleteException; +import com.google.gwt.dev.jjs.UnifiedAst; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * Create a single in-process PermutationWorker. This WorkerFactory is intended + * to be used as a fall-back in case the other PermutationWorkers are unable to + * complete. + */ +public class ThreadedPermutationWorkerFactory extends PermutationWorkerFactory { + + private static class ThreadedPermutationWorker implements PermutationWorker { + private final UnifiedAst ast; + private final int id; + + public ThreadedPermutationWorker(UnifiedAst ast, int id) { + this.ast = ast; + this.id = id; + } + + public PermutationResult compile(final TreeLogger logger, + final Permutation permutation) throws TransientWorkerException, + UnableToCompleteException { + try { + return CompilePerms.compile(logger, permutation, ast); + } catch (OutOfMemoryError e) { + logger.log(TreeLogger.ERROR, + "OutOfMemoryError: Increase heap size or lower " + + MAX_THREADS_PROPERTY); + throw new UnableToCompleteException(); + } + } + + public String getName() { + return "In-process PermutationWorker " + id; + } + + public void shutdown() { + // No-op + } + } + + /** + * A Java system property that can be used to change the number of in-process + * threads used. + */ + public static final String MAX_THREADS_PROPERTY = "gwt.jjs.maxThreads"; + + @Override + public Collection<PermutationWorker> getWorkers(TreeLogger logger, + UnifiedAst unifiedAst, int numWorkers) throws UnableToCompleteException { + logger = logger.branch(TreeLogger.SPAM, + "Creating ThreadedPermutationWorkers"); + + numWorkers = Math.min(numWorkers, Integer.getInteger(MAX_THREADS_PROPERTY, + 1)); + + if (numWorkers == 0) { + return Collections.emptyList(); + } + + // The worker will deserialize a new copy + List<PermutationWorker> toReturn = new ArrayList<PermutationWorker>( + numWorkers); + for (int i = 0; i < numWorkers; i++) { + toReturn.add(new ThreadedPermutationWorker(unifiedAst, i)); + } + return toReturn; + } + + @Override + public void init(TreeLogger logger) throws UnableToCompleteException { + logger = logger.branch(TreeLogger.SPAM, + "Initializing ThreadedPermutationWorkerFactory"); + } + + @Override + public boolean isLocal() { + return true; + } +}
diff --git a/dev/core/src/com/google/gwt/dev/TransientWorkerException.java b/dev/core/src/com/google/gwt/dev/TransientWorkerException.java new file mode 100644 index 0000000..5f4e2bc --- /dev/null +++ b/dev/core/src/com/google/gwt/dev/TransientWorkerException.java
@@ -0,0 +1,28 @@ +/* + * Copyright 2008 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev; + +/** + * Indicates that a PermutationWorker failed to complete its assignment, but + * that the permutation should be retried on another worker. This would be used + * to report errors such as OutOfMemory when compiling in-process or unexpected + * death of an out-of-process worker (network break, etc). + */ +public class TransientWorkerException extends Exception { + public TransientWorkerException(String msg, Throwable cause) { + super(msg, cause); + } +}
diff --git a/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java b/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java index 6dfc5fe..d0d2e6d 100644 --- a/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java +++ b/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java
@@ -143,6 +143,18 @@ return rebindRequests; } + /** + * Internally prepares a new AST for compilation if one is not already + * prepared. + */ + public void prepare() { + synchronized (myLockObject) { + if (initialAst == null) { + initialAst = deserializeAst(serializedAst); + } + } + } + AST getFreshAst() { synchronized (myLockObject) { if (initialAst != null) {
diff --git a/dev/core/src/com/google/gwt/dev/util/arg/ArgHandlerLocalWorkers.java b/dev/core/src/com/google/gwt/dev/util/arg/ArgHandlerLocalWorkers.java new file mode 100644 index 0000000..6800c83 --- /dev/null +++ b/dev/core/src/com/google/gwt/dev/util/arg/ArgHandlerLocalWorkers.java
@@ -0,0 +1,58 @@ +/* + * Copyright 2008 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.gwt.dev.util.arg; + +import com.google.gwt.dev.OptionLocalWorkers; +import com.google.gwt.util.tools.ArgHandlerInt; + +/** + * An arg handler to specify the number of local workers that should be used by + * the compiler. + */ +public class ArgHandlerLocalWorkers extends ArgHandlerInt { + + private final OptionLocalWorkers options; + + public ArgHandlerLocalWorkers(OptionLocalWorkers options) { + this.options = options; + } + + @Override + public String[] getDefaultArgs() { + // Default to 1 for now; we might do an "auto" later. + return new String[] {getTag(), "1"}; + } + + @Override + public String getPurpose() { + return "Specifies the number of local workers to use whe compiling permutations"; + } + + @Override + public String getTag() { + return "-localWorkers"; + } + + @Override + public String[] getTagArgs() { + return new String[] {"count"}; + } + + @Override + public void setInt(int value) { + options.setLocalWorkers(value); + } +}