Implements out-of-process compiles via -localWorkers command line option.
Patch by: bobv (+tweaks from spoon, scottb)
Review by: spoon, scottb
git-svn-id: https://google-web-toolkit.googlecode.com/svn/releases/1.6@4145 8db76d5a-ed1c-0410-87a9-c151d255dfc7
diff --git a/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardCompilationResult.java b/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardCompilationResult.java
index e8cdb0d..48a10bc 100644
--- a/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardCompilationResult.java
+++ b/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardCompilationResult.java
@@ -17,6 +17,7 @@
import com.google.gwt.core.ext.linker.CompilationResult;
import com.google.gwt.core.ext.linker.SelectionProperty;
+import com.google.gwt.dev.PermutationResult;
import com.google.gwt.dev.util.Util;
import java.io.File;
@@ -68,17 +69,17 @@
*/
public static final Comparator<SortedMap<SelectionProperty, String>> MAP_COMPARATOR = new MapComparator();
- private final File cacheFile;
+ private final File resultFile;
private transient SoftReference<String> js;
private final SortedSet<SortedMap<SelectionProperty, String>> propertyValues = new TreeSet<SortedMap<SelectionProperty, String>>(
MAP_COMPARATOR);
private final String strongName;
- public StandardCompilationResult(String js, String strongName, File cacheFile) {
+ public StandardCompilationResult(String js, String strongName, File resultFile) {
super(StandardLinkerContext.class);
this.js = new SoftReference<String>(js);
this.strongName = strongName;
- this.cacheFile = cacheFile;
+ this.resultFile = resultFile;
}
/**
@@ -98,11 +99,20 @@
toReturn = js.get();
}
if (toReturn == null) {
- toReturn = Util.readFileAsString(cacheFile);
- if (toReturn == null) {
- throw new RuntimeException("Unexpectedly unable to read JS file '"
- + cacheFile.getAbsolutePath() + "'");
+ PermutationResult result;
+ try {
+ result = Util.readFileAsObject(resultFile, PermutationResult.class);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(
+ "Unexpectedly unable to read PermutationResult '"
+ + resultFile.getAbsolutePath() + "'", e);
}
+ if (toReturn == null) {
+ throw new RuntimeException(
+ "Unexpectedly unable to read PermutationResult '"
+ + resultFile.getAbsolutePath() + "'");
+ }
+ toReturn = result.getJs();
js = new SoftReference<String>(toReturn);
}
return toReturn;
diff --git a/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardLinkerContext.java b/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardLinkerContext.java
index c5e4a06..55518a5 100644
--- a/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardLinkerContext.java
+++ b/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardLinkerContext.java
@@ -26,6 +26,7 @@
import com.google.gwt.core.ext.linker.PublicResource;
import com.google.gwt.core.ext.linker.SelectionProperty;
import com.google.gwt.core.ext.linker.LinkerOrder.Order;
+import com.google.gwt.dev.PermutationResult;
import com.google.gwt.dev.cfg.BindingProperty;
import com.google.gwt.dev.cfg.ModuleDef;
import com.google.gwt.dev.cfg.Property;
@@ -254,19 +255,27 @@
/**
* Gets or creates a CompilationResult for the given JavaScript program.
*/
- public StandardCompilationResult getCompilation(TreeLogger logger, File jsFile)
- throws UnableToCompleteException {
- byte[] bytes = Util.readFileAsBytes(jsFile);
- if (bytes == null) {
- logger.log(TreeLogger.ERROR, "Unable to read file '"
- + jsFile.getAbsolutePath() + "'");
+ public StandardCompilationResult getCompilation(TreeLogger logger,
+ File resultFile) throws UnableToCompleteException {
+ PermutationResult permutationResult;
+ try {
+ permutationResult = Util.readFileAsObject(resultFile,
+ PermutationResult.class);
+ } catch (ClassNotFoundException e) {
+ logger.log(TreeLogger.ERROR, "Unable to instantiate PermutationResult", e);
throw new UnableToCompleteException();
}
- String strongName = Util.computeStrongName(bytes);
+
+ if (permutationResult == null) {
+ logger.log(TreeLogger.ERROR, "Unable to read PermutationResult");
+ throw new UnableToCompleteException();
+ }
+
+ String strongName = Util.computeStrongName(Util.getBytes(permutationResult.getJs()));
StandardCompilationResult result = resultsByStrongName.get(strongName);
if (result == null) {
- result = new StandardCompilationResult(Util.toString(bytes), strongName,
- jsFile);
+ result = new StandardCompilationResult(permutationResult.getJs(),
+ strongName, resultFile);
resultsByStrongName.put(result.getStrongName(), result);
artifacts.add(result);
}
diff --git a/dev/core/src/com/google/gwt/dev/CompilePerms.java b/dev/core/src/com/google/gwt/dev/CompilePerms.java
index 82a08cd..94ef4c4 100644
--- a/dev/core/src/com/google/gwt/dev/CompilePerms.java
+++ b/dev/core/src/com/google/gwt/dev/CompilePerms.java
@@ -18,10 +18,10 @@
import com.google.gwt.core.ext.TreeLogger;
import com.google.gwt.core.ext.UnableToCompleteException;
import com.google.gwt.dev.CompileTaskRunner.CompileTask;
-import com.google.gwt.dev.PermutationCompiler.ResultsHandler;
-import com.google.gwt.dev.jjs.UnifiedAst;
import com.google.gwt.dev.jjs.JavaToJavaScriptCompiler;
+import com.google.gwt.dev.jjs.UnifiedAst;
import com.google.gwt.dev.util.Util;
+import com.google.gwt.dev.util.arg.ArgHandlerLocalWorkers;
import com.google.gwt.util.tools.ArgHandlerString;
import java.io.File;
@@ -29,15 +29,16 @@
import java.util.TreeSet;
/**
- * Performs the second phase of compilation, converting the Precompile's AST into
- * JavaScript outputs.
+ * Performs the second phase of compilation, converting the Precompile's AST
+ * into JavaScript outputs.
*/
public class CompilePerms {
/**
* Options for CompilePerms.
*/
- public interface CompilePermsOptions extends CompileTaskOptions, OptionPerms {
+ public interface CompilePermsOptions extends CompileTaskOptions,
+ OptionLocalWorkers, OptionPerms {
}
/**
@@ -121,6 +122,7 @@
public ArgProcessor(CompilePermsOptions options) {
super(options);
registerHandler(new ArgHandlerPerms(options));
+ registerHandler(new ArgHandlerLocalWorkers(options));
}
@Override
@@ -135,6 +137,7 @@
static class CompilePermsOptionsImpl extends CompileTaskOptionsImpl implements
CompilePermsOptions {
+ private int localWorkers;
private int[] permsToCompile = new int[0];
public CompilePermsOptionsImpl() {
@@ -146,24 +149,40 @@
public void copyFrom(CompilePermsOptions other) {
super.copyFrom(other);
-
setPermsToCompile(other.getPermsToCompile());
+ setLocalWorkers(other.getLocalWorkers());
+ }
+
+ public int getLocalWorkers() {
+ return localWorkers;
}
public int[] getPermsToCompile() {
return permsToCompile.clone();
}
+ public void setLocalWorkers(int localWorkers) {
+ this.localWorkers = localWorkers;
+ }
+
public void setPermsToCompile(int[] permsToCompile) {
this.permsToCompile = permsToCompile.clone();
}
}
- public static String compile(TreeLogger logger, Permutation permutation,
- UnifiedAst unifiedAst) {
+ /**
+ * Compile a single permutation.
+ */
+ public static PermutationResult compile(TreeLogger logger,
+ Permutation permutation, UnifiedAst unifiedAst) {
try {
- return JavaToJavaScriptCompiler.compilePermutation(logger, unifiedAst,
- permutation.getRebindAnswers());
+ final String js = JavaToJavaScriptCompiler.compilePermutation(logger,
+ unifiedAst, permutation.getRebindAnswers());
+ return new PermutationResult() {
+ public String getJs() {
+ return js;
+ }
+ };
} catch (UnableToCompleteException e) {
// We intentionally don't pass in the exception here since the real
// cause has been logged.
@@ -171,6 +190,20 @@
}
}
+ /**
+ * Compile multiple permutations.
+ */
+ public static boolean compile(TreeLogger logger,
+ Precompilation precompilation, Permutation[] perms, int localWorkers,
+ File[] resultFiles) throws UnableToCompleteException {
+ final TreeLogger branch = logger.branch(TreeLogger.INFO, "Compiling "
+ + perms.length + " permutations");
+ PermutationWorkerFactory.compilePermutations(logger, precompilation, perms,
+ localWorkers, resultFiles);
+ branch.log(TreeLogger.INFO, "Permutation compile succeeded");
+ return true;
+ }
+
public static void main(String[] args) {
/*
* NOTE: main always exits with a call to System.exit to terminate any
@@ -194,6 +227,14 @@
System.exit(1);
}
+ public static File[] makeResultFiles(File compilerWorkDir, Permutation[] perms) {
+ File[] resultFiles = new File[perms.length];
+ for (int i = 0; i < perms.length; ++i) {
+ resultFiles[i] = makePermFilename(compilerWorkDir, perms[i].getId());
+ }
+ return resultFiles;
+ }
+
/**
* Return the filename corresponding to the given permutation number,
* one-based.
@@ -229,17 +270,15 @@
+ precompilationFile.getAbsolutePath() + "'", e);
return false;
}
- Permutation[] perms = precompilation.getPermutations();
- UnifiedAst unifiedAst = precompilation.getUnifiedAst();
- int[] permsToRun = options.getPermsToCompile();
+ Permutation[] perms = precompilation.getPermutations();
+ Permutation[] subPerms;
+ int[] permsToRun = options.getPermsToCompile();
if (permsToRun.length == 0) {
- // Compile them all.
- permsToRun = new int[perms.length];
- for (int i = 0; i < permsToRun.length; ++i) {
- permsToRun[i] = i;
- }
+ subPerms = perms;
} else {
+ int i = 0;
+ subPerms = new Permutation[permsToRun.length];
// Range check the supplied perms.
for (int permToRun : permsToRun) {
if (permToRun >= perms.length) {
@@ -248,21 +287,12 @@
+ (perms.length - 1) + "'");
return false;
}
+ subPerms[i++] = perms[permToRun];
}
}
- final TreeLogger branch = logger.branch(TreeLogger.INFO, "Compiling "
- + permsToRun.length + " permutations");
- PermutationCompiler multiThread = new PermutationCompiler(branch,
- unifiedAst, perms, permsToRun);
- multiThread.go(new ResultsHandler() {
- public void addResult(Permutation permutation, int permNum, String js)
- throws UnableToCompleteException {
- Util.writeStringAsFile(branch, makePermFilename(
- options.getCompilerWorkDir(), permNum), js);
- }
- });
- branch.log(TreeLogger.INFO, "Permutation compile succeeded");
- return true;
+ File[] resultFiles = makeResultFiles(options.getCompilerWorkDir(), perms);
+ return compile(logger, precompilation, subPerms, options.getLocalWorkers(),
+ resultFiles);
}
}
diff --git a/dev/core/src/com/google/gwt/dev/CompilePermsServer.java b/dev/core/src/com/google/gwt/dev/CompilePermsServer.java
new file mode 100644
index 0000000..ace3385
--- /dev/null
+++ b/dev/core/src/com/google/gwt/dev/CompilePermsServer.java
@@ -0,0 +1,309 @@
+/*
+ * Copyright 2008 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.dev;
+
+import com.google.gwt.core.ext.TreeLogger;
+import com.google.gwt.core.ext.TreeLogger.Type;
+import com.google.gwt.dev.jjs.UnifiedAst;
+import com.google.gwt.dev.util.arg.ArgHandlerLogLevel;
+import com.google.gwt.dev.util.arg.OptionLogLevel;
+import com.google.gwt.dev.util.log.PrintWriterTreeLogger;
+import com.google.gwt.util.tools.ArgHandlerString;
+import com.google.gwt.util.tools.ToolBase;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.net.Socket;
+import java.net.UnknownHostException;
+
+/**
+ * An out-of-process implementation of CompilePerms that will connect back to an
+ * existing compiler host. This class is intended to be launched by
+ * {@link ExternalPermutationWorkerFactory} and not by users directly.
+ */
+public class CompilePermsServer {
+ /**
+ * Adds host and port information.
+ */
+ public interface CompileServerOptions extends OptionLogLevel {
+ String getCompileHost();
+
+ int getCompilePort();
+
+ String getCookie();
+
+ void setCompileHost(String host);
+
+ void setCompilePort(int port);
+
+ void setCookie(String cookie);
+ }
+
+ static final class ArgHandlerCompileHost extends ArgHandlerString {
+ private final CompileServerOptions options;
+
+ public ArgHandlerCompileHost(CompileServerOptions options) {
+ this.options = options;
+ }
+
+ @Override
+ public String getPurpose() {
+ return "The host to which the permutation server should connect";
+ }
+
+ @Override
+ public String getTag() {
+ return "-host";
+ }
+
+ @Override
+ public String[] getTagArgs() {
+ return new String[] {"hostname"};
+ }
+
+ @Override
+ public boolean isRequired() {
+ return true;
+ }
+
+ @Override
+ public boolean setString(String str) {
+ options.setCompileHost(str);
+ return true;
+ }
+ }
+
+ static final class ArgHandlerCompilePort extends ArgHandlerString {
+ private final CompileServerOptions options;
+
+ public ArgHandlerCompilePort(CompileServerOptions options) {
+ this.options = options;
+ }
+
+ @Override
+ public String getPurpose() {
+ return "The port to which the permutation server should connect";
+ }
+
+ @Override
+ public String getTag() {
+ return "-port";
+ }
+
+ @Override
+ public String[] getTagArgs() {
+ return new String[] {"1234"};
+ }
+
+ @Override
+ public boolean isRequired() {
+ return true;
+ }
+
+ @Override
+ public boolean setString(String str) {
+ Integer port = Integer.parseInt(str);
+ if (port <= 0) {
+ return false;
+ }
+ options.setCompilePort(port);
+ return true;
+ }
+ }
+
+ static final class ArgHandlerCookie extends ArgHandlerString {
+
+ private final CompileServerOptions options;
+
+ public ArgHandlerCookie(CompileServerOptions option) {
+ this.options = option;
+ }
+
+ @Override
+ public String getPurpose() {
+ return "Specifies the security cookie the server expects";
+ }
+
+ @Override
+ public String getTag() {
+ return "-cookie";
+ }
+
+ @Override
+ public String[] getTagArgs() {
+ return new String[] {"cookie"};
+ }
+
+ @Override
+ public boolean isRequired() {
+ return true;
+ }
+
+ @Override
+ public boolean setString(String str) {
+ options.setCookie(str);
+ return true;
+ }
+ }
+
+ static class ArgProcessor extends ToolBase {
+ public ArgProcessor(CompileServerOptions options) {
+ registerHandler(new ArgHandlerLogLevel(options));
+ registerHandler(new ArgHandlerCompileHost(options));
+ registerHandler(new ArgHandlerCompilePort(options));
+ registerHandler(new ArgHandlerCookie(options));
+ }
+
+ /*
+ * Overridden to make public.
+ */
+ @Override
+ public final boolean processArgs(String[] args) {
+ return super.processArgs(args);
+ }
+
+ @Override
+ protected String getName() {
+ return CompilePermsServer.class.getName();
+ }
+ }
+
+ static class CompileServerOptionsImpl implements CompileServerOptions {
+
+ private String compileHost;
+ private int compilePort;
+ private String cookie;
+ private Type logLevel;
+
+ public void copyFrom(CompileServerOptions other) {
+ setCompileHost(other.getCompileHost());
+ setCompilePort(other.getCompilePort());
+ setCookie(other.getCookie());
+ setLogLevel(other.getLogLevel());
+ }
+
+ public String getCompileHost() {
+ return compileHost;
+ }
+
+ public int getCompilePort() {
+ return compilePort;
+ }
+
+ public String getCookie() {
+ return cookie;
+ }
+
+ public Type getLogLevel() {
+ return logLevel;
+ }
+
+ public void setCompileHost(String host) {
+ assert host != null;
+ compileHost = host;
+ }
+
+ public void setCompilePort(int port) {
+ assert port > 0;
+ compilePort = port;
+ }
+
+ public void setCookie(String cookie) {
+ this.cookie = cookie;
+ }
+
+ public void setLogLevel(Type logLevel) {
+ this.logLevel = logLevel;
+ }
+ }
+
+ public static void main(String[] args) {
+ final CompileServerOptions options = new CompileServerOptionsImpl();
+ if (new ArgProcessor(options).processArgs(args)) {
+ PrintWriterTreeLogger logger = new PrintWriterTreeLogger();
+ logger.setMaxDetail(options.getLogLevel());
+ if (run(options, logger)) {
+ System.exit(0);
+ }
+ }
+ System.exit(-1);
+ }
+
+ public static boolean run(CompileServerOptions options, TreeLogger logger) {
+ try {
+ Socket s = new Socket(options.getCompileHost(), options.getCompilePort());
+ logger.log(TreeLogger.DEBUG, "Socket opened");
+
+ ObjectOutputStream out = new ObjectOutputStream(s.getOutputStream());
+ ObjectInputStream in = new ObjectInputStream(s.getInputStream());
+
+ // Write my cookie
+ out.writeUTF(options.getCookie());
+ out.flush();
+
+ // Read the File that contains the serialized UnifiedAst
+ File astFile = (File) in.readObject();
+ ObjectInputStream astIn = new ObjectInputStream(new FileInputStream(
+ astFile));
+ UnifiedAst ast = (UnifiedAst) astIn.readObject();
+ ast.prepare();
+ logger.log(TreeLogger.SPAM, "Created new UnifiedAst instance");
+
+ // Report on the amount of memory we think we're using
+ long estimatedMemory = Runtime.getRuntime().totalMemory()
+ - Runtime.getRuntime().freeMemory();
+ out.writeLong(estimatedMemory);
+ out.flush();
+
+ boolean keepGoing = in.readBoolean();
+ while (keepGoing) {
+ compilePermutation(logger, ast, in, out);
+
+ keepGoing = in.readBoolean();
+ logger.log(TreeLogger.SPAM, "keepGoing = " + keepGoing);
+ }
+
+ logger.log(TreeLogger.DEBUG, "Successfully terminating");
+ return true;
+
+ } catch (UnknownHostException e) {
+ logger.log(TreeLogger.ERROR, "Invalid hostname", e);
+ } catch (IOException e) {
+ logger.log(TreeLogger.ERROR, "Communication error", e);
+ } catch (ClassNotFoundException e) {
+ logger.log(TreeLogger.ERROR, "Probable client/server mismatch or "
+ + "classpath misconfiguration", e);
+ }
+ return false;
+ }
+
+ static void compilePermutation(TreeLogger logger, UnifiedAst ast,
+ ObjectInputStream in, ObjectOutputStream out)
+ throws ClassNotFoundException, IOException {
+ Permutation p = (Permutation) in.readObject();
+ logger.log(TreeLogger.SPAM, "Permutation read");
+
+ PermutationResult result = CompilePerms.compile(logger.branch(
+ TreeLogger.DEBUG, "Compiling"), p, ast);
+ logger.log(TreeLogger.DEBUG, "Successfully compiled permutation");
+ out.writeObject(result);
+ out.flush();
+ logger.log(TreeLogger.SPAM, "Sent result");
+ }
+}
diff --git a/dev/core/src/com/google/gwt/dev/CompileTaskOptionsImpl.java b/dev/core/src/com/google/gwt/dev/CompileTaskOptionsImpl.java
index e0b7ead..e92d1c8 100644
--- a/dev/core/src/com/google/gwt/dev/CompileTaskOptionsImpl.java
+++ b/dev/core/src/com/google/gwt/dev/CompileTaskOptionsImpl.java
@@ -24,8 +24,6 @@
*/
class CompileTaskOptionsImpl implements CompileTaskOptions {
- public static final String GWT_TMP_DIR = "gwt-tmp";
-
private Type logLevel;
private String moduleName;
private boolean useGuiLogger;
diff --git a/dev/core/src/com/google/gwt/dev/CompilerOptions.java b/dev/core/src/com/google/gwt/dev/CompilerOptions.java
index 0b568e4..382eee8 100644
--- a/dev/core/src/com/google/gwt/dev/CompilerOptions.java
+++ b/dev/core/src/com/google/gwt/dev/CompilerOptions.java
@@ -21,5 +21,6 @@
/**
* The complete set of options for the GWT compiler.
*/
-public interface CompilerOptions extends PrecompileOptions, LinkOptions {
+public interface CompilerOptions extends PrecompileOptions, LinkOptions,
+ OptionLocalWorkers {
}
diff --git a/dev/core/src/com/google/gwt/dev/ExternalPermutationWorkerFactory.java b/dev/core/src/com/google/gwt/dev/ExternalPermutationWorkerFactory.java
new file mode 100644
index 0000000..392afea
--- /dev/null
+++ b/dev/core/src/com/google/gwt/dev/ExternalPermutationWorkerFactory.java
@@ -0,0 +1,376 @@
+/*
+ * Copyright 2008 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.dev;
+
+import com.google.gwt.core.ext.TreeLogger;
+import com.google.gwt.core.ext.UnableToCompleteException;
+import com.google.gwt.dev.jjs.UnifiedAst;
+import com.google.gwt.dev.util.Util;
+
+import java.io.BufferedReader;
+import java.io.EOFException;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.lang.management.ManagementFactory;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketTimeoutException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+
+/**
+ * A PermutationWorkerFactory designed to launch instances of
+ * {@link CompilePermsServer}. The system property
+ * {@value #JAVA_COMMAND_PROPERTY} can be used to change the command used to
+ * launch the JVM. The system property {@link #JVM_ARGS_PROPERTY} can be used to
+ * override the JVM args passed to the subprocess.
+ */
+public class ExternalPermutationWorkerFactory extends PermutationWorkerFactory {
+
+ /**
+ * Allows accept() to be called a finite number of times on a ServerSocket
+ * before closing the socket.
+ */
+ private static class CountedServerSocket {
+ private int accepts;
+ private ServerSocket sock;
+
+ public CountedServerSocket(ServerSocket sock, int maxAccepts) {
+ assert sock != null;
+ assert maxAccepts >= 1;
+
+ this.accepts = maxAccepts;
+ this.sock = sock;
+ }
+
+ public synchronized Socket accept() throws IOException {
+ assert accepts >= 0;
+
+ if (accepts == 0) {
+ throw new IllegalStateException("Too many calls to accept()");
+ }
+
+ try {
+ return sock.accept();
+ } finally {
+ if (--accepts == 0) {
+ sock.close();
+ sock = null;
+ }
+ }
+ }
+ }
+
+ private static class ExternalPermutationWorker implements PermutationWorker {
+ private final File astFile;
+ private final Set<String> cookies;
+ private ObjectInputStream in;
+ private ObjectOutputStream out;
+ private final CountedServerSocket serverSocket;
+ private Socket workerSocket;
+
+ public ExternalPermutationWorker(CountedServerSocket sock, File astFile,
+ Set<String> cookies) {
+ this.astFile = astFile;
+ this.cookies = cookies;
+ this.serverSocket = sock;
+ }
+
+ public PermutationResult compile(TreeLogger logger, Permutation permutation)
+ throws TransientWorkerException, UnableToCompleteException {
+
+ // If we've just started, we need to get a connection from a subprocess
+ if (workerSocket == null) {
+ try {
+ /*
+ * We've set SO_TIMEOUT, so this may fail if the remote process never
+ * connects back.
+ */
+ workerSocket = serverSocket.accept();
+
+ in = new ObjectInputStream(workerSocket.getInputStream());
+ out = new ObjectOutputStream(workerSocket.getOutputStream());
+
+ // Verify we're talking to the right worker
+ String c = in.readUTF();
+ if (!cookies.contains(c)) {
+ throw new TransientWorkerException("Received unknown cookie " + c,
+ null);
+ }
+
+ out.writeObject(astFile);
+
+ // Get the remote worker's estimate of memory use
+ long memoryUse = in.readLong();
+ logger.log(TreeLogger.SPAM, "Remote process indicates " + memoryUse
+ + " bytes of memory used");
+
+ } catch (SocketTimeoutException e) {
+ throw new TransientWorkerException(
+ "Remote process did not connect within timeout period", e);
+ } catch (IOException e) {
+ throw new TransientWorkerException(
+ "Unable to communicate with worker", e);
+ }
+ }
+
+ try {
+ out.writeBoolean(true);
+ out.writeObject(permutation);
+ out.flush();
+ return (PermutationResult) in.readObject();
+ } catch (IOException e) {
+ logger.log(TreeLogger.WARN, "Lost communication with remote process", e);
+ throw new TransientWorkerException(
+ "Lost communication with remote process", e);
+ } catch (ClassNotFoundException e) {
+ logger.log(TreeLogger.ERROR, "Unable to receive response", e);
+ throw new UnableToCompleteException();
+ }
+ }
+
+ public String getName() {
+ return "External worker "
+ + (workerSocket != null ? workerSocket.getRemoteSocketAddress()
+ : "unconnected");
+ }
+
+ public void shutdown() {
+ if (out != null) {
+ try {
+ out.writeBoolean(false);
+ out.flush();
+ out.close();
+ } catch (IOException e) {
+ // Not much to do here
+ }
+ }
+
+ if (in != null) {
+ try {
+ in.close();
+ } catch (IOException e) {
+ // Not much to do here
+ }
+ }
+
+ if (workerSocket != null) {
+ try {
+ workerSocket.close();
+ } catch (IOException e) {
+ // Nothing to do
+ }
+ }
+ }
+ }
+
+ /**
+ * A system property that can be used to override the command used to invoke a
+ * JVM instance.
+ */
+ public static final String JAVA_COMMAND_PROPERTY = "gwt.jjs.javaCommand";
+
+ /**
+ * A system property that can be used to override the JVM args passed to the
+ * subprocess.
+ */
+ public static final String JVM_ARGS_PROPERTY = "gwt.jjs.javaArgs";
+
+ /**
+ * Launches an external worker and returns the cookie that worker should
+ * return via the network connection.
+ */
+ private static String launchExternalWorker(TreeLogger logger, int port)
+ throws UnableToCompleteException {
+
+ String javaCommand = System.getProperty(JAVA_COMMAND_PROPERTY,
+ System.getProperty("java.home") + File.separator + "bin"
+ + File.separator + "java");
+ logger.log(TreeLogger.TRACE, "javaCommand = " + javaCommand);
+
+ // Construct the arguments
+ List<String> args = new ArrayList<String>();
+ args.add(javaCommand);
+
+ // This will include -Xmx, -D, etc...
+ String userJvmArgs = System.getProperty(JVM_ARGS_PROPERTY);
+ if (userJvmArgs == null) {
+ args.addAll(ManagementFactory.getRuntimeMXBean().getInputArguments());
+ } else {
+ args.addAll(Arrays.asList(userJvmArgs.split(" ")));
+ }
+
+ // Determine the logLevel for the external program
+ TreeLogger.Type logLevel = TreeLogger.ERROR;
+ for (TreeLogger.Type t : TreeLogger.Type.values()) {
+ if (logger.isLoggable(t)) {
+ logLevel = t;
+ } else {
+ break;
+ }
+ }
+
+ byte[] cookieBytes = new byte[16];
+ (new Random()).nextBytes(cookieBytes);
+ String cookie = Util.toHexString(cookieBytes);
+
+ // Cook up the classpath, main class, and extra args
+ args.addAll(Arrays.asList("-classpath",
+ ManagementFactory.getRuntimeMXBean().getClassPath(),
+ CompilePermsServer.class.getName(), "-host", "localhost", "-port",
+ String.valueOf(port), "-logLevel", logLevel.toString(), "-cookie",
+ cookie));
+
+ // Filter undesirable arguments
+ for (Iterator<String> iter = args.iterator(); iter.hasNext();) {
+ String arg = iter.next();
+ if (arg.startsWith("-agentlib")) {
+ iter.remove();
+ }
+ }
+
+ ProcessBuilder builder = new ProcessBuilder();
+ builder.command(args);
+
+ try {
+ final Process proc = builder.start();
+ final BufferedReader bin = new BufferedReader(new InputStreamReader(
+ proc.getInputStream()));
+ final BufferedReader berr = new BufferedReader(new InputStreamReader(
+ proc.getErrorStream()));
+ final TreeLogger procLogger = logger.branch(TreeLogger.DEBUG,
+ "Process output");
+
+ // Threads to copy stdout, stderr to the logger
+ new Thread(new Runnable() {
+ public void run() {
+ while (true) {
+ try {
+ String line = bin.readLine();
+ if (line == null) {
+ break;
+ }
+ procLogger.log(TreeLogger.INFO, line);
+ } catch (EOFException e) {
+ // Ignore
+ } catch (IOException e) {
+ procLogger.log(TreeLogger.ERROR,
+ "Unable to read from subprocess", e);
+ }
+ }
+ }
+ }).start();
+
+ new Thread(new Runnable() {
+ public void run() {
+ while (true) {
+ try {
+ String line = berr.readLine();
+ if (line == null) {
+ break;
+ }
+ procLogger.log(TreeLogger.ERROR, line);
+ } catch (EOFException e) {
+ // Ignore
+ } catch (IOException e) {
+ procLogger.log(TreeLogger.ERROR,
+ "Unable to read from subprocess", e);
+ }
+ }
+ }
+ }).start();
+
+ // The child process should not outlive this JVM
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ public void run() {
+ try {
+ proc.exitValue();
+ } catch (IllegalThreadStateException e) {
+ proc.destroy();
+ }
+ }
+ }));
+
+ return cookie;
+ } catch (IOException e) {
+ logger.log(TreeLogger.ERROR, "Unable to start external process", e);
+ throw new UnableToCompleteException();
+ }
+ }
+
+ private ServerSocket sock;
+
+ @Override
+ public Collection<PermutationWorker> getWorkers(TreeLogger logger,
+ UnifiedAst unifiedAst, int numWorkers) throws UnableToCompleteException {
+ File astFile;
+ try {
+ astFile = File.createTempFile("externalPermutationWorkerFactory", ".ser");
+ astFile.deleteOnExit();
+ Util.writeObjectAsFile(logger, astFile, unifiedAst);
+ } catch (IOException e) {
+ logger.log(TreeLogger.ERROR, "Unable to create temporary file", e);
+ throw new UnableToCompleteException();
+ }
+
+ Set<String> cookies = Collections.synchronizedSet(new HashSet<String>(
+ numWorkers));
+ CountedServerSocket countedSock = new CountedServerSocket(sock, numWorkers);
+ List<PermutationWorker> toReturn = new ArrayList<PermutationWorker>(
+ numWorkers);
+
+ for (int i = 0; i < numWorkers; i++) {
+ String cookie = launchExternalWorker(logger, sock.getLocalPort());
+ cookies.add(cookie);
+ toReturn.add(new ExternalPermutationWorker(countedSock, astFile, cookies));
+ }
+
+ return toReturn;
+ }
+
+ @Override
+ public void init(TreeLogger logger) throws UnableToCompleteException {
+ try {
+ sock = new ServerSocket();
+ /*
+ * Have accept() wait no more than one minute for a connection. This
+ * prevents dead-head behavior.
+ */
+ sock.setSoTimeout(60000);
+ sock.bind(null);
+ logger.log(TreeLogger.SPAM, "Listening for external workers on port "
+ + sock.getLocalPort());
+ } catch (IOException e) {
+ logger.log(TreeLogger.ERROR, "Unable to create socket", e);
+ throw new UnableToCompleteException();
+ }
+ }
+
+ @Override
+ public boolean isLocal() {
+ return true;
+ }
+}
diff --git a/dev/core/src/com/google/gwt/dev/GWTCompiler.java b/dev/core/src/com/google/gwt/dev/GWTCompiler.java
index f331040..546e5ab 100644
--- a/dev/core/src/com/google/gwt/dev/GWTCompiler.java
+++ b/dev/core/src/com/google/gwt/dev/GWTCompiler.java
@@ -17,12 +17,15 @@
import com.google.gwt.core.ext.TreeLogger;
import com.google.gwt.core.ext.UnableToCompleteException;
-import com.google.gwt.dev.CompilePerms.CompilePermsOptionsImpl;
import com.google.gwt.dev.CompileTaskRunner.CompileTask;
import com.google.gwt.dev.Link.LinkOptionsImpl;
import com.google.gwt.dev.Precompile.PrecompileOptionsImpl;
+import com.google.gwt.dev.cfg.ModuleDef;
+import com.google.gwt.dev.cfg.ModuleDefLoader;
+import com.google.gwt.dev.util.PerfLogger;
import com.google.gwt.dev.util.Util;
import com.google.gwt.dev.util.arg.ArgHandlerExtraDir;
+import com.google.gwt.dev.util.arg.ArgHandlerLocalWorkers;
import com.google.gwt.dev.util.arg.ArgHandlerOutDir;
import com.google.gwt.dev.util.arg.ArgHandlerWorkDirOptional;
import com.google.gwt.util.tools.Utility;
@@ -43,6 +46,7 @@
registerHandler(new ArgHandlerWorkDirOptional(options));
registerHandler(new ArgHandlerExtraDir(options));
+ registerHandler(new ArgHandlerLocalWorkers(options));
registerHandler(new ArgHandlerOutDir(options));
}
@@ -56,6 +60,7 @@
CompilerOptions {
private LinkOptionsImpl linkOptions = new LinkOptionsImpl();
+ private int localWorkers;
public GWTCompilerOptionsImpl() {
}
@@ -67,12 +72,17 @@
public void copyFrom(CompilerOptions other) {
super.copyFrom(other);
linkOptions.copyFrom(other);
+ localWorkers = other.getLocalWorkers();
}
public File getExtraDir() {
return linkOptions.getExtraDir();
}
+ public int getLocalWorkers() {
+ return localWorkers;
+ }
+
public File getOutDir() {
return linkOptions.getOutDir();
}
@@ -81,6 +91,10 @@
linkOptions.setExtraDir(extraDir);
}
+ public void setLocalWorkers(int localWorkers) {
+ this.localWorkers = localWorkers;
+ }
+
public void setOutDir(File outDir) {
linkOptions.setOutDir(outDir);
}
@@ -116,9 +130,15 @@
}
public boolean run(TreeLogger logger) throws UnableToCompleteException {
+ ModuleDef module = ModuleDefLoader.loadFromClassPath(logger,
+ options.getModuleName());
+
if (options.isValidateOnly()) {
- return new Precompile(options).run(logger);
+ return Precompile.validate(logger, options, module, options.getGenDir(),
+ options.getCompilerWorkDir());
} else {
+ PerfLogger.start("compile");
+ long compileStart = System.currentTimeMillis();
logger = logger.branch(TreeLogger.INFO, "Compiling module "
+ options.getModuleName());
@@ -129,26 +149,29 @@
tempWorkDir = true;
}
- if (new Precompile(options).run(logger)) {
- /*
- * TODO: use the in-memory result of Precompile to run CompilePerms
- * instead of serializing through the file system.
- */
- CompilePermsOptionsImpl permsOptions = new CompilePermsOptionsImpl();
- permsOptions.copyFrom(options);
- if (new CompilePerms(permsOptions).run(logger)) {
- if (new Link(options).run(logger)) {
- logger.log(TreeLogger.INFO, "Compilation succeeded");
- return true;
- }
- }
- }
+ Precompilation precompilation = Precompile.precompile(logger, options,
+ module, options.getGenDir(), options.getCompilerWorkDir());
- logger.log(TreeLogger.ERROR, "Compilation failed");
+ Permutation[] allPerms = precompilation.getPermutations();
+ File[] resultFiles = CompilePerms.makeResultFiles(
+ options.getCompilerWorkDir(), allPerms);
+ CompilePerms.compile(logger, precompilation, allPerms,
+ options.getLocalWorkers(), resultFiles);
+
+ Link.link(logger.branch(TreeLogger.INFO, "Linking into "
+ + options.getOutDir().getPath()), module, precompilation,
+ resultFiles, options.getOutDir(), options.getExtraDir());
+
+ long compileDone = System.currentTimeMillis();
+ long delta = compileDone - compileStart;
+ logger.log(TreeLogger.INFO, "Compilation succeeded -- "
+ + String.format("%.3f", delta / 1000d) + "s");
+ return true;
} catch (IOException e) {
logger.log(TreeLogger.ERROR,
"Unable to create compiler work directory", e);
} finally {
+ PerfLogger.end();
if (tempWorkDir) {
Util.recursiveDelete(options.getWorkDir(), false);
}
diff --git a/dev/core/src/com/google/gwt/dev/Link.java b/dev/core/src/com/google/gwt/dev/Link.java
index d476bbc..f10d910 100644
--- a/dev/core/src/com/google/gwt/dev/Link.java
+++ b/dev/core/src/com/google/gwt/dev/Link.java
@@ -100,20 +100,20 @@
}
public static ArtifactSet link(TreeLogger logger, ModuleDef module,
- Precompilation precompilation, File[] jsFiles)
+ Precompilation precompilation, File[] resultFiles)
throws UnableToCompleteException {
StandardLinkerContext linkerContext = new StandardLinkerContext(logger,
module, precompilation.getUnifiedAst().getOptions());
- return doLink(logger, linkerContext, precompilation, jsFiles);
+ return doLink(logger, linkerContext, precompilation, resultFiles);
}
public static void link(TreeLogger logger, ModuleDef module,
- Precompilation precompilation, File[] jsFiles, File outDir, File extrasDir)
- throws UnableToCompleteException {
+ Precompilation precompilation, File[] resultFiles, File outDir,
+ File extrasDir) throws UnableToCompleteException {
StandardLinkerContext linkerContext = new StandardLinkerContext(logger,
module, precompilation.getUnifiedAst().getOptions());
ArtifactSet artifacts = doLink(logger, linkerContext, precompilation,
- jsFiles);
+ resultFiles);
doProduceOutput(logger, artifacts, linkerContext, module, outDir, extrasDir);
}
@@ -142,15 +142,15 @@
private static ArtifactSet doLink(TreeLogger logger,
StandardLinkerContext linkerContext, Precompilation precompilation,
- File[] jsFiles) throws UnableToCompleteException {
+ File[] resultFiles) throws UnableToCompleteException {
Permutation[] perms = precompilation.getPermutations();
- if (perms.length != jsFiles.length) {
+ if (perms.length != resultFiles.length) {
throw new IllegalArgumentException(
- "Mismatched jsFiles.length and permutation count");
+ "Mismatched resultFiles.length and permutation count");
}
for (int i = 0; i < perms.length; ++i) {
- finishPermuation(logger, perms[i], jsFiles[i], linkerContext);
+ finishPermuation(logger, perms[i], resultFiles[i], linkerContext);
}
linkerContext.addOrReplaceArtifacts(precompilation.getGeneratedArtifacts());
@@ -254,11 +254,11 @@
return false;
}
Permutation[] perms = precompilation.getPermutations();
- File[] jsFiles = new File[perms.length];
+ File[] resultFiles = new File[perms.length];
for (int i = 0; i < perms.length; ++i) {
- jsFiles[i] = CompilePerms.makePermFilename(options.getCompilerWorkDir(),
- i);
- if (!jsFiles[i].exists()) {
+ resultFiles[i] = CompilePerms.makePermFilename(
+ options.getCompilerWorkDir(), i);
+ if (!resultFiles[i].exists()) {
logger.log(TreeLogger.ERROR, "File not found '"
+ precompilationFile.getAbsolutePath()
+ "'; please compile all permutations");
@@ -271,7 +271,7 @@
StandardLinkerContext linkerContext = new StandardLinkerContext(branch,
module, precompilation.getUnifiedAst().getOptions());
ArtifactSet artifacts = doLink(branch, linkerContext, precompilation,
- jsFiles);
+ resultFiles);
doProduceOutput(branch, artifacts, linkerContext, module,
options.getOutDir(), options.getExtraDir());
diff --git a/dev/core/src/com/google/gwt/dev/OptionLocalWorkers.java b/dev/core/src/com/google/gwt/dev/OptionLocalWorkers.java
new file mode 100644
index 0000000..e853888
--- /dev/null
+++ b/dev/core/src/com/google/gwt/dev/OptionLocalWorkers.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2008 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.dev;
+
+/**
+ * Controls the number of local PermutationWorkers to be used.
+ */
+public interface OptionLocalWorkers {
+ /**
+ * Returns the number of workers to run on this machine.
+ */
+ int getLocalWorkers();
+
+ /**
+ * Sets the number of workers to run on this machine.
+ */
+ void setLocalWorkers(int localWorkers);
+}
diff --git a/dev/core/src/com/google/gwt/dev/Permutation.java b/dev/core/src/com/google/gwt/dev/Permutation.java
index 13ca881..3eba3f2 100644
--- a/dev/core/src/com/google/gwt/dev/Permutation.java
+++ b/dev/core/src/com/google/gwt/dev/Permutation.java
@@ -28,13 +28,19 @@
* Represents the state of a single permutation for compile.
*/
public final class Permutation implements Serializable {
+ private final int id;
private final List<StaticPropertyOracle> propertyOracles = new ArrayList<StaticPropertyOracle>();
private final SortedMap<String, String> rebindAnswers = new TreeMap<String, String>();
- public Permutation(StaticPropertyOracle propertyOracle) {
+ public Permutation(int id, StaticPropertyOracle propertyOracle) {
+ this.id = id;
this.propertyOracles.add(propertyOracle);
}
+ public int getId() {
+ return id;
+ }
+
public StaticPropertyOracle[] getPropertyOracles() {
return propertyOracles.toArray(new StaticPropertyOracle[propertyOracles.size()]);
}
diff --git a/dev/core/src/com/google/gwt/dev/PermutationResult.java b/dev/core/src/com/google/gwt/dev/PermutationResult.java
new file mode 100644
index 0000000..d0133ea
--- /dev/null
+++ b/dev/core/src/com/google/gwt/dev/PermutationResult.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2008 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.dev;
+
+import java.io.Serializable;
+
+/**
+ * An extensible return type for the results of compiling a single permutation.
+ */
+public interface PermutationResult extends Serializable {
+ /**
+ * The compiled JavaScript code.
+ */
+ String getJs();
+}
diff --git a/dev/core/src/com/google/gwt/dev/PermutationWorker.java b/dev/core/src/com/google/gwt/dev/PermutationWorker.java
new file mode 100644
index 0000000..f24d35f
--- /dev/null
+++ b/dev/core/src/com/google/gwt/dev/PermutationWorker.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2008 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.dev;
+
+import com.google.gwt.core.ext.TreeLogger;
+import com.google.gwt.core.ext.UnableToCompleteException;
+
+/**
+ * Represents a facility that can compile an individual {@link Permutation}.
+ * Instances of PermutationWorker are expected to be created via
+ * {@link PermutationWorkerFactory#getWorkers(int)}.
+ */
+interface PermutationWorker {
+
+ /**
+ * Compile a single permutation. The {@link com.google.gwt.dev.jjs.UnifiedAst}
+ * will have been provided to {@link PermutationWorkerFactory#getWorkers}
+ * method.
+ *
+ * @throws TransientWorkerException if the Permutation should be tried again
+ * on another worker
+ * @throws UnableToCompleteException due to a fatal error
+ */
+ PermutationResult compile(TreeLogger logger, Permutation permutation)
+ throws TransientWorkerException, UnableToCompleteException;
+
+ /**
+ * Returns a human-readable description of the worker instance. This may be
+ * used for error reporting.
+ */
+ String getName();
+
+ /**
+ * Release any resources associated with the worker.
+ */
+ void shutdown();
+}
diff --git a/dev/core/src/com/google/gwt/dev/PermutationWorkerFactory.java b/dev/core/src/com/google/gwt/dev/PermutationWorkerFactory.java
new file mode 100644
index 0000000..4448c3d
--- /dev/null
+++ b/dev/core/src/com/google/gwt/dev/PermutationWorkerFactory.java
@@ -0,0 +1,396 @@
+/*
+ * Copyright 2008 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.dev;
+
+import com.google.gwt.core.ext.TreeLogger;
+import com.google.gwt.core.ext.UnableToCompleteException;
+import com.google.gwt.dev.jjs.InternalCompilerException;
+import com.google.gwt.dev.jjs.UnifiedAst;
+import com.google.gwt.dev.util.Util;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ * Represents a factory for implementations of an endpoint that will invoke
+ * CompilePerms. Implementations of PermutationWorkerFactory should be
+ * default-instantiable and will have {@link #init} called immediately after
+ * construction.
+ */
+public abstract class PermutationWorkerFactory {
+
+ /**
+ * This callable is responsible for compiling one permutation. It waits for an
+ * available worker, uses it, and returns it to the pool when finished.
+ */
+ private static class CompileOnePermutation implements Callable<ResultStatus> {
+ private final BlockingQueue<PermutationWorker> availableWorkers;
+ private final TreeLogger logger;
+ private final Permutation permutation;
+ private final File resultFile;
+
+ public CompileOnePermutation(TreeLogger logger, Permutation permutation,
+ File resultFile, BlockingQueue<PermutationWorker> availableWorkers) {
+ this.logger = logger;
+ this.permutation = permutation;
+ this.resultFile = resultFile;
+ this.availableWorkers = availableWorkers;
+ }
+
+ public ResultStatus call() {
+ // Find a free worker
+ PermutationWorker worker;
+ try {
+ worker = availableWorkers.take();
+ } catch (InterruptedException e) {
+ logger.log(TreeLogger.DEBUG, "Worker interrupted", e);
+ return ResultStatus.HARD_FAILURE;
+ }
+
+ if (worker == noMoreWorkersWorker) {
+ // Shutting down
+ return ResultStatus.HARD_FAILURE;
+ }
+
+ // Invoke the worker
+ try {
+ PermutationResult result = worker.compile(logger, permutation);
+ Util.writeObjectAsFile(logger, resultFile, result);
+ logger.log(TreeLogger.DEBUG, "Successfully compiled permutation");
+ availableWorkers.add(worker);
+ return ResultStatus.SUCCESS;
+ } catch (TransientWorkerException e) {
+ logger.log(TreeLogger.DEBUG, "Worker died, will retry Permutation", e);
+ return ResultStatus.TRANSIENT_FAILURE;
+ } catch (UnableToCompleteException e) {
+ logger.log(TreeLogger.ERROR, "Unrecoverable exception, shutting down",
+ e);
+ return ResultStatus.HARD_FAILURE;
+ }
+ }
+ }
+
+ private static enum ResultStatus {
+ /**
+ * A failure bad enough to merit shutting down the compilation.
+ */
+ HARD_FAILURE,
+
+ /**
+ * A successful compile.
+ */
+ SUCCESS,
+
+ /**
+ * A worker died while processing this permutation, but it is worth trying
+ * to compile it with another worker.
+ */
+ TRANSIENT_FAILURE
+ };
+
+ /**
+ * The name of the system property used to define the workers.
+ */
+ public static final String FACTORY_IMPL_PROPERTY = "gwt.jjs.permutationWorkerFactory";
+
+ /**
+ * This value can be passed into {@link #setLocalWorkers(int)} to indicate
+ * that a heuristic should be used to determine the total number of local
+ * workers.
+ */
+ public static final int WORKERS_AUTO = 0;
+
+ private static List<PermutationWorkerFactory> lazyFactories;
+
+ private static final PermutationWorker noMoreWorkersWorker = new PermutationWorker() {
+
+ public PermutationResult compile(TreeLogger logger, Permutation permutation)
+ throws TransientWorkerException, UnableToCompleteException {
+ throw new UnableToCompleteException();
+ }
+
+ public String getName() {
+ return "Marker worker indicating no more workers";
+ }
+
+ public void shutdown() {
+ }
+ };
+
+ /**
+ * Compiles all Permutations in a Precompilation and returns an array of Files
+ * that can be consumed by Link using the system-default
+ * PermutationWorkersFactories.
+ */
+ public static void compilePermutations(TreeLogger logger,
+ Precompilation precompilation, int localWorkers, File[] resultFiles)
+ throws UnableToCompleteException {
+ compilePermutations(logger, precompilation,
+ precompilation.getPermutations(), localWorkers, resultFiles);
+ }
+
+ /**
+ * Compiles a subset of the Permutations in a Precompilation and returns an
+ * array of Files that can be consumed by Link using the system-default
+ * PermutationWorkersFactories.
+ *
+ * @param localWorkers Set the maximum number of workers that should be
+ * executed on the local system by the PermutationWorkerFactory. The
+ * value {@link #WORKERS_AUTO} will allow the
+ * PermutationWorkerFactory to apply a heuristic to determine the
+ * correct number of local workers.
+ * @param resultFiles the output files to write into; must be the same length
+ * as permutations
+ */
+ public static void compilePermutations(TreeLogger logger,
+ Precompilation precompilation, Permutation[] permutations,
+ int localWorkers, File[] resultFiles) throws UnableToCompleteException {
+ assert permutations.length == resultFiles.length;
+ assert Arrays.asList(precompilation.getPermutations()).containsAll(
+ Arrays.asList(permutations));
+
+ // We may have a mixed collection of workers from different factories
+ List<PermutationWorker> workers = new ArrayList<PermutationWorker>();
+
+ /*
+ * We can have errors below this point, there's a finally block to handle
+ * cleanup of workers.
+ */
+ try {
+ createWorkers(logger, precompilation.getUnifiedAst(),
+ permutations.length, localWorkers, workers);
+ ExecutorService executor = Executors.newFixedThreadPool(workers.size());
+
+ // List of available workers.
+ // The extra space is for inserting nulls at shutdown time
+ BlockingQueue<PermutationWorker> availableWorkers = new ArrayBlockingQueue<PermutationWorker>(
+ 2 * workers.size());
+ availableWorkers.addAll(workers);
+
+ try {
+
+ // Submit all tasks to the executor
+
+ // The permutation compiles not yet finished
+ Queue<CompileOnePermutation> tasksOutstanding = new LinkedList<CompileOnePermutation>();
+
+ // The futures for the results of those compiles
+ Queue<Future<ResultStatus>> resultFutures = new LinkedList<Future<ResultStatus>>();
+
+ for (int i = 0; i < permutations.length; ++i) {
+ TreeLogger permLogger = logger.branch(TreeLogger.DEBUG,
+ "Worker permutation " + permutations[i].getId() + " of "
+ + permutations.length);
+ CompileOnePermutation task = new CompileOnePermutation(permLogger,
+ permutations[i], resultFiles[i], availableWorkers);
+ tasksOutstanding.add(task);
+ resultFutures.add(executor.submit(task));
+ }
+
+ // Count the number of dead workers
+ int numDeadWorkers = 0;
+ int successCount = 0;
+
+ while (!resultFutures.isEmpty() && numDeadWorkers < workers.size()) {
+ assert resultFutures.size() == tasksOutstanding.size();
+
+ CompileOnePermutation task = tasksOutstanding.remove();
+ Future<ResultStatus> future = resultFutures.remove();
+ ResultStatus result;
+ try {
+ result = future.get();
+ } catch (InterruptedException e) {
+ logger.log(TreeLogger.ERROR,
+ "Exiting without results due to interruption", e);
+ throw new UnableToCompleteException();
+ } catch (ExecutionException e) {
+ logger.log(TreeLogger.ERROR, "A compilation failed", e);
+ throw new UnableToCompleteException();
+ }
+
+ if (result == ResultStatus.SUCCESS) {
+ ++successCount;
+ } else if (result == ResultStatus.TRANSIENT_FAILURE) {
+ // A worker died. Resubmit for the remaining workers.
+ ++numDeadWorkers;
+ tasksOutstanding.add(task);
+ resultFutures.add(executor.submit(task));
+ } else if (result == ResultStatus.HARD_FAILURE) {
+ // Shut down.
+ break;
+ } else {
+ throw new InternalCompilerException("Unknown result type");
+ }
+ }
+
+ // Too many permutations is a coding error
+ assert successCount <= permutations.length;
+
+ if (successCount < permutations.length) {
+ // Likely as not, all of the workers died
+ logger.log(TreeLogger.ERROR, "Not all permutation were compiled "
+ + successCount + " of " + permutations.length);
+ throw new UnableToCompleteException();
+ }
+ } finally {
+ // Shut down the executor
+ executor.shutdown();
+
+ // Inform any residual CompileOnePermutation's that there aren't any
+ // more workers
+ for (int i = 0; i < workers.size(); i++) {
+ availableWorkers.add(noMoreWorkersWorker);
+ }
+ }
+ } finally {
+ // Shut down all workers
+ for (PermutationWorker worker : workers) {
+ worker.shutdown();
+ }
+ }
+ }
+
+ /**
+ * Creates one or more implementations of worker factories. This will treat
+ * the value of the {@value #FACTORY_IMPL_PROPERTY} system property as a
+ * comma-separated list of type names.
+ */
+ private static synchronized List<PermutationWorkerFactory> createAll(
+ TreeLogger logger) throws UnableToCompleteException {
+ // NB: This is the much-derided FactoryFactory pattern
+
+ logger = logger.branch(TreeLogger.TRACE,
+ "Creating PermutationWorkerFactory instances");
+
+ if (lazyFactories != null) {
+ logger.log(TreeLogger.SPAM, "Using lazy instances");
+ return lazyFactories;
+ }
+
+ List<PermutationWorkerFactory> mutableFactories = new ArrayList<PermutationWorkerFactory>();
+ String classes = System.getProperty(FACTORY_IMPL_PROPERTY,
+ ThreadedPermutationWorkerFactory.class.getName() + ","
+ + ExternalPermutationWorkerFactory.class.getName());
+ logger.log(TreeLogger.SPAM, "Factory impl property is " + classes);
+
+ String[] classParts = classes.split(",");
+ for (String className : classParts) {
+ try {
+ Class<? extends PermutationWorkerFactory> clazz = Class.forName(
+ className).asSubclass(PermutationWorkerFactory.class);
+ PermutationWorkerFactory factory = clazz.newInstance();
+ factory.init(logger);
+ mutableFactories.add(factory);
+ logger.log(TreeLogger.SPAM, "Added PermutationWorkerFactory "
+ + clazz.getName());
+ } catch (ClassCastException e) {
+ logger.log(TreeLogger.ERROR, className + " is not a "
+ + PermutationWorkerFactory.class.getName());
+ } catch (ClassNotFoundException e) {
+ logger.log(TreeLogger.ERROR,
+ "Unable to find PermutationWorkerFactory named " + className);
+ } catch (InstantiationException e) {
+ logger.log(TreeLogger.ERROR,
+ "Unable to instantiate PermutationWorkerFactory " + className, e);
+ } catch (IllegalAccessException e) {
+ logger.log(TreeLogger.ERROR,
+ "Unable to instantiate PermutationWorkerFactory " + className, e);
+ }
+ }
+
+ if (mutableFactories.size() == 0) {
+ logger.log(TreeLogger.ERROR,
+ "No usable PermutationWorkerFactories available");
+ throw new UnableToCompleteException();
+ }
+
+ return lazyFactories = Collections.unmodifiableList(mutableFactories);
+ }
+
+ /**
+ * Create as many workers as possible to service the Permutations.
+ */
+ private static void createWorkers(TreeLogger logger, UnifiedAst unifiedAst,
+ int workersNeeded, int localWorkers, List<PermutationWorker> workers)
+ throws UnableToCompleteException {
+ if (localWorkers <= WORKERS_AUTO) {
+ // TODO: something smarter?
+ localWorkers = 1;
+ }
+
+ for (PermutationWorkerFactory factory : PermutationWorkerFactory.createAll(logger)) {
+ if (workersNeeded <= 0) {
+ break;
+ }
+
+ int wanted = factory.isLocal() ? Math.min(workersNeeded, localWorkers)
+ : workersNeeded;
+ if (wanted <= 0) {
+ continue;
+ }
+
+ Collection<PermutationWorker> newWorkers = factory.getWorkers(logger,
+ unifiedAst, wanted);
+
+ workers.addAll(newWorkers);
+ workersNeeded -= newWorkers.size();
+ if (factory.isLocal()) {
+ localWorkers -= newWorkers.size();
+ }
+ }
+
+ if (workers.size() == 0) {
+ logger.log(TreeLogger.ERROR, "No PermutationWorkers created");
+ throw new UnableToCompleteException();
+ }
+ }
+
+ /**
+ * Return some number of PermutationWorkers.
+ *
+ * @param unifiedAst a UnifiedAst
+ * @param numWorkers the desired number of workers
+ * @return a collection of PermutationWorkers, the size of which may be less
+ * than <code>numWorkers</code>
+ */
+ public abstract Collection<PermutationWorker> getWorkers(TreeLogger logger,
+ UnifiedAst unifiedAst, int numWorkers) throws UnableToCompleteException;
+
+ /**
+ * Initialize the PermutationWorkerFactory.
+ */
+ public abstract void init(TreeLogger logger) throws UnableToCompleteException;
+
+ /**
+ * Indicates if the PermutationWorkers created by the factory consume
+ * computational or memory resources on the local system, as opposed to the
+ * per-permutation work being performed on a remote system.
+ */
+ public abstract boolean isLocal();
+}
diff --git a/dev/core/src/com/google/gwt/dev/Precompile.java b/dev/core/src/com/google/gwt/dev/Precompile.java
index a5aa4ed..bb96be3 100644
--- a/dev/core/src/com/google/gwt/dev/Precompile.java
+++ b/dev/core/src/com/google/gwt/dev/Precompile.java
@@ -170,7 +170,7 @@
rebindOracles[i] = new StandardRebindOracle(compilationState,
propertyOracles[i], module, rules, genDir, generatorResourcesDir,
generatorArtifacts);
- permutations[i] = new Permutation(propertyOracles[i]);
+ permutations[i] = new Permutation(i, propertyOracles[i]);
}
}
diff --git a/dev/core/src/com/google/gwt/dev/ThreadedPermutationWorkerFactory.java b/dev/core/src/com/google/gwt/dev/ThreadedPermutationWorkerFactory.java
new file mode 100644
index 0000000..89eebf0
--- /dev/null
+++ b/dev/core/src/com/google/gwt/dev/ThreadedPermutationWorkerFactory.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2008 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.dev;
+
+import com.google.gwt.core.ext.TreeLogger;
+import com.google.gwt.core.ext.UnableToCompleteException;
+import com.google.gwt.dev.jjs.UnifiedAst;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Create a single in-process PermutationWorker. This WorkerFactory is intended
+ * to be used as a fall-back in case the other PermutationWorkers are unable to
+ * complete.
+ */
+public class ThreadedPermutationWorkerFactory extends PermutationWorkerFactory {
+
+ private static class ThreadedPermutationWorker implements PermutationWorker {
+ private final UnifiedAst ast;
+ private final int id;
+
+ public ThreadedPermutationWorker(UnifiedAst ast, int id) {
+ this.ast = ast;
+ this.id = id;
+ }
+
+ public PermutationResult compile(final TreeLogger logger,
+ final Permutation permutation) throws TransientWorkerException,
+ UnableToCompleteException {
+ try {
+ return CompilePerms.compile(logger, permutation, ast);
+ } catch (OutOfMemoryError e) {
+ logger.log(TreeLogger.ERROR,
+ "OutOfMemoryError: Increase heap size or lower "
+ + MAX_THREADS_PROPERTY);
+ throw new UnableToCompleteException();
+ }
+ }
+
+ public String getName() {
+ return "In-process PermutationWorker " + id;
+ }
+
+ public void shutdown() {
+ // No-op
+ }
+ }
+
+ /**
+ * A Java system property that can be used to change the number of in-process
+ * threads used.
+ */
+ public static final String MAX_THREADS_PROPERTY = "gwt.jjs.maxThreads";
+
+ @Override
+ public Collection<PermutationWorker> getWorkers(TreeLogger logger,
+ UnifiedAst unifiedAst, int numWorkers) throws UnableToCompleteException {
+ logger = logger.branch(TreeLogger.SPAM,
+ "Creating ThreadedPermutationWorkers");
+
+ numWorkers = Math.min(numWorkers, Integer.getInteger(MAX_THREADS_PROPERTY,
+ 1));
+
+ if (numWorkers == 0) {
+ return Collections.emptyList();
+ }
+
+ // The worker will deserialize a new copy
+ List<PermutationWorker> toReturn = new ArrayList<PermutationWorker>(
+ numWorkers);
+ for (int i = 0; i < numWorkers; i++) {
+ toReturn.add(new ThreadedPermutationWorker(unifiedAst, i));
+ }
+ return toReturn;
+ }
+
+ @Override
+ public void init(TreeLogger logger) throws UnableToCompleteException {
+ logger = logger.branch(TreeLogger.SPAM,
+ "Initializing ThreadedPermutationWorkerFactory");
+ }
+
+ @Override
+ public boolean isLocal() {
+ return true;
+ }
+}
diff --git a/dev/core/src/com/google/gwt/dev/TransientWorkerException.java b/dev/core/src/com/google/gwt/dev/TransientWorkerException.java
new file mode 100644
index 0000000..5f4e2bc
--- /dev/null
+++ b/dev/core/src/com/google/gwt/dev/TransientWorkerException.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2008 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.dev;
+
+/**
+ * Indicates that a PermutationWorker failed to complete its assignment, but
+ * that the permutation should be retried on another worker. This would be used
+ * to report errors such as OutOfMemory when compiling in-process or unexpected
+ * death of an out-of-process worker (network break, etc).
+ */
+public class TransientWorkerException extends Exception {
+ public TransientWorkerException(String msg, Throwable cause) {
+ super(msg, cause);
+ }
+}
diff --git a/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java b/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java
index 6dfc5fe..d0d2e6d 100644
--- a/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java
+++ b/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java
@@ -143,6 +143,18 @@
return rebindRequests;
}
+ /**
+ * Internally prepares a new AST for compilation if one is not already
+ * prepared.
+ */
+ public void prepare() {
+ synchronized (myLockObject) {
+ if (initialAst == null) {
+ initialAst = deserializeAst(serializedAst);
+ }
+ }
+ }
+
AST getFreshAst() {
synchronized (myLockObject) {
if (initialAst != null) {
diff --git a/dev/core/src/com/google/gwt/dev/util/arg/ArgHandlerLocalWorkers.java b/dev/core/src/com/google/gwt/dev/util/arg/ArgHandlerLocalWorkers.java
new file mode 100644
index 0000000..6800c83
--- /dev/null
+++ b/dev/core/src/com/google/gwt/dev/util/arg/ArgHandlerLocalWorkers.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2008 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.dev.util.arg;
+
+import com.google.gwt.dev.OptionLocalWorkers;
+import com.google.gwt.util.tools.ArgHandlerInt;
+
+/**
+ * An arg handler to specify the number of local workers that should be used by
+ * the compiler.
+ */
+public class ArgHandlerLocalWorkers extends ArgHandlerInt {
+
+ private final OptionLocalWorkers options;
+
+ public ArgHandlerLocalWorkers(OptionLocalWorkers options) {
+ this.options = options;
+ }
+
+ @Override
+ public String[] getDefaultArgs() {
+ // Default to 1 for now; we might do an "auto" later.
+ return new String[] {getTag(), "1"};
+ }
+
+ @Override
+ public String getPurpose() {
+ return "Specifies the number of local workers to use whe compiling permutations";
+ }
+
+ @Override
+ public String getTag() {
+ return "-localWorkers";
+ }
+
+ @Override
+ public String[] getTagArgs() {
+ return new String[] {"count"};
+ }
+
+ @Override
+ public void setInt(int value) {
+ options.setLocalWorkers(value);
+ }
+}
