Added compile server backend.

Change-Id: I4b675f7e788363f4e2bfddcd72e49329c844b639
diff --git a/.gitignore b/.gitignore
index 911fa5c..1554db4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,12 @@
 benchmarks/.project
 benchmarks/.settings/
 benchmarks/target/
+common/.classpath
+common/.project
+common/.settings/
+compileserver/.classpath
+compileserver/.project
+compileserver/.settings/
+common/target/
+compileserver/target/
 
diff --git a/benchmarks/src/main/scripts/buildSDK b/benchmarks/src/main/scripts/buildSDK
new file mode 100755
index 0000000..22080ee
--- /dev/null
+++ b/benchmarks/src/main/scripts/buildSDK
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+
+if [[ -z "${1}" ]]; then
+  echo "usage: buildSDK sourceLocation" >&2
+  exit 1
+fi
+
+SOURCE_LOCATION=${1}
+
+cd ${SOURCE_LOCATION}
+ant dist-dev
\ No newline at end of file
diff --git a/benchmarks/src/main/scripts/checkout b/benchmarks/src/main/scripts/checkout
new file mode 100755
index 0000000..b166823
--- /dev/null
+++ b/benchmarks/src/main/scripts/checkout
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+if [[ -z "${1}" || -z "${2}" ]]; then
+  echo "usage: checkout commitId sourceLocation" >&2
+  exit 1
+fi
+
+COMMIT_ID=${1}
+SOURCE_LOCATION=${2}
+
+cd ${SOURCE_LOCATION}
+git checkout ${COMMIT_ID}
\ No newline at end of file
diff --git a/benchmarks/src/main/scripts/commitDate b/benchmarks/src/main/scripts/commitDate
new file mode 100755
index 0000000..08d52f7
--- /dev/null
+++ b/benchmarks/src/main/scripts/commitDate
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+if [[ -z "${1}" || -z "${2}" ]]; then
+  echo "usage: commitDate sourceLocation commitId" >&2
+  exit 1
+fi
+
+
+SOURCE_LOCATION=${1}
+COMMIT_ID=${2}
+
+cd ${SOURCE_LOCATION}
+git show -s --format=%ci ${COMMIT_ID}
\ No newline at end of file
diff --git a/benchmarks/src/main/scripts/commitId b/benchmarks/src/main/scripts/commitId
new file mode 100755
index 0000000..a4591b0
--- /dev/null
+++ b/benchmarks/src/main/scripts/commitId
@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+if [[ -z "${1}" ]]; then
+  echo "usage: commitId sourceLocation" >&2
+  exit 1
+fi
+
+SOURCE_LOCATION=${1}
+
+cd ${SOURCE_LOCATION}
+git rev-parse HEAD
\ No newline at end of file
diff --git a/benchmarks/src/main/scripts/compileModule b/benchmarks/src/main/scripts/compileModule
new file mode 100755
index 0000000..d89fab7
--- /dev/null
+++ b/benchmarks/src/main/scripts/compileModule
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+if [[ -z "${1}" || -z "${2}" || -z "${3}" || -z "${4}" || -z "${5}" ]]; then
+  # Not sending error to &2 since GWT compiler does not do that either
+  echo "usage: compileModule moduleName dev.jar user.jar benchmark_src output_dir"
+  exit 1
+fi
+
+
+MODULE_NAME=${1}
+GWT_DEV_JAR=${2}
+GWT_USER_JAR=${3}
+BENCHMARKS_SRC=${4}
+OUTPUT_DIR=${5}
+
+java -cp ${GWT_DEV_JAR}:${GWT_USER_JAR}:${BENCHMARKS_SRC} com.google.gwt.dev.Compiler -style PRETTY -war ${OUTPUT_DIR} "${MODULE_NAME}"
\ No newline at end of file
diff --git a/benchmarks/src/main/scripts/maybe_checkout_next_commit b/benchmarks/src/main/scripts/maybe_checkout_next_commit
new file mode 100755
index 0000000..29f3d44
--- /dev/null
+++ b/benchmarks/src/main/scripts/maybe_checkout_next_commit
@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e
+
+if [[ -z "${1}" || -z "${2}" ]]; then
+  echo "usage: maybe_checkout_next_commit parentCommitId sourceLocation" >&2
+  exit 1
+fi
+
+PARENT_COMMIT=${1}
+SOURCE_LOCATION=${2}
+
+cd ${SOURCE_LOCATION}
+git fetch origin master
+git checkout origin/master
+
+NEXT_COMMIT=$(git log --ancestry-path --format=%H ${PARENT_COMMIT}..origin/master | tail -1)
+
+if [[ -z "${NEXT_COMMIT}" ]]; then
+  exit 0
+fi
+
+git checkout ${NEXT_COMMIT}
\ No newline at end of file
diff --git a/common/pom.xml b/common/pom.xml
new file mode 100644
index 0000000..b409233
--- /dev/null
+++ b/common/pom.xml
@@ -0,0 +1,106 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.google.gwt.benchmark</groupId>
+  <artifactId>gwt-benchmark-common</artifactId>
+  <version>1.0-SNAPSHOT</version>
+  <packaging>jar</packaging>
+
+  <parent>
+    <groupId>com.google.gwt.benchmark</groupId>
+    <artifactId>gwt-benchmark-parent</artifactId>
+    <version>1.0-SNAPSHOT</version>
+  </parent>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <gwtversion>2.6.0</gwtversion>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.google.gwt</groupId>
+      <artifactId>gwt-user</artifactId>
+      <version>${gwtversion}</version>
+    </dependency>
+    <dependency>
+      <groupId>javax.validation</groupId>
+      <artifactId>validation-api</artifactId>
+      <version>1.0.0.GA</version>
+    </dependency>
+    <dependency>
+      <groupId>javax.validation</groupId>
+      <artifactId>validation-api</artifactId>
+      <version>1.0.0.GA</version>
+      <classifier>sources</classifier>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.11</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <version>2.3</version>
+        <executions>
+          <execution>
+            <phase>package</phase>
+            <goals>
+              <goal>jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <version>2.5.1</version>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <source>1.7</source>
+          <target>1.7</target>
+        </configuration>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-eclipse-plugin</artifactId>
+        <version>2.8</version>
+
+        <configuration>
+          <downloadSources>true</downloadSources>
+          <downloadJavadocs>false</downloadJavadocs>
+          <buildOutputDirectory>${project.build.directory}/${project.build.finalName}/WEB-INF/classes</buildOutputDirectory>
+          <projectnatures>
+            <projectnature>org.eclipse.jdt.core.javanature</projectnature>
+            <projectnature>com.google.gdt.eclipse.core.webAppNature</projectnature>
+            <nature>com.google.gwt.eclipse.core.gwtNature</nature>
+          </projectnatures>
+          <buildcommands>
+            <buildcommand>org.eclipse.jdt.core.javabuilder</buildcommand>
+            <buildcommand>com.google.gdt.eclipse.core.webAppProjectValidator</buildcommand>
+            <buildcommand>com.google.gwt.eclipse.core.gwtProjectValidator</buildcommand>
+          </buildcommands>
+          <classpathContainers>
+            <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
+            <classpathContainer>com.google.gwt.eclipse.core.GWT_CONTAINER</classpathContainer>
+          </classpathContainers>
+          <excludes>
+            <exclude>com.google.gwt:gwt-servlet</exclude>
+            <exclude>com.google.gwt:gwt-user</exclude>
+            <exclude>com.google.gwt:gwt-dev</exclude>
+            <exclude>javax.validation:validation-api</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
+
+    </plugins>
+  </build>
+</project>
diff --git a/common/src/main/java/com/google/gwt/benchmark/common/Common.gwt.xml b/common/src/main/java/com/google/gwt/benchmark/common/Common.gwt.xml
new file mode 100644
index 0000000..939aa30
--- /dev/null
+++ b/common/src/main/java/com/google/gwt/benchmark/common/Common.gwt.xml
@@ -0,0 +1,20 @@
+<!--                                                                        -->
+<!-- Copyright 2014 Google Inc.                                             -->
+<!-- Licensed under the Apache License, Version 2.0 (the "License"); you    -->
+<!-- may not use this file except in compliance with the License. You may   -->
+<!-- may obtain a copy of the License at                                    -->
+<!--                                                                        -->
+<!-- http://www.apache.org/licenses/LICENSE-2.0                             -->
+<!--                                                                        -->
+<!-- Unless required by applicable law or agreed to in writing, software    -->
+<!-- distributed under the License is distributed on an "AS IS" BASIS,      -->
+<!-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or        -->
+<!-- implied. License for the specific language governing permissions and   -->
+<!-- limitations under the License.                                         -->
+
+<module>
+  <inherits name="com.google.gwt.core.Core" />
+  <inherits name="com.google.gwt.autobeans.AutoBeans"/>
+  <source path="client" />
+  <source path="shared" />
+</module>
\ No newline at end of file
diff --git a/common/src/main/java/com/google/gwt/benchmark/common/shared/json/BenchmarkResultJson.java b/common/src/main/java/com/google/gwt/benchmark/common/shared/json/BenchmarkResultJson.java
new file mode 100644
index 0000000..b595760
--- /dev/null
+++ b/common/src/main/java/com/google/gwt/benchmark/common/shared/json/BenchmarkResultJson.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.common.shared.json;
+
+
+/**
+ * A JSON representation of a benchmark result.
+ */
+public interface BenchmarkResultJson {
+
+  public String getBenchmarkName();
+
+  public void setBenchmarkName(String benchmarkName);
+
+  public void setRunnerId(String runnerId);
+
+  public String getRunnerId();
+
+  public double getRunsPerMinute();
+
+  public void setRunsPerMinute(double runsPerMinute);
+}
diff --git a/common/src/main/java/com/google/gwt/benchmark/common/shared/json/BenchmarkRunJson.java b/common/src/main/java/com/google/gwt/benchmark/common/shared/json/BenchmarkRunJson.java
new file mode 100644
index 0000000..e02cea2
--- /dev/null
+++ b/common/src/main/java/com/google/gwt/benchmark/common/shared/json/BenchmarkRunJson.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.common.shared.json;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A JSON representation of a set of benchmark runs.
+ */
+public interface BenchmarkRunJson {
+
+  void setResultByBenchmarkName(Map<String, List<BenchmarkResultJson>> results);
+
+  void setCommitId(String commitId);
+
+  void setCommitTime(String commitTime);
+
+  String getCommitTime();
+
+  String getCommitId();
+
+  Map<String, List<BenchmarkResultJson>> getResultByBenchmarkName();
+}
diff --git a/common/src/main/java/com/google/gwt/benchmark/common/shared/json/JsonFactory.java b/common/src/main/java/com/google/gwt/benchmark/common/shared/json/JsonFactory.java
new file mode 100644
index 0000000..8162731
--- /dev/null
+++ b/common/src/main/java/com/google/gwt/benchmark/common/shared/json/JsonFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.common.shared.json;
+
+import com.google.gwt.core.shared.GWT;
+import com.google.web.bindery.autobean.shared.AutoBean;
+import com.google.web.bindery.autobean.shared.AutoBeanFactory;
+import com.google.web.bindery.autobean.vm.AutoBeanFactorySource;
+
+/**
+ * Factory to create JSON representations.
+ */
+public final class JsonFactory {
+
+  public interface Factory extends AutoBeanFactory {
+    AutoBean<BenchmarkResultJson> result();
+
+    AutoBean<BenchmarkRunJson> run();
+  }
+
+  private static final Factory INSTANCE = (GWT.isClient() ? (Factory) GWT.create(Factory.class)
+      : AutoBeanFactorySource.create(Factory.class));
+
+  public static Factory get() {
+    return INSTANCE;
+  }
+
+  public JsonFactory() {
+  }
+}
diff --git a/compileserver/config/config.example b/compileserver/config/config.example
new file mode 100644
index 0000000..273a527
--- /dev/null
+++ b/compileserver/config/config.example
@@ -0,0 +1,56 @@
+## The compile server uses a config file to read in all of its settings.
+## The file is passed to the container running the compile server using a
+## environment variable -DconfigFile=<absolute path>.
+##
+
+## Location settings
+
+# Location of the benchmarks (benchmarks/ in the checkout)
+benchmarksDirectory = /usr/local/temp/gwt-benchmarks/benchmarks/
+
+# The output folder for the GWT compiler, this folder is also served up by jetty
+compileOutputDir = /usr/local/temp/gwt-benchmarks/compileserver/target/benchmark-js/
+
+# The directory where the compile server can store information (last commit)
+persistenceDir = /usr/local/temp/persistenceDir/
+
+# The location of the GWT SDK source
+gwtSourceLocation = /usr/local/temp/gwt/
+
+## Runners
+# Url of the selenium hub
+seleniumHubUrl = http://myhost:4444/wd/hub
+
+# How many GWT compiles / benchmark execution should happen in parallel
+threadPoolSize = 5
+
+# Mode of the benchmarking system
+# Currently the system only supports server mode. In server mode the system will
+# track a repository and recompile and run the benchmarks for every change of 
+# the tracked repository.
+mode = server
+
+## Report settings
+# should we report results
+reportResuts = false
+
+# Url of the benchmark dashboard
+reporterUrl = https://gwt-bench.appspot.com/post_result
+
+# The secret that protects the dashboard
+reporterSecret = <mysecret>
+
+
+## Mail settings
+mail.to = peopletonotify@mydomain.com
+mail.from = mygmail@gmail.com
+mail.host = smtp.gmail.com
+mail.username = mygmail@gmail.com
+mail.password = my password
+
+## Misc
+# the html file used to serve up benchmarks
+moduleTemplate = runner_template.html
+
+# The port on which the benchmarking system is running
+servletContainerPort = 8888
\ No newline at end of file
diff --git a/compileserver/config/runner_template.html b/compileserver/config/runner_template.html
new file mode 100644
index 0000000..1f0fdba
--- /dev/null
+++ b/compileserver/config/runner_template.html
@@ -0,0 +1,10 @@
+<!doctype html>
+
+<html>
+  <head>
+    <meta http-equiv="content-type" content="text/html; charset=UTF-8">
+    <script type="text/javascript" src="{module_nocache}"></script>
+  </head>
+  <body>
+  </body>
+</html>
diff --git a/compileserver/pom.xml b/compileserver/pom.xml
new file mode 100644
index 0000000..aa1c2ba
--- /dev/null
+++ b/compileserver/pom.xml
@@ -0,0 +1,138 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.google.gwt.benchmark</groupId>
+  <artifactId>gwt-benchmark-compileserver</artifactId>
+  <version>1.0-SNAPSHOT</version>
+  <packaging>war</packaging>
+
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <gwtversion>2.6.0</gwtversion>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.google.gwt.benchmark</groupId>
+      <artifactId>gwt-benchmark-common</artifactId>
+      <version>1.0-SNAPSHOT</version>
+    </dependency>
+    <dependency>
+      <groupId>org.seleniumhq.selenium</groupId>
+      <artifactId>selenium-java</artifactId>
+      <version>2.40.0</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.inject.extensions</groupId>
+      <artifactId>guice-servlet</artifactId>
+      <version>3.0</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.gwt</groupId>
+      <artifactId>gwt-user</artifactId>
+      <version>${gwtversion}</version>
+    </dependency>
+    <dependency>
+      <groupId>javax.validation</groupId>
+      <artifactId>validation-api</artifactId>
+      <version>1.0.0.GA</version>
+    </dependency>
+    <dependency>
+      <groupId>javax.validation</groupId>
+      <artifactId>validation-api</artifactId>
+      <version>1.0.0.GA</version>
+      <classifier>sources</classifier>
+    </dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>4.11</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <version>1.9.5</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.gwt.inject</groupId>
+      <artifactId>gin</artifactId>
+      <version>2.1.2</version>
+    </dependency>
+    <dependency>
+      <groupId>javax.mail</groupId>
+      <artifactId>mail</artifactId>
+      <version>1.4</version>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-webapp</artifactId>
+      <version>9.1.4.v20140401</version>
+    </dependency>
+  </dependencies>
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <version>2.5.1</version>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <source>1.7</source>
+          <target>1.7</target>
+        </configuration>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-war-plugin</artifactId>
+        <version>2.3</version>
+        <configuration>
+          <webResources>
+            <resource>
+              <directory>${basedir}/src/main/webapp/WEB-INF</directory>
+              <filtering>true</filtering>
+              <targetPath>WEB-INF</targetPath>
+            </resource>
+          </webResources>
+        </configuration>
+      </plugin>
+
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-eclipse-plugin</artifactId>
+        <version>2.8</version>
+        <configuration>
+          <downloadSources>true</downloadSources>
+          <downloadJavadocs>false</downloadJavadocs>
+          <buildOutputDirectory>${project.build.directory}/${project.build.finalName}/WEB-INF/classes</buildOutputDirectory>
+          <projectnatures>
+            <projectnature>org.eclipse.jdt.core.javanature</projectnature>
+            <projectnature>com.google.gdt.eclipse.core.webAppNature</projectnature>
+            <nature>com.google.gwt.eclipse.core.gwtNature</nature>
+          </projectnatures>
+          <buildcommands>
+            <buildcommand>org.eclipse.jdt.core.javabuilder</buildcommand>
+            <buildcommand>com.google.gdt.eclipse.core.webAppProjectValidator</buildcommand>
+            <buildcommand>com.google.gwt.eclipse.core.gwtProjectValidator</buildcommand>
+
+          </buildcommands>
+          <classpathContainers>
+            <classpathContainer>org.eclipse.jdt.launching.JRE_CONTAINER</classpathContainer>
+            <classpathContainer>com.google.gwt.eclipse.core.GWT_CONTAINER</classpathContainer>
+          </classpathContainers>
+          <excludes>
+            <exclude>com.google.gwt:gwt-servlet</exclude>
+            <exclude>com.google.gwt:gwt-user</exclude>
+            <exclude>com.google.gwt:gwt-dev</exclude>
+            <exclude>javax.validation:validation-api</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>
+
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/guice/BenchmarkModule.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/guice/BenchmarkModule.java
new file mode 100644
index 0000000..92751ea
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/guice/BenchmarkModule.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.guice;
+
+import com.google.gwt.benchmark.compileserver.server.manager.BenchmarkCompiler;
+import com.google.gwt.benchmark.compileserver.server.manager.BenchmarkReporter;
+import com.google.gwt.benchmark.compileserver.server.manager.BenchmarkWorker;
+import com.google.gwt.benchmark.compileserver.server.manager.CliInteractor;
+import com.google.gwt.benchmark.compileserver.server.manager.MailReporter.MailHelper;
+import com.google.gwt.benchmark.compileserver.server.manager.MailReporter.MailHelperProdImpl;
+import com.google.gwt.benchmark.compileserver.server.manager.Runner;
+import com.google.gwt.benchmark.compileserver.server.manager.WebDriverRunner;
+import com.google.gwt.benchmark.compileserver.server.runners.settings.MailSettings;
+import com.google.gwt.benchmark.compileserver.server.runners.settings.Settings;
+import com.google.inject.AbstractModule;
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+import com.google.inject.assistedinject.FactoryModuleBuilder;
+import com.google.inject.name.Named;
+import com.google.inject.name.Names;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * The compile server guice module.
+ */
+public class BenchmarkModule extends AbstractModule {
+
+  private final Settings settings;
+
+  public BenchmarkModule(Settings settings) {
+    this.settings = settings;
+  }
+
+  @Override
+  protected void configure() {
+    bind(BenchmarkCompiler.class).to(CliInteractor.class);
+    install(new FactoryModuleBuilder().implement(Runner.class, WebDriverRunner.class).build(
+        Runner.Factory.class));
+    install(new FactoryModuleBuilder().build(BenchmarkWorker.Factory.class));
+    install(new FactoryModuleBuilder().build(BenchmarkReporter.Factory.class));
+    bind(BenchmarkReporter.HttpURLConnectionFactory.class).to(HttpUrlConnectionProvider.class);
+    bind(MailHelper.class).to(MailHelperProdImpl.class);
+    bind(String.class).annotatedWith(Names.named("randomStringProvider"))
+        .toProvider(RandomStringProvider.class);
+
+    bind(File.class).annotatedWith(Names.named("compilerOutputDir"))
+        .toInstance(settings.getBenchmarkCompileOutputDir());
+    bind(Integer.class).annotatedWith(Names.named("poolSize"))
+        .toInstance(settings.getThreadPoolSize());
+    bind(Integer.class).annotatedWith(Names.named("port"))
+        .toInstance(settings.getServletContainerPort());
+    bind(File.class).annotatedWith(Names.named("benchmarkSourceLocation"))
+        .toInstance(new File(settings.getBenchmarkRootDirectory(), "src/main/java/"));
+    bind(ExecutorService.class).annotatedWith(Names.named("managerPoolSize"))
+        .toProvider(PoolProvider.class);
+    bind(String.class).annotatedWith(Names.named("ip")).toInstance(settings.getIpAddress());
+    bind(String.class).annotatedWith(Names.named("moduleTemplate"))
+        .toInstance(settings.getModuleTemplate());
+    bind(URL.class).annotatedWith(Names.named("hubUrl")).toInstance(settings.getHubUrl());
+    bind(File.class).annotatedWith(Names.named("scriptDirectory"))
+        .toInstance(settings.getScriptsDirectory());
+    bind(Boolean.class).annotatedWith(Names.named("useReporter"))
+        .toInstance(settings.reportResults());
+    bind(String.class).annotatedWith(Names.named("benchmarkDashboardUrl"))
+        .toInstance(settings.getReporterUrl());
+    bind(File.class).annotatedWith(Names.named("persistenceDir"))
+        .toInstance(settings.getPersistenceDir());
+    bind(File.class).annotatedWith(Names.named("gwtSourceLocation"))
+        .toInstance(settings.getGwtSourceLocation());
+    bind(MailSettings.class).toInstance(settings.getMailSettings());
+
+  }
+
+  private static class PoolProvider implements Provider<ExecutorService> {
+
+    private int poolSize;
+
+    @Inject
+    public PoolProvider(@Named("poolsize") int poolSize) {
+      this.poolSize = poolSize;
+    }
+
+    @Override
+    public ExecutorService get() {
+      return Executors.newFixedThreadPool(poolSize);
+    }
+  }
+
+  private static class HttpUrlConnectionProvider implements
+      BenchmarkReporter.HttpURLConnectionFactory {
+
+    private String url;
+
+    @Inject
+    public HttpUrlConnectionProvider(@Named("benchmarkDashboardUrl") String url) {
+      this.url = url;
+    }
+
+    @Override
+    public HttpURLConnection create() throws IOException {
+      return (HttpURLConnection) new URL(url).openConnection();
+    }
+  }
+
+  private static class RandomStringProvider implements Provider<String> {
+    @Override
+    public String get() {
+      return UUID.randomUUID().toString();
+    }
+  }
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/guice/CompileServerGuiceModule.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/guice/CompileServerGuiceModule.java
new file mode 100644
index 0000000..5da8fbd
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/guice/CompileServerGuiceModule.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.guice;
+
+import com.google.inject.Singleton;
+import com.google.inject.servlet.ServletModule;
+
+import org.eclipse.jetty.servlet.DefaultServlet;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * The guice module for all our servlets.
+ */
+public class CompileServerGuiceModule extends ServletModule {
+
+  private static Map<String, String> createServletParams(String resourceBaseAbsolutePath) {
+    Map<String, String> initParams = new HashMap<String, String>();
+    initParams.put("dirAllowed", "true"); // Allow dir listing
+    initParams.put("pathInfoOnly", "true");
+    initParams.put("resourceBase", resourceBaseAbsolutePath);
+    return initParams;
+  }
+
+  private final File benchmarkOutputDir;
+
+  public CompileServerGuiceModule(File benchmarkOutputDir) {
+    this.benchmarkOutputDir = benchmarkOutputDir;
+  }
+
+  @Override
+  protected void configureServlets() {
+    bind(DefaultServlet.class).in(Singleton.class);
+    serve("/__bench/*").with(DefaultServlet.class,
+        createServletParams(benchmarkOutputDir.getAbsolutePath()));
+  }
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/guice/GuiceServletConfig.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/guice/GuiceServletConfig.java
new file mode 100644
index 0000000..7ca4002
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/guice/GuiceServletConfig.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.guice;
+
+import com.google.gwt.benchmark.compileserver.server.runners.settings.Settings;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
+
+import java.io.File;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Servlet context listener, will be invoked by the servlet container.
+ */
+public class GuiceServletConfig extends GuiceServletContextListener {
+
+  private static final Logger logger = Logger.getLogger(CompileServerGuiceModule.class.getName());
+
+  private static Settings createSettings() {
+    try {
+      String configFile = System.getProperty("configFile");
+      if (configFile == null) {
+        logger.severe("Can not read property file. Start server with -DconfigFile=...");
+        throw new RuntimeException(
+            "Can not read property file. Start server with -DconfigFile=...");
+      }
+      return Settings.parseSettings(new File(configFile));
+    } catch (Exception e) {
+      logger.log(Level.SEVERE, "Can not parse settings from file", e);
+      throw new RuntimeException("Can not parse settings from file", e);
+    }
+  }
+
+  @Override
+  protected Injector getInjector() {
+    Settings settings = createSettings();
+    return Guice.createInjector(
+        new CompileServerGuiceModule(settings.getBenchmarkCompileOutputDir()),
+        new BenchmarkModule(settings));
+  }
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkCompiler.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkCompiler.java
new file mode 100644
index 0000000..e583088
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkCompiler.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import java.io.File;
+
+/**
+ * Implementing classes can compile a given GWT module.
+ */
+public interface BenchmarkCompiler {
+  /**
+   * Invokes the GWT compiler for the specified module.
+   */
+  public void compile(String moduleName, File outputDir) throws BenchmarkCompilerException;
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkCompilerException.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkCompilerException.java
new file mode 100644
index 0000000..c9c199a
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkCompilerException.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+/**
+ * This exception is thrown if a benchmark compile fails for some reason.
+ */
+public class BenchmarkCompilerException extends Exception {
+
+  private String output;
+
+  public BenchmarkCompilerException(String message, Exception e) {
+    super(message, e);
+    output = "";
+  }
+
+  public BenchmarkCompilerException(String message) {
+    super(message);
+    output = "";
+  }
+
+  public String getOutput() {
+    return output;
+  }
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkFinder.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkFinder.java
new file mode 100644
index 0000000..bc68f0c
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkFinder.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import com.google.inject.name.Named;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import javax.inject.Inject;
+
+/**
+ * BenchmarkFinder will traverse a directory structure and find all benchmarks in it.
+ * <p>
+ * Currently benchmarks need to be located under com.google.gwt.benchmark.benchmarks in order to get
+ * picked up by the collector and have the word 'Benchmark' in them, e.g. 'MyCoolBenchmark.gwt.xml'.
+ */
+public class BenchmarkFinder {
+
+  private File benchmarkSourceLocation;
+
+  @Inject
+  public BenchmarkFinder(@Named("benchmarkSourceLocation") File benchmarkSourceLocation) {
+    this.benchmarkSourceLocation = benchmarkSourceLocation;
+  }
+
+  /**
+   * Returns a list of names of available benchmark modules.
+   */
+  public List<String> get() {
+    return traverse(benchmarkSourceLocation);
+  }
+
+  private List<String> traverse(File file) {
+    List<String> moduleNames = new ArrayList<>();
+    traverse(file, file, moduleNames);
+    Collections.sort(moduleNames);
+    return moduleNames;
+  }
+
+  private void traverse(File root, File file, List<String> moduleNames) {
+    if (file.isDirectory()) {
+      File[] listFiles = file.listFiles();
+      for (File newFile : listFiles) {
+        traverse(root, newFile, moduleNames);
+      }
+    } else if (file.isFile()) {
+      if (file.getName().endsWith("gwt.xml") && file.getName().contains("Benchmark")) {
+        String path = file.getAbsolutePath();
+        String relative = root.toURI().relativize(new File(path).toURI()).getPath();
+        String moduleName = relative.replaceAll("\\/", "\\.");
+        moduleName = moduleName.substring(0, moduleName.length() - ".gwt.xml".length());
+
+        if (moduleName.startsWith("com.google.gwt.benchmark.benchmarks.")) {
+          moduleNames.add(moduleName);
+        }
+      }
+    }
+  }
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkManager.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkManager.java
new file mode 100644
index 0000000..e84553c
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkManager.java
@@ -0,0 +1,491 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import com.google.gwt.benchmark.compileserver.server.manager.BenchmarkReporter.Factory;
+import com.google.gwt.benchmark.compileserver.server.manager.BenchmarkReporter.ReportProgressHandler;
+import com.google.gwt.benchmark.compileserver.server.manager.BenchmarkRun.Result;
+import com.google.gwt.benchmark.compileserver.server.manager.BenchmarkWorker.ProgressHandler;
+import com.google.inject.Provider;
+import com.google.inject.Singleton;
+import com.google.inject.name.Named;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+/**
+ * BenchmarkManager is the central place for starting and stopping benchmarks.
+ *
+ * <p>
+ * BenchmarkManager interacts with several other objects to execute benchmarks, pull new changes
+ * into the local repository, build the SDK and report results or errors.
+ */
+@Singleton
+public class BenchmarkManager {
+
+  private class EventLoop implements Runnable {
+    @Override
+    public void run() {
+      runEventLoop();
+    }
+  }
+
+  private class ThreadSafeProgressHandler implements ProgressHandler {
+
+    private BenchmarkRun benchmarkRun;
+
+    public ThreadSafeProgressHandler(BenchmarkRun benchmarkRun) {
+      this.benchmarkRun = benchmarkRun;
+    }
+
+    @Override
+    public void onCompilationFailed(String message) {
+      synchronized (benchmarkRunsByNameLock) {
+        benchmarkRun.setFailedCompile(message);
+      }
+    }
+
+    @Override
+    public void onResult(RunnerConfig config, double result) {
+      synchronized (benchmarkRunsByNameLock) {
+        benchmarkRun.addResult(config, result);
+      }
+    }
+
+    @Override
+    public void failedToRunBenchmark(RunnerConfig config, String errorMessage) {
+      synchronized (benchmarkRunsByNameLock) {
+        benchmarkRun.getResults().get(config).setErrorMessage(errorMessage);
+        benchmarkRun.setFailedToRunOnServer();
+      }
+    }
+
+    @Override
+    public void onHostPageGenerationFailed(String errorMessage) {
+      synchronized (benchmarkRunsByNameLock) {
+        benchmarkRun.setFailedHostPageGenerationFailed(errorMessage);
+      }
+    }
+
+    @Override
+    public void onRunEnded() {
+      synchronized (benchmarkRunsByNameLock) {
+        if (!benchmarkRun.isFailed()) {
+          benchmarkRun.setRunEnded();
+        }
+      }
+      if (workCount.decrementAndGet() == 0) {
+        maybeReportResults(benchmarkRun.getCommitId(), benchmarkRun.getCommitDate());
+      }
+    }
+
+    @Override
+    public void onCompileDirCreationFailed() {
+      synchronized (benchmarkRunsByNameLock) {
+        benchmarkRun.setFailedToCreateDirectory();
+      }
+    }
+  }
+
+  private static final Logger logger = Logger.getLogger(BenchmarkManager.class.getName());
+
+  private static final long TICK_INTERVAL = 10 * 1000L;
+
+  private static Map<String, BenchmarkRun> deepClone(Map<String, BenchmarkRun> runMap) {
+    Map<String, BenchmarkRun> map = new HashMap<>();
+    for (Map.Entry<String, BenchmarkRun> entry : runMap.entrySet()) {
+      map.put(entry.getKey(), BenchmarkRun.from(entry.getValue()));
+    }
+    return map;
+  }
+
+  private static Collection<String> getNonSuccessfulRuns(Map<String, BenchmarkRun> results) {
+
+    List<String> list = new ArrayList<>();
+    for (BenchmarkRun benchmarkRun : results.values()) {
+
+      if (benchmarkRun.isFailed()) {
+        for (Entry<RunnerConfig, Result> result : benchmarkRun.getResults().entrySet()) {
+          list.add(benchmarkRun.getModuleName() + " " + result.getKey());
+        }
+      } else {
+        for (Entry<RunnerConfig, Result> result : benchmarkRun.getResults().entrySet()) {
+          if (result.getValue().getState() != Result.State.DONE) {
+            list.add(benchmarkRun.getModuleName() + " " + result.getKey());
+          }
+        }
+      }
+    }
+    return list;
+  }
+
+  private enum Command {
+    EXIT, CHECK_FOR_UPDATES, RUN_BENCHMARKS, SUCCESSFUL_RUN, FAILED_RUN
+  }
+
+  private enum State {
+    IDLE, RUNNING_BENCHMARKS
+  }
+
+  private BenchmarkFinder benchmarkFinder;
+
+  private Object benchmarkRunsByNameLock = new Object();
+
+  private Map<String, BenchmarkRun> benchmarkRunsByName = new HashMap<>();
+
+  private BenchmarkWorker.Factory benchmarkWorkerFactory;
+
+  private String currentCommitDate;
+
+  private boolean currentlyRunning = false;
+
+  private BlockingQueue<Command> commands = new LinkedBlockingQueue<>();
+
+  private String currentCommitId;
+
+  private MailReporter errorReporter;
+
+  private Thread eventLoop;
+
+  private String lastSuccessfulCommitId;
+
+  private ExecutorService pool;
+
+  private Provider<ExecutorService> poolProvider;
+
+  private Factory reporterFactory;
+
+  private CliInteractor cliInteractor;
+
+  private State state = State.IDLE;
+
+  private Timer timer;
+
+  private Provider<Timer> timerProvider;
+
+  private boolean useReporter;
+
+  private AtomicInteger workCount = new AtomicInteger();
+
+  @Inject
+  public BenchmarkManager(BenchmarkFinder collector,
+      BenchmarkWorker.Factory benchmarkWorkerFactory,
+      @Named("managerPoolSize") Provider<ExecutorService> poolProvider,
+      BenchmarkReporter.Factory reporterFactory,
+      @Named("useReporter") boolean useReporter,
+      CliInteractor commitReader,
+      Provider<Timer> timerProvider,
+      MailReporter errorReporter) {
+    this.benchmarkFinder = collector;
+    this.benchmarkWorkerFactory = benchmarkWorkerFactory;
+    this.poolProvider = poolProvider;
+    this.reporterFactory = reporterFactory;
+    this.useReporter = useReporter;
+    this.cliInteractor = commitReader;
+    this.errorReporter = errorReporter;
+    this.timerProvider = timerProvider;
+  }
+
+  public String getLastCommitId() {
+    synchronized (benchmarkRunsByNameLock) {
+      return lastSuccessfulCommitId;
+    }
+  }
+
+  public Map<String, BenchmarkRun> getLatestRun() {
+    synchronized (benchmarkRunsByNameLock) {
+      return getLatestRunSynchronized(benchmarkRunsByName);
+    }
+  }
+
+  public List<RunnerConfig> getAllRunners() {
+    return Arrays.asList(RunnerConfigs.FIREFOX_LINUX);
+  }
+
+  public synchronized boolean isRunning() {
+    return currentlyRunning;
+  }
+
+  public synchronized void start() {
+
+    if (isRunning()) {
+      throw new IllegalStateException();
+    }
+    commands.clear();
+    eventLoop = new Thread(new EventLoop());
+    eventLoop.start();
+
+    timer = timerProvider.get();
+    timer.scheduleAtFixedRate(new TimerTask() {
+
+      @Override
+      public void run() {
+        commands.add(Command.CHECK_FOR_UPDATES);
+      }
+    }, TICK_INTERVAL, TICK_INTERVAL);
+
+    currentlyRunning = true;
+  }
+
+  public synchronized void stop() {
+    if (!isRunning()) {
+      throw new IllegalStateException();
+    }
+
+    commands.add(Command.EXIT);
+    try {
+      eventLoop.join();
+    } catch (InterruptedException e) {
+      // Our framework does not make use of thread.interrupt() so this must mean the JVM is trying
+      // to gracefully shut down in response to an external signal. Let it happen.
+    }
+
+    timer.cancel();
+    commands.clear();
+
+    currentlyRunning = false;
+  }
+
+  protected void addBenchmarkRun(BenchmarkRun br) {
+    synchronized (benchmarkRunsByNameLock) {
+      addBenchmarkRunSynchronized(benchmarkRunsByName, br);
+    }
+  }
+
+  private BenchmarkRun createBenchmarkRunForModule(String moduleName, String commitId,
+      String commitDate) {
+    BenchmarkRun br = new BenchmarkRun(moduleName, commitId, commitDate);
+    br.addRunner(RunnerConfigs.FIREFOX_LINUX);
+    return br;
+  }
+
+  private void maybeReportResults(String commitId, String commitDate) {
+    Map<String, BenchmarkRun> results;
+    synchronized (benchmarkRunsByNameLock) {
+      results = deepClone(benchmarkRunsByName);
+    }
+
+    Collection<String> runs = getNonSuccessfulRuns(results);
+    if (!runs.isEmpty()) {
+      StringBuilder builder = new StringBuilder();
+      builder.append("Failed Benchmarks: \n");
+      for (String errorModule : runs) {
+        builder.append(String.format("%s \n", errorModule));
+      }
+      logger.severe(
+          String.format("Benchmarks failed executing - stopping system\n%s", builder.toString()));
+      reportError("Benchmarks failed executing - stopping system");
+      stop();
+      return;
+    }
+
+    if (!useReporter) {
+      commands.add(Command.SUCCESSFUL_RUN);
+      return;
+    }
+
+    ReportProgressHandler p = new ReportProgressHandler() {
+
+      @Override
+      public void onPermanentFailure() {
+        reportError("Reporter failed to report results, shutting down the system");
+        stop();
+      }
+
+      @Override
+      public void onCommitReported() {
+        commands.add(Command.SUCCESSFUL_RUN);
+      }
+    };
+    new Thread(reporterFactory.create(results, commitId, commitDate, p)).start();
+  }
+
+  private Command getNextCommand() {
+    try {
+      return commands.take();
+    } catch (InterruptedException e) {
+      // Our framework does not make use of thread.interrupt() so this must mean the JVM is trying
+      // to gracefully shut down in response to an external signal. Let it happen.
+      return Command.EXIT;
+    }
+  }
+
+  private void runEventLoop() {
+    state = State.IDLE;
+    // check out last successful commit
+    try {
+      logger.info("Getting last commit");
+      setLastCommit(cliInteractor.getLastCommitId());
+      currentCommitId = getLastCommitId();
+      logger.info(String.format("Last commit was %s", currentCommitId));
+      currentCommitDate = cliInteractor.getDateForCommit(currentCommitId);
+      logger.info("Checking out last commit");
+      cliInteractor.checkout(getLastCommitId());
+
+    } catch (BenchmarkManagerException e) {
+      logger.log(Level.WARNING, "Can not checkout commit - shutting down", e);
+      reportError("Can not update git repo");
+      new Thread(new Runnable() {
+
+        @Override
+        public void run() {
+          stop();
+        }
+      }).start();
+      return;
+    }
+
+    while (true) {
+
+      Command command = getNextCommand();
+      switch (command) {
+        case CHECK_FOR_UPDATES:
+          // do not check for updates if we are already running a benchmark
+          if (state == State.RUNNING_BENCHMARKS) {
+            continue;
+          }
+
+          boolean hasUpdates = false;
+          try {
+            cliInteractor.maybeCheckoutNextCommit(getLastCommitId());
+            String commitId = cliInteractor.getCurrentCommitId();
+            hasUpdates = !currentCommitId.equals(commitId);
+            currentCommitId = commitId;
+
+          } catch (BenchmarkManagerException e) {
+            logger.log(Level.WARNING, "Can not update repository", e);
+            reportError("Can not update git repo");
+            break;
+          }
+
+          try {
+            if (hasUpdates) {
+              logger.info(String.format("found a new commit %s", currentCommitId));
+
+              logger.info("Getting its commit date");
+              currentCommitDate = cliInteractor.getDateForCommit(currentCommitId);
+
+              logger.info("Building SDK");
+              cliInteractor.buildSDK();
+              logger.info("Starting benchmark runners");
+              startBenchmarkingAllForCommit(currentCommitId, currentCommitDate);
+              state = State.RUNNING_BENCHMARKS;
+            }
+
+          } catch (BenchmarkManagerException e) {
+            logger.log(Level.WARNING, "Can not build SDK", e);
+            reportError("Can not build SDK");
+            break;
+          }
+          break;
+        case EXIT:
+          if (state == State.RUNNING_BENCHMARKS) {
+            try {
+              pool.shutdown();
+              pool.awaitTermination(30, TimeUnit.SECONDS);
+            } catch (InterruptedException e) {
+            }
+          }
+          // end this thread
+          commands.clear();
+          return;
+
+        case RUN_BENCHMARKS:
+          if (state == State.RUNNING_BENCHMARKS) {
+            logger.warning(
+                "asked to run benchmarks, but we are already running them, should not happen");
+            continue;
+          }
+          state = State.RUNNING_BENCHMARKS;
+          startBenchmarkingAllForCommit(currentCommitId, currentCommitDate);
+          break;
+
+        case SUCCESSFUL_RUN:
+          setLastCommit(currentCommitId);
+          state = State.IDLE;
+          break;
+        default:
+          logger.severe("Hit default case");
+          break;
+      }
+    }
+  }
+
+  private void reportError(String message) {
+    errorReporter.sendEmail(message);
+  }
+
+  private void setLastCommit(String commitId) {
+    synchronized (benchmarkRunsByNameLock) {
+      lastSuccessfulCommitId = commitId;
+    }
+  }
+
+  private void startBenchmarkingAllForCommit(String commitId, String commitDate) {
+
+    pool = poolProvider.get();
+
+    List<String> benchmarkModuleNames = benchmarkFinder.get();
+
+    for (String benchmarkModuleName : benchmarkModuleNames) {
+
+      // we are currently ignoring D8 benchmarks until we have a V8 runner.
+      if (benchmarkModuleName.endsWith("D8")) {
+        continue;
+      }
+
+      BenchmarkRun br = createBenchmarkRunForModule(benchmarkModuleName, commitId, commitDate);
+      addBenchmarkRun(br);
+
+      ProgressHandler progressHandler = new ThreadSafeProgressHandler(br);
+
+      BenchmarkWorker worker =
+          benchmarkWorkerFactory.create(BenchmarkWorkerConfig.from(br), progressHandler);
+      workCount.incrementAndGet();
+
+      pool.execute(worker);
+    }
+  }
+
+  // Visible for testings
+  void addBenchmarkRunSynchronized(Map<String, BenchmarkRun> runsByName, BenchmarkRun br) {
+    runsByName.put(br.getModuleName(), br);
+  }
+
+  // Visible for testings
+  Map<String, BenchmarkRun> getLatestRunSynchronized(Map<String, BenchmarkRun> runsByName) {
+    return deepClone(runsByName);
+  }
+
+  // Visible for testings
+  boolean isEventLoopAlive() {
+    return eventLoop.isAlive();
+  }
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkManagerException.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkManagerException.java
new file mode 100644
index 0000000..d38e08b
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkManagerException.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import java.io.IOException;
+
+/**
+ * Exception being thrown by components of the {@link BenchmarkManager}.
+ */
+public class BenchmarkManagerException extends Exception {
+
+  public BenchmarkManagerException(String message) {
+    super(message);
+  }
+
+  public BenchmarkManagerException(String message, IOException e) {
+    super(message, e);
+  }
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkReporter.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkReporter.java
new file mode 100644
index 0000000..a04cc56
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkReporter.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import com.google.gwt.benchmark.common.shared.json.BenchmarkResultJson;
+import com.google.gwt.benchmark.common.shared.json.BenchmarkRunJson;
+import com.google.gwt.benchmark.common.shared.json.JsonFactory;
+import com.google.gwt.benchmark.compileserver.server.manager.BenchmarkRun.Result;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import com.google.web.bindery.autobean.shared.AutoBean;
+import com.google.web.bindery.autobean.shared.AutoBeanCodex;
+import com.google.web.bindery.autobean.shared.AutoBeanUtils;
+
+import org.apache.commons.io.IOUtils;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * BenchmarkReporter reports benchmark results to a remote URL.
+ * <p>
+ * It does a HTTP Put with JSON in the body using the AutoBeans interfaces from the common project.
+ * On Failure it retries a couple of times before reporting a permanent failure.
+ */
+public class BenchmarkReporter implements Runnable {
+
+  public interface HttpURLConnectionFactory {
+    HttpURLConnection create() throws IOException;
+  }
+
+  public interface ReportProgressHandler {
+    void onCommitReported();
+
+    void onPermanentFailure();
+  }
+
+  public interface Factory {
+    BenchmarkReporter create(Map<String, BenchmarkRun> results,
+        @Assisted("commitId") String commitId, @Assisted("commitDate") String commitDate,
+        ReportProgressHandler reportProgressHandler);
+  }
+
+  public static final int[] WAITING_TIME_SECONDS = new int[] {1, 10, 100, 1000, 1000};
+
+  private static final int HTTP_OK = 200;
+  private static Logger logger = Logger.getLogger(BenchmarkReporter.class.getName());
+
+  private final Map<String, BenchmarkRun> results;
+  private final String commitId;
+  private final HttpURLConnectionFactory httpURLConnectionFactory;
+
+  private final ReportProgressHandler reportProgressHandler;
+
+  private final String commitDate;
+
+  @Inject
+  public BenchmarkReporter(HttpURLConnectionFactory httpURLConnectionFactory,
+      @Assisted Map<String, BenchmarkRun> results, @Assisted("commitId") String commitId,
+      @Assisted("commitDate") String commitDate,
+      @Assisted ReportProgressHandler reportProgressHandler) {
+    this.httpURLConnectionFactory = httpURLConnectionFactory;
+    this.results = results;
+    this.commitId = commitId;
+    this.commitDate = commitDate;
+    this.reportProgressHandler = reportProgressHandler;
+  }
+
+  @Override
+  public void run() {
+    AutoBean<BenchmarkRunJson> bean = AutoBeanUtils.getAutoBean(createBenchmarkRunJson());
+    String jsonString = AutoBeanCodex.encode(bean).getPayload();
+
+    boolean sent = false;
+    for (int count = 0; count < WAITING_TIME_SECONDS.length; count++) {
+      if (postResultToServer(jsonString)) {
+        sent = true;
+
+        break;
+      }
+      logger.warning(String.format("Could not post results to dashboard retrying in %d seconds.",
+          WAITING_TIME_SECONDS[count]));
+      if (!sleep(WAITING_TIME_SECONDS[count])) {
+        break;
+      }
+    }
+
+    if (!sent) {
+      reportProgressHandler.onPermanentFailure();
+    } else {
+      reportProgressHandler.onCommitReported();
+    }
+  }
+
+  private BenchmarkRunJson createBenchmarkRunJson() {
+    JsonFactory.Factory factory = JsonFactory.get();
+
+    BenchmarkRunJson runJSON = factory.run().as();
+    runJSON.setCommitId(commitId);
+    runJSON.setCommitTime(commitDate);
+    Map<String, List<BenchmarkResultJson>> results = new LinkedHashMap<>();
+
+    for (Entry<String, BenchmarkRun> br : this.results.entrySet()) {
+
+      String moduleName = br.getKey();
+      List<BenchmarkResultJson> list = new ArrayList<>();
+      results.put(moduleName, list);
+
+      for (Entry<RunnerConfig, Result> entry : br.getValue().getResults().entrySet()) {
+        Result result = entry.getValue();
+        RunnerConfig runnerConfig = entry.getKey();
+        BenchmarkResultJson resultJSON = factory.result().as();
+        resultJSON.setBenchmarkName(moduleName);
+        resultJSON.setRunnerId(runnerConfig.toString());
+        resultJSON.setRunsPerMinute(result.getRunsPerSecond());
+        list.add(resultJSON);
+      }
+
+    }
+    runJSON.setResultByBenchmarkName(results);
+    return runJSON;
+  }
+
+  private boolean postResultToServer(String json) {
+    OutputStream out = null;
+    try {
+
+      HttpURLConnection httpCon = httpURLConnectionFactory.create();
+      httpCon.setDoOutput(true);
+      httpCon.setRequestMethod("PUT");
+
+      out = httpCon.getOutputStream();
+      out.write(json.getBytes("UTF-8"));
+
+      if (httpCon.getResponseCode() == HTTP_OK) {
+        return true;
+      }
+
+    } catch (IOException e) {
+      logger.log(Level.WARNING, "Could not post results to server", e);
+    } finally {
+      IOUtils.closeQuietly(out);
+    }
+    return false;
+  }
+
+  // Visible for testing
+  boolean sleep(int seconds) {
+    try {
+      Thread.sleep(1000L * seconds);
+      return true;
+    } catch (InterruptedException e) {
+      // Our framework does not make use of thread.interrupt() so this must mean the JVM is trying
+      // to gracefully shut down in response to an external signal. Let it happen.
+      return false;
+    }
+  }
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkRun.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkRun.java
new file mode 100644
index 0000000..6172e3e
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkRun.java
@@ -0,0 +1,179 @@
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * BenchmarkRun contains all relevant information for one benchmark being run.
+ *
+ * <p>
+ * It has a state to keep track of execution progress. Since one benchmark module will be run by
+ * multiple Runners it also keeps track of each individual result.
+ */
+public class BenchmarkRun {
+
+  /**
+   * BenchmarkResult contains the result for one module being run on one exact runner.
+   */
+  public static class Result {
+
+    public static Result copyOf(Result other) {
+      Result result = new Result();
+      result.runsPerSecond = other.runsPerSecond;
+      result.state = other.state;
+      result.errorMessage = other.errorMessage;
+      return result;
+    }
+
+    private State state;
+    private double runsPerSecond;
+    private String errorMessage;
+
+    public enum State {
+      NOT_RUN, FAILED_RUN, DONE
+    }
+
+    public Result() {
+      state = State.NOT_RUN;
+    }
+
+    public void setRunsPerSecond(double runsPerSecond) {
+      state = State.DONE;
+      this.runsPerSecond = runsPerSecond;
+    }
+
+    public double getRunsPerSecond() {
+      return runsPerSecond;
+    }
+
+    public State getState() {
+      return state;
+    }
+
+    public void setErrorMessage(String errorMessage) {
+      this.errorMessage = errorMessage;
+    }
+
+    public String getErrorMessage() {
+      return errorMessage;
+    }
+  }
+
+  public enum State {
+    NOT_RUN, COMPILING, FAILED_COMPILE, FAILED_TO_GENERATE_HOST_PAGE, FAILED_TO_RUN_ON_RUNNER,
+    DONE, FAILED_TO_CREATE_DIR,
+  }
+
+  public static BenchmarkRun from(BenchmarkRun other) {
+    BenchmarkRun clone = new BenchmarkRun(other.moduleName, other.commitId, other.commitDate);
+    clone.runners = other.getRunConfigs();
+    clone.results = deepClone(other.results);
+    clone.state = other.state;
+    clone.errorMessage = other.errorMessage;
+    return clone;
+  }
+
+  private static Map<RunnerConfig, Result> deepClone(Map<RunnerConfig, Result> runMap) {
+    Map<RunnerConfig, Result> map = new HashMap<>();
+    for (Map.Entry<RunnerConfig, Result> entry : runMap.entrySet()) {
+      map.put(entry.getKey(), Result.copyOf(entry.getValue()));
+    }
+    return map;
+  }
+
+  private State state;
+
+  private List<RunnerConfig> runners = new ArrayList<>();
+
+  private Map<RunnerConfig, Result> results = new LinkedHashMap<>();
+
+  private final String moduleName;
+
+  private final String commitId;
+
+  private String errorMessage;
+
+  private final String commitDate;
+
+  public BenchmarkRun(String moduleName, String commitId, String commitDate) {
+    this.moduleName = moduleName;
+    this.commitId = commitId;
+    this.commitDate = commitDate;
+    state = State.NOT_RUN;
+  }
+
+  public void addRunner(RunnerConfig config) {
+    runners.add(config);
+    results.put(config, new Result());
+  }
+
+  public void addResult(RunnerConfig config, double runsPerSecond) {
+    if (!runners.contains(config)) {
+      throw new IllegalStateException();
+    }
+    if (!results.containsKey(config)) {
+      throw new IllegalStateException();
+    }
+
+    Result result = results.get(config);
+    result.setRunsPerSecond(runsPerSecond);
+  }
+
+  public String getCommitDate() {
+    return commitDate;
+  }
+
+  public String getCommitId() {
+    return commitId;
+  }
+
+  public String getErrorMessage() {
+    return errorMessage;
+  }
+
+  public String getModuleName() {
+    return moduleName;
+  }
+
+  public Map<RunnerConfig, Result> getResults() {
+    return results;
+  }
+
+  public List<RunnerConfig> getRunConfigs() {
+    return Collections.unmodifiableList(runners);
+  }
+
+  public State getState() {
+    return state;
+  }
+
+  public boolean isFailed() {
+    return !(state == State.DONE || state == State.NOT_RUN);
+  }
+
+  public void setFailedCompile(String message) {
+    this.errorMessage = message;
+    state = State.FAILED_COMPILE;
+  }
+
+  public void setFailedHostPageGenerationFailed(String errorMessage) {
+    this.errorMessage = errorMessage;
+    state = State.FAILED_TO_GENERATE_HOST_PAGE;
+  }
+
+  public void setFailedToCreateDirectory() {
+    state = State.FAILED_TO_CREATE_DIR;
+  }
+
+  public void setFailedToRunOnServer() {
+    state = State.FAILED_TO_RUN_ON_RUNNER;
+  }
+
+  public void setRunEnded() {
+    state = State.DONE;
+  }
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkWorker.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkWorker.java
new file mode 100644
index 0000000..1c94546
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkWorker.java
@@ -0,0 +1,161 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import com.google.inject.Provider;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.name.Named;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.inject.Inject;
+
+/**
+ * BenchmarkWorker compiles a single module, writes the host page and asks a Runner to execute the
+ * benchmark handing back results.
+ */
+public class BenchmarkWorker implements Runnable {
+
+  public interface Factory {
+    BenchmarkWorker create(BenchmarkWorkerConfig benchmarkWorkerConfig,
+        ProgressHandler progressHandler);
+  }
+
+  public interface ProgressHandler {
+    void onCompilationFailed(String message);
+
+    void onResult(RunnerConfig config, double result);
+
+    void failedToRunBenchmark(RunnerConfig config, String errorMessage);
+
+    void onHostPageGenerationFailed(String errorMessage);
+
+    void onRunEnded();
+
+    void onCompileDirCreationFailed();
+  }
+
+  private final BenchmarkCompiler compiler;
+  private final Runner.Factory runnerProvider;
+  private final BenchmarkWorkerConfig benchmarkData;
+  private final ProgressHandler progressHandler;
+  private final String ip;
+  private final String moduleTemplate;
+  private File compilerOutputDir;
+  private int port;
+  private Provider<String> randomStringProvider;
+
+  @Inject
+  public BenchmarkWorker(BenchmarkCompiler compiler,
+      Runner.Factory runnerProvider,
+      @Named("moduleTemplate") String moduleTemplate,
+      @Assisted BenchmarkWorkerConfig benchmarkData,
+      @Assisted ProgressHandler progressHandler,
+      @Named("ip") String ip,
+      @Named("port") int port,
+      @Named("compilerOutputDir") File compilerOutputDir,
+      @Named("randomStringProvider") Provider<String> randomStringProvider) {
+    this.compiler = compiler;
+    this.runnerProvider = runnerProvider;
+    this.moduleTemplate = moduleTemplate;
+    this.benchmarkData = benchmarkData;
+    this.progressHandler = progressHandler;
+    this.ip = ip;
+    this.port = port;
+    this.compilerOutputDir = compilerOutputDir;
+    this.randomStringProvider = randomStringProvider;
+  }
+
+  @Override
+  public void run() {
+    // create working dir
+    String randomDirName = randomStringProvider.get();
+    File outputDir = new File(compilerOutputDir, randomDirName);
+    if (!outputDir.mkdirs()) {
+      progressHandler.onCompileDirCreationFailed();
+      progressHandler.onRunEnded();
+      return;
+    }
+
+    try {
+      compiler.compile(benchmarkData.getModuleName(), outputDir);
+    } catch (BenchmarkCompilerException e) {
+      cleanupDirectory(outputDir);
+      progressHandler.onCompilationFailed(e.getMessage() + " " + e.getOutput());
+      progressHandler.onRunEnded();
+      return;
+    }
+
+    try {
+      writeHostPage(outputDir, benchmarkData.getModuleName());
+    } catch (IOException e) {
+      cleanupDirectory(outputDir);
+      progressHandler.onHostPageGenerationFailed("Failed to write host Page: " + e.getMessage());
+      progressHandler.onRunEnded();
+      return;
+    }
+
+    List<Runner> runners = new ArrayList<>();
+
+    for (RunnerConfig config : benchmarkData.getRunners()) {
+      String url = getUrl(this.port, randomDirName, benchmarkData.getModuleName());
+      Runner r = runnerProvider.create(config, url);
+      runners.add(r);
+      // TODO currently executing runners in sequence, switch to parallel execution.
+      r.run();
+    }
+    for (Runner runner : runners) {
+      if (runner.isFailed()) {
+        progressHandler.failedToRunBenchmark(runner.getConfig(), runner.getErrorMessage());
+      } else {
+        progressHandler.onResult(runner.getConfig(), runner.getResult());
+      }
+    }
+    cleanupDirectory(outputDir);
+    progressHandler.onRunEnded();
+  }
+
+  private void writeHostPage(File outputDir, String moduleName) throws IOException {
+    String tpl =
+        moduleTemplate.replace("{module_nocache}", moduleName + "/" + moduleName + ".nocache.js");
+    FileOutputStream stream = null;
+    try {
+      stream = new FileOutputStream(new File(outputDir, moduleName + ".html"));
+      IOUtils.write(tpl.getBytes("UTF-8"), stream);
+    } finally {
+      IOUtils.closeQuietly(stream);
+    }
+  }
+
+  private String getUrl(int port, String randomDirName, String module) {
+    String url = "http://{host}:{port}/__bench/{dirName}/{module}.html";
+    url = url.replace("{host}", ip);
+    url = url.replace("{port}", "" + port);
+    url = url.replace("{dirName}", randomDirName);
+    url = url.replace("{module}", module);
+    return url;
+  }
+
+  // Visible for Testing
+  void cleanupDirectory(File outputDir) {
+    FileUtils.deleteQuietly(outputDir);
+  }
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkWorkerConfig.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkWorkerConfig.java
new file mode 100644
index 0000000..debafcd
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkWorkerConfig.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * BenchmarkWorkerConfig contains the moduleName for a worker to compile and the configurations for
+ * runners that the worker should launch.
+ */
+public class BenchmarkWorkerConfig {
+  public static BenchmarkWorkerConfig from(BenchmarkRun run) {
+    return new BenchmarkWorkerConfig(run.getModuleName(), run.getRunConfigs());
+  }
+
+  private final String moduleName;
+  private final List<RunnerConfig> runnerConfigs;
+
+  public BenchmarkWorkerConfig(String moduleName, List<RunnerConfig> runners) {
+    this.moduleName = moduleName;
+    this.runnerConfigs = new ArrayList<>(runners);
+  }
+
+  public String getModuleName() {
+    return moduleName;
+  }
+
+  public List<RunnerConfig> getRunners() {
+    return Collections.unmodifiableList(runnerConfigs);
+  }
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/CliInteractor.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/CliInteractor.java
new file mode 100644
index 0000000..ee51a8d
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/CliInteractor.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.google.inject.name.Named;
+
+import org.apache.commons.io.IOUtils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * All low level interactions with scripts are done by this class.
+ */
+@Singleton
+public class CliInteractor implements BenchmarkCompiler {
+
+  private static final Logger logger = Logger.getLogger(CliInteractor.class.getName());
+
+  private File scriptDirectory;
+
+  private File persistenceDir;
+
+  private File gwtSourceLocation;
+
+  private File benchmarkSourceLocation;
+
+  @Inject
+  public CliInteractor(@Named("scriptDirectory") File scriptDirectory,
+      @Named("persistenceDir") File persistenceDir,
+      @Named("gwtSourceLocation") File gwtSourceLocation,
+      @Named("benchmarkSourceLocation") File benchmarkSourceLocation) {
+    this.scriptDirectory = scriptDirectory;
+    this.persistenceDir = persistenceDir;
+    this.gwtSourceLocation = gwtSourceLocation;
+    this.benchmarkSourceLocation = benchmarkSourceLocation;
+  }
+
+  public void buildSDK() throws BenchmarkManagerException {
+    File pullChangesScript = new File(scriptDirectory, "buildSDK");
+    runCommand(pullChangesScript.getAbsolutePath() + " " + gwtSourceLocation.getAbsolutePath());
+  }
+
+  public void checkout(String commitId) throws BenchmarkManagerException {
+    File pullChangesScript = new File(scriptDirectory, "checkout");
+    runCommand(pullChangesScript.getAbsolutePath() + " " + commitId + " "
+        + gwtSourceLocation.getAbsolutePath());
+  }
+
+  @Override
+  public void compile(String moduleName, File compilerOutputDir) throws BenchmarkCompilerException {
+    logger.info("compiling: " + moduleName);
+    File compileScript = new File(scriptDirectory, "compileModule");
+
+    String devjar =
+        new File(gwtSourceLocation, "build/staging/gwt-0.0.0/gwt-dev.jar").getAbsolutePath();
+    String userjar =
+        new File(gwtSourceLocation, "build/staging/gwt-0.0.0/gwt-user.jar").getAbsolutePath();
+    String bsl = benchmarkSourceLocation.getAbsolutePath();
+    if (!bsl.endsWith("/")) {
+      bsl += "/";
+    }
+
+    String outputDir = compilerOutputDir.getAbsolutePath();
+    try {
+      runCommand(compileScript.getAbsolutePath() + " " + moduleName + " " + devjar + " " + userjar
+          + " " + bsl + " " + outputDir, false);
+    } catch (BenchmarkManagerException e) {
+      throw new BenchmarkCompilerException("failed compile", e);
+    }
+  }
+
+  public String getCurrentCommitId() throws BenchmarkManagerException {
+    File gitCommitScript = new File(scriptDirectory, "commitId");
+    return runCommand(
+        gitCommitScript.getAbsolutePath() + " " + gwtSourceLocation.getAbsolutePath());
+  }
+
+  public String getDateForCommit(String currentCommitId) throws BenchmarkManagerException {
+    File commitDateScript = new File(scriptDirectory, "commitDate");
+    return runCommand(commitDateScript.getAbsolutePath() + " " + gwtSourceLocation.getAbsolutePath()
+        + " " + currentCommitId);
+  }
+
+  public String getLastCommitId() throws BenchmarkManagerException {
+    Properties prop = new Properties();
+    FileInputStream stream = null;
+    try {
+      stream = new FileInputStream(new File(persistenceDir, "store"));
+      prop.load(stream);
+      String commitId = prop.getProperty("commitId");
+      if (commitId == null) {
+        logger.severe("can not load last commitId from store");
+        throw new BenchmarkManagerException("can not load last commitId from store");
+      }
+      return commitId;
+    } catch (IOException e) {
+      logger.log(Level.WARNING, "Can not read commit from store file", e);
+      throw new BenchmarkManagerException("Can not read commit from store file", e);
+    } finally {
+      IOUtils.closeQuietly(stream);
+    }
+  }
+
+  public void maybeCheckoutNextCommit(String baseCommitId) throws BenchmarkManagerException {
+    File pullChangesScript = new File(scriptDirectory, "maybe_checkout_next_commit");
+    runCommand(pullChangesScript.getAbsolutePath() + " " + baseCommitId + " "
+        + gwtSourceLocation.getAbsolutePath());
+  }
+
+  public void storeCommitId(String commitId) throws BenchmarkManagerException {
+
+    Properties prop = new Properties();
+    prop.setProperty("commitId", commitId);
+
+    FileOutputStream stream = null;
+    try {
+      stream = new FileOutputStream(new File(persistenceDir, "store"));
+      prop.store(stream, null);
+
+    } catch (IOException e) {
+      logger.log(Level.WARNING, "Can not read commit from store file", e);
+      throw new BenchmarkManagerException("Can not read commit from store file", e);
+    } finally {
+      IOUtils.closeQuietly(stream);
+    }
+  }
+
+  private String runCommand(String command) throws BenchmarkManagerException {
+    return runCommand(command, true);
+  }
+
+  private String runCommand(String command, boolean useErrorSteam)
+      throws BenchmarkManagerException {
+    InputStream stream = null;
+    try {
+      Process process = Runtime.getRuntime().exec(command);
+      int exitValue = process.waitFor();
+
+      if (exitValue != 0) {
+        stream = useErrorSteam ? process.getErrorStream() : process.getInputStream();
+        String error =
+            "Command returned with " + exitValue + " " + IOUtils.toString(stream, "UTF-8");
+        logger.warning(error);
+        throw new BenchmarkManagerException(error);
+      }
+
+      stream = process.getInputStream();
+      return IOUtils.toString(stream, "UTF-8");
+
+    } catch (IOException | InterruptedException e) {
+      logger.log(Level.WARNING, "Can not run command", e);
+      throw new BenchmarkManagerException("Can not run command");
+    } finally {
+      IOUtils.closeQuietly(stream);
+    }
+  }
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/MailReporter.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/MailReporter.java
new file mode 100644
index 0000000..87ad80c
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/MailReporter.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import com.google.gwt.benchmark.compileserver.server.runners.settings.MailSettings;
+import com.google.inject.Inject;
+
+import java.util.Properties;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.mail.Authenticator;
+import javax.mail.Message;
+import javax.mail.MessagingException;
+import javax.mail.PasswordAuthentication;
+import javax.mail.Session;
+import javax.mail.Transport;
+import javax.mail.internet.InternetAddress;
+import javax.mail.internet.MimeMessage;
+
+/**
+ * MailReporter can send emails.
+ */
+public class MailReporter {
+
+  public static class PasswordAuthenticator extends Authenticator {
+
+    private final String username;
+    private final String password;
+
+    public PasswordAuthenticator(String username, String password) {
+      this.username = username;
+      this.password = password;
+    }
+
+    @Override
+    public PasswordAuthentication getPasswordAuthentication() {
+      return new PasswordAuthentication(username, password);
+    }
+  }
+
+  public interface MailHelper {
+    Session create(Properties properties, PasswordAuthenticator authenticator);
+
+    void send(MimeMessage message) throws MessagingException;
+  }
+
+  public static class MailHelperProdImpl implements MailHelper {
+
+    @Override
+    public Session create(Properties properties, PasswordAuthenticator authenticator) {
+      return Session.getInstance(properties, authenticator);
+    }
+
+    @Override
+    public void send(MimeMessage message) throws MessagingException {
+      Transport.send(message);
+    }
+  }
+
+  private static Logger logger = Logger.getLogger(MailReporter.class.getName());
+  private MailSettings settings;
+  private MailHelper mailHelper;
+
+  @Inject
+  public MailReporter(MailHelper mailHelper, MailSettings settings) {
+    this.mailHelper = mailHelper;
+    this.settings = settings;
+  }
+
+  public void sendEmail(String messageToSend) {
+
+    // Get system properties
+    Properties properties = new Properties();
+
+    // Setup mail server
+    properties.setProperty("mail.smtp.host", settings.getHost());
+    properties.put("mail.smtp.port", "465");
+    properties.put("mail.smtp.socketFactory.port", "465");
+    properties.put("mail.smtp.socketFactory.class", "javax.net.ssl.SSLSocketFactory");
+    properties.put("mail.smtp.socketFactory.fallback", "false");
+    properties.put("mail.smtp.auth", "true");
+
+    Session session = mailHelper.create(properties,
+        new PasswordAuthenticator(settings.getUsername(), settings.getPassword()));
+
+    try {
+      MimeMessage message = new MimeMessage(session);
+      message.setFrom(new InternetAddress(settings.getFrom()));
+      message.addRecipient(Message.RecipientType.TO, new InternetAddress(settings.getTo()));
+      message.setSubject("Error in the benchmarking system that needs attention");
+      message.setContent(messageToSend, "text/html");
+      mailHelper.send(message);
+    } catch (MessagingException e) {
+      logger.log(Level.SEVERE, "Can not send email to report an error", e);
+      logger.severe(messageToSend);
+    }
+  }
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/Runner.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/Runner.java
new file mode 100644
index 0000000..2057a87
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/Runner.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+/**
+ * A Runner executes a compiled benchmark and makes the result available.
+ */
+public interface Runner extends Runnable {
+
+  /**
+   * Factory for creating runners.
+   */
+  public interface Factory {
+    /**
+     * Create a new runner with a certain config and a url to invoke.
+     */
+    Runner create(RunnerConfig runnerConfig, String url);
+  }
+
+  /**
+   * Get the configuration of this runner.
+   *
+   * @return the configuration of the runner.
+   */
+  RunnerConfig getConfig();
+
+  /**
+   * Get the error message describing the error that happened while running the benchmark.
+   *
+   * @return the error message if {@link #isFailed()} returns true, otherwise null
+   */
+  String getErrorMessage();
+
+  /**
+   * The result of the run (runs per second)
+   * <p>
+   * Note: this is only available if {@link #isDone()} returns true.
+   *
+   * @return the result of the run
+   */
+  double getResult();
+
+  /**
+   * Is the runner done with execution of the benchmark.
+   *
+   * @return true if runner is done with the execution of the benchmark
+   */
+  boolean isDone();
+
+  /**
+   * Is the run failed
+   *
+   * @return true if the run failed
+   */
+  boolean isFailed();
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/RunnerConfig.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/RunnerConfig.java
new file mode 100644
index 0000000..0ac7478
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/RunnerConfig.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+/**
+ * A RunnerConfig describes all attributes of a runner like browser, OS and version.
+ */
+public interface RunnerConfig {
+
+  /**
+   * Enum for all supported browsers.
+   */
+  public enum Browser {
+    CHROME("chrome"), FIREFOX("firefox"), INTERNET_EXPLORER("ie"), SAFARI("safari");
+    private String value;
+
+    Browser(String value) {
+      this.value = value;
+    }
+
+    public String getValue() {
+      return value;
+    }
+
+    @Override
+    public String toString() {
+      return value;
+    }
+  }
+
+  /**
+   * Enum for all supported operating systems.
+   */
+  public enum OS {
+    ANDROID("android"), IOS("ios"), LINUX("linux"), OSX("osx"), WINDOWS("windows");
+    private String value;
+
+    OS(String value) {
+      this.value = value;
+    }
+
+    public String getValue() {
+      return value;
+    }
+
+    @Override
+    public String toString() {
+      return value;
+    }
+  }
+
+  /**
+   * Get the browser for this config.
+   *
+   * @return the {@link Browser} for this config.
+   */
+  Browser getBrowser();
+
+  /**
+   * Get the operating system for this config.
+   *
+   * @return the operating system for this config.
+   */
+  OS getOS();
+
+  /**
+   * Get the version of the browser for this config.
+   *
+   * @return the version of the browser for this config.
+   */
+  String getBrowserVersion();
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/RunnerConfigs.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/RunnerConfigs.java
new file mode 100644
index 0000000..7919ae3
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/RunnerConfigs.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import com.google.gwt.benchmark.compileserver.server.manager.RunnerConfig.Browser;
+import com.google.gwt.benchmark.compileserver.server.manager.RunnerConfig.OS;
+
+/**
+ * Global Object containing all available {@link RunnerConfig RunnerConfigs}.
+ */
+public final class RunnerConfigs {
+
+  private static class RunnerConfigImpl implements RunnerConfig {
+
+    private Browser browser;
+    private OS os;
+    private String version;
+
+    public RunnerConfigImpl(Browser browser, OS os, String version) {
+      this.browser = browser;
+      this.os = os;
+      this.version = version;
+    }
+
+    @Override
+    public Browser getBrowser() {
+      return browser;
+    }
+
+    @Override
+    public OS getOS() {
+      return os;
+    }
+
+    @Override
+    public String getBrowserVersion() {
+      return version;
+    }
+
+    @Override
+    public String toString() {
+      return os + " " + browser;
+    }
+  }
+
+  /** Chrome on linux */
+  public static final RunnerConfig CHROME_LINUX =
+      new RunnerConfigImpl(Browser.CHROME, OS.LINUX, "");
+  /** Firefox on linux */
+  public static final RunnerConfig FIREFOX_LINUX =
+      new RunnerConfigImpl(Browser.FIREFOX, OS.LINUX, "");
+
+  private RunnerConfigs() {}
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/WebDriverRunner.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/WebDriverRunner.java
new file mode 100644
index 0000000..0ecb12f
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/manager/WebDriverRunner.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.name.Named;
+
+import org.openqa.selenium.remote.DesiredCapabilities;
+import org.openqa.selenium.remote.RemoteWebDriver;
+
+import java.net.URL;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.inject.Inject;
+
+/**
+ * WebDriverRunner uses Webdriver to remotely control a browser to run a benchmark.
+ */
+public class WebDriverRunner implements Runner {
+  private static final String IS_READY_JS = "return !!window.__gwt__benchmarking__ran";
+  private static final String IS_FAILED_JS = "return !!window.__gwt__benchmarking__failed";
+  private static final String GET_RESULT_JS = "return window.__gwt__benchmarking__result";
+  private static final int TIMEOUT_MS = 60000;
+  private static final Logger logger = Logger.getLogger(WebDriverRunner.class.getName());
+
+  private boolean done;
+  private double result;
+  private String errorMessage;
+  private String url;
+  private RunnerConfig config;
+  private URL hubURL;
+  private boolean failed = false;
+
+  @Inject
+  public WebDriverRunner(@Assisted RunnerConfig config, @Assisted String url,
+      @Named("hubUrl") URL hubUrl) {
+    this.config = config;
+    this.url = url;
+    this.hubURL = hubUrl;
+  }
+
+  @Override
+  public void run() {
+    logger.info("Starting webdriver for " + url);
+    // TODO use right capabilities
+    DesiredCapabilities capabilities = DesiredCapabilities.firefox();
+    RemoteWebDriver driver = null;
+    try {
+      driver = new RemoteWebDriver(hubURL, capabilities);
+      driver.navigate().to(url);
+
+      boolean isReady = (Boolean) driver.executeScript(IS_READY_JS, new Object[] {});
+      long startMs = System.currentTimeMillis();
+
+      // Wait till the benchmark has finished running.
+      while (!isReady) {
+        try {
+          Thread.sleep(100);
+        } catch (InterruptedException ignored) {
+        }
+        isReady = (Boolean) driver.executeScript(IS_READY_JS, new Object[] {});
+
+        if (System.currentTimeMillis() - startMs > TIMEOUT_MS) {
+          this.failed = true;
+          logger.info("Timeout webdriver for " + url);
+
+          failed = true;
+          errorMessage = "Timeout";
+          return;
+        }
+      }
+
+      // Read and report status.
+      boolean failed = (Boolean) driver.executeScript(IS_FAILED_JS, new Object[] {});
+      if (failed) {
+        this.failed = true;
+        this.errorMessage =
+            "Benchmark failed to run in browser - Benchmarkframework reported a failure";
+        logger.info("Benchmark failed to run for " + url);
+      } else {
+
+        result = ((Number) driver.executeScript(GET_RESULT_JS, new Object[] {})).doubleValue();
+        done = true;
+      }
+    } catch (Exception e) {
+      logger.log(Level.INFO, "Error while running webdriver for " + url, e);
+      failed = true;
+      errorMessage = "Unexpected excpetion during webdriver run: " + e.getMessage();
+    } finally {
+      if (driver != null) {
+        driver.quit();
+      }
+    }
+  }
+
+  @Override
+  public double getResult() {
+    return result;
+  }
+
+  @Override
+  public boolean isDone() {
+    return done;
+  }
+
+  @Override
+  public String getErrorMessage() {
+    return errorMessage;
+  }
+
+  @Override
+  public RunnerConfig getConfig() {
+    return config;
+  }
+
+  @Override
+  public boolean isFailed() {
+    return failed;
+  }
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/runners/settings/MailSettings.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/runners/settings/MailSettings.java
new file mode 100644
index 0000000..df4e3e2
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/runners/settings/MailSettings.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.runners.settings;
+
+/**
+ * Settings relevant for sending an email.
+ */
+public class MailSettings {
+  private String to;
+  private String from;
+  private String host;
+  private String username;
+  private String password;
+
+  public MailSettings(String from, String to, String host, String username, String password) {
+    this.from = from;
+    this.to = to;
+    this.host = host;
+    this.username = username;
+    this.password = password;
+  }
+
+  public String getHost() {
+    return host;
+  }
+
+  public String getUsername() {
+    return username;
+  }
+
+  public String getPassword() {
+    return password;
+  }
+
+  public String getFrom() {
+    return from;
+  }
+
+  public String getTo() {
+    return to;
+  }
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/runners/settings/ManagerMode.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/runners/settings/ManagerMode.java
new file mode 100644
index 0000000..a313085
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/runners/settings/ManagerMode.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.runners.settings;
+
+/**
+ * The mode in which the system is running in.
+ *
+ * <p>
+ * Note: This is currently not being used since the system currently always runs in server mode, but
+ * will be eventually extended to be run locally as well.
+ */
+public enum ManagerMode {
+  SERVER, LOCAL
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/runners/settings/Settings.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/runners/settings/Settings.java
new file mode 100644
index 0000000..d08d288
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/runners/settings/Settings.java
@@ -0,0 +1,151 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.runners.settings;
+
+import org.apache.commons.io.IOUtils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Properties;
+
+/**
+ * Setting class containing all the configuration for the benchmarking system.
+ */
+public class Settings {
+
+  public static Settings parseSettings(File settingsFile) throws Exception {
+    Settings settings = new Settings();
+    Properties prop = new Properties();
+    FileInputStream stream = null;
+    try {
+      stream = new FileInputStream(settingsFile);
+      // load a properties file from class path, inside static method
+      prop.load(stream);
+
+      // get the property value and print it out
+      settings.benchmarkRootDirectory = new File(prop.getProperty("benchmarksDirectory"));
+
+      settings.moduleTemplate =
+          loadModuleTemplate(settingsFile.getParentFile(), prop.getProperty("moduleTemplate"));
+      settings.hubUrl = new URL(prop.getProperty("seleniumHubUrl"));
+      settings.benchmarkCompileOutputDir = new File(prop.getProperty("compileOutputDir"));
+      settings.threadPoolSize = Integer.parseInt(prop.getProperty("threadPoolSize"));
+      settings.servletContainerPort = Integer.parseInt(prop.getProperty("servletContainerPort"));
+      settings.ipAddress = Util.getFirstNonLoopbackAddress().getHostAddress();
+      settings.reportResults = prop.getProperty("reportResuts").equals("true");
+      settings.reporterUrl = prop.getProperty("reporterUrl");
+      settings.mode =
+          prop.getProperty("mode").equals("server") ? ManagerMode.SERVER : ManagerMode.LOCAL;
+      settings.persistenceDir = new File(prop.getProperty("persistenceDir"));
+      settings.gwtSourceLocation = new File(prop.getProperty("gwtSourceLocation"));
+
+      String mailTo = prop.getProperty("mail.to");
+      String mailFrom = prop.getProperty("mail.from");
+      String mailHost = prop.getProperty("mail.host");
+      String mailUsername = prop.getProperty("mail.username");
+      String mailPassword = prop.getProperty("mail.password");
+
+      settings.mailSettings =
+          new MailSettings(mailFrom, mailTo, mailHost, mailUsername, mailPassword);
+    } finally {
+      IOUtils.closeQuietly(stream);
+    }
+    return settings;
+  }
+
+  private static String loadModuleTemplate(File parent, String fileName) throws IOException {
+    FileInputStream inputStream = null;
+
+    try {
+      inputStream = new FileInputStream(new File(parent, fileName));
+      return IOUtils.toString(inputStream, "UTF-8");
+    } finally {
+      IOUtils.closeQuietly(inputStream);
+    }
+  }
+
+  private File benchmarkRootDirectory;
+  private String moduleTemplate;
+  private String ipAddress;
+  private URL hubUrl;
+  private File benchmarkCompileOutputDir;
+  private int threadPoolSize;
+  private boolean reportResults;
+  private String reporterUrl;
+  private File persistenceDir;
+  private ManagerMode mode;
+  private File gwtSourceLocation;
+  private MailSettings mailSettings;
+  private int servletContainerPort;
+
+  public File getBenchmarkRootDirectory() {
+    return benchmarkRootDirectory;
+  }
+
+  public String getModuleTemplate() {
+    return moduleTemplate;
+  }
+
+  public File getScriptsDirectory() {
+    return new File(getBenchmarkRootDirectory(), "src/main/scripts/");
+  }
+
+  public String getIpAddress() {
+    return ipAddress;
+  }
+
+  public URL getHubUrl() {
+    return hubUrl;
+  }
+
+  public File getBenchmarkCompileOutputDir() {
+    return benchmarkCompileOutputDir;
+  }
+
+  public int getThreadPoolSize() {
+    return threadPoolSize;
+  }
+
+  public boolean reportResults() {
+    return reportResults;
+  }
+
+  public String getReporterUrl() {
+    return reporterUrl;
+  }
+
+  public ManagerMode getMode() {
+    return mode;
+  }
+
+  public File getPersistenceDir() {
+    return persistenceDir;
+  }
+
+  public File getGwtSourceLocation() {
+    return gwtSourceLocation;
+  }
+
+  public MailSettings getMailSettings() {
+    return mailSettings;
+  }
+
+  public int getServletContainerPort() {
+    return servletContainerPort;
+  }
+
+  private Settings() {}
+}
diff --git a/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/runners/settings/Util.java b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/runners/settings/Util.java
new file mode 100644
index 0000000..91af3c4
--- /dev/null
+++ b/compileserver/src/main/java/com/google/gwt/benchmark/compileserver/server/runners/settings/Util.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.runners.settings;
+
+import java.net.Inet4Address;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.util.Enumeration;
+
+public class Util {
+  public static InetAddress getFirstNonLoopbackAddress() throws Exception {
+    Enumeration<NetworkInterface> en = NetworkInterface.getNetworkInterfaces();
+    while (en.hasMoreElements()) {
+      NetworkInterface i = en.nextElement();
+      for (Enumeration<InetAddress> en2 = i.getInetAddresses(); en2.hasMoreElements();) {
+        InetAddress addr = en2.nextElement();
+        if (!addr.isLoopbackAddress()) {
+          if (addr instanceof Inet4Address) {
+            return addr;
+          }
+        }
+      }
+    }
+    throw new Exception("could not find any interface");
+  }
+}
diff --git a/compileserver/src/main/webapp/WEB-INF/web.xml b/compileserver/src/main/webapp/WEB-INF/web.xml
new file mode 100644
index 0000000..02e51c4
--- /dev/null
+++ b/compileserver/src/main/webapp/WEB-INF/web.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd">
+
+  <welcome-file-list>
+    <welcome-file>index.html</welcome-file>
+  </welcome-file-list>
+
+  <filter>
+    <filter-name>guiceFilter</filter-name>
+    <filter-class>com.google.inject.servlet.GuiceFilter</filter-class>
+  </filter>
+
+  <filter-mapping>
+    <filter-name>guiceFilter</filter-name>
+    <url-pattern>/*</url-pattern>
+  </filter-mapping>
+
+  <listener>
+    <listener-class>com.google.gwt.benchmark.compileserver.server.guice.GuiceServletConfig</listener-class>
+  </listener>
+</web-app>
diff --git a/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkFinderTest.java b/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkFinderTest.java
new file mode 100644
index 0000000..45d1e2b
--- /dev/null
+++ b/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkFinderTest.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.List;
+
+/**
+ * Tests for {@link BenchmarkFinder}.
+ */
+public class BenchmarkFinderTest {
+
+  @Test
+  public void testCollectionFromFileSystem() {
+    BenchmarkFinder finder = new BenchmarkFinder(new File("./src/test/resources/collector-test/"));
+    List<String> list = finder.get();
+
+    Assert.assertEquals(2, list.size());
+    Assert.assertTrue(list.contains("com.google.gwt.benchmark.benchmarks.TestBenchmark"));
+    Assert.assertTrue(list.contains("com.google.gwt.benchmark.benchmarks.package.TestBenchmark2"));
+  }
+}
diff --git a/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkManagerTest.java b/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkManagerTest.java
new file mode 100644
index 0000000..0649b71
--- /dev/null
+++ b/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkManagerTest.java
@@ -0,0 +1,576 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import com.google.gwt.benchmark.compileserver.server.manager.BenchmarkReporter.ReportProgressHandler;
+import com.google.gwt.benchmark.compileserver.server.manager.BenchmarkWorker.Factory;
+import com.google.gwt.benchmark.compileserver.server.manager.BenchmarkWorker.ProgressHandler;
+import com.google.inject.Provider;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+import org.mockito.verification.VerificationWithTimeout;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+
+/**
+ * Test for {@link BenchmarkManager}.
+ */
+public class BenchmarkManagerTest {
+
+  private interface Condition {
+    boolean condition();
+  }
+
+  private static class BenchmarkManagerWithPublicAdd extends BenchmarkManager {
+    public BenchmarkManagerWithPublicAdd(BenchmarkFinder collector,
+        Factory benchmarkWorkerFactory,
+        Provider<ExecutorService> poolProvider,
+        com.google.gwt.benchmark.compileserver.server.manager.BenchmarkReporter.Factory reporterFactory,
+        boolean useReporter,
+        CliInteractor commitReader,
+        Provider<Timer> timerProvider,
+        MailReporter errorReporter) {
+      super(collector,
+          benchmarkWorkerFactory,
+          poolProvider,
+          reporterFactory,
+          useReporter,
+          commitReader,
+          timerProvider,
+          errorReporter);
+    }
+
+    @Override
+    public void addBenchmarkRun(BenchmarkRun br) {
+      super.addBenchmarkRun(br);
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  public static <T> T cast(Object a) {
+    return (T) a;
+  }
+
+  private BenchmarkFinder collector;
+  private BenchmarkWorker.Factory benchmarkWorkerFactory;
+  private Provider<ExecutorService> poolProvider;
+  private Provider<Timer> timerProvider;
+  private BenchmarkReporter.Factory reporterFactory;
+  private BenchmarkManager manager;
+  private CliInteractor commitReader;
+  private ThreadPoolExecutor threadPoolExecutor;
+  private BenchmarkWorker benchmarkWorker;
+  private BenchmarkReporter benchmarkReporter;
+  private MailReporter errorReporter;
+  private Timer timer;
+
+  @Before
+  public void setup() {
+    collector = Mockito.mock(BenchmarkFinder.class);
+    benchmarkWorkerFactory = Mockito.mock(BenchmarkWorker.Factory.class);
+    poolProvider = cast(Mockito.mock(Provider.class));
+    reporterFactory = Mockito.mock(BenchmarkReporter.Factory.class);
+    commitReader = Mockito.mock(CliInteractor.class);
+    timerProvider = cast(Mockito.mock(Provider.class));
+    threadPoolExecutor = Mockito.mock(ThreadPoolExecutor.class);
+    benchmarkWorker = Mockito.mock(BenchmarkWorker.class);
+    benchmarkReporter = Mockito.mock(BenchmarkReporter.class);
+    errorReporter = Mockito.mock(MailReporter.class);
+    timer = Mockito.mock(Timer.class);
+  }
+
+  @Test
+  public void testSuccessfulRun() throws BenchmarkManagerException, InterruptedException {
+
+    Mockito.when(timerProvider.get()).thenReturn(timer);
+
+    manager = new BenchmarkManager(collector,
+        benchmarkWorkerFactory,
+        poolProvider,
+        reporterFactory,
+        true,
+        commitReader,
+        timerProvider,
+        errorReporter);
+
+    Mockito.when(commitReader.getLastCommitId()).thenReturn("commit1");
+    Mockito.when(commitReader.getCurrentCommitId()).thenReturn("commit2");
+    Mockito.when(poolProvider.get()).thenReturn(threadPoolExecutor);
+    Mockito.when(collector.get()).thenReturn(Arrays.asList("module1", "module2"));
+
+    ArgumentCaptor<ProgressHandler> progressHandlerCaptor =
+        ArgumentCaptor.forClass(ProgressHandler.class);
+
+    ArgumentCaptor<BenchmarkWorkerConfig> workerConfigCapture =
+        ArgumentCaptor.forClass(BenchmarkWorkerConfig.class);
+
+    Mockito.when(benchmarkWorkerFactory.create(workerConfigCapture.capture(),
+        progressHandlerCaptor.capture())).thenReturn(benchmarkWorker);
+
+    ArgumentCaptor<Map<String, BenchmarkRun>> resultCaptor =
+        cast(ArgumentCaptor.forClass(Map.class));
+
+    ArgumentCaptor<ReportProgressHandler> reportProgressHandlerCaptor =
+        ArgumentCaptor.forClass(ReportProgressHandler.class);
+
+    Mockito.when(reporterFactory.create(resultCaptor.capture(), Mockito.anyString(),
+        Mockito.anyString(), reportProgressHandlerCaptor.capture())).thenReturn(benchmarkReporter);
+
+    Assert.assertFalse(manager.isRunning());
+
+    manager.start();
+
+    Assert.assertTrue(manager.isRunning());
+    Assert.assertTrue(manager.isEventLoopAlive());
+
+    ArgumentCaptor<TimerTask> captor = ArgumentCaptor.forClass(TimerTask.class);
+
+    Mockito.verify(timer).scheduleAtFixedRate(captor.capture(), Mockito.anyLong(),
+        Mockito.anyLong());
+
+    TimerTask timerTask = captor.getValue();
+    timerTask.run();
+
+    VerificationWithTimeout timeout = Mockito.timeout(200);
+    Mockito.verify(commitReader, timeout).checkout("commit1");
+    Mockito.verify(commitReader, timeout).maybeCheckoutNextCommit("commit1");
+    Mockito.verify(poolProvider, timeout).get();
+    Mockito.verify(collector, timeout).get();
+    Mockito.verify(threadPoolExecutor, timeout.times(2)).execute(benchmarkWorker);
+
+    List<ProgressHandler> progressHandlers = progressHandlerCaptor.getAllValues();
+    Assert.assertEquals(2, progressHandlers.size());
+    List<BenchmarkWorkerConfig> workerConfigs = workerConfigCapture.getAllValues();
+    Assert.assertEquals(2, workerConfigs.size());
+
+    // TODO right now there is only one runner, needs updating
+    // simulate benchmarks done
+    progressHandlers.get(0).onResult(workerConfigs.get(0).getRunners().get(0), 1);
+    progressHandlers.get(0).onRunEnded();
+    progressHandlers.get(1).onResult(workerConfigs.get(0).getRunners().get(0), 2);
+    progressHandlers.get(1).onRunEnded();
+
+    Mockito.verify(benchmarkReporter, timeout).run();
+
+    Map<String, BenchmarkRun> map = resultCaptor.getValue();
+    Assert.assertEquals(2, map.size());
+
+    Assert.assertTrue(map.containsKey("module1"));
+    Assert.assertTrue(map.containsKey("module2"));
+
+    BenchmarkRun benchmarkRun = map.get("module1");
+
+    Assert.assertEquals("commit2", benchmarkRun.getCommitId());
+    Assert.assertEquals("module1", benchmarkRun.getModuleName());
+    Assert.assertEquals(BenchmarkRun.State.DONE, benchmarkRun.getState());
+    Assert.assertEquals(1,
+        benchmarkRun.getResults().get(benchmarkRun.getRunConfigs().get(0)).getRunsPerSecond(),
+        0.0001);
+
+    benchmarkRun = map.get("module2");
+    Assert.assertEquals("commit2", benchmarkRun.getCommitId());
+    Assert.assertEquals("module2", benchmarkRun.getModuleName());
+    Assert.assertEquals(BenchmarkRun.State.DONE, benchmarkRun.getState());
+    Assert.assertEquals(2,
+        benchmarkRun.getResults().get(benchmarkRun.getRunConfigs().get(0)).getRunsPerSecond(),
+        0.0001);
+
+    // tell benchmark manager that we reported successfully
+    ReportProgressHandler progressHandler = reportProgressHandlerCaptor.getValue();
+    progressHandler.onCommitReported();
+
+    // wait for benchmark manager to change to the next commit
+    waitFor(new Condition() {
+      @Override
+      public boolean condition() {
+        return manager.getLastCommitId().equals("commit2");
+      }
+    }, 200);
+
+    Assert.assertTrue(manager.isRunning());
+    Assert.assertTrue(manager.isEventLoopAlive());
+
+    manager.stop();
+
+    Assert.assertFalse(manager.isRunning());
+
+    waitFor(new Condition() {
+      @Override
+      public boolean condition() {
+        return !manager.isEventLoopAlive();
+      }
+    }, 200);
+  }
+
+  @Test
+  public void testEventLoopFailsOnStartupWhileGettingCommitId() throws BenchmarkManagerException,
+      InterruptedException {
+    Mockito.when(timerProvider.get()).thenReturn(timer);
+    manager = new BenchmarkManager(collector,
+        benchmarkWorkerFactory,
+        poolProvider,
+        reporterFactory,
+        true,
+        commitReader,
+        timerProvider,
+        errorReporter);
+
+    Mockito.when(commitReader.getLastCommitId()).thenThrow(new BenchmarkManagerException(""));
+
+    Assert.assertFalse(manager.isRunning());
+    manager.start();
+    VerificationWithTimeout timeout = Mockito.timeout(200);
+    Mockito.verify(errorReporter, timeout).sendEmail("Can not update git repo");
+    waitFor(new Condition() {
+
+      @Override
+      public boolean condition() {
+        return !manager.isRunning();
+      }
+    }, 200);
+    Assert.assertFalse(manager.isRunning());
+    Assert.assertFalse(manager.isEventLoopAlive());
+  }
+
+  @Test
+  public void testEventLoopFailsOnStartupWhileGettingCommitDate() throws BenchmarkManagerException,
+      InterruptedException {
+    Mockito.when(timerProvider.get()).thenReturn(timer);
+    manager = new BenchmarkManager(collector,
+        benchmarkWorkerFactory,
+        poolProvider,
+        reporterFactory,
+        true,
+        commitReader,
+        timerProvider,
+        errorReporter);
+
+    Mockito.when(commitReader.getDateForCommit(Mockito.anyString())).thenThrow(
+        new BenchmarkManagerException(""));
+
+    Assert.assertFalse(manager.isRunning());
+    manager.start();
+    VerificationWithTimeout timeout = Mockito.timeout(200);
+    Mockito.verify(errorReporter, timeout).sendEmail("Can not update git repo");
+    waitFor(new Condition() {
+
+      @Override
+      public boolean condition() {
+        return !manager.isRunning();
+      }
+    }, 200);
+    Assert.assertFalse(manager.isRunning());
+    Assert.assertFalse(manager.isEventLoopAlive());
+  }
+
+  @Test
+  public void testEventLoopFailsOnStartupWhileCheckingOutCurrentCommit()
+      throws BenchmarkManagerException, InterruptedException {
+    Mockito.when(timerProvider.get()).thenReturn(timer);
+    manager = new BenchmarkManager(collector,
+        benchmarkWorkerFactory,
+        poolProvider,
+        reporterFactory,
+        true,
+        commitReader,
+        timerProvider,
+        errorReporter);
+
+    Mockito.doThrow(new BenchmarkManagerException("")).when(commitReader)
+        .checkout(Mockito.anyString());
+
+    Assert.assertFalse(manager.isRunning());
+    manager.start();
+    VerificationWithTimeout timeout = Mockito.timeout(200);
+    Mockito.verify(errorReporter, timeout).sendEmail("Can not update git repo");
+    waitFor(new Condition() {
+
+      @Override
+      public boolean condition() {
+        return !manager.isRunning();
+      }
+    }, 200);
+    Assert.assertFalse(manager.isRunning());
+    Assert.assertFalse(manager.isEventLoopAlive());
+  }
+
+  @Test
+  public void testFailedToCompileBenchmark() throws BenchmarkManagerException,
+      InterruptedException {
+
+    Mockito.when(timerProvider.get()).thenReturn(timer);
+
+    manager = new BenchmarkManager(collector,
+        benchmarkWorkerFactory,
+        poolProvider,
+        reporterFactory,
+        true,
+        commitReader,
+        timerProvider,
+        errorReporter);
+
+    Mockito.when(commitReader.getLastCommitId()).thenReturn("commit1");
+    Mockito.when(commitReader.getCurrentCommitId()).thenReturn("commit2");
+    Mockito.when(poolProvider.get()).thenReturn(threadPoolExecutor);
+    Mockito.when(collector.get()).thenReturn(Arrays.asList("module1", "module2"));
+
+    ArgumentCaptor<ProgressHandler> progressHandlerCaptor =
+        ArgumentCaptor.forClass(ProgressHandler.class);
+
+    ArgumentCaptor<BenchmarkWorkerConfig> benchmarkWorkerConfigCaptor =
+        ArgumentCaptor.forClass(BenchmarkWorkerConfig.class);
+
+    Mockito.when(benchmarkWorkerFactory.create(benchmarkWorkerConfigCaptor.capture(),
+        progressHandlerCaptor.capture())).thenReturn(benchmarkWorker);
+
+    ArgumentCaptor<Map<String, BenchmarkRun>> resultCaptor =
+        cast(ArgumentCaptor.forClass(Map.class));
+
+    ArgumentCaptor<ReportProgressHandler> reportProgressHandlerCaptor =
+        ArgumentCaptor.forClass(ReportProgressHandler.class);
+
+    Mockito.when(reporterFactory.create(resultCaptor.capture(), Mockito.anyString(),
+        Mockito.anyString(), reportProgressHandlerCaptor.capture())).thenReturn(benchmarkReporter);
+
+    Assert.assertFalse(manager.isRunning());
+
+    manager.start();
+
+    Assert.assertTrue(manager.isRunning());
+    Assert.assertTrue(manager.isEventLoopAlive());
+
+    ArgumentCaptor<TimerTask> captor = ArgumentCaptor.forClass(TimerTask.class);
+
+    Mockito.verify(timer).scheduleAtFixedRate(captor.capture(), Mockito.anyLong(),
+        Mockito.anyLong());
+
+    TimerTask timerTask = captor.getValue();
+    timerTask.run();
+
+    VerificationWithTimeout timeout = Mockito.timeout(200);
+    Mockito.verify(commitReader, timeout).checkout("commit1");
+    Mockito.verify(commitReader, timeout).maybeCheckoutNextCommit("commit1");
+    Mockito.verify(poolProvider, timeout).get();
+    Mockito.verify(collector, timeout).get();
+    Mockito.verify(threadPoolExecutor, timeout.times(2)).execute(benchmarkWorker);
+
+    List<ProgressHandler> progressHandlers = progressHandlerCaptor.getAllValues();
+    Assert.assertEquals(2, progressHandlers.size());
+    List<BenchmarkWorkerConfig> workerConfigs = benchmarkWorkerConfigCaptor.getAllValues();
+    Assert.assertEquals(2, workerConfigs.size());
+
+    // TODO right now there is only one runner, needs updating
+    // simulate benchmarks done
+    progressHandlers.get(0).onResult(workerConfigs.get(0).getRunners().get(0), 1);
+    progressHandlers.get(0).onCompilationFailed("bad module1");
+    progressHandlers.get(0).onRunEnded();
+    progressHandlers.get(1).onResult(workerConfigs.get(0).getRunners().get(0), 2);
+    progressHandlers.get(1).onCompilationFailed("bad module2");
+    progressHandlers.get(1).onRunEnded();
+
+    Mockito.verifyZeroInteractions(benchmarkReporter);
+
+    Map<String, BenchmarkRun> map = manager.getLatestRun();
+    Assert.assertEquals(2, map.size());
+
+    Assert.assertTrue(map.containsKey("module1"));
+    Assert.assertTrue(map.containsKey("module2"));
+
+    BenchmarkRun benchmarkRun = map.get("module1");
+    Assert.assertEquals(BenchmarkRun.State.FAILED_COMPILE, benchmarkRun.getState());
+    benchmarkRun = map.get("module2");
+    Assert.assertEquals(BenchmarkRun.State.FAILED_COMPILE, benchmarkRun.getState());
+
+    Mockito.verify(errorReporter).sendEmail(Mockito.anyString());
+
+    waitFor(new Condition() {
+
+      @Override
+      public boolean condition() {
+        return !manager.isRunning();
+      }
+    }, 200);
+    Assert.assertFalse(manager.isRunning());
+    Assert.assertFalse(manager.isEventLoopAlive());
+  }
+
+  @Test
+  public void testOneFailedRunner() throws BenchmarkManagerException, InterruptedException {
+
+    Mockito.when(timerProvider.get()).thenReturn(timer);
+
+    manager = new BenchmarkManager(collector,
+        benchmarkWorkerFactory,
+        poolProvider,
+        reporterFactory,
+        true,
+        commitReader,
+        timerProvider,
+        errorReporter);
+
+    Mockito.when(commitReader.getLastCommitId()).thenReturn("commit1");
+    Mockito.when(commitReader.getCurrentCommitId()).thenReturn("commit2");
+    Mockito.when(poolProvider.get()).thenReturn(threadPoolExecutor);
+    Mockito.when(collector.get()).thenReturn(Arrays.asList("module1", "module2"));
+
+    ArgumentCaptor<ProgressHandler> progressHandlerCaptor =
+        ArgumentCaptor.forClass(ProgressHandler.class);
+
+    ArgumentCaptor<BenchmarkWorkerConfig> workerConfigCaptor =
+        ArgumentCaptor.forClass(BenchmarkWorkerConfig.class);
+
+    Mockito.when(benchmarkWorkerFactory.create(workerConfigCaptor.capture(),
+        progressHandlerCaptor.capture())).thenReturn(benchmarkWorker);
+
+    ArgumentCaptor<Map<String, BenchmarkRun>> resultCaptor =
+        cast(ArgumentCaptor.forClass(Map.class));
+
+    ArgumentCaptor<ReportProgressHandler> reportProgressHandlerCaptor =
+        ArgumentCaptor.forClass(ReportProgressHandler.class);
+
+    Mockito.when(reporterFactory.create(resultCaptor.capture(), Mockito.anyString(),
+        Mockito.anyString(), reportProgressHandlerCaptor.capture())).thenReturn(benchmarkReporter);
+
+    Assert.assertFalse(manager.isRunning());
+
+    manager.start();
+
+    Assert.assertTrue(manager.isRunning());
+    Assert.assertTrue(manager.isEventLoopAlive());
+
+    ArgumentCaptor<TimerTask> captor = ArgumentCaptor.forClass(TimerTask.class);
+
+    Mockito.verify(timer).scheduleAtFixedRate(captor.capture(), Mockito.anyLong(),
+        Mockito.anyLong());
+
+    TimerTask timerTask = captor.getValue();
+    timerTask.run();
+
+    VerificationWithTimeout timeout = Mockito.timeout(200);
+    Mockito.verify(commitReader, timeout).checkout("commit1");
+    Mockito.verify(commitReader, timeout).maybeCheckoutNextCommit("commit1");
+    Mockito.verify(poolProvider, timeout).get();
+    Mockito.verify(collector, timeout).get();
+    Mockito.verify(threadPoolExecutor, timeout.times(2)).execute(benchmarkWorker);
+
+    List<ProgressHandler> progressHandlers = progressHandlerCaptor.getAllValues();
+    Assert.assertEquals(2, progressHandlers.size());
+    List<BenchmarkWorkerConfig> workerConfig = workerConfigCaptor.getAllValues();
+    Assert.assertEquals(2, workerConfig.size());
+
+    // TODO right now there is only one runner, needs updating
+    // simulate benchmarks done
+    progressHandlers.get(0).onResult(workerConfig.get(0).getRunners().get(0), 1);
+    progressHandlers.get(0).failedToRunBenchmark(workerConfig.get(0).getRunners().get(0),
+        "testerror");
+    progressHandlers.get(0).onRunEnded();
+    progressHandlers.get(1).onResult(workerConfig.get(0).getRunners().get(0), 2);
+    progressHandlers.get(1).onResult(workerConfig.get(1).getRunners().get(0), 23);
+    progressHandlers.get(1).onRunEnded();
+
+    Mockito.verifyZeroInteractions(benchmarkReporter);
+
+    Map<String, BenchmarkRun> map = manager.getLatestRun();
+    Assert.assertEquals(2, map.size());
+
+    Assert.assertTrue(map.containsKey("module1"));
+    Assert.assertTrue(map.containsKey("module2"));
+
+    BenchmarkRun benchmarkRun = map.get("module1");
+    Assert.assertEquals(BenchmarkRun.State.FAILED_TO_RUN_ON_RUNNER, benchmarkRun.getState());
+    benchmarkRun = map.get("module2");
+    Assert.assertEquals(BenchmarkRun.State.DONE, benchmarkRun.getState());
+
+    Mockito.verify(errorReporter).sendEmail(Mockito.anyString());
+
+    waitFor(new Condition() {
+
+      @Override
+      public boolean condition() {
+        return !manager.isRunning();
+      }
+    }, 200);
+    Assert.assertFalse(manager.isRunning());
+    Assert.assertFalse(manager.isEventLoopAlive());
+  }
+
+  @Test
+  public void testMapIsLocked() throws InterruptedException {
+
+    final List<String> list = new ArrayList<>();
+
+    final BenchmarkManagerWithPublicAdd manager = new BenchmarkManagerWithPublicAdd(collector,
+        benchmarkWorkerFactory,
+        poolProvider,
+        reporterFactory,
+        true,
+        commitReader,
+        timerProvider,
+        errorReporter) {
+
+      @Override
+      void addBenchmarkRunSynchronized(Map<String, BenchmarkRun> runsByName, BenchmarkRun br) {
+        try {
+          Thread.sleep(40);
+        } catch (InterruptedException ignored) {
+        }
+        list.add("add");
+      }
+
+      @Override
+      Map<String, BenchmarkRun> getLatestRunSynchronized(Map<String, BenchmarkRun> runsByName) {
+        list.add("get");
+        return null;
+      }
+    };
+
+    Thread thread = new Thread(new Runnable() {
+
+      @Override
+      public void run() {
+        manager.addBenchmarkRun(null);
+      }
+    });
+
+    thread.start();
+
+    Thread.sleep(50);
+    manager.getLatestRun();
+
+    Assert.assertEquals(2, list.size());
+    Assert.assertEquals("add", list.get(0));
+    Assert.assertEquals("get", list.get(1));
+  }
+
+  private void waitFor(Condition c, long timeout) throws InterruptedException {
+    long endMs = System.currentTimeMillis() + timeout;
+    while (!c.condition()) {
+      if (System.currentTimeMillis() > endMs) {
+        throw new RuntimeException("timeout while waiting for condition");
+      }
+      Thread.sleep(5);
+    }
+  }
+}
diff --git a/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkReporterTest.java b/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkReporterTest.java
new file mode 100644
index 0000000..8e12795
--- /dev/null
+++ b/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkReporterTest.java
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import com.google.gwt.benchmark.common.shared.json.BenchmarkResultJson;
+import com.google.gwt.benchmark.common.shared.json.BenchmarkRunJson;
+import com.google.gwt.benchmark.common.shared.json.JsonFactory;
+import com.google.gwt.benchmark.compileserver.server.manager.BenchmarkReporter.HttpURLConnectionFactory;
+import com.google.gwt.benchmark.compileserver.server.manager.BenchmarkReporter.ReportProgressHandler;
+import com.google.web.bindery.autobean.shared.AutoBean;
+import com.google.web.bindery.autobean.shared.AutoBeanCodex;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Test for {@link BenchmarkReporter}.
+ */
+public class BenchmarkReporterTest {
+
+  private BenchmarkReporter reporter;
+  private HashMap<String, BenchmarkRun> results;
+  private HttpURLConnectionFactory urlFactory;
+  private HttpURLConnection urlConnection;
+  private OutputStream outputStream;
+  private String commitId;
+  private String commitDate;
+  private ReportProgressHandler reportProgressHandler;
+
+  @Before
+  public void setup() {
+    commitId = "commitId1";
+    commitDate = "my commit date";
+
+    results = new HashMap<>();
+    BenchmarkRun benchmarkRun = new BenchmarkRun("module1", commitId, commitDate);
+    benchmarkRun.addRunner(RunnerConfigs.CHROME_LINUX);
+    benchmarkRun.addRunner(RunnerConfigs.FIREFOX_LINUX);
+    benchmarkRun.addResult(RunnerConfigs.CHROME_LINUX, 2);
+    benchmarkRun.addResult(RunnerConfigs.FIREFOX_LINUX, 3);
+    results.put("module1", benchmarkRun);
+    BenchmarkRun benchmarkRun1 = new BenchmarkRun("module2", commitId, commitDate);
+    benchmarkRun1.addRunner(RunnerConfigs.CHROME_LINUX);
+    benchmarkRun1.addRunner(RunnerConfigs.FIREFOX_LINUX);
+    benchmarkRun1.addResult(RunnerConfigs.CHROME_LINUX, 4);
+    benchmarkRun1.addResult(RunnerConfigs.FIREFOX_LINUX, 5);
+    results.put("module2", benchmarkRun1);
+
+    urlFactory = Mockito.mock(BenchmarkReporter.HttpURLConnectionFactory.class);
+
+    urlConnection = Mockito.mock(HttpURLConnection.class);
+
+    outputStream = Mockito.mock(OutputStream.class);
+
+    reportProgressHandler = Mockito.mock(ReportProgressHandler.class);
+
+  }
+
+  @Test
+  public void testSuccessFulPostToServer() throws IOException {
+
+    Mockito.when(urlFactory.create()).thenReturn(urlConnection);
+    Mockito.when(urlConnection.getOutputStream()).thenReturn(outputStream);
+    Mockito.when(urlConnection.getResponseCode()).thenReturn(200);
+
+    reporter =
+        new BenchmarkReporter(urlFactory, results, commitId, commitDate, reportProgressHandler);
+
+    reporter.run();
+
+    Mockito.verify(outputStream).close();
+    Mockito.verify(urlConnection).setRequestMethod("PUT");
+    ArgumentCaptor<byte[]> captor = ArgumentCaptor.forClass(byte[].class);
+    Mockito.verify(outputStream).write(captor.capture());
+
+    byte[] jsonBytes = captor.getValue();
+    String jsonString = new String(jsonBytes, "UTF-8");
+
+    AutoBean<BenchmarkRunJson> bean =
+        AutoBeanCodex.decode(JsonFactory.get(), BenchmarkRunJson.class, jsonString);
+    BenchmarkRunJson benchmarkRunJSON = bean.as();
+
+    Assert.assertEquals(commitId, benchmarkRunJSON.getCommitId());
+    Assert.assertEquals(commitDate, benchmarkRunJSON.getCommitTime());
+
+    Map<String, List<BenchmarkResultJson>> resultsJSON =
+        benchmarkRunJSON.getResultByBenchmarkName();
+
+    Assert.assertEquals(2, resultsJSON.size());
+    List<BenchmarkResultJson> module1List = resultsJSON.get("module1");
+    Assert.assertEquals(2, module1List.size());
+    Assert.assertEquals("module1", module1List.get(0).getBenchmarkName());
+    Assert.assertEquals(2, module1List.get(0).getRunsPerMinute(), 0.0001);
+    Assert.assertEquals(RunnerConfigs.CHROME_LINUX.toString(),
+        module1List.get(0).getRunnerId().toString());
+    Assert.assertEquals("module1", module1List.get(1).getBenchmarkName());
+    Assert.assertEquals(3, module1List.get(1).getRunsPerMinute(), 0.0001);
+    Assert.assertEquals(RunnerConfigs.FIREFOX_LINUX.toString(),
+        module1List.get(1).getRunnerId().toString());
+
+    List<BenchmarkResultJson> module1List2 = resultsJSON.get("module2");
+    Assert.assertEquals("module2", module1List2.get(0).getBenchmarkName());
+    Assert.assertEquals(4, module1List2.get(0).getRunsPerMinute(), 0.0001);
+    Assert.assertEquals(RunnerConfigs.CHROME_LINUX.toString(),
+        module1List2.get(0).getRunnerId().toString());
+    Assert.assertEquals("module2", module1List2.get(1).getBenchmarkName());
+    Assert.assertEquals(5, module1List2.get(1).getRunsPerMinute(), 0.0001);
+    Assert.assertEquals(RunnerConfigs.FIREFOX_LINUX.toString(),
+        module1List2.get(1).getRunnerId().toString());
+
+    Mockito.verify(reportProgressHandler).onCommitReported();
+  }
+
+  @Test
+  public void testFailingRetries() throws IOException {
+
+    Mockito.when(urlFactory.create()).thenReturn(urlConnection);
+    Mockito.when(urlConnection.getOutputStream()).thenReturn(outputStream);
+    Mockito.when(urlConnection.getResponseCode()).thenReturn(500);
+
+    final List<Integer> waitingTimes = new ArrayList<>();
+
+    reporter =
+        new BenchmarkReporter(urlFactory, results, commitId, commitDate, reportProgressHandler) {
+
+          @Override
+          boolean sleep(int seconds) {
+            waitingTimes.add(seconds);
+            return true;
+          }
+        };
+
+    reporter.run();
+
+    Mockito.verify(outputStream, Mockito.times(BenchmarkReporter.WAITING_TIME_SECONDS.length))
+        .close();
+    Mockito.verify(urlConnection, Mockito.times(BenchmarkReporter.WAITING_TIME_SECONDS.length))
+        .setRequestMethod("PUT");
+
+    Assert.assertEquals(BenchmarkReporter.WAITING_TIME_SECONDS.length, waitingTimes.size());
+
+    for (int i = 0; i < BenchmarkReporter.WAITING_TIME_SECONDS.length; i++) {
+      Assert.assertEquals(BenchmarkReporter.WAITING_TIME_SECONDS[i],
+          waitingTimes.get(i).intValue());
+    }
+
+    Mockito.verify(reportProgressHandler).onPermanentFailure();
+  }
+}
diff --git a/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkWorkerTest.java b/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkWorkerTest.java
new file mode 100644
index 0000000..7b01ff4
--- /dev/null
+++ b/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/BenchmarkWorkerTest.java
@@ -0,0 +1,222 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import com.google.gwt.benchmark.compileserver.server.manager.BenchmarkWorker.ProgressHandler;
+import com.google.gwt.benchmark.compileserver.server.manager.Runner.Factory;
+import com.google.inject.Provider;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Test for {@link BenchmarkWorker}.
+ */
+public class BenchmarkWorkerTest {
+
+  private String ip;
+  private int port;
+  private RunnerConfig runnerConfig;
+  private BenchmarkCompiler compiler;
+  private Factory runnerProvider;
+  private Runner runner;
+  private String moduleName;
+  private BenchmarkWorker worker;
+  private File benchmarkCompileOutputDir;
+  private ProgressHandler progressHandler;
+  private Provider<String> randomStringProvider;
+  private String moduleTemplate;
+  private BenchmarkWorkerConfig benchmarkData;
+
+  @Before
+  public void setup() {
+    moduleName = "moduleName1";
+
+    benchmarkCompileOutputDir = new File("./target/test/TestBenchmarkWorker/");
+    if (!benchmarkCompileOutputDir.exists()) {
+      if (!benchmarkCompileOutputDir.mkdirs()) {
+        Assert.fail("failed to create dirs");
+      }
+    }
+
+    ip = "127.0.0.1";
+    port = 8080;
+
+    runnerConfig = RunnerConfigs.CHROME_LINUX;
+
+    compiler = Mockito.mock(BenchmarkCompiler.class);
+    runnerProvider = Mockito.mock(Runner.Factory.class);
+
+    runner = Mockito.mock(Runner.class);
+
+    Mockito.when(runnerProvider.create(RunnerConfigs.CHROME_LINUX,
+        "http://" + ip + ":" + port + "/__bench/" + moduleName + ".html")).thenReturn(runner);
+
+    moduleTemplate = "{module_nocache}";
+
+    benchmarkData = new BenchmarkWorkerConfig(moduleName, Arrays.asList(runnerConfig));
+
+    progressHandler = Mockito.mock(ProgressHandler.class);
+
+    randomStringProvider = BenchmarkManagerTest.cast(Mockito.mock(Provider.class));
+
+    worker = new BenchmarkWorker(compiler, runnerProvider, moduleTemplate, benchmarkData,
+        progressHandler, ip, port, benchmarkCompileOutputDir, randomStringProvider);
+  }
+
+  @After
+  public void tearDown() throws IOException {
+    FileUtils.deleteDirectory(benchmarkCompileOutputDir);
+  }
+
+  @Test
+  public void testCompilerError() throws BenchmarkCompilerException {
+
+    BenchmarkCompiler compiler = Mockito.mock(BenchmarkCompiler.class);
+    Factory runnerProvider = Mockito.mock(Runner.Factory.class);
+
+    Mockito.when(randomStringProvider.get()).thenReturn("randomDir1");
+
+    String moduleTemplate = "mytemplate [{module_nocache}]";
+    String moduleName = "moduleName1";
+    BenchmarkWorkerConfig benchmarkData =
+        new BenchmarkWorkerConfig(moduleName, Arrays.asList(RunnerConfigs.CHROME_LINUX));
+
+    ProgressHandler progressHandler = Mockito.mock(ProgressHandler.class);
+    String ip = "127.0.0.1";
+    File benchmarkCompileOutputDir = new File("./target/test/TestBenchmarkWorker");
+    File workDir = new File(benchmarkCompileOutputDir, "randomDir1");
+
+    BenchmarkWorker worker = new BenchmarkWorker(compiler, runnerProvider, moduleTemplate,
+        benchmarkData, progressHandler, ip, 8080, benchmarkCompileOutputDir, randomStringProvider);
+
+    Mockito.doThrow(new BenchmarkCompilerException("test")).when(compiler)
+        .compile(moduleName, workDir);
+
+    worker.run();
+
+    Mockito.verify(compiler).compile(Mockito.eq(moduleName), Mockito.<File>anyObject());
+    Mockito.verify(progressHandler).onCompilationFailed(Mockito.anyString());
+    Mockito.verifyZeroInteractions(runnerProvider);
+    Assert.assertFalse(workDir.exists());
+  }
+
+  @Test
+  public void testBenchmarkWorker() throws BenchmarkCompilerException {
+
+    Mockito.when(runner.isFailed()).thenReturn(false);
+    Mockito.when(runner.getResult()).thenReturn(Double.valueOf(1337));
+    Mockito.when(runner.getConfig()).thenReturn(runnerConfig);
+
+    Mockito.when(randomStringProvider.get()).thenReturn("randomDir1");
+
+    Mockito.when(runnerProvider.create(RunnerConfigs.CHROME_LINUX,
+        "http://" + ip + ":" + port + "/__bench/randomDir1/" + moduleName + ".html")).thenReturn(
+        runner);
+
+    worker.run();
+
+    File workDir = new File(benchmarkCompileOutputDir, "randomDir1");
+
+    Mockito.verify(compiler).compile(moduleName, workDir);
+
+    Mockito.verify(runnerProvider).create(runnerConfig,
+        "http://" + ip + ":" + port + "/__bench/randomDir1/" + moduleName + ".html");
+
+    Mockito.verify(progressHandler).onResult(runnerConfig, 1337);
+    Mockito.verify(progressHandler).onRunEnded();
+
+    Assert.assertFalse(workDir.exists());
+  }
+
+  @Test
+  public void testBenchmarkWorkerWithFailingRuns() throws BenchmarkCompilerException {
+
+    Mockito.when(runner.isFailed()).thenReturn(true);
+    Mockito.when(runner.getResult()).thenReturn(Double.valueOf(1337));
+    Mockito.when(runner.getConfig()).thenReturn(runnerConfig);
+    Mockito.when(randomStringProvider.get()).thenReturn("randomDir1");
+
+    Mockito.when(runnerProvider.create(RunnerConfigs.CHROME_LINUX,
+        "http://" + ip + ":" + port + "/__bench/randomDir1/" + moduleName + ".html")).thenReturn(
+        runner);
+
+    worker.run();
+    File workDir = new File(benchmarkCompileOutputDir, "randomDir1");
+
+    Mockito.verify(compiler).compile(moduleName, workDir);
+
+    Mockito.verify(runnerProvider).create(runnerConfig,
+        "http://" + ip + ":" + port + "/__bench/randomDir1/" + moduleName + ".html");
+
+    Mockito.verify(progressHandler).failedToRunBenchmark(Mockito.eq(runnerConfig),
+        Mockito.anyString());
+    Mockito.verify(progressHandler).onRunEnded();
+
+    Assert.assertFalse(workDir.exists());
+  }
+
+  @Test
+  public void testModuleFileIsBeingWritten() throws BenchmarkCompilerException,
+      FileNotFoundException, IOException {
+
+    worker = new BenchmarkWorker(compiler, runnerProvider, moduleTemplate, benchmarkData,
+        progressHandler, ip, port, benchmarkCompileOutputDir, randomStringProvider) {
+      @Override
+      void cleanupDirectory(File outputDir) {
+        // do nothing so we can see if the directory has the right content
+      }
+    };
+
+    Mockito.when(runner.isFailed()).thenReturn(false);
+    Mockito.when(runner.getResult()).thenReturn(Double.valueOf(1337));
+    Mockito.when(runner.getConfig()).thenReturn(runnerConfig);
+
+    Mockito.when(randomStringProvider.get()).thenReturn("randomDir1");
+
+    Mockito.when(runnerProvider.create(RunnerConfigs.CHROME_LINUX,
+        "http://" + ip + ":" + port + "/__bench/randomDir1/" + moduleName + ".html")).thenReturn(
+        runner);
+
+    worker.run();
+
+    File workDir = new File(benchmarkCompileOutputDir, "randomDir1");
+
+    Mockito.verify(compiler).compile(moduleName, workDir);
+
+    Mockito.verify(runnerProvider).create(runnerConfig,
+        "http://" + ip + ":" + port + "/__bench/randomDir1/" + moduleName + ".html");
+
+    Mockito.verify(progressHandler).onResult(runnerConfig, 1337);
+    Mockito.verify(progressHandler).onRunEnded();
+
+    Assert.assertTrue(workDir.exists());
+
+    String hostPageContent =
+        IOUtils.toString(new FileInputStream(new File(workDir, moduleName + ".html")), "UTF-8");
+
+    Assert.assertEquals(moduleName + "/" + moduleName + ".nocache.js", hostPageContent);
+  }
+}
diff --git a/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/CliInteractorTest.java b/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/CliInteractorTest.java
new file mode 100644
index 0000000..26ed5ca
--- /dev/null
+++ b/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/CliInteractorTest.java
@@ -0,0 +1,223 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import org.apache.commons.io.IOUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+
+/**
+ * Test for {@link CliInteractor}.
+ */
+public class CliInteractorTest {
+
+  private static String getTestOutput() throws IOException {
+    FileInputStream inputStream = null;
+    try {
+      inputStream = new FileInputStream(new File("./target/test-out"));
+      String out = IOUtils.toString(inputStream);
+      // Cut off new line char
+      return out.substring(0, out.length() - 1);
+    } finally {
+      IOUtils.closeQuietly(inputStream);
+    }
+  }
+
+  private File scriptDirectory;
+  private File persistenceDir;
+
+  private File gwtSourceLocation;
+  private File benchmarkSourceLocation;
+  private File compilerOutputDir;
+  private CliInteractor scriptInteractor;
+  private File scriptDirectoryFail;
+
+  @Before
+  public void setup() {
+
+    scriptDirectory = new File("./src/test/resources/scripts-working/");
+    scriptDirectoryFail = new File("./src/test/resources/scripts-fail/");
+    persistenceDir = new File("./target/");
+    gwtSourceLocation = new File("./target/fakesource/");
+    benchmarkSourceLocation = new File("./target/fakebenchmark/");
+    compilerOutputDir = new File("./target/compilerout/");
+
+    scriptInteractor = new CliInteractor(scriptDirectory, persistenceDir, gwtSourceLocation,
+        benchmarkSourceLocation);
+
+  }
+
+  @Test
+  public void testCompileModule() throws BenchmarkCompilerException, IOException {
+    scriptInteractor.compile("myModule1", compilerOutputDir);
+
+    FileInputStream inputStream = null;
+    try {
+      inputStream = new FileInputStream(new File("./target/test-out"));
+      String out = IOUtils.toString(inputStream);
+      // Cut off new line char
+      out = out.substring(0, out.length() - 1);
+
+      String[] split = out.split(";");
+      Assert.assertEquals(5, split.length);
+
+      Assert.assertEquals("myModule1", split[0]);
+      Assert.assertEquals(
+          new File(gwtSourceLocation, "build/staging/gwt-0.0.0/gwt-dev.jar").getAbsolutePath(),
+          new File(split[1]).getAbsolutePath());
+
+      Assert.assertEquals(
+          new File(gwtSourceLocation, "build/staging/gwt-0.0.0/gwt-user.jar").getAbsolutePath(),
+          new File(split[2]).getAbsolutePath());
+
+      Assert.assertEquals(benchmarkSourceLocation.getAbsolutePath(),
+          new File(split[3]).getAbsolutePath());
+      Assert.assertEquals(compilerOutputDir.getAbsolutePath(),
+          new File(split[4]).getAbsolutePath());
+    } finally {
+      IOUtils.closeQuietly(inputStream);
+    }
+  }
+
+  @Test
+  public void testFailingDueToGWTCompilerFail() {
+    scriptInteractor = new CliInteractor(scriptDirectoryFail, persistenceDir, gwtSourceLocation,
+        benchmarkSourceLocation);
+    try {
+      scriptInteractor.compile("myModule1", compilerOutputDir);
+      Assert.fail("Expected exception did not occur");
+    } catch (BenchmarkCompilerException e) {
+      Assert.assertEquals("Command returned with 1 This is my errormessage!\n",
+          e.getCause().getMessage());
+    }
+  }
+
+  @Test
+  public void testGetCurrentCommitId() throws BenchmarkManagerException {
+    String commitId = scriptInteractor.getCurrentCommitId();
+    // Cut off new line char
+    commitId = commitId.substring(0, commitId.length() - 1);
+
+    Assert.assertEquals(gwtSourceLocation.getAbsolutePath(), new File(commitId).getAbsolutePath());
+  }
+
+  @Test
+  public void testGetCurrentCommitIdFailing() {
+    scriptInteractor = new CliInteractor(scriptDirectoryFail, persistenceDir, gwtSourceLocation,
+        benchmarkSourceLocation);
+    try {
+      scriptInteractor.getCurrentCommitId();
+      Assert.fail("Expected exception did not occur");
+    } catch (BenchmarkManagerException e) {
+      Assert.assertEquals("Command returned with 1 commitId: This is my errormessage!\n",
+          e.getMessage());
+    }
+  }
+
+  @Test
+  public void testGetDateForCommit() throws BenchmarkManagerException {
+    String date = scriptInteractor.getDateForCommit("asdf1");
+    // Cut off new line char
+    date = date.substring(0, date.length() - 1);
+
+    String[] split = date.split(";");
+    Assert.assertEquals(2, split.length);
+    Assert.assertEquals(gwtSourceLocation.getAbsolutePath(), split[0]);
+    Assert.assertEquals("asdf1", split[1]);
+  }
+
+  @Test
+  public void testGetDateForCommitFailing() {
+    scriptInteractor = new CliInteractor(scriptDirectoryFail, persistenceDir, gwtSourceLocation,
+        benchmarkSourceLocation);
+    try {
+      scriptInteractor.getDateForCommit("commitId1");
+      Assert.fail("Expected exception did not occur");
+    } catch (BenchmarkManagerException e) {
+      Assert.assertEquals("Command returned with 1 commitDate: This is my errormessage!\n",
+          e.getMessage());
+    }
+  }
+
+  @Test
+  public void testBuildSDK() throws BenchmarkManagerException, IOException {
+    scriptInteractor.buildSDK();
+    Assert.assertEquals(gwtSourceLocation.getAbsolutePath(),
+        new File(getTestOutput()).getAbsolutePath());
+  }
+
+  @Test
+  public void testBuildSDKFailing() {
+    scriptInteractor = new CliInteractor(scriptDirectoryFail, persistenceDir, gwtSourceLocation,
+        benchmarkSourceLocation);
+    try {
+      scriptInteractor.buildSDK();
+      Assert.fail("Expected exception did not occur");
+    } catch (BenchmarkManagerException e) {
+      Assert.assertEquals("Command returned with 1 buildSDK: This is my errormessage!\n",
+          e.getMessage());
+    }
+  }
+
+  @Test
+  public void testCheckout() throws BenchmarkManagerException, IOException {
+    scriptInteractor.checkout("commit12");
+    String[] split = getTestOutput().split(";");
+    Assert.assertEquals(2, split.length);
+    Assert.assertEquals("commit12", split[0]);
+    Assert.assertEquals(gwtSourceLocation.getAbsolutePath(), new File(split[1]).getAbsolutePath());
+  }
+
+  @Test
+  public void testCheckoutFailing() {
+    scriptInteractor = new CliInteractor(scriptDirectoryFail, persistenceDir, gwtSourceLocation,
+        benchmarkSourceLocation);
+    try {
+      scriptInteractor.buildSDK();
+      Assert.fail("Expected exception did not occur");
+    } catch (BenchmarkManagerException e) {
+      Assert.assertEquals("Command returned with 1 buildSDK: This is my errormessage!\n",
+          e.getMessage());
+    }
+  }
+
+  @Test
+  public void testCheckoutNextCommit() throws BenchmarkManagerException, IOException {
+    scriptInteractor.maybeCheckoutNextCommit("baseCommit1");
+
+    String[] split = getTestOutput().split(";");
+    Assert.assertEquals(2, split.length);
+
+    Assert.assertEquals("baseCommit1", split[0]);
+    Assert.assertEquals(gwtSourceLocation.getAbsolutePath(), new File(split[1]).getAbsolutePath());
+  }
+
+  @Test
+  public void testCheckoutNextCommitFailing() {
+    scriptInteractor = new CliInteractor(scriptDirectoryFail, persistenceDir, gwtSourceLocation,
+        benchmarkSourceLocation);
+    try {
+      scriptInteractor.maybeCheckoutNextCommit("doesntmatter");
+      Assert.fail("Expected exception did not occur");
+    } catch (BenchmarkManagerException e) {
+      Assert.assertEquals("Command returned with 1 maybe: This is my errormessage!\n",
+          e.getMessage());
+    }
+  }
+}
diff --git a/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/MailReporterTest.java b/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/MailReporterTest.java
new file mode 100644
index 0000000..342b647
--- /dev/null
+++ b/compileserver/src/test/java/com/google/gwt/benchmark/compileserver/server/manager/MailReporterTest.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2014 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.google.gwt.benchmark.compileserver.server.manager;
+
+import com.google.gwt.benchmark.compileserver.server.manager.MailReporter.MailHelper;
+import com.google.gwt.benchmark.compileserver.server.manager.MailReporter.PasswordAuthenticator;
+import com.google.gwt.benchmark.compileserver.server.runners.settings.MailSettings;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import javax.mail.Message.RecipientType;
+import javax.mail.MessagingException;
+import javax.mail.internet.MimeMessage;
+
+/**
+ * Test for {@link MailReporter}.
+ */
+public class MailReporterTest {
+
+
+  private MailSettings mailSettings;
+  private MailHelper mailHelper;
+  private MailReporter errorReporter;
+
+  @Before
+  public void setup() {
+    mailSettings = new MailSettings("from1", "to1", "host1", "username1", "password1");
+    mailHelper = Mockito.mock(MailHelper.class);
+    errorReporter = new MailReporter(mailHelper, mailSettings);
+  }
+
+  @Test
+  public void testSendEmail() throws MessagingException, IOException {
+
+    errorReporter.sendEmail("my cool message");
+
+    ArgumentCaptor<Properties> captor = ArgumentCaptor.forClass(Properties.class);
+    ArgumentCaptor<PasswordAuthenticator> authenticatorCaptor =
+        ArgumentCaptor.forClass(PasswordAuthenticator.class);
+
+    Mockito.verify(mailHelper).create(captor.capture(), authenticatorCaptor.capture());
+    Properties properties = captor.getValue();
+
+    Assert.assertEquals(mailSettings.getHost(), properties.get("mail.smtp.host"));
+    Assert.assertEquals("465", properties.get("mail.smtp.port"));
+    Assert.assertEquals("465", properties.get("mail.smtp.socketFactory.port"));
+    Assert.assertEquals("javax.net.ssl.SSLSocketFactory",
+        properties.get("mail.smtp.socketFactory.class"));
+    Assert.assertEquals("false", properties.get("mail.smtp.socketFactory.fallback"));
+    Assert.assertEquals("true", properties.get("mail.smtp.auth"));
+
+    PasswordAuthenticator authenticator = authenticatorCaptor.getValue();
+    Assert.assertEquals(mailSettings.getUsername(),
+        authenticator.getPasswordAuthentication().getUserName());
+    Assert.assertEquals(mailSettings.getPassword(),
+        authenticator.getPasswordAuthentication().getPassword());
+
+    ArgumentCaptor<MimeMessage> messageCaptor = ArgumentCaptor.forClass(MimeMessage.class);
+
+    Mockito.verify(mailHelper).send(messageCaptor.capture());
+    MimeMessage message = messageCaptor.getValue();
+
+    Assert.assertEquals(mailSettings.getFrom(), message.getFrom()[0].toString());
+    Assert.assertEquals(mailSettings.getTo(),
+        message.getRecipients(RecipientType.TO)[0].toString());
+    Assert.assertEquals("Error in the benchmarking system that needs attention",
+        message.getSubject());
+    Assert.assertEquals("my cool message", message.getContent());
+  }
+
+  @Test
+  public void testFailedToSendMail() throws MessagingException {
+    Mockito.doThrow(new MessagingException()).when(mailHelper)
+        .send(Mockito.<MimeMessage>anyObject());
+
+    errorReporter.sendEmail("my cool message");
+  }
+}
diff --git a/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/ShouldNotBeIncludedBenchmark.gwt.xml b/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/ShouldNotBeIncludedBenchmark.gwt.xml
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/ShouldNotBeIncludedBenchmark.gwt.xml
diff --git a/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/benchmarks/IgnoredTest1.gwt.xml b/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/benchmarks/IgnoredTest1.gwt.xml
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/benchmarks/IgnoredTest1.gwt.xml
diff --git a/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/benchmarks/TestBenchmark.gwt.xml b/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/benchmarks/TestBenchmark.gwt.xml
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/benchmarks/TestBenchmark.gwt.xml
diff --git a/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/benchmarks/otherfileBenchmark.txt b/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/benchmarks/otherfileBenchmark.txt
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/benchmarks/otherfileBenchmark.txt
diff --git a/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/benchmarks/package/IgnoredTest2.gwt.xml b/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/benchmarks/package/IgnoredTest2.gwt.xml
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/benchmarks/package/IgnoredTest2.gwt.xml
diff --git a/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/benchmarks/package/TestBenchmark2.gwt.xml b/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/benchmarks/package/TestBenchmark2.gwt.xml
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/compileserver/src/test/resources/collector-test/com/google/gwt/benchmark/benchmarks/package/TestBenchmark2.gwt.xml
diff --git a/compileserver/src/test/resources/scripts-fail/buildSDK b/compileserver/src/test/resources/scripts-fail/buildSDK
new file mode 100755
index 0000000..d56f2bd
--- /dev/null
+++ b/compileserver/src/test/resources/scripts-fail/buildSDK
@@ -0,0 +1,2 @@
+echo "buildSDK: This is my errormessage!" >&2
+exit 1;
\ No newline at end of file
diff --git a/compileserver/src/test/resources/scripts-fail/checkout b/compileserver/src/test/resources/scripts-fail/checkout
new file mode 100755
index 0000000..71aea3c
--- /dev/null
+++ b/compileserver/src/test/resources/scripts-fail/checkout
@@ -0,0 +1,2 @@
+echo "checkout: This is my errormessage!" >&2
+exit 1;
\ No newline at end of file
diff --git a/compileserver/src/test/resources/scripts-fail/commitDate b/compileserver/src/test/resources/scripts-fail/commitDate
new file mode 100755
index 0000000..8970899
--- /dev/null
+++ b/compileserver/src/test/resources/scripts-fail/commitDate
@@ -0,0 +1,2 @@
+echo "commitDate: This is my errormessage!" >&2
+exit 1;
\ No newline at end of file
diff --git a/compileserver/src/test/resources/scripts-fail/commitId b/compileserver/src/test/resources/scripts-fail/commitId
new file mode 100755
index 0000000..b75442c
--- /dev/null
+++ b/compileserver/src/test/resources/scripts-fail/commitId
@@ -0,0 +1,2 @@
+echo "commitId: This is my errormessage!" >&2
+exit 1;
\ No newline at end of file
diff --git a/compileserver/src/test/resources/scripts-fail/compileModule b/compileserver/src/test/resources/scripts-fail/compileModule
new file mode 100755
index 0000000..385fb23
--- /dev/null
+++ b/compileserver/src/test/resources/scripts-fail/compileModule
@@ -0,0 +1,2 @@
+echo "This is my errormessage!"
+exit 1;
\ No newline at end of file
diff --git a/compileserver/src/test/resources/scripts-fail/maybe_checkout_next_commit b/compileserver/src/test/resources/scripts-fail/maybe_checkout_next_commit
new file mode 100755
index 0000000..9c3536b
--- /dev/null
+++ b/compileserver/src/test/resources/scripts-fail/maybe_checkout_next_commit
@@ -0,0 +1,2 @@
+echo "maybe: This is my errormessage!" >&2
+exit 1;
\ No newline at end of file
diff --git a/compileserver/src/test/resources/scripts-working/buildSDK b/compileserver/src/test/resources/scripts-working/buildSDK
new file mode 100755
index 0000000..e936e61
--- /dev/null
+++ b/compileserver/src/test/resources/scripts-working/buildSDK
@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+if [[ -z "${1}" ]]; then
+  echo "usage: buildSDK sourceLocation" >&2
+  exit 1
+fi
+
+SOURCE_LOCATION=${1}
+echo "${SOURCE_LOCATION}" > target/test-out
\ No newline at end of file
diff --git a/compileserver/src/test/resources/scripts-working/checkout b/compileserver/src/test/resources/scripts-working/checkout
new file mode 100755
index 0000000..e5f30d1
--- /dev/null
+++ b/compileserver/src/test/resources/scripts-working/checkout
@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+if [[ -z "${1}" && -z "${2}" ]]; then
+  echo "usage: checkout commitId sourceLocation" >&2
+  exit 1
+fi
+
+COMMIT_ID=${1}
+SOURCE_LOCATION=${2}
+echo "${COMMIT_ID};${SOURCE_LOCATION}" > target/test-out
\ No newline at end of file
diff --git a/compileserver/src/test/resources/scripts-working/commitDate b/compileserver/src/test/resources/scripts-working/commitDate
new file mode 100755
index 0000000..87042bf
--- /dev/null
+++ b/compileserver/src/test/resources/scripts-working/commitDate
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+
+if [[ -z "${1}" && -z "${2}" ]]; then
+  echo "usage: commitDate sourceLocation commitId" >&2
+  exit 1
+fi
+
+
+SOURCE_LOCATION=${1}
+COMMIT_ID=${2}
+echo "${SOURCE_LOCATION};${COMMIT_ID}"
\ No newline at end of file
diff --git a/compileserver/src/test/resources/scripts-working/commitId b/compileserver/src/test/resources/scripts-working/commitId
new file mode 100755
index 0000000..8abf007
--- /dev/null
+++ b/compileserver/src/test/resources/scripts-working/commitId
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+if [[ -z "${1}" ]]; then
+  echo "usage: commitId sourceLocation" >&2
+  exit 1
+fi
+
+SOURCE_LOCATION=${1}
+echo "${SOURCE_LOCATION}"
\ No newline at end of file
diff --git a/compileserver/src/test/resources/scripts-working/compileModule b/compileserver/src/test/resources/scripts-working/compileModule
new file mode 100755
index 0000000..c722a21
--- /dev/null
+++ b/compileserver/src/test/resources/scripts-working/compileModule
@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+if [[ -z "${1}" && -z "${2}" && -z "${3}" && -z "${4}" && -z "${5}" ]]; then
+  echo "usage: compileModule moduleName dev.jar user.jar benchmark_src output_dir"
+  exit 1
+fi
+
+
+MODULE_NAME=${1}
+GWT_DEV_JAR=${2}
+GWT_USER_JAR=${3}
+BENCHMARKS_SRC=${4}
+OUTPUT_DIR=${5}
+
+#Echo back all the parameters so that we can verify them in test
+echo "${MODULE_NAME};${GWT_DEV_JAR};${GWT_USER_JAR};${BENCHMARKS_SRC};${OUTPUT_DIR}" > target/test-out
diff --git a/compileserver/src/test/resources/scripts-working/maybe_checkout_next_commit b/compileserver/src/test/resources/scripts-working/maybe_checkout_next_commit
new file mode 100755
index 0000000..1bb978b
--- /dev/null
+++ b/compileserver/src/test/resources/scripts-working/maybe_checkout_next_commit
@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+
+if [[ -z "${1}" && -z "${2}" ]]; then
+  echo "usage: maybe_checkout_next_commit parentCommitId sourceLocation" >&2
+  exit 1
+fi
+
+PARENT_COMMIT=${1}
+SOURCE_LOCATION=${2}
+
+echo "${PARENT_COMMIT};${SOURCE_LOCATION}" > target/test-out
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 40eded3..cab7126 100644
--- a/pom.xml
+++ b/pom.xml
@@ -12,5 +12,7 @@
 
     <modules>
         <module>benchmarks</module>
+        <module>common</module>
+        <module>compileserver</module>
     </modules>
 </project>