IO improvements to DiskCache and UnifiedAST.
- Renamed "writeTo" to "transferToStream" to avoid the confusion that it actually does the opposite of the other "write" methods, and added its inverse "transferFromStream".
- Factored out moveToEndPosition() which fixes a bug where atEnd would *never* be true, forcing an unnecessary seek on every write.
- Updated some call sites to take advantage of transferFromStream(), which is more efficient than buffering the whole data stream into a byte buffer.
- Updated UnifiedAST to take advantage of DiskCache and the newer serialization techniques, which should reduce memory usage.
Review by: spoon
git-svn-id: https://google-web-toolkit.googlecode.com/svn/trunk@6511 8db76d5a-ed1c-0410-87a9-c151d255dfc7
diff --git a/dev/core/src/com/google/gwt/core/ext/linker/SyntheticArtifact.java b/dev/core/src/com/google/gwt/core/ext/linker/SyntheticArtifact.java
index d65a214..561f04a 100644
--- a/dev/core/src/com/google/gwt/core/ext/linker/SyntheticArtifact.java
+++ b/dev/core/src/com/google/gwt/core/ext/linker/SyntheticArtifact.java
@@ -19,10 +19,8 @@
import com.google.gwt.core.ext.TreeLogger;
import com.google.gwt.core.ext.UnableToCompleteException;
import com.google.gwt.dev.util.DiskCache;
-import com.google.gwt.dev.util.Util;
import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
@@ -65,19 +63,17 @@
@Override
public void writeTo(TreeLogger logger, OutputStream out)
throws UnableToCompleteException {
- diskCache.writeTo(token, out);
+ diskCache.transferToStream(token, out);
}
private void readObject(ObjectInputStream stream) throws IOException,
ClassNotFoundException {
stream.defaultReadObject();
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- Util.copyNoClose(stream, baos);
- token = diskCache.writeByteArray(baos.toByteArray());
+ token = diskCache.transferFromStream(stream);
}
private void writeObject(ObjectOutputStream stream) throws IOException {
stream.defaultWriteObject();
- diskCache.writeTo(token, stream);
+ diskCache.transferToStream(token, stream);
}
}
diff --git a/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardGeneratedResource.java b/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardGeneratedResource.java
index 5283ed6..e10e51a 100644
--- a/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardGeneratedResource.java
+++ b/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardGeneratedResource.java
@@ -20,11 +20,11 @@
import com.google.gwt.core.ext.UnableToCompleteException;
import com.google.gwt.core.ext.linker.GeneratedResource;
import com.google.gwt.dev.util.DiskCache;
-import com.google.gwt.dev.util.Util;
import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
@@ -45,7 +45,12 @@
String partialPath, File file) {
super(StandardLinkerContext.class, generatorType, partialPath);
this.lastModified = file.lastModified();
- this.token = diskCache.writeByteArray(Util.readFileAsBytes(file));
+ try {
+ this.token = diskCache.transferFromStream(new FileInputStream(file));
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException("Unable to open file '"
+ + file.getAbsolutePath() + "'", e);
+ }
}
@Override
@@ -62,19 +67,17 @@
@Override
public void writeTo(TreeLogger logger, OutputStream out)
throws UnableToCompleteException {
- diskCache.writeTo(token, out);
+ diskCache.transferToStream(token, out);
}
private void readObject(ObjectInputStream stream) throws IOException,
ClassNotFoundException {
stream.defaultReadObject();
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- Util.copyNoClose(stream, baos);
- token = diskCache.writeByteArray(baos.toByteArray());
+ token = diskCache.transferFromStream(stream);
}
private void writeObject(ObjectOutputStream stream) throws IOException {
stream.defaultWriteObject();
- diskCache.writeTo(token, stream);
+ diskCache.transferToStream(token, stream);
}
}
diff --git a/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java b/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java
index 8f3bfb4..79deb6d 100644
--- a/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java
+++ b/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java
@@ -20,10 +20,9 @@
import com.google.gwt.dev.Permutation;
import com.google.gwt.dev.jjs.ast.JProgram;
import com.google.gwt.dev.js.ast.JsProgram;
-import com.google.gwt.dev.util.PerfLogger;
+import com.google.gwt.dev.util.DiskCache;
+import com.google.gwt.dev.util.Util;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
@@ -42,9 +41,9 @@
/**
* Encapsulates the combined programs.
*/
- static final class AST {
- private JProgram jProgram;
- private JsProgram jsProgram;
+ static final class AST implements Serializable {
+ private final JProgram jProgram;
+ private final JsProgram jsProgram;
public AST(JProgram jProgram, JsProgram jsProgram) {
this.jProgram = jProgram;
@@ -60,42 +59,7 @@
}
}
- private static AST deserializeAst(byte[] serializedAst) {
- try {
- PerfLogger.start("deserialize");
- ByteArrayInputStream bais = new ByteArrayInputStream(serializedAst);
- ObjectInputStream is;
- is = new ObjectInputStream(bais);
- JProgram jprogram = (JProgram) is.readObject();
- JsProgram jsProgram = (JsProgram) is.readObject();
- return new AST(jprogram, jsProgram);
- } catch (IOException e) {
- throw new RuntimeException(
- "Should be impossible for memory based streams", e);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException(
- "Should be impossible when deserializing in process", e);
- } finally {
- PerfLogger.end();
- }
- }
-
- private static byte[] serializeAst(AST ast) {
- try {
- PerfLogger.start("serialize");
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- ObjectOutputStream os = new ObjectOutputStream(baos);
- os.writeObject(ast.getJProgram());
- os.writeObject(ast.getJsProgram());
- os.close();
- return baos.toByteArray();
- } catch (IOException e) {
- throw new RuntimeException(
- "Should be impossible for memory based streams", e);
- } finally {
- PerfLogger.end();
- }
- }
+ private static final DiskCache diskCache = new DiskCache();
/**
* The original AST; nulled out once consumed (by the first call to
@@ -121,7 +85,7 @@
/**
* The serialized AST.
*/
- private byte[] serializedAst;
+ private transient long serializedAstToken;
public UnifiedAst(JJSOptions options, AST initialAst,
boolean singlePermutation, Set<String> rebindRequests) {
@@ -129,7 +93,8 @@
this.initialAst = initialAst;
this.rebindRequests = Collections.unmodifiableSortedSet(new TreeSet<String>(
rebindRequests));
- this.serializedAst = singlePermutation ? null : serializeAst(initialAst);
+ this.serializedAstToken = singlePermutation ? -1
+ : diskCache.writeObject(initialAst);
}
/**
@@ -140,7 +105,7 @@
this.initialAst = other.initialAst;
other.initialAst = null; // steal its copy
this.rebindRequests = other.rebindRequests;
- this.serializedAst = other.serializedAst;
+ this.serializedAstToken = other.serializedAstToken;
}
/**
@@ -179,7 +144,7 @@
public void prepare() {
synchronized (myLockObject) {
if (initialAst == null) {
- initialAst = deserializeAst(serializedAst);
+ initialAst = diskCache.readObject(serializedAstToken, AST.class);
}
}
}
@@ -191,36 +156,39 @@
initialAst = null;
return result;
} else {
- if (serializedAst == null) {
+ if (serializedAstToken < 0) {
throw new IllegalStateException(
"No serialized AST was cached and AST was already consumed.");
}
- return deserializeAst(serializedAst);
+ return diskCache.readObject(serializedAstToken, AST.class);
}
}
}
/**
- * Re-initialize lock object.
+ * Re-initialize lock object; copy serialized AST straight to cache.
*/
- private Object readResolve() {
+ private void readObject(ObjectInputStream stream) throws IOException,
+ ClassNotFoundException {
+ stream.defaultReadObject();
myLockObject = new Object();
- return this;
+ serializedAstToken = diskCache.transferFromStream(stream);
}
/**
* Force byte serialization of AST before writing.
*/
- private Object writeReplace() {
- if (serializedAst == null) {
- synchronized (myLockObject) {
- if (initialAst == null) {
- throw new IllegalStateException(
- "No serialized AST was cached and AST was already consumed.");
- }
- serializedAst = serializeAst(initialAst);
- }
+ private void writeObject(ObjectOutputStream stream) throws IOException {
+ stream.defaultWriteObject();
+ if (serializedAstToken >= 0) {
+ // Copy the bytes.
+ diskCache.transferToStream(serializedAstToken, stream);
+ } else if (initialAst != null) {
+ // Serialize into raw bytes.
+ Util.writeObjectToStream(stream, stream);
+ } else {
+ throw new IllegalStateException(
+ "No serialized AST was cached and AST was already consumed.");
}
- return this;
}
}
diff --git a/dev/core/src/com/google/gwt/dev/util/DiskCache.java b/dev/core/src/com/google/gwt/dev/util/DiskCache.java
index 250d08f..0b3bdfd 100644
--- a/dev/core/src/com/google/gwt/dev/util/DiskCache.java
+++ b/dev/core/src/com/google/gwt/dev/util/DiskCache.java
@@ -19,6 +19,7 @@
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.lang.ref.WeakReference;
@@ -125,16 +126,75 @@
}
/**
+ * Write the rest of the data in an input stream to disk.
+ *
+ * @return a handle to retrieve it later
+ */
+ public synchronized long transferFromStream(InputStream in) {
+ byte[] buf = Util.takeThreadLocalBuf();
+ try {
+ long position = moveToEndPosition();
+
+ // Placeholder, we don't know the length yet.
+ file.writeInt(-1);
+
+ // Transfer all the bytes.
+ int length = 0;
+ int bytesRead;
+ while ((bytesRead = in.read(buf)) != -1) {
+ file.write(buf, 0, bytesRead);
+ length += bytesRead;
+ }
+
+ // Now go back and fill in the length.
+ file.seek(position);
+ file.writeInt(length);
+ // Don't eagerly seek the end, the next operation might be a read.
+ atEnd = false;
+ return position;
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to read from byte cache", e);
+ } finally {
+ Util.releaseThreadLocalBuf(buf);
+ }
+ }
+
+ /**
+ * Reads bytes of data back from disk and writes them into the specified
+ * output stream.
+ */
+ public synchronized void transferToStream(long token, OutputStream out) {
+ byte[] buf = Util.takeThreadLocalBuf();
+ try {
+ atEnd = false;
+ file.seek(token);
+ int length = file.readInt();
+ int bufLen = buf.length;
+ while (length > bufLen) {
+ int read = file.read(buf, 0, bufLen);
+ length -= read;
+ out.write(buf, 0, read);
+ }
+ while (length > 0) {
+ int read = file.read(buf, 0, length);
+ length -= read;
+ out.write(buf, 0, read);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to read from byte cache", e);
+ } finally {
+ Util.releaseThreadLocalBuf(buf);
+ }
+ }
+
+ /**
* Write a byte array to disk.
*
* @return a handle to retrieve it later
*/
public synchronized long writeByteArray(byte[] bytes) {
try {
- if (!atEnd) {
- file.seek(file.length());
- }
- long position = file.getFilePointer();
+ long position = moveToEndPosition();
file.writeInt(bytes.length);
file.write(bytes);
return position;
@@ -163,34 +223,6 @@
return writeByteArray(Util.getBytes(str));
}
- /**
- * Reads bytes of data back from disk and writes them into the specified
- * output stream.
- */
- public synchronized void writeTo(long token, OutputStream out) {
- byte[] buf = Util.takeThreadLocalBuf();
- try {
- atEnd = false;
- file.seek(token);
- int length = file.readInt();
- int bufLen = buf.length;
- while (length > bufLen) {
- int read = file.read(buf, 0, bufLen);
- length -= read;
- out.write(buf, 0, read);
- }
- while (length > 0) {
- int read = file.read(buf, 0, length);
- length -= read;
- out.write(buf, 0, read);
- }
- } catch (IOException e) {
- throw new RuntimeException("Unable to read from byte cache", e);
- } finally {
- Util.releaseThreadLocalBuf(buf);
- }
- }
-
@Override
protected synchronized void finalize() throws Throwable {
close();
@@ -203,4 +235,23 @@
file = null;
}
}
+
+ /**
+ * Moves to the end of the file if necessary and returns the offset position.
+ * Caller must synchronize.
+ *
+ * @return the offset position of the end of the file
+ * @throws IOException
+ */
+ private long moveToEndPosition() throws IOException {
+ // Get an end pointer.
+ if (atEnd) {
+ return file.getFilePointer();
+ } else {
+ long position = file.length();
+ file.seek(position);
+ atEnd = true;
+ return position;
+ }
+ }
}
\ No newline at end of file
