IO improvements to DiskCache and UnifiedAST. - Renamed "writeTo" to "transferToStream" to avoid the confusion that it actually does the opposite of the other "write" methods, and added its inverse "transferFromStream". - Factored out moveToEndPosition() which fixes a bug where atEnd would *never* be true, forcing an unnecessary seek on every write. - Updated some call sites to take advantage of transferFromStream(), which is more efficient than buffering the whole data stream into a byte buffer. - Updated UnifiedAST to take advantage of DiskCache and the newer serialization techniques, which should reduce memory usage. Review by: spoon git-svn-id: https://google-web-toolkit.googlecode.com/svn/trunk@6511 8db76d5a-ed1c-0410-87a9-c151d255dfc7
diff --git a/dev/core/src/com/google/gwt/core/ext/linker/SyntheticArtifact.java b/dev/core/src/com/google/gwt/core/ext/linker/SyntheticArtifact.java index d65a214..561f04a 100644 --- a/dev/core/src/com/google/gwt/core/ext/linker/SyntheticArtifact.java +++ b/dev/core/src/com/google/gwt/core/ext/linker/SyntheticArtifact.java
@@ -19,10 +19,8 @@ import com.google.gwt.core.ext.TreeLogger; import com.google.gwt.core.ext.UnableToCompleteException; import com.google.gwt.dev.util.DiskCache; -import com.google.gwt.dev.util.Util; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; @@ -65,19 +63,17 @@ @Override public void writeTo(TreeLogger logger, OutputStream out) throws UnableToCompleteException { - diskCache.writeTo(token, out); + diskCache.transferToStream(token, out); } private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException { stream.defaultReadObject(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - Util.copyNoClose(stream, baos); - token = diskCache.writeByteArray(baos.toByteArray()); + token = diskCache.transferFromStream(stream); } private void writeObject(ObjectOutputStream stream) throws IOException { stream.defaultWriteObject(); - diskCache.writeTo(token, stream); + diskCache.transferToStream(token, stream); } }
diff --git a/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardGeneratedResource.java b/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardGeneratedResource.java index 5283ed6..e10e51a 100644 --- a/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardGeneratedResource.java +++ b/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardGeneratedResource.java
@@ -20,11 +20,11 @@ import com.google.gwt.core.ext.UnableToCompleteException; import com.google.gwt.core.ext.linker.GeneratedResource; import com.google.gwt.dev.util.DiskCache; -import com.google.gwt.dev.util.Util; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; @@ -45,7 +45,12 @@ String partialPath, File file) { super(StandardLinkerContext.class, generatorType, partialPath); this.lastModified = file.lastModified(); - this.token = diskCache.writeByteArray(Util.readFileAsBytes(file)); + try { + this.token = diskCache.transferFromStream(new FileInputStream(file)); + } catch (FileNotFoundException e) { + throw new RuntimeException("Unable to open file '" + + file.getAbsolutePath() + "'", e); + } } @Override @@ -62,19 +67,17 @@ @Override public void writeTo(TreeLogger logger, OutputStream out) throws UnableToCompleteException { - diskCache.writeTo(token, out); + diskCache.transferToStream(token, out); } private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException { stream.defaultReadObject(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - Util.copyNoClose(stream, baos); - token = diskCache.writeByteArray(baos.toByteArray()); + token = diskCache.transferFromStream(stream); } private void writeObject(ObjectOutputStream stream) throws IOException { stream.defaultWriteObject(); - diskCache.writeTo(token, stream); + diskCache.transferToStream(token, stream); } }
diff --git a/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java b/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java index 8f3bfb4..79deb6d 100644 --- a/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java +++ b/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java
@@ -20,10 +20,9 @@ import com.google.gwt.dev.Permutation; import com.google.gwt.dev.jjs.ast.JProgram; import com.google.gwt.dev.js.ast.JsProgram; -import com.google.gwt.dev.util.PerfLogger; +import com.google.gwt.dev.util.DiskCache; +import com.google.gwt.dev.util.Util; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; @@ -42,9 +41,9 @@ /** * Encapsulates the combined programs. */ - static final class AST { - private JProgram jProgram; - private JsProgram jsProgram; + static final class AST implements Serializable { + private final JProgram jProgram; + private final JsProgram jsProgram; public AST(JProgram jProgram, JsProgram jsProgram) { this.jProgram = jProgram; @@ -60,42 +59,7 @@ } } - private static AST deserializeAst(byte[] serializedAst) { - try { - PerfLogger.start("deserialize"); - ByteArrayInputStream bais = new ByteArrayInputStream(serializedAst); - ObjectInputStream is; - is = new ObjectInputStream(bais); - JProgram jprogram = (JProgram) is.readObject(); - JsProgram jsProgram = (JsProgram) is.readObject(); - return new AST(jprogram, jsProgram); - } catch (IOException e) { - throw new RuntimeException( - "Should be impossible for memory based streams", e); - } catch (ClassNotFoundException e) { - throw new RuntimeException( - "Should be impossible when deserializing in process", e); - } finally { - PerfLogger.end(); - } - } - - private static byte[] serializeAst(AST ast) { - try { - PerfLogger.start("serialize"); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ObjectOutputStream os = new ObjectOutputStream(baos); - os.writeObject(ast.getJProgram()); - os.writeObject(ast.getJsProgram()); - os.close(); - return baos.toByteArray(); - } catch (IOException e) { - throw new RuntimeException( - "Should be impossible for memory based streams", e); - } finally { - PerfLogger.end(); - } - } + private static final DiskCache diskCache = new DiskCache(); /** * The original AST; nulled out once consumed (by the first call to @@ -121,7 +85,7 @@ /** * The serialized AST. */ - private byte[] serializedAst; + private transient long serializedAstToken; public UnifiedAst(JJSOptions options, AST initialAst, boolean singlePermutation, Set<String> rebindRequests) { @@ -129,7 +93,8 @@ this.initialAst = initialAst; this.rebindRequests = Collections.unmodifiableSortedSet(new TreeSet<String>( rebindRequests)); - this.serializedAst = singlePermutation ? null : serializeAst(initialAst); + this.serializedAstToken = singlePermutation ? -1 + : diskCache.writeObject(initialAst); } /** @@ -140,7 +105,7 @@ this.initialAst = other.initialAst; other.initialAst = null; // steal its copy this.rebindRequests = other.rebindRequests; - this.serializedAst = other.serializedAst; + this.serializedAstToken = other.serializedAstToken; } /** @@ -179,7 +144,7 @@ public void prepare() { synchronized (myLockObject) { if (initialAst == null) { - initialAst = deserializeAst(serializedAst); + initialAst = diskCache.readObject(serializedAstToken, AST.class); } } } @@ -191,36 +156,39 @@ initialAst = null; return result; } else { - if (serializedAst == null) { + if (serializedAstToken < 0) { throw new IllegalStateException( "No serialized AST was cached and AST was already consumed."); } - return deserializeAst(serializedAst); + return diskCache.readObject(serializedAstToken, AST.class); } } } /** - * Re-initialize lock object. + * Re-initialize lock object; copy serialized AST straight to cache. */ - private Object readResolve() { + private void readObject(ObjectInputStream stream) throws IOException, + ClassNotFoundException { + stream.defaultReadObject(); myLockObject = new Object(); - return this; + serializedAstToken = diskCache.transferFromStream(stream); } /** * Force byte serialization of AST before writing. */ - private Object writeReplace() { - if (serializedAst == null) { - synchronized (myLockObject) { - if (initialAst == null) { - throw new IllegalStateException( - "No serialized AST was cached and AST was already consumed."); - } - serializedAst = serializeAst(initialAst); - } + private void writeObject(ObjectOutputStream stream) throws IOException { + stream.defaultWriteObject(); + if (serializedAstToken >= 0) { + // Copy the bytes. + diskCache.transferToStream(serializedAstToken, stream); + } else if (initialAst != null) { + // Serialize into raw bytes. + Util.writeObjectToStream(stream, stream); + } else { + throw new IllegalStateException( + "No serialized AST was cached and AST was already consumed."); } - return this; } }
diff --git a/dev/core/src/com/google/gwt/dev/util/DiskCache.java b/dev/core/src/com/google/gwt/dev/util/DiskCache.java index 250d08f..0b3bdfd 100644 --- a/dev/core/src/com/google/gwt/dev/util/DiskCache.java +++ b/dev/core/src/com/google/gwt/dev/util/DiskCache.java
@@ -19,6 +19,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.io.RandomAccessFile; import java.lang.ref.WeakReference; @@ -125,16 +126,75 @@ } /** + * Write the rest of the data in an input stream to disk. + * + * @return a handle to retrieve it later + */ + public synchronized long transferFromStream(InputStream in) { + byte[] buf = Util.takeThreadLocalBuf(); + try { + long position = moveToEndPosition(); + + // Placeholder, we don't know the length yet. + file.writeInt(-1); + + // Transfer all the bytes. + int length = 0; + int bytesRead; + while ((bytesRead = in.read(buf)) != -1) { + file.write(buf, 0, bytesRead); + length += bytesRead; + } + + // Now go back and fill in the length. + file.seek(position); + file.writeInt(length); + // Don't eagerly seek the end, the next operation might be a read. + atEnd = false; + return position; + } catch (IOException e) { + throw new RuntimeException("Unable to read from byte cache", e); + } finally { + Util.releaseThreadLocalBuf(buf); + } + } + + /** + * Reads bytes of data back from disk and writes them into the specified + * output stream. + */ + public synchronized void transferToStream(long token, OutputStream out) { + byte[] buf = Util.takeThreadLocalBuf(); + try { + atEnd = false; + file.seek(token); + int length = file.readInt(); + int bufLen = buf.length; + while (length > bufLen) { + int read = file.read(buf, 0, bufLen); + length -= read; + out.write(buf, 0, read); + } + while (length > 0) { + int read = file.read(buf, 0, length); + length -= read; + out.write(buf, 0, read); + } + } catch (IOException e) { + throw new RuntimeException("Unable to read from byte cache", e); + } finally { + Util.releaseThreadLocalBuf(buf); + } + } + + /** * Write a byte array to disk. * * @return a handle to retrieve it later */ public synchronized long writeByteArray(byte[] bytes) { try { - if (!atEnd) { - file.seek(file.length()); - } - long position = file.getFilePointer(); + long position = moveToEndPosition(); file.writeInt(bytes.length); file.write(bytes); return position; @@ -163,34 +223,6 @@ return writeByteArray(Util.getBytes(str)); } - /** - * Reads bytes of data back from disk and writes them into the specified - * output stream. - */ - public synchronized void writeTo(long token, OutputStream out) { - byte[] buf = Util.takeThreadLocalBuf(); - try { - atEnd = false; - file.seek(token); - int length = file.readInt(); - int bufLen = buf.length; - while (length > bufLen) { - int read = file.read(buf, 0, bufLen); - length -= read; - out.write(buf, 0, read); - } - while (length > 0) { - int read = file.read(buf, 0, length); - length -= read; - out.write(buf, 0, read); - } - } catch (IOException e) { - throw new RuntimeException("Unable to read from byte cache", e); - } finally { - Util.releaseThreadLocalBuf(buf); - } - } - @Override protected synchronized void finalize() throws Throwable { close(); @@ -203,4 +235,23 @@ file = null; } } + + /** + * Moves to the end of the file if necessary and returns the offset position. + * Caller must synchronize. + * + * @return the offset position of the end of the file + * @throws IOException + */ + private long moveToEndPosition() throws IOException { + // Get an end pointer. + if (atEnd) { + return file.getFilePointer(); + } else { + long position = file.length(); + file.seek(position); + atEnd = true; + return position; + } + } } \ No newline at end of file