IO improvements to DiskCache and UnifiedAST.

- Renamed "writeTo" to "transferToStream" to avoid the confusion that it actually does the opposite of the other "write" methods, and added its inverse "transferFromStream".

- Factored out moveToEndPosition() which fixes a bug where atEnd would *never* be true, forcing an unnecessary seek on every write.

- Updated some call sites to take advantage of transferFromStream(), which is more efficient than buffering the whole data stream into a byte buffer.

- Updated UnifiedAST to take advantage of DiskCache and the newer serialization techniques, which should reduce memory usage.

Review by: spoon



git-svn-id: https://google-web-toolkit.googlecode.com/svn/trunk@6511 8db76d5a-ed1c-0410-87a9-c151d255dfc7
diff --git a/dev/core/src/com/google/gwt/core/ext/linker/SyntheticArtifact.java b/dev/core/src/com/google/gwt/core/ext/linker/SyntheticArtifact.java
index d65a214..561f04a 100644
--- a/dev/core/src/com/google/gwt/core/ext/linker/SyntheticArtifact.java
+++ b/dev/core/src/com/google/gwt/core/ext/linker/SyntheticArtifact.java
@@ -19,10 +19,8 @@
 import com.google.gwt.core.ext.TreeLogger;
 import com.google.gwt.core.ext.UnableToCompleteException;
 import com.google.gwt.dev.util.DiskCache;
-import com.google.gwt.dev.util.Util;
 
 import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.ObjectInputStream;
@@ -65,19 +63,17 @@
   @Override
   public void writeTo(TreeLogger logger, OutputStream out)
       throws UnableToCompleteException {
-    diskCache.writeTo(token, out);
+    diskCache.transferToStream(token, out);
   }
 
   private void readObject(ObjectInputStream stream) throws IOException,
       ClassNotFoundException {
     stream.defaultReadObject();
-    ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    Util.copyNoClose(stream, baos);
-    token = diskCache.writeByteArray(baos.toByteArray());
+    token = diskCache.transferFromStream(stream);
   }
 
   private void writeObject(ObjectOutputStream stream) throws IOException {
     stream.defaultWriteObject();
-    diskCache.writeTo(token, stream);
+    diskCache.transferToStream(token, stream);
   }
 }
diff --git a/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardGeneratedResource.java b/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardGeneratedResource.java
index 5283ed6..e10e51a 100644
--- a/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardGeneratedResource.java
+++ b/dev/core/src/com/google/gwt/core/ext/linker/impl/StandardGeneratedResource.java
@@ -20,11 +20,11 @@
 import com.google.gwt.core.ext.UnableToCompleteException;
 import com.google.gwt.core.ext.linker.GeneratedResource;
 import com.google.gwt.dev.util.DiskCache;
-import com.google.gwt.dev.util.Util;
 
 import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.ObjectInputStream;
@@ -45,7 +45,12 @@
       String partialPath, File file) {
     super(StandardLinkerContext.class, generatorType, partialPath);
     this.lastModified = file.lastModified();
-    this.token = diskCache.writeByteArray(Util.readFileAsBytes(file));
+    try {
+      this.token = diskCache.transferFromStream(new FileInputStream(file));
+    } catch (FileNotFoundException e) {
+      throw new RuntimeException("Unable to open file '"
+          + file.getAbsolutePath() + "'", e);
+    }
   }
 
   @Override
@@ -62,19 +67,17 @@
   @Override
   public void writeTo(TreeLogger logger, OutputStream out)
       throws UnableToCompleteException {
-    diskCache.writeTo(token, out);
+    diskCache.transferToStream(token, out);
   }
 
   private void readObject(ObjectInputStream stream) throws IOException,
       ClassNotFoundException {
     stream.defaultReadObject();
-    ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    Util.copyNoClose(stream, baos);
-    token = diskCache.writeByteArray(baos.toByteArray());
+    token = diskCache.transferFromStream(stream);
   }
 
   private void writeObject(ObjectOutputStream stream) throws IOException {
     stream.defaultWriteObject();
-    diskCache.writeTo(token, stream);
+    diskCache.transferToStream(token, stream);
   }
 }
diff --git a/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java b/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java
index 8f3bfb4..79deb6d 100644
--- a/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java
+++ b/dev/core/src/com/google/gwt/dev/jjs/UnifiedAst.java
@@ -20,10 +20,9 @@
 import com.google.gwt.dev.Permutation;
 import com.google.gwt.dev.jjs.ast.JProgram;
 import com.google.gwt.dev.js.ast.JsProgram;
-import com.google.gwt.dev.util.PerfLogger;
+import com.google.gwt.dev.util.DiskCache;
+import com.google.gwt.dev.util.Util;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
@@ -42,9 +41,9 @@
   /**
    * Encapsulates the combined programs.
    */
-  static final class AST {
-    private JProgram jProgram;
-    private JsProgram jsProgram;
+  static final class AST implements Serializable {
+    private final JProgram jProgram;
+    private final JsProgram jsProgram;
 
     public AST(JProgram jProgram, JsProgram jsProgram) {
       this.jProgram = jProgram;
@@ -60,42 +59,7 @@
     }
   }
 
-  private static AST deserializeAst(byte[] serializedAst) {
-    try {
-      PerfLogger.start("deserialize");
-      ByteArrayInputStream bais = new ByteArrayInputStream(serializedAst);
-      ObjectInputStream is;
-      is = new ObjectInputStream(bais);
-      JProgram jprogram = (JProgram) is.readObject();
-      JsProgram jsProgram = (JsProgram) is.readObject();
-      return new AST(jprogram, jsProgram);
-    } catch (IOException e) {
-      throw new RuntimeException(
-          "Should be impossible for memory based streams", e);
-    } catch (ClassNotFoundException e) {
-      throw new RuntimeException(
-          "Should be impossible when deserializing in process", e);
-    } finally {
-      PerfLogger.end();
-    }
-  }
-
-  private static byte[] serializeAst(AST ast) {
-    try {
-      PerfLogger.start("serialize");
-      ByteArrayOutputStream baos = new ByteArrayOutputStream();
-      ObjectOutputStream os = new ObjectOutputStream(baos);
-      os.writeObject(ast.getJProgram());
-      os.writeObject(ast.getJsProgram());
-      os.close();
-      return baos.toByteArray();
-    } catch (IOException e) {
-      throw new RuntimeException(
-          "Should be impossible for memory based streams", e);
-    } finally {
-      PerfLogger.end();
-    }
-  }
+  private static final DiskCache diskCache = new DiskCache();
 
   /**
    * The original AST; nulled out once consumed (by the first call to
@@ -121,7 +85,7 @@
   /**
    * The serialized AST.
    */
-  private byte[] serializedAst;
+  private transient long serializedAstToken;
 
   public UnifiedAst(JJSOptions options, AST initialAst,
       boolean singlePermutation, Set<String> rebindRequests) {
@@ -129,7 +93,8 @@
     this.initialAst = initialAst;
     this.rebindRequests = Collections.unmodifiableSortedSet(new TreeSet<String>(
         rebindRequests));
-    this.serializedAst = singlePermutation ? null : serializeAst(initialAst);
+    this.serializedAstToken = singlePermutation ? -1
+        : diskCache.writeObject(initialAst);
   }
 
   /**
@@ -140,7 +105,7 @@
     this.initialAst = other.initialAst;
     other.initialAst = null; // steal its copy
     this.rebindRequests = other.rebindRequests;
-    this.serializedAst = other.serializedAst;
+    this.serializedAstToken = other.serializedAstToken;
   }
 
   /**
@@ -179,7 +144,7 @@
   public void prepare() {
     synchronized (myLockObject) {
       if (initialAst == null) {
-        initialAst = deserializeAst(serializedAst);
+        initialAst = diskCache.readObject(serializedAstToken, AST.class);
       }
     }
   }
@@ -191,36 +156,39 @@
         initialAst = null;
         return result;
       } else {
-        if (serializedAst == null) {
+        if (serializedAstToken < 0) {
           throw new IllegalStateException(
               "No serialized AST was cached and AST was already consumed.");
         }
-        return deserializeAst(serializedAst);
+        return diskCache.readObject(serializedAstToken, AST.class);
       }
     }
   }
 
   /**
-   * Re-initialize lock object.
+   * Re-initialize lock object; copy serialized AST straight to cache.
    */
-  private Object readResolve() {
+  private void readObject(ObjectInputStream stream) throws IOException,
+      ClassNotFoundException {
+    stream.defaultReadObject();
     myLockObject = new Object();
-    return this;
+    serializedAstToken = diskCache.transferFromStream(stream);
   }
 
   /**
    * Force byte serialization of AST before writing.
    */
-  private Object writeReplace() {
-    if (serializedAst == null) {
-      synchronized (myLockObject) {
-        if (initialAst == null) {
-          throw new IllegalStateException(
-              "No serialized AST was cached and AST was already consumed.");
-        }
-        serializedAst = serializeAst(initialAst);
-      }
+  private void writeObject(ObjectOutputStream stream) throws IOException {
+    stream.defaultWriteObject();
+    if (serializedAstToken >= 0) {
+      // Copy the bytes.
+      diskCache.transferToStream(serializedAstToken, stream);
+    } else if (initialAst != null) {
+      // Serialize into raw bytes.
+      Util.writeObjectToStream(stream, stream);
+    } else {
+      throw new IllegalStateException(
+          "No serialized AST was cached and AST was already consumed.");
     }
-    return this;
   }
 }
diff --git a/dev/core/src/com/google/gwt/dev/util/DiskCache.java b/dev/core/src/com/google/gwt/dev/util/DiskCache.java
index 250d08f..0b3bdfd 100644
--- a/dev/core/src/com/google/gwt/dev/util/DiskCache.java
+++ b/dev/core/src/com/google/gwt/dev/util/DiskCache.java
@@ -19,6 +19,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.RandomAccessFile;
 import java.lang.ref.WeakReference;
@@ -125,16 +126,75 @@
   }
 
   /**
+   * Write the rest of the data in an input stream to disk.
+   * 
+   * @return a handle to retrieve it later
+   */
+  public synchronized long transferFromStream(InputStream in) {
+    byte[] buf = Util.takeThreadLocalBuf();
+    try {
+      long position = moveToEndPosition();
+
+      // Placeholder, we don't know the length yet.
+      file.writeInt(-1);
+
+      // Transfer all the bytes.
+      int length = 0;
+      int bytesRead;
+      while ((bytesRead = in.read(buf)) != -1) {
+        file.write(buf, 0, bytesRead);
+        length += bytesRead;
+      }
+
+      // Now go back and fill in the length.
+      file.seek(position);
+      file.writeInt(length);
+      // Don't eagerly seek the end, the next operation might be a read.
+      atEnd = false;
+      return position;
+    } catch (IOException e) {
+      throw new RuntimeException("Unable to read from byte cache", e);
+    } finally {
+      Util.releaseThreadLocalBuf(buf);
+    }
+  }
+
+  /**
+   * Reads bytes of data back from disk and writes them into the specified
+   * output stream.
+   */
+  public synchronized void transferToStream(long token, OutputStream out) {
+    byte[] buf = Util.takeThreadLocalBuf();
+    try {
+      atEnd = false;
+      file.seek(token);
+      int length = file.readInt();
+      int bufLen = buf.length;
+      while (length > bufLen) {
+        int read = file.read(buf, 0, bufLen);
+        length -= read;
+        out.write(buf, 0, read);
+      }
+      while (length > 0) {
+        int read = file.read(buf, 0, length);
+        length -= read;
+        out.write(buf, 0, read);
+      }
+    } catch (IOException e) {
+      throw new RuntimeException("Unable to read from byte cache", e);
+    } finally {
+      Util.releaseThreadLocalBuf(buf);
+    }
+  }
+
+  /**
    * Write a byte array to disk.
    * 
    * @return a handle to retrieve it later
    */
   public synchronized long writeByteArray(byte[] bytes) {
     try {
-      if (!atEnd) {
-        file.seek(file.length());
-      }
-      long position = file.getFilePointer();
+      long position = moveToEndPosition();
       file.writeInt(bytes.length);
       file.write(bytes);
       return position;
@@ -163,34 +223,6 @@
     return writeByteArray(Util.getBytes(str));
   }
 
-  /**
-   * Reads bytes of data back from disk and writes them into the specified
-   * output stream.
-   */
-  public synchronized void writeTo(long token, OutputStream out) {
-    byte[] buf = Util.takeThreadLocalBuf();
-    try {
-      atEnd = false;
-      file.seek(token);
-      int length = file.readInt();
-      int bufLen = buf.length;
-      while (length > bufLen) {
-        int read = file.read(buf, 0, bufLen);
-        length -= read;
-        out.write(buf, 0, read);
-      }
-      while (length > 0) {
-        int read = file.read(buf, 0, length);
-        length -= read;
-        out.write(buf, 0, read);
-      }
-    } catch (IOException e) {
-      throw new RuntimeException("Unable to read from byte cache", e);
-    } finally {
-      Util.releaseThreadLocalBuf(buf);
-    }
-  }
-
   @Override
   protected synchronized void finalize() throws Throwable {
     close();
@@ -203,4 +235,23 @@
       file = null;
     }
   }
+
+  /**
+   * Moves to the end of the file if necessary and returns the offset position.
+   * Caller must synchronize.
+   * 
+   * @return the offset position of the end of the file
+   * @throws IOException
+   */
+  private long moveToEndPosition() throws IOException {
+    // Get an end pointer.
+    if (atEnd) {
+      return file.getFilePointer();
+    } else {
+      long position = file.length();
+      file.seek(position);
+      atEnd = true;
+      return position;
+    }
+  }
 }
\ No newline at end of file