dev/core/src/com/google/gwt/dev/javac/PersistentUnitCache.java - gwt - Git at Google

 /*
  * Copyright 2011 Google Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not
  * use this file except in compliance with the License. You may obtain a copy of
  * the License at
  *
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  * License for the specific language governing permissions and limitations under
  * the License.
  */
 package com.google.gwt.dev.javac;

 import com.google.gwt.core.ext.TreeLogger;
 import com.google.gwt.core.ext.UnableToCompleteException;
 import com.google.gwt.dev.jjs.InternalCompilerException;
 import com.google.gwt.dev.jjs.ast.JNode;
 import com.google.gwt.dev.jjs.impl.GwtAstBuilder;
 import com.google.gwt.dev.util.log.speedtracer.DevModeEventType;
 import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger;
 import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger.Event;
 import com.google.gwt.util.tools.Utility;
 import com.google.gwt.thirdparty.guava.common.hash.Hashing;
 import com.google.gwt.thirdparty.guava.common.io.Files;

 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
 import java.io.EOFException;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.net.JarURLConnection;
 import java.net.URLConnection;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicBoolean;

 /**
  * A class that manages a persistent cache of {@link CompilationUnit} instances.
  * Writes out {@CompilationUnit} instances to a cache in a
  * background thread.
  * <p>
  * The persistent cache is implemented as a directory of log files with a date
  * timestamp. A new log file gets created each time a new PersistentUnitCache is
  * instantiated, (once per invocation of the compiler or DevMode). The design is
  * intended to support only a single PersistentUnitCache instance in the
  * compiler at a time.
  * <p>
  * As new units are compiled, the cache data is appended to a log. This allows
  * Java serialization to efficiently store references. The next time the cache
  * is started, all logs are replayed and loaded into the cache in chronological
  * order, with newer units taking precedence. A new cache file is created for
  * any newly compiled units in this session. After a threshold of a certain
  * number of files in the directory is reached
  * {@link PersistentUnitCache#CACHE_FILE_THRESHOLD} , the cache files are
  * consolidated back into a single file.
  *
  * <p>
  * System Properties (see {@link UnitCacheFactory}).
  *
  * <ul>
  * <li>gwt.persistentunitcache : enables the persistent cache (eventually will
  * be default)</li>
  * <li>gwt.persistentunitcachedir=<dir>: sets or overrides the cache directory</li>
  * </ul>
  *
  * <p>
  * Known Issues:
  *
  * <ul>
  * <li>This design uses an eager cache to load every unit in the cache on the
  * first reference to find() or add(). When the cache is large (10000 units), it
  * uses lots of heap and takes 5-10 seconds. Once the PersistentUnitCache is
  * created, it starts eagerly loading the cache in a background thread).</li>
  *
  * <li>Although units logged to disk with the same resource path are eventually
  * cleaned up, the most recently compiled unit stays in the cache forever. This
  * means that stale units that are no longer referenced will never be purged,
  * unless by some external action (e.g. ant clean).</li>
  *
  * <li>Unless ant builds are made aware of the cache directory, the cache will
  * persist if a user does an ant clean.</li>
  * </ul>
  *
  */
 class PersistentUnitCache extends MemoryUnitCache {

   /**
    * If there are more than this many files in the cache, clean up the old
    * files.
    */
   static final int CACHE_FILE_THRESHOLD = 40;

   /**
    * Common prefix for creating directories and cache files.
    */
   static final String UNIT_CACHE_PREFIX = "gwt-unitCache";
   static final String CACHE_FILE_PREFIX = UNIT_CACHE_PREFIX + "-";
   static final String CURRENT_VERSION_CACHE_FILE_PREFIX = computePersistentCacheFilenamePrefix();

   /**
    * Creates a new file with a name based on the current system time.
    */
   private static File createCacheFile(TreeLogger logger, File cacheDirectory)
       throws UnableToCompleteException {
     File newFile = null;
     long timestamp = System.currentTimeMillis();
     try {
       do {
         newFile = new File(cacheDirectory, CURRENT_VERSION_CACHE_FILE_PREFIX +
             String.format("%016X", timestamp++));
       } while (!newFile.createNewFile());
     } catch (IOException ex) {
       logger.log(TreeLogger.WARN, "Unable to create new cache log file "
           + (newFile == null ? "<not created>" : newFile.getAbsolutePath()) + ".", ex);
       throw new UnableToCompleteException();
     }

     if (!newFile.canWrite()) {
       logger.log(TreeLogger.WARN, "Unable to write to new cache log file "
           + newFile.getAbsolutePath() + ".");
       throw new UnableToCompleteException();
     }

     return newFile;
   }

   /**
    * Finds all files matching a pattern in the cache directory.
    *
    * @return an array of sorted filenames. The file name pattern is such that
    *         sorting them alphabetically also sorts the files by age.
    */
   private static File[] getCacheFiles(File cacheDirectory, boolean currentVersion) {
     if (cacheDirectory.isDirectory()) {
       File[] files = cacheDirectory.listFiles();
       List<File> cacheFiles = new ArrayList<File>();
       for (File file : files) {
         if (file.getName().startsWith(currentVersion ? CURRENT_VERSION_CACHE_FILE_PREFIX :
             CACHE_FILE_PREFIX)) {
           cacheFiles.add(file);
         }
       }
       File[] retFiles = cacheFiles.toArray(new File[cacheFiles.size()]);
       Arrays.sort(retFiles);
       return retFiles;
     }
     return new File[0];
   }

   /**
    * There is no significance in the return value, we just want to be able
    * to tell if the purgeOldCacheFilesTask has completed.
    */
   private Future<Boolean> purgeTaskStatus;
   private AtomicBoolean purgeInProgress = new AtomicBoolean(false);

   private final Runnable purgeOldCacheFilesTask = new Runnable() {
     @Override
     public void run() {
       try {
         // Delete all cache files in the directory except for the currently open
         // file.
         SpeedTracerLogger.Event deleteEvent = SpeedTracerLogger.start(DevModeEventType.DELETE_CACHE);
         File[] filesToDelete = getCacheFiles(cacheDirectory, false);
         logger.log(TreeLogger.TRACE, "Purging cache files from " + cacheDirectory);
         for (File toDelete : filesToDelete) {
           if (!currentCacheFile.equals(toDelete)) {
             if (!toDelete.delete()) {
               logger.log(TreeLogger.WARN, "Couldn't delete file: " + toDelete);
             }
           }
         }
         deleteEvent.end();

         rotateCurrentCacheFile();
       } catch (UnableToCompleteException e) {
         backgroundService.shutdownNow();
       } finally {
         purgeInProgress.set(false);
       }
     }
   };

   private final Runnable rotateCacheFilesTask = new Runnable() {
     @Override
     public void run() {
       try {
         rotateCurrentCacheFile();
       } catch (UnableToCompleteException e) {
         backgroundService.shutdownNow();
       }
       assert (currentCacheFile != null);
     }
   };

   private final Runnable shutdownThreadTask = new Runnable() {
     @Override
     public void run() {
       assert (currentCacheFile != null);
       closeCurrentCacheFile(currentCacheFile, currentCacheFileStream);
       logger.log(TreeLogger.TRACE, "Shutting down PersistentUnitCache thread");
       backgroundService.shutdownNow();
     }
   };

   /**
    * Saved to be able to wait for UNIT_MAP_LOAD_TASK to complete.
    */
   private Future<Boolean> unitMapLoadStatus;

   private final Runnable unitMapLoadTask = new Runnable() {
     @Override
     public void run() {
       loadUnitMap(logger, currentCacheFile);
     }
   };

   /**
    * Used to execute the above Runnables in a background thread.
    */
   private final ExecutorService backgroundService;

   private int unitsWritten = 0;

   private int addedSinceLastCleanup = 0;

   /**
    * A directory to store the cache files that should persist between
    * invocations.
    */
   private final File cacheDirectory;

   /**
    * Current file and stream being written to.
    */
   private File currentCacheFile;
   private ObjectOutputStream currentCacheFileStream;

   private final TreeLogger logger;

   PersistentUnitCache(final TreeLogger logger, File cacheDir) throws UnableToCompleteException {
     assert cacheDir != null;
     this.logger = logger;

     /*
      * We must canonicalize the path here, otherwise we might set cacheDirectory
      * to something like "/path/to/x/../gwt-unitCache". If this were to happen,
      * the mkdirs() call below would create "/path/to/gwt-unitCache" but
      * not "/path/to/x".
      * Further accesses via the uncanonicalized path will fail, if "/path/to/x"
      * had not been created by other means.
      *
      * Fixes issue 6443
      */
     try {
       this.cacheDirectory = new File(cacheDir.getCanonicalFile(), UNIT_CACHE_PREFIX);
     } catch (IOException ex) {
       logger.log(TreeLogger.WARN, "Unable to create canonical file of "
           + cacheDir.getAbsolutePath() + ".", ex);
       throw new UnableToCompleteException();
     }

     if (logger.isLoggable(TreeLogger.TRACE)) {
       logger.log(TreeLogger.TRACE, "Persistent unit cache dir set to: "
           + this.cacheDirectory.getAbsolutePath());
     }

     if (!cacheDirectory.isDirectory() && !cacheDirectory.mkdirs()) {
       logger.log(TreeLogger.WARN, "Unable to initialize cache. Couldn't create directory "
           + cacheDirectory.getAbsolutePath() + ".");
       throw new UnableToCompleteException();
     }

     currentCacheFile = createCacheFile(logger, cacheDirectory);

     backgroundService = Executors.newSingleThreadExecutor();
     Runtime.getRuntime().addShutdownHook(new Thread() {
       @Override
       public void run() {
         try {
           Future<Boolean> status = backgroundService.submit(shutdownThreadTask, Boolean.TRUE);
           // Don't let the shutdown hang more than 5 seconds
           status.get(5, TimeUnit.SECONDS);
         } catch (InterruptedException e) {
           // ignore
         } catch (RejectedExecutionException e) {
           // already shutdown, ignore
         } catch (ExecutionException e) {
           logger.log(TreeLogger.ERROR, "Error during shutdown", e);
         } catch (TimeoutException e) {
           // ignore
         } finally {
           backgroundService.shutdownNow();
         }
       }
     });

     /**
      * Load up cached units from the persistent store in the background. The
      * {@link #add(CompilationUnit)} and {@link #find(String)} methods block if
      * invoked before this thread finishes.
      */
     unitMapLoadStatus = backgroundService.submit(unitMapLoadTask, Boolean.TRUE);

     FileOutputStream fstream = null;
     BufferedOutputStream bstream = null;

     try {
       fstream = new FileOutputStream(currentCacheFile);
       bstream = new BufferedOutputStream(fstream);
       currentCacheFileStream = new ObjectOutputStream(bstream);
     } catch (IOException ex) {
       closeCurrentCacheFile(currentCacheFile, currentCacheFileStream);
       logger.log(TreeLogger.ERROR, "Error creating cache " + currentCacheFile
           + ". Disabling cache.", ex);
       backgroundService.shutdownNow();
       throw new UnableToCompleteException();
     }
   }

   /**
    * Enqueue a unit to be written by the background thread.
    */
   @Override
   public void add(CompilationUnit newUnit) {
     awaitUnitCacheMapLoad();
     addedSinceLastCleanup++;
     super.add(newUnit);
     addImpl(unitMap.get(newUnit.getResourcePath()));
   }

   /**
    * Cleans up old cache files in the directory, migrating everything previously
    * loaded in them to the current cache file.
    *
    * Normally, only newly compiled units are written to the current log, but
    * when it is time to cleanup, valid units from older log files need to be
    * re-written.
    */
   @Override
   public void cleanup(TreeLogger logger) {
     awaitUnitCacheMapLoad();

     if (backgroundService.isShutdown()) {
       return;
     }
     boolean shouldRotate = addedSinceLastCleanup > 0;
     logger.log(TreeLogger.TRACE, "Added " + addedSinceLastCleanup +
         " units to cache since last cleanup.");
     addedSinceLastCleanup = 0;
     try {
       File[] cacheFiles = getCacheFiles(cacheDirectory, true);
       if (cacheFiles.length < CACHE_FILE_THRESHOLD) {
         if (shouldRotate) {
           backgroundService.execute(rotateCacheFilesTask);
         }
         return;
       }

       // Check to see if the previous purge task finished.
       boolean inProgress = purgeInProgress.getAndSet(true);
       if (inProgress) {
         try {
           purgeTaskStatus.get(0, TimeUnit.NANOSECONDS);
         } catch (InterruptedException ex) {
           Thread.currentThread().interrupt();
         } catch (TimeoutException ex) {
           // purge is currently in progress.
           return;
         }
       }

       /*
        * Resend all units read in from the in-memory cache to the background
        * thread. They will be re-written out and the old cache files removed.
        */
       synchronized (unitMap) {
         for (UnitCacheEntry unitCacheEntry : unitMap.values()) {
           if (unitCacheEntry.getOrigin() == UnitOrigin.PERSISTENT) {
             addImpl(unitCacheEntry);
           }
         }
       }

       purgeTaskStatus = backgroundService.submit(purgeOldCacheFilesTask, Boolean.TRUE);

     } catch (ExecutionException ex) {
       throw new InternalCompilerException("Error purging cache", ex);
     } catch (RejectedExecutionException ex) {
       // Cache background thread is not running - ignore
     }
   }

   @Override
   public CompilationUnit find(ContentId contentId) {
     awaitUnitCacheMapLoad();
     return super.find(contentId);
   }

   @Override
   public CompilationUnit find(String resourcePath) {
     awaitUnitCacheMapLoad();
     return super.find(resourcePath);
   }

   public void rotateCurrentCacheFile() throws UnableToCompleteException {
     if (logger.isLoggable(TreeLogger.TRACE)) {
       logger.log(TreeLogger.TRACE, "Wrote " + unitsWritten + " units to persistent cache.");
     }

     // Close and re-open a new log file to drop object references kept
     // alive in the existing file by Java serialization.
     closeCurrentCacheFile(currentCacheFile, currentCacheFileStream);
     unitsWritten = 0;
     currentCacheFile = createCacheFile(logger, cacheDirectory);
     FileOutputStream fstream = null;
     BufferedOutputStream bstream = null;
     try {
       fstream = new FileOutputStream(currentCacheFile);
       bstream = new BufferedOutputStream(fstream);
       currentCacheFileStream = new ObjectOutputStream(bstream);
     } catch (IOException ex) {
       // Close all 3 streams, not sure where the exception occurred.
       Utility.close(bstream);
       Utility.close(fstream);
       closeCurrentCacheFile(currentCacheFile, currentCacheFileStream);
       logger.log(TreeLogger.ERROR, "Error rotating file.  Shutting down cache thread.", ex);
       throw new UnableToCompleteException();
     }
   }

   /**
    * For Unit testing - shutdown the persistent cache.
    *
    * @throws ExecutionException
    * @throws InterruptedException
    */
   void shutdown() throws InterruptedException, ExecutionException {
     try {
       Future<Runnable> future = backgroundService.submit(shutdownThreadTask, shutdownThreadTask);
       backgroundService.shutdown();
       future.get();
     } catch (RejectedExecutionException ex) {
       // background thread is not running - ignore
     }
   }

   private static String computePersistentCacheFilenamePrefix() {
     try {
       URLConnection urlConnection =
           JNode.class.getResource("JNode.class").openConnection();
       if (urlConnection instanceof JarURLConnection) {
         String gwtdevJar = ((JarURLConnection) urlConnection).getJarFile().getName();
         return CACHE_FILE_PREFIX +
             Files.hash(new File(gwtdevJar), Hashing.sha1()).toString() + "-";
       }
       System.err.println("Could not find the GWT compiler jarfile. "
           + "Serialization errors might occur when accessing the persistent unit cache.");
     } catch (IOException e) {
       System.err.println("Could not compute the hash for the GWT compiler jarfile."
           + "Serialization errors might occur when accessing the persistent unit cache.");
       e.printStackTrace();
     }
     return CACHE_FILE_PREFIX;
   }

   private void addImpl(final UnitCacheEntry entry) {
     try {
       backgroundService.execute(new Runnable() {
         @Override
         public void run() {
           try {
             assert entry.getOrigin() != UnitOrigin.ARCHIVE;
             CompilationUnit unit = entry.getUnit();
             assert unit != null;
             currentCacheFileStream.writeObject(unit);
             unitsWritten++;
           } catch (IOException ex) {
             backgroundService.shutdownNow();
             if (logger.isLoggable(TreeLogger.TRACE)) {
               logger.log(TreeLogger.TRACE, "Error saving unit to cache in: "
                   + cacheDirectory.getAbsolutePath(), ex);
             }
           }
         }
       });
     } catch (RejectedExecutionException ex) {
       // background thread is not running, ignore
     }
   }

   private synchronized void awaitUnitCacheMapLoad() {
     // wait on initial load of unit map to complete.
     try {
       if (unitMapLoadStatus != null) {
         unitMapLoadStatus.get();
         // no need to check any more.
         unitMapLoadStatus = null;
       }
     } catch (InterruptedException e) {
       throw new InternalCompilerException("Interrupted waiting for unit cache map to load.", e);
     } catch (ExecutionException e) {
       logger.log(TreeLogger.ERROR, "Failure in unit cache map load.", e);
       // keep going
       unitMapLoadStatus = null;
     }
   }

   private void closeCurrentCacheFile(File openFile, ObjectOutputStream stream) {
     Utility.close(stream);
     if (unitsWritten == 0) {
       // Remove useless empty file.
       openFile.delete();
     }
   }

   /**
    * Load everything cached on disk into memory.
    */
   private void loadUnitMap(TreeLogger logger, File currentCacheFile) {
     Event loadPersistentUnitEvent =
         SpeedTracerLogger.start(DevModeEventType.LOAD_PERSISTENT_UNIT_CACHE);
     if (logger.isLoggable(TreeLogger.TRACE)) {
       logger.log(TreeLogger.TRACE, "Looking for previously cached Compilation Units in "
           + cacheDirectory.getAbsolutePath());
     }
     try {
       if (cacheDirectory.isDirectory() && cacheDirectory.canRead()) {
         File[] files = getCacheFiles(cacheDirectory, true);
         for (File cacheFile : files) {
           FileInputStream fis = null;
           BufferedInputStream bis = null;
           ObjectInputStream inputStream = null;
           if (cacheFile.equals(currentCacheFile)) {
             continue;
           }
           boolean deleteCacheFile = false;
           try {
             fis = new FileInputStream(cacheFile);
             bis = new BufferedInputStream(fis);
             /*
              * It is possible for the next call to throw an exception, leaving
              * inputStream null and fis still live.
              */
             inputStream = new ObjectInputStream(bis);
             while (true) {
               CachedCompilationUnit unit = (CachedCompilationUnit) inputStream.readObject();
               if (unit == null) {
                 break;
               }
               if (unit.getTypesSerializedVersion() != GwtAstBuilder.getSerializationVersion()) {
                 continue;
               }
               UnitCacheEntry entry = new UnitCacheEntry(unit, UnitOrigin.PERSISTENT);
               UnitCacheEntry existingEntry = unitMap.get(unit.getResourcePath());
               /*
                * Don't assume that an existing entry is stale - an entry might
                * have been loaded already from another source like a
                * CompilationUnitArchive that is more up to date. If the
                * timestamps are the same, accept the latest version. If it turns
                * out to be stale, it will be recompiled and the updated unit
                * will win this test the next time the session starts.
                */
               if (existingEntry != null
                   && unit.getLastModified() >= existingEntry.getUnit().getLastModified()) {
                 super.remove(existingEntry.getUnit());
                 unitMap.put(unit.getResourcePath(), entry);
                 unitMapByContentId.put(unit.getContentId(), entry);
               } else if (existingEntry == null) {
                 unitMap.put(unit.getResourcePath(), entry);
                 unitMapByContentId.put(unit.getContentId(), entry);
               }
             }
           } catch (EOFException ex) {
             // Go on to the next file.
           } catch (IOException ex) {
             deleteCacheFile = true;
             if (logger.isLoggable(TreeLogger.TRACE)) {
               logger.log(TreeLogger.TRACE, "Ignoring and deleting cache log "
                   + cacheFile.getAbsolutePath() + " due to read error.", ex);
             }
           } catch (ClassNotFoundException ex) {
             deleteCacheFile = true;
             if (logger.isLoggable(TreeLogger.TRACE)) {
               logger.log(TreeLogger.TRACE, "Ignoring and deleting cache log "
                   + cacheFile.getAbsolutePath() + " due to deserialization error.", ex);
             }
           } finally {
             Utility.close(inputStream);
             Utility.close(bis);
             Utility.close(fis);
           }
           if (deleteCacheFile) {
             cacheFile.delete();
           } else {
             if (logger.isLoggable(TreeLogger.TRACE)) {
               logger.log(TreeLogger.TRACE, cacheFile.getName() + ": Load complete");
             }
           }
         }
       } else {
         logger
             .log(TreeLogger.TRACE,
                 "Starting with empty Cache: CompilationUnit cache "
                     + "directory does not exist or is not readable.");
       }
     } finally {
       loadPersistentUnitEvent.end();
     }
   }
 }
	/*
	* Copyright 2011 Google Inc.
	*
	* Licensed under the Apache License, Version 2.0 (the "License"); you may not
	* use this file except in compliance with the License. You may obtain a copy of
	* the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
	* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
	* License for the specific language governing permissions and limitations under
	* the License.
	*/
	package com.google.gwt.dev.javac;

	import com.google.gwt.core.ext.TreeLogger;
	import com.google.gwt.core.ext.UnableToCompleteException;
	import com.google.gwt.dev.jjs.InternalCompilerException;
	import com.google.gwt.dev.jjs.ast.JNode;
	import com.google.gwt.dev.jjs.impl.GwtAstBuilder;
	import com.google.gwt.dev.util.log.speedtracer.DevModeEventType;
	import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger;
	import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger.Event;
	import com.google.gwt.util.tools.Utility;
	import com.google.gwt.thirdparty.guava.common.hash.Hashing;
	import com.google.gwt.thirdparty.guava.common.io.Files;

	import java.io.BufferedInputStream;
	import java.io.BufferedOutputStream;
	import java.io.EOFException;
	import java.io.File;
	import java.io.FileInputStream;
	import java.io.FileOutputStream;
	import java.io.IOException;
	import java.io.ObjectInputStream;
	import java.io.ObjectOutputStream;
	import java.net.JarURLConnection;
	import java.net.URLConnection;
	import java.util.ArrayList;
	import java.util.Arrays;
	import java.util.List;
	import java.util.concurrent.ExecutionException;
	import java.util.concurrent.ExecutorService;
	import java.util.concurrent.Executors;
	import java.util.concurrent.Future;
	import java.util.concurrent.RejectedExecutionException;
	import java.util.concurrent.TimeUnit;
	import java.util.concurrent.TimeoutException;
	import java.util.concurrent.atomic.AtomicBoolean;

	/**
	* A class that manages a persistent cache of {@link CompilationUnit} instances.
	* Writes out {@CompilationUnit} instances to a cache in a
	* background thread.
	* <p>
	* The persistent cache is implemented as a directory of log files with a date
	* timestamp. A new log file gets created each time a new PersistentUnitCache is
	* instantiated, (once per invocation of the compiler or DevMode). The design is
	* intended to support only a single PersistentUnitCache instance in the
	* compiler at a time.
	* <p>
	* As new units are compiled, the cache data is appended to a log. This allows
	* Java serialization to efficiently store references. The next time the cache
	* is started, all logs are replayed and loaded into the cache in chronological
	* order, with newer units taking precedence. A new cache file is created for
	* any newly compiled units in this session. After a threshold of a certain
	* number of files in the directory is reached
	* {@link PersistentUnitCache#CACHE_FILE_THRESHOLD} , the cache files are
	* consolidated back into a single file.
	*
	* <p>
	* System Properties (see {@link UnitCacheFactory}).
	*
	* <ul>
	* <li>gwt.persistentunitcache : enables the persistent cache (eventually will
	* be default)</li>
	* <li>gwt.persistentunitcachedir=<dir>: sets or overrides the cache directory</li>
	* </ul>
	*
	* <p>
	* Known Issues:
	*
	* <ul>
	* <li>This design uses an eager cache to load every unit in the cache on the
	* first reference to find() or add(). When the cache is large (10000 units), it
	* uses lots of heap and takes 5-10 seconds. Once the PersistentUnitCache is
	* created, it starts eagerly loading the cache in a background thread).</li>
	*
	* <li>Although units logged to disk with the same resource path are eventually
	* cleaned up, the most recently compiled unit stays in the cache forever. This
	* means that stale units that are no longer referenced will never be purged,
	* unless by some external action (e.g. ant clean).</li>
	*
	* <li>Unless ant builds are made aware of the cache directory, the cache will
	* persist if a user does an ant clean.</li>
	* </ul>
	*
	*/
	class PersistentUnitCache extends MemoryUnitCache {

	/**
	* If there are more than this many files in the cache, clean up the old
	* files.
	*/
	static final int CACHE_FILE_THRESHOLD = 40;

	/**
	* Common prefix for creating directories and cache files.
	*/
	static final String UNIT_CACHE_PREFIX = "gwt-unitCache";
	static final String CACHE_FILE_PREFIX = UNIT_CACHE_PREFIX + "-";
	static final String CURRENT_VERSION_CACHE_FILE_PREFIX = computePersistentCacheFilenamePrefix();

	/**
	* Creates a new file with a name based on the current system time.
	*/
	private static File createCacheFile(TreeLogger logger, File cacheDirectory)
	throws UnableToCompleteException {
	File newFile = null;
	long timestamp = System.currentTimeMillis();
	try {
	do {
	newFile = new File(cacheDirectory, CURRENT_VERSION_CACHE_FILE_PREFIX +
	String.format("%016X", timestamp++));
	} while (!newFile.createNewFile());
	} catch (IOException ex) {
	logger.log(TreeLogger.WARN, "Unable to create new cache log file "
	+ (newFile == null ? "<not created>" : newFile.getAbsolutePath()) + ".", ex);
	throw new UnableToCompleteException();
	}

	if (!newFile.canWrite()) {
	logger.log(TreeLogger.WARN, "Unable to write to new cache log file "
	+ newFile.getAbsolutePath() + ".");
	throw new UnableToCompleteException();
	}

	return newFile;
	}

	/**
	* Finds all files matching a pattern in the cache directory.
	*
	* @return an array of sorted filenames. The file name pattern is such that
	* sorting them alphabetically also sorts the files by age.
	*/
	private static File[] getCacheFiles(File cacheDirectory, boolean currentVersion) {
	if (cacheDirectory.isDirectory()) {
	File[] files = cacheDirectory.listFiles();
	List<File> cacheFiles = new ArrayList<File>();
	for (File file : files) {
	if (file.getName().startsWith(currentVersion ? CURRENT_VERSION_CACHE_FILE_PREFIX :
	CACHE_FILE_PREFIX)) {
	cacheFiles.add(file);
	}
	}
	File[] retFiles = cacheFiles.toArray(new File[cacheFiles.size()]);
	Arrays.sort(retFiles);
	return retFiles;
	}
	return new File[0];
	}

	/**
	* There is no significance in the return value, we just want to be able
	* to tell if the purgeOldCacheFilesTask has completed.
	*/
	private Future<Boolean> purgeTaskStatus;
	private AtomicBoolean purgeInProgress = new AtomicBoolean(false);

	private final Runnable purgeOldCacheFilesTask = new Runnable() {
	@Override
	public void run() {
	try {
	// Delete all cache files in the directory except for the currently open
	// file.
	SpeedTracerLogger.Event deleteEvent = SpeedTracerLogger.start(DevModeEventType.DELETE_CACHE);
	File[] filesToDelete = getCacheFiles(cacheDirectory, false);
	logger.log(TreeLogger.TRACE, "Purging cache files from " + cacheDirectory);
	for (File toDelete : filesToDelete) {
	if (!currentCacheFile.equals(toDelete)) {
	if (!toDelete.delete()) {
	logger.log(TreeLogger.WARN, "Couldn't delete file: " + toDelete);
	}
	}
	}
	deleteEvent.end();

	rotateCurrentCacheFile();
	} catch (UnableToCompleteException e) {
	backgroundService.shutdownNow();
	} finally {
	purgeInProgress.set(false);
	}
	}
	};

	private final Runnable rotateCacheFilesTask = new Runnable() {
	@Override
	public void run() {
	try {
	rotateCurrentCacheFile();
	} catch (UnableToCompleteException e) {
	backgroundService.shutdownNow();
	}
	assert (currentCacheFile != null);
	}
	};

	private final Runnable shutdownThreadTask = new Runnable() {
	@Override
	public void run() {
	assert (currentCacheFile != null);
	closeCurrentCacheFile(currentCacheFile, currentCacheFileStream);
	logger.log(TreeLogger.TRACE, "Shutting down PersistentUnitCache thread");
	backgroundService.shutdownNow();
	}
	};

	/**
	* Saved to be able to wait for UNIT_MAP_LOAD_TASK to complete.
	*/
	private Future<Boolean> unitMapLoadStatus;

	private final Runnable unitMapLoadTask = new Runnable() {
	@Override
	public void run() {
	loadUnitMap(logger, currentCacheFile);
	}
	};

	/**
	* Used to execute the above Runnables in a background thread.
	*/
	private final ExecutorService backgroundService;

	private int unitsWritten = 0;

	private int addedSinceLastCleanup = 0;

	/**
	* A directory to store the cache files that should persist between
	* invocations.
	*/
	private final File cacheDirectory;

	/**
	* Current file and stream being written to.
	*/
	private File currentCacheFile;
	private ObjectOutputStream currentCacheFileStream;

	private final TreeLogger logger;

	PersistentUnitCache(final TreeLogger logger, File cacheDir) throws UnableToCompleteException {
	assert cacheDir != null;
	this.logger = logger;

	/*
	* We must canonicalize the path here, otherwise we might set cacheDirectory
	* to something like "/path/to/x/../gwt-unitCache". If this were to happen,
	* the mkdirs() call below would create "/path/to/gwt-unitCache" but
	* not "/path/to/x".
	* Further accesses via the uncanonicalized path will fail, if "/path/to/x"
	* had not been created by other means.
	*
	* Fixes issue 6443
	*/
	try {
	this.cacheDirectory = new File(cacheDir.getCanonicalFile(), UNIT_CACHE_PREFIX);
	} catch (IOException ex) {
	logger.log(TreeLogger.WARN, "Unable to create canonical file of "
	+ cacheDir.getAbsolutePath() + ".", ex);
	throw new UnableToCompleteException();
	}

	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger.log(TreeLogger.TRACE, "Persistent unit cache dir set to: "
	+ this.cacheDirectory.getAbsolutePath());
	}

	if (!cacheDirectory.isDirectory() && !cacheDirectory.mkdirs()) {
	logger.log(TreeLogger.WARN, "Unable to initialize cache. Couldn't create directory "
	+ cacheDirectory.getAbsolutePath() + ".");
	throw new UnableToCompleteException();
	}

	currentCacheFile = createCacheFile(logger, cacheDirectory);

	backgroundService = Executors.newSingleThreadExecutor();
	Runtime.getRuntime().addShutdownHook(new Thread() {
	@Override
	public void run() {
	try {
	Future<Boolean> status = backgroundService.submit(shutdownThreadTask, Boolean.TRUE);
	// Don't let the shutdown hang more than 5 seconds
	status.get(5, TimeUnit.SECONDS);
	} catch (InterruptedException e) {
	// ignore
	} catch (RejectedExecutionException e) {
	// already shutdown, ignore
	} catch (ExecutionException e) {
	logger.log(TreeLogger.ERROR, "Error during shutdown", e);
	} catch (TimeoutException e) {
	// ignore
	} finally {
	backgroundService.shutdownNow();
	}
	}
	});

	/**
	* Load up cached units from the persistent store in the background. The
	* {@link #add(CompilationUnit)} and {@link #find(String)} methods block if
	* invoked before this thread finishes.
	*/
	unitMapLoadStatus = backgroundService.submit(unitMapLoadTask, Boolean.TRUE);

	FileOutputStream fstream = null;
	BufferedOutputStream bstream = null;

	try {
	fstream = new FileOutputStream(currentCacheFile);
	bstream = new BufferedOutputStream(fstream);
	currentCacheFileStream = new ObjectOutputStream(bstream);
	} catch (IOException ex) {
	closeCurrentCacheFile(currentCacheFile, currentCacheFileStream);
	logger.log(TreeLogger.ERROR, "Error creating cache " + currentCacheFile
	+ ". Disabling cache.", ex);
	backgroundService.shutdownNow();
	throw new UnableToCompleteException();
	}
	}

	/**
	* Enqueue a unit to be written by the background thread.
	*/
	@Override
	public void add(CompilationUnit newUnit) {
	awaitUnitCacheMapLoad();
	addedSinceLastCleanup++;
	super.add(newUnit);
	addImpl(unitMap.get(newUnit.getResourcePath()));
	}

	/**
	* Cleans up old cache files in the directory, migrating everything previously
	* loaded in them to the current cache file.
	*
	* Normally, only newly compiled units are written to the current log, but
	* when it is time to cleanup, valid units from older log files need to be
	* re-written.
	*/
	@Override
	public void cleanup(TreeLogger logger) {
	awaitUnitCacheMapLoad();

	if (backgroundService.isShutdown()) {
	return;
	}
	boolean shouldRotate = addedSinceLastCleanup > 0;
	logger.log(TreeLogger.TRACE, "Added " + addedSinceLastCleanup +
	" units to cache since last cleanup.");
	addedSinceLastCleanup = 0;
	try {
	File[] cacheFiles = getCacheFiles(cacheDirectory, true);
	if (cacheFiles.length < CACHE_FILE_THRESHOLD) {
	if (shouldRotate) {
	backgroundService.execute(rotateCacheFilesTask);
	}
	return;
	}

	// Check to see if the previous purge task finished.
	boolean inProgress = purgeInProgress.getAndSet(true);
	if (inProgress) {
	try {
	purgeTaskStatus.get(0, TimeUnit.NANOSECONDS);
	} catch (InterruptedException ex) {
	Thread.currentThread().interrupt();
	} catch (TimeoutException ex) {
	// purge is currently in progress.
	return;
	}
	}

	/*
	* Resend all units read in from the in-memory cache to the background
	* thread. They will be re-written out and the old cache files removed.
	*/
	synchronized (unitMap) {
	for (UnitCacheEntry unitCacheEntry : unitMap.values()) {
	if (unitCacheEntry.getOrigin() == UnitOrigin.PERSISTENT) {
	addImpl(unitCacheEntry);
	}
	}
	}

	purgeTaskStatus = backgroundService.submit(purgeOldCacheFilesTask, Boolean.TRUE);

	} catch (ExecutionException ex) {
	throw new InternalCompilerException("Error purging cache", ex);
	} catch (RejectedExecutionException ex) {
	// Cache background thread is not running - ignore
	}
	}

	@Override
	public CompilationUnit find(ContentId contentId) {
	awaitUnitCacheMapLoad();
	return super.find(contentId);
	}

	@Override
	public CompilationUnit find(String resourcePath) {
	awaitUnitCacheMapLoad();
	return super.find(resourcePath);
	}

	public void rotateCurrentCacheFile() throws UnableToCompleteException {
	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger.log(TreeLogger.TRACE, "Wrote " + unitsWritten + " units to persistent cache.");
	}

	// Close and re-open a new log file to drop object references kept
	// alive in the existing file by Java serialization.
	closeCurrentCacheFile(currentCacheFile, currentCacheFileStream);
	unitsWritten = 0;
	currentCacheFile = createCacheFile(logger, cacheDirectory);
	FileOutputStream fstream = null;
	BufferedOutputStream bstream = null;
	try {
	fstream = new FileOutputStream(currentCacheFile);
	bstream = new BufferedOutputStream(fstream);
	currentCacheFileStream = new ObjectOutputStream(bstream);
	} catch (IOException ex) {
	// Close all 3 streams, not sure where the exception occurred.
	Utility.close(bstream);
	Utility.close(fstream);
	closeCurrentCacheFile(currentCacheFile, currentCacheFileStream);
	logger.log(TreeLogger.ERROR, "Error rotating file. Shutting down cache thread.", ex);
	throw new UnableToCompleteException();
	}
	}

	/**
	* For Unit testing - shutdown the persistent cache.
	*
	* @throws ExecutionException
	* @throws InterruptedException
	*/
	void shutdown() throws InterruptedException, ExecutionException {
	try {
	Future<Runnable> future = backgroundService.submit(shutdownThreadTask, shutdownThreadTask);
	backgroundService.shutdown();
	future.get();
	} catch (RejectedExecutionException ex) {
	// background thread is not running - ignore
	}
	}

	private static String computePersistentCacheFilenamePrefix() {
	try {
	URLConnection urlConnection =
	JNode.class.getResource("JNode.class").openConnection();
	if (urlConnection instanceof JarURLConnection) {
	String gwtdevJar = ((JarURLConnection) urlConnection).getJarFile().getName();
	return CACHE_FILE_PREFIX +
	Files.hash(new File(gwtdevJar), Hashing.sha1()).toString() + "-";
	}
	System.err.println("Could not find the GWT compiler jarfile. "
	+ "Serialization errors might occur when accessing the persistent unit cache.");
	} catch (IOException e) {
	System.err.println("Could not compute the hash for the GWT compiler jarfile."
	+ "Serialization errors might occur when accessing the persistent unit cache.");
	e.printStackTrace();
	}
	return CACHE_FILE_PREFIX;
	}

	private void addImpl(final UnitCacheEntry entry) {
	try {
	backgroundService.execute(new Runnable() {
	@Override
	public void run() {
	try {
	assert entry.getOrigin() != UnitOrigin.ARCHIVE;
	CompilationUnit unit = entry.getUnit();
	assert unit != null;
	currentCacheFileStream.writeObject(unit);
	unitsWritten++;
	} catch (IOException ex) {
	backgroundService.shutdownNow();
	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger.log(TreeLogger.TRACE, "Error saving unit to cache in: "
	+ cacheDirectory.getAbsolutePath(), ex);
	}
	}
	}
	});
	} catch (RejectedExecutionException ex) {
	// background thread is not running, ignore
	}
	}

	private synchronized void awaitUnitCacheMapLoad() {
	// wait on initial load of unit map to complete.
	try {
	if (unitMapLoadStatus != null) {
	unitMapLoadStatus.get();
	// no need to check any more.
	unitMapLoadStatus = null;
	}
	} catch (InterruptedException e) {
	throw new InternalCompilerException("Interrupted waiting for unit cache map to load.", e);
	} catch (ExecutionException e) {
	logger.log(TreeLogger.ERROR, "Failure in unit cache map load.", e);
	// keep going
	unitMapLoadStatus = null;
	}
	}

	private void closeCurrentCacheFile(File openFile, ObjectOutputStream stream) {
	Utility.close(stream);
	if (unitsWritten == 0) {
	// Remove useless empty file.
	openFile.delete();
	}
	}

	/**
	* Load everything cached on disk into memory.
	*/
	private void loadUnitMap(TreeLogger logger, File currentCacheFile) {
	Event loadPersistentUnitEvent =
	SpeedTracerLogger.start(DevModeEventType.LOAD_PERSISTENT_UNIT_CACHE);
	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger.log(TreeLogger.TRACE, "Looking for previously cached Compilation Units in "
	+ cacheDirectory.getAbsolutePath());
	}
	try {
	if (cacheDirectory.isDirectory() && cacheDirectory.canRead()) {
	File[] files = getCacheFiles(cacheDirectory, true);
	for (File cacheFile : files) {
	FileInputStream fis = null;
	BufferedInputStream bis = null;
	ObjectInputStream inputStream = null;
	if (cacheFile.equals(currentCacheFile)) {
	continue;
	}
	boolean deleteCacheFile = false;
	try {
	fis = new FileInputStream(cacheFile);
	bis = new BufferedInputStream(fis);
	/*
	* It is possible for the next call to throw an exception, leaving
	* inputStream null and fis still live.
	*/
	inputStream = new ObjectInputStream(bis);
	while (true) {
	CachedCompilationUnit unit = (CachedCompilationUnit) inputStream.readObject();
	if (unit == null) {
	break;
	}
	if (unit.getTypesSerializedVersion() != GwtAstBuilder.getSerializationVersion()) {
	continue;
	}
	UnitCacheEntry entry = new UnitCacheEntry(unit, UnitOrigin.PERSISTENT);
	UnitCacheEntry existingEntry = unitMap.get(unit.getResourcePath());
	/*
	* Don't assume that an existing entry is stale - an entry might
	* have been loaded already from another source like a
	* CompilationUnitArchive that is more up to date. If the
	* timestamps are the same, accept the latest version. If it turns
	* out to be stale, it will be recompiled and the updated unit
	* will win this test the next time the session starts.
	*/
	if (existingEntry != null
	&& unit.getLastModified() >= existingEntry.getUnit().getLastModified()) {
	super.remove(existingEntry.getUnit());
	unitMap.put(unit.getResourcePath(), entry);
	unitMapByContentId.put(unit.getContentId(), entry);
	} else if (existingEntry == null) {
	unitMap.put(unit.getResourcePath(), entry);
	unitMapByContentId.put(unit.getContentId(), entry);
	}
	}
	} catch (EOFException ex) {
	// Go on to the next file.
	} catch (IOException ex) {
	deleteCacheFile = true;
	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger.log(TreeLogger.TRACE, "Ignoring and deleting cache log "
	+ cacheFile.getAbsolutePath() + " due to read error.", ex);
	}
	} catch (ClassNotFoundException ex) {
	deleteCacheFile = true;
	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger.log(TreeLogger.TRACE, "Ignoring and deleting cache log "
	+ cacheFile.getAbsolutePath() + " due to deserialization error.", ex);
	}
	} finally {
	Utility.close(inputStream);
	Utility.close(bis);
	Utility.close(fis);
	}
	if (deleteCacheFile) {
	cacheFile.delete();
	} else {
	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger.log(TreeLogger.TRACE, cacheFile.getName() + ": Load complete");
	}
	}
	}
	} else {
	logger
	.log(TreeLogger.TRACE,
	"Starting with empty Cache: CompilationUnit cache "
	+ "directory does not exist or is not readable.");
	}
	} finally {
	loadPersistentUnitEvent.end();
	}
	}
	}