2.4/dev/core/src/com/google/gwt/dev/javac/PersistentUnitCache.java - gwt - Git at Google

 /*
  * Copyright 2011 Google Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you may not
  * use this file except in compliance with the License. You may obtain a copy of
  * the License at
  *
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  * License for the specific language governing permissions and limitations under
  * the License.
  */
 package com.google.gwt.dev.javac;

 import com.google.gwt.core.ext.TreeLogger;
 import com.google.gwt.core.ext.UnableToCompleteException;
 import com.google.gwt.dev.jjs.impl.GwtAstBuilder;
 import com.google.gwt.dev.util.log.speedtracer.DevModeEventType;
 import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger;
 import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger.Event;
 import com.google.gwt.util.tools.Utility;

 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
 import java.io.EOFException;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;

 /**
  * A class that manages a persistent cache of {@link CompilationUnit} instances.
  * Writes out {@CompilationUnit} instances to a cache in a
  * background thread.
  * <p>
  * The persistent cache is implemented as a directory of log files with a date
  * timestamp. A new log file gets created each time a new PersistentUnitCache is
  * instantiated, (once per invocation of the compiler or DevMode). The design is
  * intended to support only a single PersistentUnitCache instance in the
  * compiler at a time.
  * <p>
  * As new units are compiled, the cache data is appended to a log. This allows
  * Java serialization to efficiently store references. The next time the cache
  * is started, all logs are replayed and loaded into the cache in chronological
  * order, with newer units taking precedence. A new cache file is created for
  * any newly compiled units in this session. After a threshold of a certain
  * number of files in the directory is reached
  * {@link PersistentUnitCache#CACHE_FILE_THRESHOLD} , the cache files are
  * consolidated back into a single file.
  *
  * <p>
  * System Properties (see {@link UnitCacheFactory}).
  *
  * <ul>
  * <li>gwt.persistentunitcache : enables the persistent cache (eventually will
  * be default)</li>
  * <li>gwt.persistentunitcachedir=<dir>: sets or overrides the cache directory</li>
  * </ul>
  *
  * <p>
  * Known Issues:
  *
  * <ul>
  * <li>This design uses an eager cache to load every unit in the cache on the
  * first reference to find() or add(). When the cache is large (10000 units), it
  * uses lots of heap and takes 5-10 seconds. Once the PersistentUnitCache is
  * created, it starts eagerly loading the cache in a background thread).</li>
  *
  * <li>Although units logged to disk with the same resource path are eventually
  * cleaned up, the most recently compiled unit stays in the cache forever. This
  * means that stale units that are no longer referenced will never be purged,
  * unless by some external action (e.g. ant clean).</li>
  *
  * <li>Unless ant builds are made aware of the cache directory, the cache will
  * persist if a user does an ant clean.</li>
  * </ul>
  *
  */
 class PersistentUnitCache extends MemoryUnitCache {
   /**
    * A thread used when the cache is instantiated to load up cached units from
    * the persistent store in the background. The
    * {@link UnitCacheFactory#addUnit(CompilationUnit)} and
    * {@link UnitCacheFactory#findUnit(String)} methods block if invoked before
    * this thread finishes.
    */
   private class UnitCacheMapLoader extends Thread {
     private final CountDownLatch loadCompleteLatch = new CountDownLatch(1);
     private final TreeLogger logger;

     public UnitCacheMapLoader(TreeLogger logger) {
       this.logger = logger;
       setDaemon(true);
       setName("UnitCacheLoader");
       setPriority(Thread.NORM_PRIORITY);
     }

     public void await() {
       try {
         loadCompleteLatch.await();
       } catch (InterruptedException ex) {
         logger.log(TreeLogger.ERROR, "Interrupted waiting for PersistentUnitCache to load.", ex);
       }
     }

     @Override
     public void run() {
       try {
         loadUnitMap(logger);
       } finally {
         loadCompleteLatch.countDown();
         if (logger.isLoggable(TreeLogger.TRACE)) {
           logger
               .log(TreeLogger.TRACE, "Loaded " + unitMap.size() + " units from persistent store.");
         }
       }
     }
   }

   /**
    * Used to pass messages to the unitWriteThread.
    */
   private static class UnitWriteMessage {
     private static final UnitWriteMessage DELETE_OLD_CACHE_FILES = new UnitWriteMessage();
     private static final UnitWriteMessage SHUTDOWN_THREAD = new UnitWriteMessage();
     private final UnitCacheEntry unitCacheEntry;

     public UnitWriteMessage() {
       unitCacheEntry = null;
     }

     public UnitWriteMessage(UnitCacheEntry unitCacheEntry) {
       this.unitCacheEntry = unitCacheEntry;
     }
   }

   /**
    * Thread that reads units from a queue and writes out to a cache file for
    * this session.
    */
   private class UnitWriter extends Thread {
     private final CountDownLatch shutDownLatch = new CountDownLatch(1);
     private final TreeLogger logger;
     private boolean errorLogged = false;
     private Thread shutdownHook = new Thread() {
       @Override
       public void run() {
         try {
           doShutdown();
         } catch (InterruptedException ex) {
           // ignore
         }
       }
     };

     public UnitWriter(TreeLogger logger) {
       this.logger = logger;
       setDaemon(true);
       setName("UnitWriteThread");
       setPriority(Thread.MIN_PRIORITY);

       Runtime.getRuntime().addShutdownHook(shutdownHook);
     }

     @Override
     public void run() {
       logger.log(TreeLogger.TRACE, "Starting UnitWriteThread.");

       FileOutputStream fstream = null;
       BufferedOutputStream bstream = null;
       ObjectOutputStream stream = null;
       try {
         fstream = new FileOutputStream(currentCacheFile);
         bstream = new BufferedOutputStream(fstream);
         stream = new ObjectOutputStream(bstream);
       } catch (IOException ex) {
         logger.log(TreeLogger.ERROR, "Error creating cache " + currentCacheFile
             + ". Disabling cache.", ex);
       }
       int recentUnitsWritten = 0;
       int totalUnitsWritten = 0;
       try {
         while (true) {
           UnitWriteMessage msg = null;
           try {
             msg = unitWriteQueue.take();
           } catch (InterruptedException e) {
             // Allow shutdown to interrupt
             break;
           }
           if (stream == null) {
             // if there is no output stream, just ignore the unit
             continue;
           }

           try {
             if (msg != null) {
               if (msg == UnitWriteMessage.DELETE_OLD_CACHE_FILES) {
                 if (logger.isLoggable(TreeLogger.TRACE)) {
                   logger.log(TreeLogger.TRACE, "Wrote " + recentUnitsWritten
                       + " units to persistent cache.");
                 }
                 recentUnitsWritten = 0;
                 deleteOldCacheFiles(logger, currentCacheFile);
               } else if (msg == UnitWriteMessage.SHUTDOWN_THREAD) {
                 stream.flush();
                 assert unitWriteQueue.size() == 0;
                 break;
               } else {
                 assert msg.unitCacheEntry.getOrigin() != UnitOrigin.ARCHIVE;
                 CompilationUnit unit = msg.unitCacheEntry.getUnit();
                 assert unit != null;
                 stream.writeObject(unit);
                 recentUnitsWritten++;
                 totalUnitsWritten++;
               }
             }

             if (unitWriteQueue.isEmpty()) {
               stream.flush();
             }
           } catch (IOException ex) {
             if (!errorLogged) {
               errorLogged = true;
               if (logger.isLoggable(TreeLogger.TRACE)) {
                 logger.log(TreeLogger.TRACE, "Error saving unit to file: "
                     + currentCacheFile.getAbsolutePath(), ex);
               }
             }
           }
         }
       } finally {
         Utility.close(stream);
         // Paranoia - close all streams
         Utility.close(bstream);
         Utility.close(fstream);
         if (totalUnitsWritten == 0) {
           // Remove useless empty output.
           currentCacheFile.delete();
         }
         shutDownLatch.countDown();
         logger.log(TreeLogger.TRACE, "Shutting down PersistentUnitCache thread");
       }
     }

     /**
      * Shutdown the thread and wait for it.
      */
     private void doShutdown() throws InterruptedException {
       // force the shutdown to finish after 5 seconds
       unitWriteQueue.add(UnitWriteMessage.SHUTDOWN_THREAD);
       // wait for shutdown
       shutDownLatch.await(5000, TimeUnit.MILLISECONDS);
       try {
         Runtime.getRuntime().removeShutdownHook(shutdownHook);
       } catch (IllegalStateException ex) {
         // ignore.
       }
     }
   }

   /**
    * Common prefix for creating directories and cache files.
    */
   static final String UNIT_CACHE_PREFIX = "gwt-unitCache";

   static final String CACHE_FILE_PREFIX = UNIT_CACHE_PREFIX + "-";

   /**
    * If there are more than this many files in the cache, clean up the old
    * files.
    */
   static final int CACHE_FILE_THRESHOLD = 10;

   /**
    * Used for communication to the unit write thread.
    */
   private final BlockingQueue<UnitWriteMessage> unitWriteQueue =
       new LinkedBlockingQueue<UnitWriteMessage>();
   private final AtomicInteger addCount = new AtomicInteger(0);
   private final UnitCacheMapLoader unitCacheMapLoader;
   private final UnitWriter unitWriter;
   private boolean cleanupHasRun = false;

   /**
    * A directory that ideally persists between invocations.
    */
   private final File cacheDirectory;

   /**
    * Cache log file currently being written to.
    */
   private File currentCacheFile;

   PersistentUnitCache(TreeLogger logger, File cacheDir) throws UnableToCompleteException {
     assert cacheDir != null;

     this.cacheDirectory = new File(cacheDir, UNIT_CACHE_PREFIX);
     if (logger.isLoggable(TreeLogger.TRACE)) {
       logger.log(TreeLogger.TRACE, "Persistent unit cache dir set to: "
           + this.cacheDirectory.getAbsolutePath());
     }

     if (!cacheDirectory.isDirectory() && !cacheDirectory.mkdirs()) {
       logger.log(TreeLogger.WARN, "Unable to initialize cache. Couldn't create directory "
           + cacheDirectory.getAbsolutePath() + ".");
       throw new UnableToCompleteException();
     }

     long timestamp = System.currentTimeMillis();
     do {
       currentCacheFile =
           new File(cacheDirectory, CACHE_FILE_PREFIX + String.format("%016X", timestamp++));
     } while (currentCacheFile.exists());

     // This isn't 100% reliable if multiple processes are in contention
     try {
       currentCacheFile.createNewFile();
     } catch (IOException ex) {
       logger.log(TreeLogger.WARN, "Unable to create new cache log file "
           + currentCacheFile.getAbsolutePath() + ".", ex);
       throw new UnableToCompleteException();
     }

     unitCacheMapLoader = new UnitCacheMapLoader(logger);
     unitCacheMapLoader.start();
     unitWriter = new UnitWriter(logger);
     unitWriter.start();
   }

   /**
    * Enqueue a unit to be written by the background thread.
    */
   @Override
   public void add(CompilationUnit newUnit) {
     unitCacheMapLoader.await();
     super.add(newUnit);
     UnitCacheEntry entry = unitMap.get(newUnit.getResourcePath());
     addCount.getAndIncrement();
     unitWriteQueue.add(new UnitWriteMessage(entry));
   }

   /**
    * Cleans up old cache files in the directory, migrating everything previously
    * loaded in them to the current cache file.
    *
    * Normally, only newly compiled units are written to the current log, but
    * when it is time to cleanup, valid units from older log files need to be
    * re-written.
    */
   @Override
   public void cleanup(TreeLogger logger) {
     if (logger.isLoggable(TreeLogger.TRACE)) {
       logger.log(TreeLogger.TRACE, "Added " + addCount.intValue() + " units to persistent cache.");
     }
     addCount.set(0);

     if (cleanupHasRun) {
       return;
     }

     cleanupHasRun = true;
     unitCacheMapLoader.await();
     File[] cacheFiles = getCacheFiles();

     if (cacheFiles.length < CACHE_FILE_THRESHOLD) {
       return;
     }

     /*
      * Resend all units read in from the in-memory cache to the writer thread.
      * They will be re-written out and the old cache files removed.
      */
     synchronized (unitMap) {
       for (UnitCacheEntry unitCacheEntry : unitMap.values()) {
         if (unitCacheEntry.getOrigin() == UnitOrigin.PERSISTENT) {
           unitWriteQueue.add(new UnitWriteMessage(unitCacheEntry));
         }
       }
     }
     unitWriteQueue.add(UnitWriteMessage.DELETE_OLD_CACHE_FILES);
   }

   @Override
   public CompilationUnit find(ContentId contentId) {
     unitCacheMapLoader.await();
     return super.find(contentId);
   }

   @Override
   public CompilationUnit find(String resourcePath) {
     unitCacheMapLoader.await();
     return super.find(resourcePath);
   }

   /**
    * Delete all cache files in the directory except for the currently open file.
    *
    * @param current Specifies the currently open cache file which will not be
    *          deleted.
    */
   void deleteOldCacheFiles(TreeLogger logger, File current) {
     assert current != null;

     SpeedTracerLogger.Event deleteEvent = SpeedTracerLogger.start(DevModeEventType.DELETE_CACHE);
     File[] filesToDelete = getCacheFiles();
     if (filesToDelete == null) {
       return;
     }
     if (logger.isLoggable(TreeLogger.INFO)) {
       logger.log(TreeLogger.TRACE, "Purging cache files from " + cacheDirectory);
     }
     for (File toDelete : filesToDelete) {
       if (!current.equals(toDelete)) {
         toDelete.delete();
       }
     }
     deleteEvent.end();
   }

   /**
    * Finds all files matching a pattern in the cache directory.
    *
    * @return an array of sorted filenames. The file name pattern is such that
    *         sorting them alphabetically also sorts the files by age.
    */
   File[] getCacheFiles() {
     if (cacheDirectory.isDirectory()) {
       File[] files = cacheDirectory.listFiles();
       List<File> cacheFiles = new ArrayList<File>();
       for (File file : files) {
         if (file.getName().startsWith(CACHE_FILE_PREFIX)) {
           cacheFiles.add(file);
         }
       }
       File[] retFiles = cacheFiles.toArray(new File[cacheFiles.size()]);
       Arrays.sort(retFiles);
       return retFiles;
     }
     return new File[0];
   }

   /**
    * For Unit testing - shutdown the persistent cache.
    */
   void shutdown() throws InterruptedException {
     unitWriter.doShutdown();
   }

   /**
    * Load everything cached on disk into memory.
    */
   private void loadUnitMap(TreeLogger logger) {
     Event loadPersistentUnitEvent =
         SpeedTracerLogger.start(DevModeEventType.LOAD_PERSISTENT_UNIT_CACHE);
     if (logger.isLoggable(TreeLogger.TRACE)) {
       logger.log(TreeLogger.TRACE, "Looking for previously cached Compilation Units in "
           + cacheDirectory.getAbsolutePath());
     }
     try {
       if (cacheDirectory.isDirectory() && cacheDirectory.canRead()) {
         File[] files = getCacheFiles();
         for (File cacheFile : files) {
           FileInputStream fis = null;
           BufferedInputStream bis = null;
           ObjectInputStream inputStream = null;
           if (cacheFile.equals(currentCacheFile)) {
             continue;
           }
           boolean deleteCacheFile = false;
           try {
             fis = new FileInputStream(cacheFile);
             bis = new BufferedInputStream(fis);
             /*
              * It is possible for the next call to throw an exception, leaving
              * inputStream null and fis still live.
              */
             inputStream = new ObjectInputStream(bis);
             while (true) {
               CachedCompilationUnit unit = (CachedCompilationUnit) inputStream.readObject();
               if (unit == null) {
                 break;
               }
               if (GwtAstBuilder.ENABLED) {
                 if (unit.getTypesSerializedVersion() != GwtAstBuilder.getSerializationVersion()) {
                   continue;
                 }
               }
               UnitCacheEntry entry = new UnitCacheEntry(unit, UnitOrigin.PERSISTENT);
               UnitCacheEntry oldEntry = unitMap.get(unit.getResourcePath());
               if (oldEntry != null && unit.getLastModified() > oldEntry.getUnit().getLastModified()) {
                 super.remove(oldEntry.getUnit());
                 unitMap.put(unit.getResourcePath(), entry);
                 unitMapByContentId.put(unit.getContentId(), entry);
               } else if (oldEntry == null) {
                 unitMap.put(unit.getResourcePath(), entry);
                 unitMapByContentId.put(unit.getContentId(), entry);
               }
             }
           } catch (EOFException ex) {
             // Go on to the next file.
           } catch (IOException ex) {
             deleteCacheFile = true;
             if (logger.isLoggable(TreeLogger.TRACE)) {
               logger.log(TreeLogger.TRACE, "Ignoring and deleting cache log "
                   + cacheFile.getAbsolutePath() + " due to read error.", ex);
             }
           } catch (ClassNotFoundException ex) {
             deleteCacheFile = true;
             if (logger.isLoggable(TreeLogger.TRACE)) {
               logger.log(TreeLogger.TRACE, "Ignoring and deleting cache log "
                   + cacheFile.getAbsolutePath() + " due to deserialization error.", ex);
             }
           } finally {
             Utility.close(inputStream);
             Utility.close(bis);
             Utility.close(fis);
           }
           if (deleteCacheFile) {
             cacheFile.delete();
           } else {
             if (logger.isLoggable(TreeLogger.TRACE)) {
               logger.log(TreeLogger.TRACE, cacheFile.getName() + ": Load complete");
             }
           }
         }
       } else {
         logger
             .log(TreeLogger.TRACE,
                 "Starting with empty Cache: CompilationUnit cache directory does not exist or is not readable.");
       }
     } finally {
       loadPersistentUnitEvent.end();
     }
   }
 }
	/*
	* Copyright 2011 Google Inc.
	*
	* Licensed under the Apache License, Version 2.0 (the "License"); you may not
	* use this file except in compliance with the License. You may obtain a copy of
	* the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
	* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
	* License for the specific language governing permissions and limitations under
	* the License.
	*/
	package com.google.gwt.dev.javac;

	import com.google.gwt.core.ext.TreeLogger;
	import com.google.gwt.core.ext.UnableToCompleteException;
	import com.google.gwt.dev.jjs.impl.GwtAstBuilder;
	import com.google.gwt.dev.util.log.speedtracer.DevModeEventType;
	import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger;
	import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger.Event;
	import com.google.gwt.util.tools.Utility;

	import java.io.BufferedInputStream;
	import java.io.BufferedOutputStream;
	import java.io.EOFException;
	import java.io.File;
	import java.io.FileInputStream;
	import java.io.FileOutputStream;
	import java.io.IOException;
	import java.io.ObjectInputStream;
	import java.io.ObjectOutputStream;
	import java.util.ArrayList;
	import java.util.Arrays;
	import java.util.List;
	import java.util.concurrent.BlockingQueue;
	import java.util.concurrent.CountDownLatch;
	import java.util.concurrent.LinkedBlockingQueue;
	import java.util.concurrent.TimeUnit;
	import java.util.concurrent.atomic.AtomicInteger;

	/**
	* A class that manages a persistent cache of {@link CompilationUnit} instances.
	* Writes out {@CompilationUnit} instances to a cache in a
	* background thread.
	* <p>
	* The persistent cache is implemented as a directory of log files with a date
	* timestamp. A new log file gets created each time a new PersistentUnitCache is
	* instantiated, (once per invocation of the compiler or DevMode). The design is
	* intended to support only a single PersistentUnitCache instance in the
	* compiler at a time.
	* <p>
	* As new units are compiled, the cache data is appended to a log. This allows
	* Java serialization to efficiently store references. The next time the cache
	* is started, all logs are replayed and loaded into the cache in chronological
	* order, with newer units taking precedence. A new cache file is created for
	* any newly compiled units in this session. After a threshold of a certain
	* number of files in the directory is reached
	* {@link PersistentUnitCache#CACHE_FILE_THRESHOLD} , the cache files are
	* consolidated back into a single file.
	*
	* <p>
	* System Properties (see {@link UnitCacheFactory}).
	*
	* <ul>
	* <li>gwt.persistentunitcache : enables the persistent cache (eventually will
	* be default)</li>
	* <li>gwt.persistentunitcachedir=<dir>: sets or overrides the cache directory</li>
	* </ul>
	*
	* <p>
	* Known Issues:
	*
	* <ul>
	* <li>This design uses an eager cache to load every unit in the cache on the
	* first reference to find() or add(). When the cache is large (10000 units), it
	* uses lots of heap and takes 5-10 seconds. Once the PersistentUnitCache is
	* created, it starts eagerly loading the cache in a background thread).</li>
	*
	* <li>Although units logged to disk with the same resource path are eventually
	* cleaned up, the most recently compiled unit stays in the cache forever. This
	* means that stale units that are no longer referenced will never be purged,
	* unless by some external action (e.g. ant clean).</li>
	*
	* <li>Unless ant builds are made aware of the cache directory, the cache will
	* persist if a user does an ant clean.</li>
	* </ul>
	*
	*/
	class PersistentUnitCache extends MemoryUnitCache {
	/**
	* A thread used when the cache is instantiated to load up cached units from
	* the persistent store in the background. The
	* {@link UnitCacheFactory#addUnit(CompilationUnit)} and
	* {@link UnitCacheFactory#findUnit(String)} methods block if invoked before
	* this thread finishes.
	*/
	private class UnitCacheMapLoader extends Thread {
	private final CountDownLatch loadCompleteLatch = new CountDownLatch(1);
	private final TreeLogger logger;

	public UnitCacheMapLoader(TreeLogger logger) {
	this.logger = logger;
	setDaemon(true);
	setName("UnitCacheLoader");
	setPriority(Thread.NORM_PRIORITY);
	}

	public void await() {
	try {
	loadCompleteLatch.await();
	} catch (InterruptedException ex) {
	logger.log(TreeLogger.ERROR, "Interrupted waiting for PersistentUnitCache to load.", ex);
	}
	}

	@Override
	public void run() {
	try {
	loadUnitMap(logger);
	} finally {
	loadCompleteLatch.countDown();
	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger
	.log(TreeLogger.TRACE, "Loaded " + unitMap.size() + " units from persistent store.");
	}
	}
	}
	}

	/**
	* Used to pass messages to the unitWriteThread.
	*/
	private static class UnitWriteMessage {
	private static final UnitWriteMessage DELETE_OLD_CACHE_FILES = new UnitWriteMessage();
	private static final UnitWriteMessage SHUTDOWN_THREAD = new UnitWriteMessage();
	private final UnitCacheEntry unitCacheEntry;

	public UnitWriteMessage() {
	unitCacheEntry = null;
	}

	public UnitWriteMessage(UnitCacheEntry unitCacheEntry) {
	this.unitCacheEntry = unitCacheEntry;
	}
	}

	/**
	* Thread that reads units from a queue and writes out to a cache file for
	* this session.
	*/
	private class UnitWriter extends Thread {
	private final CountDownLatch shutDownLatch = new CountDownLatch(1);
	private final TreeLogger logger;
	private boolean errorLogged = false;
	private Thread shutdownHook = new Thread() {
	@Override
	public void run() {
	try {
	doShutdown();
	} catch (InterruptedException ex) {
	// ignore
	}
	}
	};

	public UnitWriter(TreeLogger logger) {
	this.logger = logger;
	setDaemon(true);
	setName("UnitWriteThread");
	setPriority(Thread.MIN_PRIORITY);

	Runtime.getRuntime().addShutdownHook(shutdownHook);
	}

	@Override
	public void run() {
	logger.log(TreeLogger.TRACE, "Starting UnitWriteThread.");

	FileOutputStream fstream = null;
	BufferedOutputStream bstream = null;
	ObjectOutputStream stream = null;
	try {
	fstream = new FileOutputStream(currentCacheFile);
	bstream = new BufferedOutputStream(fstream);
	stream = new ObjectOutputStream(bstream);
	} catch (IOException ex) {
	logger.log(TreeLogger.ERROR, "Error creating cache " + currentCacheFile
	+ ". Disabling cache.", ex);
	}
	int recentUnitsWritten = 0;
	int totalUnitsWritten = 0;
	try {
	while (true) {
	UnitWriteMessage msg = null;
	try {
	msg = unitWriteQueue.take();
	} catch (InterruptedException e) {
	// Allow shutdown to interrupt
	break;
	}
	if (stream == null) {
	// if there is no output stream, just ignore the unit
	continue;
	}

	try {
	if (msg != null) {
	if (msg == UnitWriteMessage.DELETE_OLD_CACHE_FILES) {
	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger.log(TreeLogger.TRACE, "Wrote " + recentUnitsWritten
	+ " units to persistent cache.");
	}
	recentUnitsWritten = 0;
	deleteOldCacheFiles(logger, currentCacheFile);
	} else if (msg == UnitWriteMessage.SHUTDOWN_THREAD) {
	stream.flush();
	assert unitWriteQueue.size() == 0;
	break;
	} else {
	assert msg.unitCacheEntry.getOrigin() != UnitOrigin.ARCHIVE;
	CompilationUnit unit = msg.unitCacheEntry.getUnit();
	assert unit != null;
	stream.writeObject(unit);
	recentUnitsWritten++;
	totalUnitsWritten++;
	}
	}

	if (unitWriteQueue.isEmpty()) {
	stream.flush();
	}
	} catch (IOException ex) {
	if (!errorLogged) {
	errorLogged = true;
	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger.log(TreeLogger.TRACE, "Error saving unit to file: "
	+ currentCacheFile.getAbsolutePath(), ex);
	}
	}
	}
	}
	} finally {
	Utility.close(stream);
	// Paranoia - close all streams
	Utility.close(bstream);
	Utility.close(fstream);
	if (totalUnitsWritten == 0) {
	// Remove useless empty output.
	currentCacheFile.delete();
	}
	shutDownLatch.countDown();
	logger.log(TreeLogger.TRACE, "Shutting down PersistentUnitCache thread");
	}
	}

	/**
	* Shutdown the thread and wait for it.
	*/
	private void doShutdown() throws InterruptedException {
	// force the shutdown to finish after 5 seconds
	unitWriteQueue.add(UnitWriteMessage.SHUTDOWN_THREAD);
	// wait for shutdown
	shutDownLatch.await(5000, TimeUnit.MILLISECONDS);
	try {
	Runtime.getRuntime().removeShutdownHook(shutdownHook);
	} catch (IllegalStateException ex) {
	// ignore.
	}
	}
	}

	/**
	* Common prefix for creating directories and cache files.
	*/
	static final String UNIT_CACHE_PREFIX = "gwt-unitCache";

	static final String CACHE_FILE_PREFIX = UNIT_CACHE_PREFIX + "-";

	/**
	* If there are more than this many files in the cache, clean up the old
	* files.
	*/
	static final int CACHE_FILE_THRESHOLD = 10;

	/**
	* Used for communication to the unit write thread.
	*/
	private final BlockingQueue<UnitWriteMessage> unitWriteQueue =
	new LinkedBlockingQueue<UnitWriteMessage>();
	private final AtomicInteger addCount = new AtomicInteger(0);
	private final UnitCacheMapLoader unitCacheMapLoader;
	private final UnitWriter unitWriter;
	private boolean cleanupHasRun = false;

	/**
	* A directory that ideally persists between invocations.
	*/
	private final File cacheDirectory;

	/**
	* Cache log file currently being written to.
	*/
	private File currentCacheFile;

	PersistentUnitCache(TreeLogger logger, File cacheDir) throws UnableToCompleteException {
	assert cacheDir != null;

	this.cacheDirectory = new File(cacheDir, UNIT_CACHE_PREFIX);
	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger.log(TreeLogger.TRACE, "Persistent unit cache dir set to: "
	+ this.cacheDirectory.getAbsolutePath());
	}

	if (!cacheDirectory.isDirectory() && !cacheDirectory.mkdirs()) {
	logger.log(TreeLogger.WARN, "Unable to initialize cache. Couldn't create directory "
	+ cacheDirectory.getAbsolutePath() + ".");
	throw new UnableToCompleteException();
	}

	long timestamp = System.currentTimeMillis();
	do {
	currentCacheFile =
	new File(cacheDirectory, CACHE_FILE_PREFIX + String.format("%016X", timestamp++));
	} while (currentCacheFile.exists());

	// This isn't 100% reliable if multiple processes are in contention
	try {
	currentCacheFile.createNewFile();
	} catch (IOException ex) {
	logger.log(TreeLogger.WARN, "Unable to create new cache log file "
	+ currentCacheFile.getAbsolutePath() + ".", ex);
	throw new UnableToCompleteException();
	}

	unitCacheMapLoader = new UnitCacheMapLoader(logger);
	unitCacheMapLoader.start();
	unitWriter = new UnitWriter(logger);
	unitWriter.start();
	}

	/**
	* Enqueue a unit to be written by the background thread.
	*/
	@Override
	public void add(CompilationUnit newUnit) {
	unitCacheMapLoader.await();
	super.add(newUnit);
	UnitCacheEntry entry = unitMap.get(newUnit.getResourcePath());
	addCount.getAndIncrement();
	unitWriteQueue.add(new UnitWriteMessage(entry));
	}

	/**
	* Cleans up old cache files in the directory, migrating everything previously
	* loaded in them to the current cache file.
	*
	* Normally, only newly compiled units are written to the current log, but
	* when it is time to cleanup, valid units from older log files need to be
	* re-written.
	*/
	@Override
	public void cleanup(TreeLogger logger) {
	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger.log(TreeLogger.TRACE, "Added " + addCount.intValue() + " units to persistent cache.");
	}
	addCount.set(0);

	if (cleanupHasRun) {
	return;
	}

	cleanupHasRun = true;
	unitCacheMapLoader.await();
	File[] cacheFiles = getCacheFiles();

	if (cacheFiles.length < CACHE_FILE_THRESHOLD) {
	return;
	}

	/*
	* Resend all units read in from the in-memory cache to the writer thread.
	* They will be re-written out and the old cache files removed.
	*/
	synchronized (unitMap) {
	for (UnitCacheEntry unitCacheEntry : unitMap.values()) {
	if (unitCacheEntry.getOrigin() == UnitOrigin.PERSISTENT) {
	unitWriteQueue.add(new UnitWriteMessage(unitCacheEntry));
	}
	}
	}
	unitWriteQueue.add(UnitWriteMessage.DELETE_OLD_CACHE_FILES);
	}

	@Override
	public CompilationUnit find(ContentId contentId) {
	unitCacheMapLoader.await();
	return super.find(contentId);
	}

	@Override
	public CompilationUnit find(String resourcePath) {
	unitCacheMapLoader.await();
	return super.find(resourcePath);
	}

	/**
	* Delete all cache files in the directory except for the currently open file.
	*
	* @param current Specifies the currently open cache file which will not be
	* deleted.
	*/
	void deleteOldCacheFiles(TreeLogger logger, File current) {
	assert current != null;

	SpeedTracerLogger.Event deleteEvent = SpeedTracerLogger.start(DevModeEventType.DELETE_CACHE);
	File[] filesToDelete = getCacheFiles();
	if (filesToDelete == null) {
	return;
	}
	if (logger.isLoggable(TreeLogger.INFO)) {
	logger.log(TreeLogger.TRACE, "Purging cache files from " + cacheDirectory);
	}
	for (File toDelete : filesToDelete) {
	if (!current.equals(toDelete)) {
	toDelete.delete();
	}
	}
	deleteEvent.end();
	}

	/**
	* Finds all files matching a pattern in the cache directory.
	*
	* @return an array of sorted filenames. The file name pattern is such that
	* sorting them alphabetically also sorts the files by age.
	*/
	File[] getCacheFiles() {
	if (cacheDirectory.isDirectory()) {
	File[] files = cacheDirectory.listFiles();
	List<File> cacheFiles = new ArrayList<File>();
	for (File file : files) {
	if (file.getName().startsWith(CACHE_FILE_PREFIX)) {
	cacheFiles.add(file);
	}
	}
	File[] retFiles = cacheFiles.toArray(new File[cacheFiles.size()]);
	Arrays.sort(retFiles);
	return retFiles;
	}
	return new File[0];
	}

	/**
	* For Unit testing - shutdown the persistent cache.
	*/
	void shutdown() throws InterruptedException {
	unitWriter.doShutdown();
	}

	/**
	* Load everything cached on disk into memory.
	*/
	private void loadUnitMap(TreeLogger logger) {
	Event loadPersistentUnitEvent =
	SpeedTracerLogger.start(DevModeEventType.LOAD_PERSISTENT_UNIT_CACHE);
	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger.log(TreeLogger.TRACE, "Looking for previously cached Compilation Units in "
	+ cacheDirectory.getAbsolutePath());
	}
	try {
	if (cacheDirectory.isDirectory() && cacheDirectory.canRead()) {
	File[] files = getCacheFiles();
	for (File cacheFile : files) {
	FileInputStream fis = null;
	BufferedInputStream bis = null;
	ObjectInputStream inputStream = null;
	if (cacheFile.equals(currentCacheFile)) {
	continue;
	}
	boolean deleteCacheFile = false;
	try {
	fis = new FileInputStream(cacheFile);
	bis = new BufferedInputStream(fis);
	/*
	* It is possible for the next call to throw an exception, leaving
	* inputStream null and fis still live.
	*/
	inputStream = new ObjectInputStream(bis);
	while (true) {
	CachedCompilationUnit unit = (CachedCompilationUnit) inputStream.readObject();
	if (unit == null) {
	break;
	}
	if (GwtAstBuilder.ENABLED) {
	if (unit.getTypesSerializedVersion() != GwtAstBuilder.getSerializationVersion()) {
	continue;
	}
	}
	UnitCacheEntry entry = new UnitCacheEntry(unit, UnitOrigin.PERSISTENT);
	UnitCacheEntry oldEntry = unitMap.get(unit.getResourcePath());
	if (oldEntry != null && unit.getLastModified() > oldEntry.getUnit().getLastModified()) {
	super.remove(oldEntry.getUnit());
	unitMap.put(unit.getResourcePath(), entry);
	unitMapByContentId.put(unit.getContentId(), entry);
	} else if (oldEntry == null) {
	unitMap.put(unit.getResourcePath(), entry);
	unitMapByContentId.put(unit.getContentId(), entry);
	}
	}
	} catch (EOFException ex) {
	// Go on to the next file.
	} catch (IOException ex) {
	deleteCacheFile = true;
	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger.log(TreeLogger.TRACE, "Ignoring and deleting cache log "
	+ cacheFile.getAbsolutePath() + " due to read error.", ex);
	}
	} catch (ClassNotFoundException ex) {
	deleteCacheFile = true;
	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger.log(TreeLogger.TRACE, "Ignoring and deleting cache log "
	+ cacheFile.getAbsolutePath() + " due to deserialization error.", ex);
	}
	} finally {
	Utility.close(inputStream);
	Utility.close(bis);
	Utility.close(fis);
	}
	if (deleteCacheFile) {
	cacheFile.delete();
	} else {
	if (logger.isLoggable(TreeLogger.TRACE)) {
	logger.log(TreeLogger.TRACE, cacheFile.getName() + ": Load complete");
	}
	}
	}
	} else {
	logger
	.log(TreeLogger.TRACE,
	"Starting with empty Cache: CompilationUnit cache directory does not exist or is not readable.");
	}
	} finally {
	loadPersistentUnitEvent.end();
	}
	}
	}