| /* |
| * Copyright 2011 Google Inc. |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| * use this file except in compliance with the License. You may obtain a copy of |
| * the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| * License for the specific language governing permissions and limitations under |
| * the License. |
| */ |
| package com.google.gwt.dev.javac; |
| |
| import com.google.gwt.core.ext.TreeLogger; |
| import com.google.gwt.core.ext.TreeLogger.Type; |
| import com.google.gwt.core.ext.UnableToCompleteException; |
| import com.google.gwt.dev.jjs.InternalCompilerException; |
| import com.google.gwt.thirdparty.guava.common.annotations.VisibleForTesting; |
| import com.google.gwt.thirdparty.guava.common.base.Preconditions; |
| import com.google.gwt.thirdparty.guava.common.collect.Lists; |
| |
| import java.io.File; |
| import java.util.List; |
| import java.util.concurrent.ExecutionException; |
| import java.util.concurrent.ExecutorService; |
| import java.util.concurrent.Executors; |
| import java.util.concurrent.Future; |
| import java.util.concurrent.RejectedExecutionException; |
| import java.util.concurrent.Semaphore; |
| import java.util.concurrent.TimeUnit; |
| import java.util.concurrent.TimeoutException; |
| import java.util.concurrent.atomic.AtomicInteger; |
| |
| /** |
| * A class that manages a persistent cache of {@link CompilationUnit} instances. |
| * Writes out {@link CompilationUnit} instances to a cache in a |
| * background thread. |
| * <p> |
| * The persistent cache is implemented as a directory of log files with a date |
| * timestamp. A new log file gets created each time a new PersistentUnitCache is |
| * instantiated, (once per invocation of the compiler or DevMode). The design is |
| * intended to support only a single PersistentUnitCache instance in the |
| * compiler at a time. |
| * <p> |
| * As new units are compiled, the cache data is appended to a log. This allows |
| * Java serialization to efficiently store references. The next time the cache |
| * is started, all logs are replayed and loaded into the cache in chronological |
| * order, with newer units taking precedence. A new cache file is created for |
| * any newly compiled units in this session. After a threshold of a certain |
| * number of files in the directory is reached |
| * {@link PersistentUnitCache#CACHE_FILE_THRESHOLD} , the cache files are |
| * consolidated back into a single file. |
| * |
| * <p> |
| * System Properties (see {@link UnitCacheSingleton}). |
| * |
| * <ul> |
| * <li>gwt.persistentunitcache : enables the persistent cache (eventually will |
| * be default)</li> |
| * <li>gwt.persistentunitcachedir=<dir>: sets or overrides the cache directory</li> |
| * </ul> |
| * |
| * <p> |
| * Known Issues: |
| * |
| * <ul> |
| * <li>This design uses an eager cache to load every unit in the cache on the |
| * first reference to find() or add(). When the cache is large (10000 units), it |
| * uses lots of heap and takes 5-10 seconds. Once the PersistentUnitCache is |
| * created, it starts eagerly loading the cache in a background thread).</li> |
| * |
| * <li>Although units logged to disk with the same resource path are eventually |
| * cleaned up, the most recently compiled unit stays in the cache forever. This |
| * means that stale units that are no longer referenced will never be purged, |
| * unless by some external action (e.g. ant clean).</li> |
| * |
| * <li>Unless ant builds are made aware of the cache directory, the cache will |
| * persist if a user does an ant clean.</li> |
| * </ul> |
| * |
| */ |
| class PersistentUnitCache extends MemoryUnitCache { |
| |
| /** |
| * If there are more than this many files in the cache, clean up the old |
| * files. |
| */ |
| static final int CACHE_FILE_THRESHOLD = 40; |
| |
| /** |
| * Note: to avoid deadlock, methods on backgroundService should not be called from |
| * within a synchronized method. (The BackgroundService lock should be acquired first.) |
| */ |
| private final BackgroundService backgroundService; |
| |
| private Semaphore cleanupInProgress = new Semaphore(1); |
| private AtomicInteger newUnitsSinceLastCleanup = new AtomicInteger(); |
| |
| PersistentUnitCache(final TreeLogger logger, File parentDir) throws UnableToCompleteException { |
| this.backgroundService = new BackgroundService(logger, parentDir, this); |
| } |
| |
| /** |
| * Enqueue a unit to be written by the background thread. |
| */ |
| @Override |
| public void add(CompilationUnit newUnit) { |
| internalAdd(newUnit); |
| } |
| |
| @VisibleForTesting |
| Future<?> internalAdd(CompilationUnit newUnit) { |
| Preconditions.checkNotNull(newUnit); |
| backgroundService.waitForCacheToLoad(); |
| addNewUnit(newUnit); |
| return backgroundService.asyncWriteUnit(newUnit); |
| } |
| |
| @Override |
| public void clear() throws UnableToCompleteException { |
| backgroundService.asyncClearCache(); |
| backgroundService.finishAndShutdown(); |
| synchronized (this) { |
| super.clear(); |
| } |
| backgroundService.start(); |
| } |
| |
| /** |
| * Rotates to a new file and/or starts garbage collection if needed after a compile is finished. |
| * |
| * Normally, only newly compiled units are written to the current log, but |
| * when it is time to cleanup, valid units from older log files need to be |
| * re-written. |
| */ |
| @Override |
| public void cleanup(TreeLogger logger) { |
| logger.log(Type.TRACE, "PersistentUnitCache cleanup requested"); |
| backgroundService.waitForCacheToLoad(); |
| |
| if (backgroundService.isShutdown()) { |
| logger.log(TreeLogger.TRACE, "Skipped PersistentUnitCache cleanup because it's shut down"); |
| return; |
| } |
| |
| if (!cleanupInProgress.tryAcquire()) { |
| return; // some other thread is already doing this. |
| } |
| |
| int addCallCount = newUnitsSinceLastCleanup.getAndSet(0); |
| logger.log(TreeLogger.TRACE, "Added " + addCallCount + |
| " units to PersistentUnitCache since last cleanup"); |
| if (addCallCount == 0) { |
| // Don't clean up until we compiled something. |
| logger.log(TreeLogger.TRACE, "Skipped PersistentUnitCache because no units were added"); |
| cleanupInProgress.release(); |
| return; |
| } |
| |
| int closedCount = backgroundService.getClosedCacheFileCount(); |
| if (closedCount < CACHE_FILE_THRESHOLD) { |
| // Not enough files yet, so just rotate to a new file. |
| logger.log(TreeLogger.TRACE, "Rotating PersistentUnitCache file because only " + |
| closedCount + " files were added."); |
| backgroundService.asyncRotate(cleanupInProgress); |
| return; |
| } |
| |
| logger.log(Type.TRACE, "Compacting persistent unit cache files"); |
| backgroundService.asyncCompact(getUnitsToSaveToDisk(), cleanupInProgress); |
| } |
| |
| /** |
| * Waits for any cleanup in progress to finish. |
| */ |
| @VisibleForTesting |
| void waitForCleanup() throws InterruptedException { |
| cleanupInProgress.acquire(); |
| cleanupInProgress.release(); |
| } |
| |
| @VisibleForTesting |
| void shutdown() throws InterruptedException, ExecutionException { |
| backgroundService.shutdown(); |
| } |
| |
| // Methods that read or write the in-memory cache |
| |
| @Override |
| public CompilationUnit find(ContentId contentId) { |
| backgroundService.waitForCacheToLoad(); |
| synchronized (this) { |
| return super.find(contentId); |
| } |
| } |
| |
| @Override |
| public CompilationUnit find(String resourcePath) { |
| backgroundService.waitForCacheToLoad(); |
| synchronized (this) { |
| return super.find(resourcePath); |
| } |
| } |
| |
| @Override |
| public synchronized void remove(CompilationUnit unit) { |
| super.remove(unit); |
| } |
| |
| /** |
| * Saves a newly compiled unit to the in-memory cache. |
| */ |
| private synchronized void addNewUnit(CompilationUnit unit) { |
| newUnitsSinceLastCleanup.incrementAndGet(); |
| super.add(unit); |
| } |
| |
| /** |
| * Adds a compilation unit from disk into the in-memory cache. |
| * (Callback from {@link PersistentUnitCacheDir}.) |
| */ |
| synchronized void maybeAddLoadedUnit(CachedCompilationUnit unit) { |
| UnitCacheEntry entry = new UnitCacheEntry(unit, UnitOrigin.PERSISTENT); |
| UnitCacheEntry existingEntry = unitMap.get(unit.getResourcePath()); |
| /* |
| * Don't assume that an existing entry is stale - an entry might have been loaded already from |
| * another source that is more up to date. If the timestamps are the same, accept the latest |
| * version. If it turns out to be stale, it will be recompiled and the updated unit will win |
| * this test the next time the session starts. |
| */ |
| if (existingEntry != null |
| && unit.getLastModified() >= existingEntry.getUnit().getLastModified()) { |
| super.remove(existingEntry.getUnit()); |
| unitMap.put(unit.getResourcePath(), entry); |
| unitMapByContentId.put(unit.getContentId(), entry); |
| } else if (existingEntry == null) { |
| unitMap.put(unit.getResourcePath(), entry); |
| unitMapByContentId.put(unit.getContentId(), entry); |
| } |
| } |
| |
| private synchronized List<CompilationUnit> getUnitsToSaveToDisk() { |
| List<CompilationUnit> result = Lists.newArrayList(); |
| for (UnitCacheEntry entry : unitMap.values()) { |
| result.add(Preconditions.checkNotNull(entry.getUnit())); |
| } |
| return result; |
| } |
| |
| /** |
| * Implements async methods that run in the background. |
| */ |
| private static class BackgroundService { |
| |
| private final TreeLogger logger; |
| private final PersistentUnitCacheDir cacheDir; |
| private ExecutorService service; |
| private PersistentUnitCache cacheToLoad; |
| |
| /** |
| * Non-null while the unit cache is loading. |
| */ |
| private Future<?> loadingDone; |
| |
| /** |
| * Starts the background thread and starts loading the given unit cache in the background. |
| */ |
| BackgroundService(TreeLogger logger, File parentDir, final PersistentUnitCache cacheToLoad) |
| throws UnableToCompleteException { |
| this.logger = logger; |
| this.cacheDir = new PersistentUnitCacheDir(logger, parentDir); |
| this.cacheToLoad = cacheToLoad; |
| |
| start(); |
| } |
| |
| /** |
| * Blocks addition of any further tasks and waits for current tasks to finish. |
| */ |
| public void finishAndShutdown() throws UnableToCompleteException { |
| service.shutdown(); |
| try { |
| if (!service.awaitTermination(30, TimeUnit.SECONDS)) { |
| logger.log(TreeLogger.WARN, |
| "Persistent Unit Cache shutdown tasks took longer than 30 seconds to complete."); |
| throw new UnableToCompleteException(); |
| } |
| } catch (InterruptedException e) { |
| // JVM is shutting down, ignore it. |
| } |
| } |
| |
| private void start() { |
| assert service == null || service.isTerminated(); |
| service = Executors.newSingleThreadExecutor(); |
| Runtime.getRuntime().addShutdownHook(new Thread() { |
| @Override |
| public void run() { |
| try { |
| Future<?> status = asyncShutdown(); |
| // Don't let the shutdown hang more than 5 seconds |
| status.get(5, TimeUnit.SECONDS); |
| } catch (InterruptedException e) { |
| // ignore |
| } catch (RejectedExecutionException e) { |
| // already shutdown, ignore |
| } catch (ExecutionException e) { |
| BackgroundService.this.logger.log(TreeLogger.ERROR, "Error during shutdown", e); |
| } catch (TimeoutException e) { |
| // ignore |
| } finally { |
| shutdownNow(); |
| } |
| } |
| }); |
| |
| /** |
| * Load up cached units from the persistent store in the background. The |
| * {@link #add(CompilationUnit)} and {@link #find(String)} methods block if |
| * invoked before this thread finishes. |
| */ |
| loadingDone = service.submit(new Runnable() { |
| @Override |
| public void run() { |
| cacheDir.loadUnitMap(cacheToLoad); |
| } |
| }); |
| } |
| |
| /** |
| * Blocks until the background service is done loading units into the in-memory cache. |
| * Note: don't call this from a synchronized method on PersistentUnitCache. |
| */ |
| synchronized void waitForCacheToLoad() { |
| if (loadingDone == null) { |
| return; // fast path |
| } |
| |
| try { |
| loadingDone.get(); |
| loadingDone = null; |
| } catch (InterruptedException e) { |
| throw new InternalCompilerException( |
| "Interrupted waiting for PersistentUnitCache to load.", e); |
| } catch (ExecutionException e) { |
| logger.log(TreeLogger.ERROR, "Failed to load PersistentUnitCache.", e); |
| // Keep going. We didn't load anything but will still save units to the cache. |
| loadingDone = null; |
| } |
| } |
| |
| boolean isShutdown() { |
| return service.isShutdown(); |
| } |
| |
| @VisibleForTesting |
| void shutdown() throws InterruptedException, ExecutionException { |
| logger.log(Type.INFO, "PersistentUnitCache shutdown requested"); |
| try { |
| asyncShutdown().get(); |
| } catch (RejectedExecutionException ex) { |
| // background thread is not running - ignore |
| } |
| } |
| |
| int getClosedCacheFileCount() { |
| return cacheDir.getClosedCacheFileCount(); |
| } |
| |
| /** |
| * Rotates to a new file. |
| * @param cleanupInProgress a semaphore to release when done. |
| * (The permit must already be acquired.) |
| */ |
| Future<?> asyncRotate(final Semaphore cleanupInProgress) { |
| return service.submit(new Runnable() { |
| @Override |
| public void run() { |
| try { |
| cacheDir.rotate(); |
| } catch (UnableToCompleteException e) { |
| shutdownNow(); |
| } finally { |
| cleanupInProgress.release(); |
| } |
| } |
| }); |
| } |
| |
| /** |
| * Compacts the persistent unit cache and then rotates to a new file. |
| * There will be one closed file and one empty, open file when done. |
| * @param unitsToSave all compilation units to keep |
| * @param cleanupInProgress a semaphore to release when done. |
| * (The permit must already be acquired.) |
| */ |
| Future<?> asyncCompact(final List<CompilationUnit> unitsToSave, |
| final Semaphore cleanupInProgress) { |
| |
| return service.submit(new Runnable() { |
| @Override |
| public void run() { |
| try { |
| for (CompilationUnit unit : unitsToSave) { |
| cacheDir.writeUnit(unit); |
| } |
| cacheDir.deleteClosedCacheFiles(); |
| cacheDir.rotate(); // Move to a new, empty file. |
| } catch (UnableToCompleteException e) { |
| shutdownNow(); |
| } finally { |
| cleanupInProgress.release(); |
| } |
| } |
| }); |
| } |
| |
| Future<?> asyncClearCache() { |
| Future<?> status = service.submit(new Runnable() { |
| @Override |
| public void run() { |
| cacheDir.closeCurrentFile(); |
| cacheDir.deleteClosedCacheFiles(); |
| } |
| }); |
| service.shutdown(); // Don't allow more tasks to be scheduled. |
| return status; |
| } |
| |
| Future<?> asyncWriteUnit(final CompilationUnit unit) { |
| try { |
| return service.submit(new Runnable() { |
| @Override |
| public void run() { |
| try { |
| cacheDir.writeUnit(unit); |
| } catch (UnableToCompleteException e) { |
| shutdownNow(); |
| } |
| } |
| }); |
| } catch (RejectedExecutionException ex) { |
| // background thread is not running, ignore |
| return null; |
| } |
| } |
| |
| Future<?> asyncShutdown() { |
| Future<?> status = service.submit(new Runnable() { |
| @Override |
| public void run() { |
| cacheDir.closeCurrentFile(); |
| shutdownNow(); |
| } |
| }); |
| service.shutdown(); // Don't allow more tasks to be scheduled. |
| return status; |
| } |
| |
| private void shutdownNow() { |
| logger.log(TreeLogger.TRACE, "Shutting down PersistentUnitCache thread"); |
| service.shutdownNow(); |
| } |
| } |
| } |