/*
 * Copyright 2011 Google Inc.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */
package com.google.gwt.dev.javac;

import com.google.gwt.core.ext.TreeLogger;
import com.google.gwt.core.ext.UnableToCompleteException;
import com.google.gwt.dev.jjs.InternalCompilerException;
import com.google.gwt.dev.jjs.ast.JNode;
import com.google.gwt.dev.jjs.impl.GwtAstBuilder;
import com.google.gwt.dev.util.log.speedtracer.DevModeEventType;
import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger;
import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger.Event;
import com.google.gwt.util.tools.Utility;
import com.google.gwt.thirdparty.guava.common.hash.Hashing;
import com.google.gwt.thirdparty.guava.common.io.Files;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.net.JarURLConnection;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;

/**
 * A class that manages a persistent cache of {@link CompilationUnit} instances.
 * Writes out {@CompilationUnit} instances to a cache in a
 * background thread.
 * <p>
 * The persistent cache is implemented as a directory of log files with a date
 * timestamp. A new log file gets created each time a new PersistentUnitCache is
 * instantiated, (once per invocation of the compiler or DevMode). The design is
 * intended to support only a single PersistentUnitCache instance in the
 * compiler at a time.
 * <p>
 * As new units are compiled, the cache data is appended to a log. This allows
 * Java serialization to efficiently store references. The next time the cache
 * is started, all logs are replayed and loaded into the cache in chronological
 * order, with newer units taking precedence. A new cache file is created for
 * any newly compiled units in this session. After a threshold of a certain
 * number of files in the directory is reached
 * {@link PersistentUnitCache#CACHE_FILE_THRESHOLD} , the cache files are
 * consolidated back into a single file.
 *
 * <p>
 * System Properties (see {@link UnitCacheFactory}).
 *
 * <ul>
 * <li>gwt.persistentunitcache : enables the persistent cache (eventually will
 * be default)</li>
 * <li>gwt.persistentunitcachedir=<dir>: sets or overrides the cache directory</li>
 * </ul>
 *
 * <p>
 * Known Issues:
 *
 * <ul>
 * <li>This design uses an eager cache to load every unit in the cache on the
 * first reference to find() or add(). When the cache is large (10000 units), it
 * uses lots of heap and takes 5-10 seconds. Once the PersistentUnitCache is
 * created, it starts eagerly loading the cache in a background thread).</li>
 *
 * <li>Although units logged to disk with the same resource path are eventually
 * cleaned up, the most recently compiled unit stays in the cache forever. This
 * means that stale units that are no longer referenced will never be purged,
 * unless by some external action (e.g. ant clean).</li>
 *
 * <li>Unless ant builds are made aware of the cache directory, the cache will
 * persist if a user does an ant clean.</li>
 * </ul>
 *
 */
class PersistentUnitCache extends MemoryUnitCache {

  /**
   * If there are more than this many files in the cache, clean up the old
   * files.
   */
  static final int CACHE_FILE_THRESHOLD = 40;

  /**
   * Common prefix for creating directories and cache files.
   */
  static final String UNIT_CACHE_PREFIX = "gwt-unitCache";
  static final String CACHE_FILE_PREFIX = UNIT_CACHE_PREFIX + "-";
  static final String CURRENT_VERSION_CACHE_FILE_PREFIX = computePersistentCacheFilenamePrefix();

  /**
   * Creates a new file with a name based on the current system time.
   */
  private static File createCacheFile(TreeLogger logger, File cacheDirectory)
      throws UnableToCompleteException {
    File newFile = null;
    long timestamp = System.currentTimeMillis();
    try {
      do {
        newFile = new File(cacheDirectory, CURRENT_VERSION_CACHE_FILE_PREFIX +
            String.format("%016X", timestamp++));
      } while (!newFile.createNewFile());
    } catch (IOException ex) {
      logger.log(TreeLogger.WARN, "Unable to create new cache log file "
          + (newFile == null ? "<not created>" : newFile.getAbsolutePath()) + ".", ex);
      throw new UnableToCompleteException();
    }

    if (!newFile.canWrite()) {
      logger.log(TreeLogger.WARN, "Unable to write to new cache log file "
          + newFile.getAbsolutePath() + ".");
      throw new UnableToCompleteException();
    }

    return newFile;
  }

  /**
   * Finds all files matching a pattern in the cache directory.
   *
   * @return an array of sorted filenames. The file name pattern is such that
   *         sorting them alphabetically also sorts the files by age.
   */
  private static File[] getCacheFiles(File cacheDirectory, boolean currentVersion) {
    if (cacheDirectory.isDirectory()) {
      File[] files = cacheDirectory.listFiles();
      List<File> cacheFiles = new ArrayList<File>();
      for (File file : files) {
        if (file.getName().startsWith(currentVersion ? CURRENT_VERSION_CACHE_FILE_PREFIX :
            CACHE_FILE_PREFIX)) {
          cacheFiles.add(file);
        }
      }
      File[] retFiles = cacheFiles.toArray(new File[cacheFiles.size()]);
      Arrays.sort(retFiles);
      return retFiles;
    }
    return new File[0];
  }

  /**
   * There is no significance in the return value, we just want to be able
   * to tell if the purgeOldCacheFilesTask has completed.
   */
  private Future<Boolean> purgeTaskStatus;
  private AtomicBoolean purgeInProgress = new AtomicBoolean(false);

  private final Runnable purgeOldCacheFilesTask = new Runnable() {
    @Override
    public void run() {
      try {
        // Delete all cache files in the directory except for the currently open
        // file.
        SpeedTracerLogger.Event deleteEvent = SpeedTracerLogger.start(DevModeEventType.DELETE_CACHE);
        File[] filesToDelete = getCacheFiles(cacheDirectory, false);
        logger.log(TreeLogger.TRACE, "Purging cache files from " + cacheDirectory);
        for (File toDelete : filesToDelete) {
          if (!currentCacheFile.equals(toDelete)) {
            if (!toDelete.delete()) {
              logger.log(TreeLogger.WARN, "Couldn't delete file: " + toDelete);
            }
          }
        }
        deleteEvent.end();

        rotateCurrentCacheFile();
      } catch (UnableToCompleteException e) {
        backgroundService.shutdownNow();
      } finally {
        purgeInProgress.set(false);
      }
    }
  };

  private final Runnable rotateCacheFilesTask = new Runnable() {
    @Override
    public void run() {
      try {
        rotateCurrentCacheFile();
      } catch (UnableToCompleteException e) {
        backgroundService.shutdownNow();
      }
      assert (currentCacheFile != null);
    }
  };

  private final Runnable shutdownThreadTask = new Runnable() {
    @Override
    public void run() {
      assert (currentCacheFile != null);
      closeCurrentCacheFile(currentCacheFile, currentCacheFileStream);
      logger.log(TreeLogger.TRACE, "Shutting down PersistentUnitCache thread");
      backgroundService.shutdownNow();
    }
  };

  /**
   * Saved to be able to wait for UNIT_MAP_LOAD_TASK to complete.
   */
  private Future<Boolean> unitMapLoadStatus;

  private final Runnable unitMapLoadTask = new Runnable() {
    @Override
    public void run() {
      loadUnitMap(logger, currentCacheFile);
    }
  };

  /**
   * Used to execute the above Runnables in a background thread.
   */
  private final ExecutorService backgroundService;

  private int unitsWritten = 0;

  private int addedSinceLastCleanup = 0;

  /**
   * A directory to store the cache files that should persist between
   * invocations.
   */
  private final File cacheDirectory;

  /**
   * Current file and stream being written to.
   */
  private File currentCacheFile;
  private ObjectOutputStream currentCacheFileStream;

  private final TreeLogger logger;

  PersistentUnitCache(final TreeLogger logger, File cacheDir) throws UnableToCompleteException {
    assert cacheDir != null;
    this.logger = logger;

    /*
     * We must canonicalize the path here, otherwise we might set cacheDirectory
     * to something like "/path/to/x/../gwt-unitCache". If this were to happen,
     * the mkdirs() call below would create "/path/to/gwt-unitCache" but
     * not "/path/to/x".
     * Further accesses via the uncanonicalized path will fail, if "/path/to/x"
     * had not been created by other means.
     *
     * Fixes issue 6443
     */
    try {
      this.cacheDirectory = new File(cacheDir.getCanonicalFile(), UNIT_CACHE_PREFIX);
    } catch (IOException ex) {
      logger.log(TreeLogger.WARN, "Unable to create canonical file of "
          + cacheDir.getAbsolutePath() + ".", ex);
      throw new UnableToCompleteException();
    }

    if (logger.isLoggable(TreeLogger.TRACE)) {
      logger.log(TreeLogger.TRACE, "Persistent unit cache dir set to: "
          + this.cacheDirectory.getAbsolutePath());
    }

    if (!cacheDirectory.isDirectory() && !cacheDirectory.mkdirs()) {
      logger.log(TreeLogger.WARN, "Unable to initialize cache. Couldn't create directory "
          + cacheDirectory.getAbsolutePath() + ".");
      throw new UnableToCompleteException();
    }

    currentCacheFile = createCacheFile(logger, cacheDirectory);

    backgroundService = Executors.newSingleThreadExecutor();
    Runtime.getRuntime().addShutdownHook(new Thread() {
      @Override
      public void run() {
        try {
          Future<Boolean> status = backgroundService.submit(shutdownThreadTask, Boolean.TRUE);
          // Don't let the shutdown hang more than 5 seconds
          status.get(5, TimeUnit.SECONDS);
        } catch (InterruptedException e) {
          // ignore
        } catch (RejectedExecutionException e) {
          // already shutdown, ignore
        } catch (ExecutionException e) {
          logger.log(TreeLogger.ERROR, "Error during shutdown", e);
        } catch (TimeoutException e) {
          // ignore
        } finally {
          backgroundService.shutdownNow();
        }
      }
    });

    /**
     * Load up cached units from the persistent store in the background. The
     * {@link #add(CompilationUnit)} and {@link #find(String)} methods block if
     * invoked before this thread finishes.
     */
    unitMapLoadStatus = backgroundService.submit(unitMapLoadTask, Boolean.TRUE);

    FileOutputStream fstream = null;
    BufferedOutputStream bstream = null;

    try {
      fstream = new FileOutputStream(currentCacheFile);
      bstream = new BufferedOutputStream(fstream);
      currentCacheFileStream = new ObjectOutputStream(bstream);
    } catch (IOException ex) {
      closeCurrentCacheFile(currentCacheFile, currentCacheFileStream);
      logger.log(TreeLogger.ERROR, "Error creating cache " + currentCacheFile
          + ". Disabling cache.", ex);
      backgroundService.shutdownNow();
      throw new UnableToCompleteException();
    }
  }

  /**
   * Enqueue a unit to be written by the background thread.
   */
  @Override
  public void add(CompilationUnit newUnit) {
    awaitUnitCacheMapLoad();
    addedSinceLastCleanup++;
    super.add(newUnit);
    addImpl(unitMap.get(newUnit.getResourcePath()));
  }

  /**
   * Cleans up old cache files in the directory, migrating everything previously
   * loaded in them to the current cache file.
   *
   * Normally, only newly compiled units are written to the current log, but
   * when it is time to cleanup, valid units from older log files need to be
   * re-written.
   */
  @Override
  public void cleanup(TreeLogger logger) {
    awaitUnitCacheMapLoad();

    if (backgroundService.isShutdown()) {
      return;
    }
    boolean shouldRotate = addedSinceLastCleanup > 0;
    logger.log(TreeLogger.TRACE, "Added " + addedSinceLastCleanup +
        " units to cache since last cleanup.");
    addedSinceLastCleanup = 0;
    try {
      File[] cacheFiles = getCacheFiles(cacheDirectory, true);
      if (cacheFiles.length < CACHE_FILE_THRESHOLD) {
        if (shouldRotate) {
          backgroundService.execute(rotateCacheFilesTask);
        }
        return;
      }

      // Check to see if the previous purge task finished.
      boolean inProgress = purgeInProgress.getAndSet(true);
      if (inProgress) {
        try {
          purgeTaskStatus.get(0, TimeUnit.NANOSECONDS);
        } catch (InterruptedException ex) {
          Thread.currentThread().interrupt();
        } catch (TimeoutException ex) {
          // purge is currently in progress.
          return;
        }
      }

      /*
       * Resend all units read in from the in-memory cache to the background
       * thread. They will be re-written out and the old cache files removed.
       */
      synchronized (unitMap) {
        for (UnitCacheEntry unitCacheEntry : unitMap.values()) {
          if (unitCacheEntry.getOrigin() == UnitOrigin.PERSISTENT) {
            addImpl(unitCacheEntry);
          }
        }
      }

      purgeTaskStatus = backgroundService.submit(purgeOldCacheFilesTask, Boolean.TRUE);

    } catch (ExecutionException ex) {
      throw new InternalCompilerException("Error purging cache", ex);
    } catch (RejectedExecutionException ex) {
      // Cache background thread is not running - ignore
    }
  }

  @Override
  public CompilationUnit find(ContentId contentId) {
    awaitUnitCacheMapLoad();
    return super.find(contentId);
  }

  @Override
  public CompilationUnit find(String resourcePath) {
    awaitUnitCacheMapLoad();
    return super.find(resourcePath);
  }

  public void rotateCurrentCacheFile() throws UnableToCompleteException {
    if (logger.isLoggable(TreeLogger.TRACE)) {
      logger.log(TreeLogger.TRACE, "Wrote " + unitsWritten + " units to persistent cache.");
    }

    // Close and re-open a new log file to drop object references kept
    // alive in the existing file by Java serialization.
    closeCurrentCacheFile(currentCacheFile, currentCacheFileStream);
    unitsWritten = 0;
    currentCacheFile = createCacheFile(logger, cacheDirectory);
    FileOutputStream fstream = null;
    BufferedOutputStream bstream = null;
    try {
      fstream = new FileOutputStream(currentCacheFile);
      bstream = new BufferedOutputStream(fstream);
      currentCacheFileStream = new ObjectOutputStream(bstream);
    } catch (IOException ex) {
      // Close all 3 streams, not sure where the exception occurred.
      Utility.close(bstream);
      Utility.close(fstream);
      closeCurrentCacheFile(currentCacheFile, currentCacheFileStream);
      logger.log(TreeLogger.ERROR, "Error rotating file.  Shutting down cache thread.", ex);
      throw new UnableToCompleteException();
    }
  }

  /**
   * For Unit testing - shutdown the persistent cache.
   *
   * @throws ExecutionException
   * @throws InterruptedException
   */
  void shutdown() throws InterruptedException, ExecutionException {
    try {
      Future<Runnable> future = backgroundService.submit(shutdownThreadTask, shutdownThreadTask);
      backgroundService.shutdown();
      future.get();
    } catch (RejectedExecutionException ex) {
      // background thread is not running - ignore
    }
  }

  private static String computePersistentCacheFilenamePrefix() {
    try {
      URLConnection urlConnection =
          JNode.class.getResource("JNode.class").openConnection();
      if (urlConnection instanceof JarURLConnection) {
        String gwtdevJar = ((JarURLConnection) urlConnection).getJarFile().getName();
        return CACHE_FILE_PREFIX +
            Files.hash(new File(gwtdevJar), Hashing.sha1()).toString() + "-";
      }
      System.err.println("Could not find the GWT compiler jarfile. "
          + "Serialization errors might occur when accessing the persistent unit cache.");
    } catch (IOException e) {
      System.err.println("Could not compute the hash for the GWT compiler jarfile."
          + "Serialization errors might occur when accessing the persistent unit cache.");
      e.printStackTrace();
    }
    return CACHE_FILE_PREFIX;
  }

  private void addImpl(final UnitCacheEntry entry) {
    try {
      backgroundService.execute(new Runnable() {
        @Override
        public void run() {
          try {
            assert entry.getOrigin() != UnitOrigin.ARCHIVE;
            CompilationUnit unit = entry.getUnit();
            assert unit != null;
            currentCacheFileStream.writeObject(unit);
            unitsWritten++;
          } catch (IOException ex) {
            backgroundService.shutdownNow();
            if (logger.isLoggable(TreeLogger.TRACE)) {
              logger.log(TreeLogger.TRACE, "Error saving unit to cache in: "
                  + cacheDirectory.getAbsolutePath(), ex);
            }
          }
        }
      });
    } catch (RejectedExecutionException ex) {
      // background thread is not running, ignore
    }
  }

  private synchronized void awaitUnitCacheMapLoad() {
    // wait on initial load of unit map to complete.
    try {
      if (unitMapLoadStatus != null) {
        unitMapLoadStatus.get();
        // no need to check any more.
        unitMapLoadStatus = null;
      }
    } catch (InterruptedException e) {
      throw new InternalCompilerException("Interrupted waiting for unit cache map to load.", e);
    } catch (ExecutionException e) {
      logger.log(TreeLogger.ERROR, "Failure in unit cache map load.", e);
      // keep going
      unitMapLoadStatus = null;
    }
  }

  private void closeCurrentCacheFile(File openFile, ObjectOutputStream stream) {
    Utility.close(stream);
    if (unitsWritten == 0) {
      // Remove useless empty file.
      openFile.delete();
    }
  }

  /**
   * Load everything cached on disk into memory.
   */
  private void loadUnitMap(TreeLogger logger, File currentCacheFile) {
    Event loadPersistentUnitEvent =
        SpeedTracerLogger.start(DevModeEventType.LOAD_PERSISTENT_UNIT_CACHE);
    if (logger.isLoggable(TreeLogger.TRACE)) {
      logger.log(TreeLogger.TRACE, "Looking for previously cached Compilation Units in "
          + cacheDirectory.getAbsolutePath());
    }
    try {
      if (cacheDirectory.isDirectory() && cacheDirectory.canRead()) {
        File[] files = getCacheFiles(cacheDirectory, true);
        for (File cacheFile : files) {
          FileInputStream fis = null;
          BufferedInputStream bis = null;
          ObjectInputStream inputStream = null;
          if (cacheFile.equals(currentCacheFile)) {
            continue;
          }
          boolean deleteCacheFile = false;
          try {
            fis = new FileInputStream(cacheFile);
            bis = new BufferedInputStream(fis);
            /*
             * It is possible for the next call to throw an exception, leaving
             * inputStream null and fis still live.
             */
            inputStream = new ObjectInputStream(bis);
            while (true) {
              CachedCompilationUnit unit = (CachedCompilationUnit) inputStream.readObject();
              if (unit == null) {
                break;
              }
              if (unit.getTypesSerializedVersion() != GwtAstBuilder.getSerializationVersion()) {
                continue;
              }
              UnitCacheEntry entry = new UnitCacheEntry(unit, UnitOrigin.PERSISTENT);
              UnitCacheEntry existingEntry = unitMap.get(unit.getResourcePath());
              /*
               * Don't assume that an existing entry is stale - an entry might
               * have been loaded already from another source like a
               * CompilationUnitArchive that is more up to date. If the
               * timestamps are the same, accept the latest version. If it turns
               * out to be stale, it will be recompiled and the updated unit
               * will win this test the next time the session starts.
               */
              if (existingEntry != null
                  && unit.getLastModified() >= existingEntry.getUnit().getLastModified()) {
                super.remove(existingEntry.getUnit());
                unitMap.put(unit.getResourcePath(), entry);
                unitMapByContentId.put(unit.getContentId(), entry);
              } else if (existingEntry == null) {
                unitMap.put(unit.getResourcePath(), entry);
                unitMapByContentId.put(unit.getContentId(), entry);
              }
            }
          } catch (EOFException ex) {
            // Go on to the next file.
          } catch (IOException ex) {
            deleteCacheFile = true;
            if (logger.isLoggable(TreeLogger.TRACE)) {
              logger.log(TreeLogger.TRACE, "Ignoring and deleting cache log "
                  + cacheFile.getAbsolutePath() + " due to read error.", ex);
            }
          } catch (ClassNotFoundException ex) {
            deleteCacheFile = true;
            if (logger.isLoggable(TreeLogger.TRACE)) {
              logger.log(TreeLogger.TRACE, "Ignoring and deleting cache log "
                  + cacheFile.getAbsolutePath() + " due to deserialization error.", ex);
            }
          } finally {
            Utility.close(inputStream);
            Utility.close(bis);
            Utility.close(fis);
          }
          if (deleteCacheFile) {
            cacheFile.delete();
          } else {
            if (logger.isLoggable(TreeLogger.TRACE)) {
              logger.log(TreeLogger.TRACE, cacheFile.getName() + ": Load complete");
            }
          }
        }
      } else {
        logger
            .log(TreeLogger.TRACE,
                "Starting with empty Cache: CompilationUnit cache "
                    + "directory does not exist or is not readable.");
      }
    } finally {
      loadPersistentUnitEvent.end();
    }
  }
}