API and plumbing for indexing file metadata (length + timestamp)

When SimpleCache uses a CacheFileMetadataIndex, it will be able to avoid
querying file.length() and renaming files, both of which are expensive
operations on some file systems.

PiperOrigin-RevId: 232664255

library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheFileMetadata.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.upstream.cache;
+
+/** Metadata associated with a cache file. */
+/* package */ final class CacheFileMetadata {
+
+  public final long length;
+  public final long lastAccessTimestamp;
+
+  public CacheFileMetadata(long length, long lastAccessTimestamp) {
+    this.length = length;
+    this.lastAccessTimestamp = lastAccessTimestamp;
+  }
+}

library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheFileMetadataIndex.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.upstream.cache;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+/** Maintains an index of cache file metadata. */
+/* package */ class CacheFileMetadataIndex {
+
+  /**
+   * Returns all file metadata keyed by file name. The returned map is mutable and may be modified
+   * by the caller.
+   */
+  public Map<String, CacheFileMetadata> getAll() {
+    return Collections.emptyMap();
+  }
+
+  /**
+   * Sets metadata for a given file.
+   *
+   * @param name The name of the file.
+   * @param length The file length.
+   * @param lastAccessTimestamp The file last access timestamp.
+   * @return Whether the index was updated successfully.
+   */
+  public boolean set(String name, long length, long lastAccessTimestamp) {
+    // TODO.
+    return false;
+  }
+
+  /**
+   * Removes metadata.
+   *
+   * @param name The name of the file whose metadata is to be removed.
+   */
+  public void remove(String name) {
+    // TODO.
+  }
+
+  /**
+   * Removes metadata.
+   *
+   * @param names The names of the files whose metadata is to be removed.
+   */
+  public void removeAll(Set<String> names) {
+    // TODO.
+  }
+}

library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CachedContent.java

@@ -16,13 +16,16 @@
 package com.google.android.exoplayer2.upstream.cache;
 
 import android.support.annotation.Nullable;
-import com.google.android.exoplayer2.upstream.cache.Cache.CacheException;
 import com.google.android.exoplayer2.util.Assertions;
+import com.google.android.exoplayer2.util.Log;
+import java.io.File;
 import java.util.TreeSet;
 
 /** Defines the cached content for a single stream. */
 /* package */ final class CachedContent {
 
+  private static final String TAG = "CachedContent";
+
   /** The cache file id that uniquely identifies the original stream. */
   public final int id;
   /** The cache key that uniquely identifies the original stream. */
@@ -138,21 +141,30 @@ import java.util.TreeSet;
   }
 
   /**
-   * Copies the given span with an updated last access time. Passed span becomes invalid after this
-   * call.
+   * Sets the given span's last access timestamp. The passed span becomes invalid after this call.
    *
    * @param cacheSpan Span to be copied and updated.
-   * @return a span with the updated last access time.
-   * @throws CacheException If renaming of the underlying span file failed.
+   * @param lastAccessTimestamp The new last access timestamp.
+   * @param updateFile Whether the span file should be renamed to have its timestamp match the new
+   *     last access time.
+   * @return A span with the updated last access timestamp.
    */
-  public SimpleCacheSpan touch(SimpleCacheSpan cacheSpan) throws CacheException {
-    SimpleCacheSpan newCacheSpan = cacheSpan.copyWithUpdatedLastAccessTime(id);
-    if (!cacheSpan.file.renameTo(newCacheSpan.file)) {
-      throw new CacheException("Renaming of " + cacheSpan.file + " to " + newCacheSpan.file
-          + " failed.");
-    }
-    // Replace the in-memory representation of the span.
+  public SimpleCacheSpan setLastAccessTimestamp(
+      SimpleCacheSpan cacheSpan, long lastAccessTimestamp, boolean updateFile) {
     Assertions.checkState(cachedSpans.remove(cacheSpan));
+    File file = cacheSpan.file;
+    if (updateFile) {
+      File directory = file.getParentFile();
+      long position = cacheSpan.position;
+      File newFile = SimpleCacheSpan.getCacheFile(directory, id, position, lastAccessTimestamp);
+      if (file.renameTo(newFile)) {
+        file = newFile;
+      } else {
+        Log.w(TAG, "Failed to rename " + file + " to " + newFile + ".");
+      }
+    }
+    SimpleCacheSpan newCacheSpan =
+        cacheSpan.copyWithFileAndLastAccessTimestamp(file, lastAccessTimestamp);
     cachedSpans.add(newCacheSpan);
     return newCacheSpan;
   }

library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/SimpleCache.java

@@ -25,6 +25,7 @@ import java.io.File;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.NavigableSet;
 import java.util.Random;
 import java.util.Set;
@@ -51,7 +52,8 @@ public final class SimpleCache implements Cache {
 
   private final File cacheDir;
   private final CacheEvictor evictor;
-  private final CachedContentIndex index;
+  private final CachedContentIndex contentIndex;
+  @Nullable private final CacheFileMetadataIndex fileIndex;
   private final HashMap<String, ArrayList<Listener>> listeners;
   private final Random random;
 
@@ -128,16 +130,17 @@ public final class SimpleCache implements Cache {
    *
    * @param cacheDir A dedicated cache directory.
    * @param evictor The evictor to be used.
-   * @param index The CachedContentIndex to be used.
+   * @param contentIndex The content index to be used.
    */
-  /* package */ SimpleCache(File cacheDir, CacheEvictor evictor, CachedContentIndex index) {
+  /* package */ SimpleCache(File cacheDir, CacheEvictor evictor, CachedContentIndex contentIndex) {
     if (!lockFolder(cacheDir)) {
       throw new IllegalStateException("Another SimpleCache instance uses the folder: " + cacheDir);
     }
 
     this.cacheDir = cacheDir;
     this.evictor = evictor;
-    this.index = index;
+    this.contentIndex = contentIndex;
+    this.fileIndex = null;
     listeners = new HashMap<>();
     random = new Random();
 
@@ -164,11 +167,11 @@ public final class SimpleCache implements Cache {
     listeners.clear();
     removeStaleSpans();
     try {
-      index.store();
+      contentIndex.store();
     } catch (CacheException e) {
       Log.e(TAG, "Storing index file failed", e);
     } finally {
-      index.release();
+      contentIndex.release();
       unlockFolder(cacheDir);
       released = true;
     }
@@ -204,7 +207,7 @@ public final class SimpleCache implements Cache {
   @Override
   public synchronized NavigableSet<CacheSpan> getCachedSpans(String key) {
     Assertions.checkState(!released);
-    CachedContent cachedContent = index.get(key);
+    CachedContent cachedContent = contentIndex.get(key);
     return cachedContent == null || cachedContent.isEmpty()
         ? new TreeSet<>()
         : new TreeSet<CacheSpan>(cachedContent.getSpans());
@@ -213,7 +216,7 @@ public final class SimpleCache implements Cache {
   @Override
   public synchronized Set<String> getKeys() {
     Assertions.checkState(!released);
-    return new HashSet<>(index.getKeys());
+    return new HashSet<>(contentIndex.getKeys());
   }
 
   @Override
@@ -243,27 +246,33 @@ public final class SimpleCache implements Cache {
   public synchronized @Nullable SimpleCacheSpan startReadWriteNonBlocking(String key, long position)
       throws CacheException {
     Assertions.checkState(!released);
-    SimpleCacheSpan cacheSpan = getSpan(key, position);
+    SimpleCacheSpan span = getSpan(key, position);
 
     // Read case.
-    if (cacheSpan.isCached) {
-      try {
-        // Obtain a new span with updated last access timestamp.
-        SimpleCacheSpan newCacheSpan = index.get(key).touch(cacheSpan);
-        notifySpanTouched(cacheSpan, newCacheSpan);
-        return newCacheSpan;
-      } catch (CacheException e) {
-        // Ignore. In worst case the cache span is evicted early.
-        // This happens very rarely [Internal: b/38351639]
-        return cacheSpan;
+    if (span.isCached) {
+      String fileName = span.file.getName();
+      long length = span.length;
+      long lastAccessTimestamp = System.currentTimeMillis();
+      // Updating the file itself to incorporate the new last access timestamp is much slower than
+      // updating the file index. Hence we only update the file if we don't have a file index, or if
+      // updating the file index failed.
+      boolean updateFile;
+      if (fileIndex != null) {
+        updateFile = !fileIndex.set(fileName, length, lastAccessTimestamp);
+      } else {
+        updateFile = true;
       }
+      SimpleCacheSpan newSpan =
+          contentIndex.get(key).setLastAccessTimestamp(span, lastAccessTimestamp, updateFile);
+      notifySpanTouched(span, newSpan);
+      return newSpan;
     }
 
-    CachedContent cachedContent = index.getOrAdd(key);
+    CachedContent cachedContent = contentIndex.getOrAdd(key);
     if (!cachedContent.isLocked()) {
       // Write case, lock available.
       cachedContent.setLocked(true);
-      return cacheSpan;
+      return span;
     }
 
     // Write case, lock not available.
@@ -273,7 +282,7 @@ public final class SimpleCache implements Cache {
   @Override
   public synchronized File startFile(String key, long position, long length) throws CacheException {
     Assertions.checkState(!released);
-    CachedContent cachedContent = index.get(key);
+    CachedContent cachedContent = contentIndex.get(key);
     Assertions.checkNotNull(cachedContent);
     Assertions.checkState(cachedContent.isLocked());
     if (!cacheDir.exists()) {
@@ -301,29 +310,35 @@ public final class SimpleCache implements Cache {
       file.delete();
       return;
     }
-    SimpleCacheSpan span = SimpleCacheSpan.createCacheEntry(file, length, index);
+
+    SimpleCacheSpan span = SimpleCacheSpan.createCacheEntry(file, length, contentIndex);
     Assertions.checkState(span != null);
-    CachedContent cachedContent = index.get(span.key);
+    CachedContent cachedContent = contentIndex.get(span.key);
     Assertions.checkNotNull(cachedContent);
     Assertions.checkState(cachedContent.isLocked());
+
     // Check if the span conflicts with the set content length
     long contentLength = ContentMetadata.getContentLength(cachedContent.getMetadata());
     if (contentLength != C.LENGTH_UNSET) {
       Assertions.checkState((span.position + span.length) <= contentLength);
     }
+
+    if (fileIndex != null) {
+      fileIndex.set(file.getName(), span.length, span.lastAccessTimestamp);
+    }
     addSpan(span);
-    index.store();
+    contentIndex.store();
     notifyAll();
   }
 
   @Override
   public synchronized void releaseHoleSpan(CacheSpan holeSpan) {
     Assertions.checkState(!released);
-    CachedContent cachedContent = index.get(holeSpan.key);
+    CachedContent cachedContent = contentIndex.get(holeSpan.key);
     Assertions.checkNotNull(cachedContent);
     Assertions.checkState(cachedContent.isLocked());
     cachedContent.setLocked(false);
-    index.maybeRemove(cachedContent.key);
+    contentIndex.maybeRemove(cachedContent.key);
     notifyAll();
   }
 
@@ -336,14 +351,14 @@ public final class SimpleCache implements Cache {
   @Override
   public synchronized boolean isCached(String key, long position, long length) {
     Assertions.checkState(!released);
-    CachedContent cachedContent = index.get(key);
+    CachedContent cachedContent = contentIndex.get(key);
     return cachedContent != null && cachedContent.getCachedBytesLength(position, length) >= length;
   }
 
   @Override
   public synchronized long getCachedLength(String key, long position, long length) {
     Assertions.checkState(!released);
-    CachedContent cachedContent = index.get(key);
+    CachedContent cachedContent = contentIndex.get(key);
     return cachedContent != null ? cachedContent.getCachedBytesLength(position, length) : -length;
   }
 
@@ -351,14 +366,14 @@ public final class SimpleCache implements Cache {
   public synchronized void applyContentMetadataMutations(
       String key, ContentMetadataMutations mutations) throws CacheException {
     Assertions.checkState(!released);
-    index.applyContentMetadataMutations(key, mutations);
-    index.store();
+    contentIndex.applyContentMetadataMutations(key, mutations);
+    contentIndex.store();
   }
 
   @Override
   public synchronized ContentMetadata getContentMetadata(String key) {
     Assertions.checkState(!released);
-    return index.getContentMetadata(key);
+    return contentIndex.getContentMetadata(key);
   }
 
   /**
@@ -375,7 +390,7 @@ public final class SimpleCache implements Cache {
    * @return The corresponding cache {@link SimpleCacheSpan}.
    */
   private SimpleCacheSpan getSpan(String key, long position) throws CacheException {
-    CachedContent cachedContent = index.get(key);
+    CachedContent cachedContent = contentIndex.get(key);
     if (cachedContent == null) {
       return SimpleCacheSpan.createOpenHole(key, position);
     }
@@ -398,40 +413,63 @@ public final class SimpleCache implements Cache {
       return;
     }
 
-    index.load();
-    loadDirectory(cacheDir, /* isRootDirectory= */ true);
-    index.removeEmpty();
+    contentIndex.load();
+    if (fileIndex != null) {
+      Map<String, CacheFileMetadata> fileMetadata = fileIndex.getAll();
+      loadDirectory(cacheDir, /* isRoot= */ true, fileMetadata);
+      fileIndex.removeAll(fileMetadata.keySet());
+    } else {
+      loadDirectory(cacheDir, /* isRoot= */ true, /* fileMetadata= */ null);
+    }
+    contentIndex.removeEmpty();
 
     try {
-      index.store();
+      contentIndex.store();
     } catch (CacheException e) {
       Log.e(TAG, "Storing index file failed", e);
     }
   }
 
-  private void loadDirectory(File directory, boolean isRootDirectory) {
+  /**
+   * Loads a cache directory. If the root directory is passed, also loads any subdirectories.
+   *
+   * @param directory The directory to load.
+   * @param isRoot Whether the directory is the root directory.
+   * @param fileMetadata A mutable map containing cache file metadata, keyed by file name. The map
+   *     is modified by removing entries for all loaded files. When the method call returns, the map
+   *     will contain only metadata that was unused. May be null if no file metadata is available.
+   */
+  private void loadDirectory(
+      File directory, boolean isRoot, @Nullable Map<String, CacheFileMetadata> fileMetadata) {
     File[] files = directory.listFiles();
     if (files == null) {
       // Not a directory.
       return;
     }
-    if (!isRootDirectory && files.length == 0) {
+    if (!isRoot && files.length == 0) {
       // Empty non-root directory.
       directory.delete();
       return;
     }
     for (File file : files) {
       String fileName = file.getName();
-      if (isRootDirectory && fileName.indexOf('.') == -1) {
-        loadDirectory(file, /* isRootDirectory= */ false);
+      if (isRoot && fileName.indexOf('.') == -1) {
+        loadDirectory(file, /* isRoot= */ false, fileMetadata);
       } else {
-        if (isRootDirectory && CachedContentIndex.isIndexFile(fileName)) {
+        if (isRoot && CachedContentIndex.isIndexFile(fileName)) {
           // Skip the (expected) index files in the root directory.
           continue;
         }
-        long fileLength = file.length();
+        CacheFileMetadata metadata =
+            fileMetadata != null ? fileMetadata.remove(file.getName()) : null;
+        long length = C.LENGTH_UNSET;
+        long lastAccessTimestamp = C.TIME_UNSET;
+        if (metadata != null) {
+          length = metadata.length;
+          lastAccessTimestamp = metadata.lastAccessTimestamp;
+        }
         SimpleCacheSpan span =
-            fileLength > 0 ? SimpleCacheSpan.createCacheEntry(file, fileLength, index) : null;
+            SimpleCacheSpan.createCacheEntry(file, length, lastAccessTimestamp, contentIndex);
         if (span != null) {
           addSpan(span);
         } else {
@@ -447,18 +485,21 @@ public final class SimpleCache implements Cache {
    * @param span The span to be added.
    */
   private void addSpan(SimpleCacheSpan span) {
-    index.getOrAdd(span.key).addSpan(span);
+    contentIndex.getOrAdd(span.key).addSpan(span);
     totalSpace += span.length;
     notifySpanAdded(span);
   }
 
   private void removeSpanInternal(CacheSpan span) {
-    CachedContent cachedContent = index.get(span.key);
+    CachedContent cachedContent = contentIndex.get(span.key);
     if (cachedContent == null || !cachedContent.removeSpan(span)) {
       return;
     }
     totalSpace -= span.length;
-    index.maybeRemove(cachedContent.key);
+    if (fileIndex != null) {
+      fileIndex.remove(span.file.getName());
+    }
+    contentIndex.maybeRemove(cachedContent.key);
     notifySpanRemoved(span);
   }
 
@@ -468,7 +509,7 @@ public final class SimpleCache implements Cache {
    */
   private void removeStaleSpans() {
     ArrayList<CacheSpan> spansToBeRemoved = new ArrayList<>();
-    for (CachedContent cachedContent : index.getAll()) {
+    for (CachedContent cachedContent : contentIndex.getAll()) {
       for (CacheSpan span : cachedContent.getSpans()) {
         if (!span.file.exists()) {
           spansToBeRemoved.add(span);

library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/SimpleCacheSpan.java

@@ -38,16 +38,16 @@ import java.util.regex.Pattern;
 
   /**
    * Returns a new {@link File} instance from {@code cacheDir}, {@code id}, {@code position}, {@code
-   * lastAccessTimestamp}.
+   * timestamp}.
    *
    * @param cacheDir The parent abstract pathname.
    * @param id The cache file id.
    * @param position The position of the stored data in the original stream.
-   * @param lastAccessTimestamp The last access timestamp.
+   * @param timestamp The file timestamp.
    * @return The cache file.
    */
-  public static File getCacheFile(File cacheDir, int id, long position, long lastAccessTimestamp) {
-    return new File(cacheDir, id + "." + position + "." + lastAccessTimestamp + SUFFIX);
+  public static File getCacheFile(File cacheDir, int id, long position, long timestamp) {
+    return new File(cacheDir, id + "." + position + "." + timestamp + SUFFIX);
   }
 
   /**
@@ -84,22 +84,36 @@ import java.util.regex.Pattern;
     return new SimpleCacheSpan(key, position, length, C.TIME_UNSET, null);
   }
 
-  /*
-   * Note: {@code fileLength} is equivalent to {@code file.length()}, but passing it as an explicit
-   * argument can reduce the number of calls to this method if the calling code already knows the
-   * file length. This is preferable because calling {@code file.length()} can be expensive. See:
-   * https://github.com/google/ExoPlayer/issues/4253#issuecomment-451593889.
-   */
   /**
    * Creates a cache span from an underlying cache file. Upgrades the file if necessary.
    *
    * @param file The cache file.
-   * @param length The length of the cache file in bytes.
+   * @param length The length of the cache file in bytes, or {@link C#LENGTH_UNSET} to query the
+   *     underlying file system. Querying the underlying file system can be expensive, so callers
+   *     that already know the length of the file should pass it explicitly.
    * @return The span, or null if the file name is not correctly formatted, or if the id is not
-   *     present in the content index.
+   *     present in the content index, or if the length is 0.
    */
   @Nullable
   public static SimpleCacheSpan createCacheEntry(File file, long length, CachedContentIndex index) {
+    return createCacheEntry(file, length, /* lastAccessTimestamp= */ C.TIME_UNSET, index);
+  }
+
+  /**
+   * Creates a cache span from an underlying cache file. Upgrades the file if necessary.
+   *
+   * @param file The cache file.
+   * @param length The length of the cache file in bytes, or {@link C#LENGTH_UNSET} to query the
+   *     underlying file system. Querying the underlying file system can be expensive, so callers
+   *     that already know the length of the file should pass it explicitly.
+   * @param lastAccessTimestamp The last access timestamp, or {@link C#TIME_UNSET} to use the file
+   *     timestamp.
+   * @return The span, or null if the file name is not correctly formatted, or if the id is not
+   *     present in the content index, or if the length is 0.
+   */
+  @Nullable
+  public static SimpleCacheSpan createCacheEntry(
+      File file, long length, long lastAccessTimestamp, CachedContentIndex index) {
     String name = file.getName();
     if (!name.endsWith(SUFFIX)) {
       file = upgradeFile(file, index);
@@ -120,9 +134,18 @@ import java.util.regex.Pattern;
       return null;
     }
 
+    if (length == C.LENGTH_UNSET) {
+      length = file.length();
+    }
+    if (length == 0) {
+      return null;
+    }
+
     long position = Long.parseLong(matcher.group(2));
-    long lastAccessTime = Long.parseLong(matcher.group(3));
-    return new SimpleCacheSpan(key, position, length, lastAccessTime, file);
+    if (lastAccessTimestamp == C.TIME_UNSET) {
+      lastAccessTimestamp = Long.parseLong(matcher.group(3));
+    }
+    return new SimpleCacheSpan(key, position, length, lastAccessTimestamp, file);
   }
 
   /**
@@ -174,18 +197,16 @@ import java.util.regex.Pattern;
   }
 
   /**
-   * Returns a copy of this CacheSpan whose last access time stamp is set to current time. This
-   * doesn't copy or change the underlying cache file.
+   * Returns a copy of this CacheSpan with a new file and last access timestamp.
    *
-   * @param id The cache file id.
-   * @return A {@link SimpleCacheSpan} with updated last access time stamp.
+   * @param file The new file.
+   * @param lastAccessTimestamp The new last access time.
+   * @return A copy with the new file and last access timestamp.
    * @throws IllegalStateException If called on a non-cached span (i.e. {@link #isCached} is false).
    */
-  public SimpleCacheSpan copyWithUpdatedLastAccessTime(int id) {
+  public SimpleCacheSpan copyWithFileAndLastAccessTimestamp(File file, long lastAccessTimestamp) {
     Assertions.checkState(isCached);
-    long now = System.currentTimeMillis();
-    File newCacheFile = getCacheFile(file.getParentFile(), id, position, now);
-    return new SimpleCacheSpan(key, position, length, now, newCacheFile);
+    return new SimpleCacheSpan(key, position, length, lastAccessTimestamp, file);
   }
 
 }