From bdc87a4fc7fc07ca4446c790a644f5d60e42109d Mon Sep 17 00:00:00 2001 From: olly Date: Wed, 6 Feb 2019 14:26:02 +0000 Subject: [PATCH] API and plumbing for indexing file metadata (length + timestamp) When SimpleCache uses a CacheFileMetadataIndex, it will be able to avoid querying file.length() and renaming files, both of which are expensive operations on some file systems. PiperOrigin-RevId: 232664255 --- .../upstream/cache/CacheFileMetadata.java | 28 ++++ .../cache/CacheFileMetadataIndex.java | 63 ++++++++ .../upstream/cache/CachedContent.java | 36 +++-- .../upstream/cache/SimpleCache.java | 137 ++++++++++++------ .../upstream/cache/SimpleCacheSpan.java | 65 ++++++--- 5 files changed, 247 insertions(+), 82 deletions(-) create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheFileMetadata.java create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheFileMetadataIndex.java diff --git a/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheFileMetadata.java b/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheFileMetadata.java new file mode 100644 index 0000000000..492b98a0de --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheFileMetadata.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.upstream.cache; + +/** Metadata associated with a cache file. */ +/* package */ final class CacheFileMetadata { + + public final long length; + public final long lastAccessTimestamp; + + public CacheFileMetadata(long length, long lastAccessTimestamp) { + this.length = length; + this.lastAccessTimestamp = lastAccessTimestamp; + } +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheFileMetadataIndex.java b/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheFileMetadataIndex.java new file mode 100644 index 0000000000..b25eb91810 --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheFileMetadataIndex.java @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.upstream.cache; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +/** Maintains an index of cache file metadata. */ +/* package */ class CacheFileMetadataIndex { + + /** + * Returns all file metadata keyed by file name. The returned map is mutable and may be modified + * by the caller. + */ + public Map getAll() { + return Collections.emptyMap(); + } + + /** + * Sets metadata for a given file. + * + * @param name The name of the file. + * @param length The file length. + * @param lastAccessTimestamp The file last access timestamp. + * @return Whether the index was updated successfully. + */ + public boolean set(String name, long length, long lastAccessTimestamp) { + // TODO. + return false; + } + + /** + * Removes metadata. + * + * @param name The name of the file whose metadata is to be removed. + */ + public void remove(String name) { + // TODO. + } + + /** + * Removes metadata. + * + * @param names The names of the files whose metadata is to be removed. + */ + public void removeAll(Set names) { + // TODO. + } +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CachedContent.java b/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CachedContent.java index 64ef33e3c9..80b50d862a 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CachedContent.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CachedContent.java @@ -16,13 +16,16 @@ package com.google.android.exoplayer2.upstream.cache; import android.support.annotation.Nullable; -import com.google.android.exoplayer2.upstream.cache.Cache.CacheException; import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.Log; +import java.io.File; import java.util.TreeSet; /** Defines the cached content for a single stream. */ /* package */ final class CachedContent { + private static final String TAG = "CachedContent"; + /** The cache file id that uniquely identifies the original stream. */ public final int id; /** The cache key that uniquely identifies the original stream. */ @@ -138,21 +141,30 @@ import java.util.TreeSet; } /** - * Copies the given span with an updated last access time. Passed span becomes invalid after this - * call. + * Sets the given span's last access timestamp. The passed span becomes invalid after this call. * * @param cacheSpan Span to be copied and updated. - * @return a span with the updated last access time. - * @throws CacheException If renaming of the underlying span file failed. + * @param lastAccessTimestamp The new last access timestamp. + * @param updateFile Whether the span file should be renamed to have its timestamp match the new + * last access time. + * @return A span with the updated last access timestamp. */ - public SimpleCacheSpan touch(SimpleCacheSpan cacheSpan) throws CacheException { - SimpleCacheSpan newCacheSpan = cacheSpan.copyWithUpdatedLastAccessTime(id); - if (!cacheSpan.file.renameTo(newCacheSpan.file)) { - throw new CacheException("Renaming of " + cacheSpan.file + " to " + newCacheSpan.file - + " failed."); - } - // Replace the in-memory representation of the span. + public SimpleCacheSpan setLastAccessTimestamp( + SimpleCacheSpan cacheSpan, long lastAccessTimestamp, boolean updateFile) { Assertions.checkState(cachedSpans.remove(cacheSpan)); + File file = cacheSpan.file; + if (updateFile) { + File directory = file.getParentFile(); + long position = cacheSpan.position; + File newFile = SimpleCacheSpan.getCacheFile(directory, id, position, lastAccessTimestamp); + if (file.renameTo(newFile)) { + file = newFile; + } else { + Log.w(TAG, "Failed to rename " + file + " to " + newFile + "."); + } + } + SimpleCacheSpan newCacheSpan = + cacheSpan.copyWithFileAndLastAccessTimestamp(file, lastAccessTimestamp); cachedSpans.add(newCacheSpan); return newCacheSpan; } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/SimpleCache.java b/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/SimpleCache.java index 72b39e24d1..dcdedfc32d 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/SimpleCache.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/SimpleCache.java @@ -25,6 +25,7 @@ import java.io.File; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.NavigableSet; import java.util.Random; import java.util.Set; @@ -51,7 +52,8 @@ public final class SimpleCache implements Cache { private final File cacheDir; private final CacheEvictor evictor; - private final CachedContentIndex index; + private final CachedContentIndex contentIndex; + @Nullable private final CacheFileMetadataIndex fileIndex; private final HashMap> listeners; private final Random random; @@ -128,16 +130,17 @@ public final class SimpleCache implements Cache { * * @param cacheDir A dedicated cache directory. * @param evictor The evictor to be used. - * @param index The CachedContentIndex to be used. + * @param contentIndex The content index to be used. */ - /* package */ SimpleCache(File cacheDir, CacheEvictor evictor, CachedContentIndex index) { + /* package */ SimpleCache(File cacheDir, CacheEvictor evictor, CachedContentIndex contentIndex) { if (!lockFolder(cacheDir)) { throw new IllegalStateException("Another SimpleCache instance uses the folder: " + cacheDir); } this.cacheDir = cacheDir; this.evictor = evictor; - this.index = index; + this.contentIndex = contentIndex; + this.fileIndex = null; listeners = new HashMap<>(); random = new Random(); @@ -164,11 +167,11 @@ public final class SimpleCache implements Cache { listeners.clear(); removeStaleSpans(); try { - index.store(); + contentIndex.store(); } catch (CacheException e) { Log.e(TAG, "Storing index file failed", e); } finally { - index.release(); + contentIndex.release(); unlockFolder(cacheDir); released = true; } @@ -204,7 +207,7 @@ public final class SimpleCache implements Cache { @Override public synchronized NavigableSet getCachedSpans(String key) { Assertions.checkState(!released); - CachedContent cachedContent = index.get(key); + CachedContent cachedContent = contentIndex.get(key); return cachedContent == null || cachedContent.isEmpty() ? new TreeSet<>() : new TreeSet(cachedContent.getSpans()); @@ -213,7 +216,7 @@ public final class SimpleCache implements Cache { @Override public synchronized Set getKeys() { Assertions.checkState(!released); - return new HashSet<>(index.getKeys()); + return new HashSet<>(contentIndex.getKeys()); } @Override @@ -243,27 +246,33 @@ public final class SimpleCache implements Cache { public synchronized @Nullable SimpleCacheSpan startReadWriteNonBlocking(String key, long position) throws CacheException { Assertions.checkState(!released); - SimpleCacheSpan cacheSpan = getSpan(key, position); + SimpleCacheSpan span = getSpan(key, position); // Read case. - if (cacheSpan.isCached) { - try { - // Obtain a new span with updated last access timestamp. - SimpleCacheSpan newCacheSpan = index.get(key).touch(cacheSpan); - notifySpanTouched(cacheSpan, newCacheSpan); - return newCacheSpan; - } catch (CacheException e) { - // Ignore. In worst case the cache span is evicted early. - // This happens very rarely [Internal: b/38351639] - return cacheSpan; + if (span.isCached) { + String fileName = span.file.getName(); + long length = span.length; + long lastAccessTimestamp = System.currentTimeMillis(); + // Updating the file itself to incorporate the new last access timestamp is much slower than + // updating the file index. Hence we only update the file if we don't have a file index, or if + // updating the file index failed. + boolean updateFile; + if (fileIndex != null) { + updateFile = !fileIndex.set(fileName, length, lastAccessTimestamp); + } else { + updateFile = true; } + SimpleCacheSpan newSpan = + contentIndex.get(key).setLastAccessTimestamp(span, lastAccessTimestamp, updateFile); + notifySpanTouched(span, newSpan); + return newSpan; } - CachedContent cachedContent = index.getOrAdd(key); + CachedContent cachedContent = contentIndex.getOrAdd(key); if (!cachedContent.isLocked()) { // Write case, lock available. cachedContent.setLocked(true); - return cacheSpan; + return span; } // Write case, lock not available. @@ -273,7 +282,7 @@ public final class SimpleCache implements Cache { @Override public synchronized File startFile(String key, long position, long length) throws CacheException { Assertions.checkState(!released); - CachedContent cachedContent = index.get(key); + CachedContent cachedContent = contentIndex.get(key); Assertions.checkNotNull(cachedContent); Assertions.checkState(cachedContent.isLocked()); if (!cacheDir.exists()) { @@ -301,29 +310,35 @@ public final class SimpleCache implements Cache { file.delete(); return; } - SimpleCacheSpan span = SimpleCacheSpan.createCacheEntry(file, length, index); + + SimpleCacheSpan span = SimpleCacheSpan.createCacheEntry(file, length, contentIndex); Assertions.checkState(span != null); - CachedContent cachedContent = index.get(span.key); + CachedContent cachedContent = contentIndex.get(span.key); Assertions.checkNotNull(cachedContent); Assertions.checkState(cachedContent.isLocked()); + // Check if the span conflicts with the set content length long contentLength = ContentMetadata.getContentLength(cachedContent.getMetadata()); if (contentLength != C.LENGTH_UNSET) { Assertions.checkState((span.position + span.length) <= contentLength); } + + if (fileIndex != null) { + fileIndex.set(file.getName(), span.length, span.lastAccessTimestamp); + } addSpan(span); - index.store(); + contentIndex.store(); notifyAll(); } @Override public synchronized void releaseHoleSpan(CacheSpan holeSpan) { Assertions.checkState(!released); - CachedContent cachedContent = index.get(holeSpan.key); + CachedContent cachedContent = contentIndex.get(holeSpan.key); Assertions.checkNotNull(cachedContent); Assertions.checkState(cachedContent.isLocked()); cachedContent.setLocked(false); - index.maybeRemove(cachedContent.key); + contentIndex.maybeRemove(cachedContent.key); notifyAll(); } @@ -336,14 +351,14 @@ public final class SimpleCache implements Cache { @Override public synchronized boolean isCached(String key, long position, long length) { Assertions.checkState(!released); - CachedContent cachedContent = index.get(key); + CachedContent cachedContent = contentIndex.get(key); return cachedContent != null && cachedContent.getCachedBytesLength(position, length) >= length; } @Override public synchronized long getCachedLength(String key, long position, long length) { Assertions.checkState(!released); - CachedContent cachedContent = index.get(key); + CachedContent cachedContent = contentIndex.get(key); return cachedContent != null ? cachedContent.getCachedBytesLength(position, length) : -length; } @@ -351,14 +366,14 @@ public final class SimpleCache implements Cache { public synchronized void applyContentMetadataMutations( String key, ContentMetadataMutations mutations) throws CacheException { Assertions.checkState(!released); - index.applyContentMetadataMutations(key, mutations); - index.store(); + contentIndex.applyContentMetadataMutations(key, mutations); + contentIndex.store(); } @Override public synchronized ContentMetadata getContentMetadata(String key) { Assertions.checkState(!released); - return index.getContentMetadata(key); + return contentIndex.getContentMetadata(key); } /** @@ -375,7 +390,7 @@ public final class SimpleCache implements Cache { * @return The corresponding cache {@link SimpleCacheSpan}. */ private SimpleCacheSpan getSpan(String key, long position) throws CacheException { - CachedContent cachedContent = index.get(key); + CachedContent cachedContent = contentIndex.get(key); if (cachedContent == null) { return SimpleCacheSpan.createOpenHole(key, position); } @@ -398,40 +413,63 @@ public final class SimpleCache implements Cache { return; } - index.load(); - loadDirectory(cacheDir, /* isRootDirectory= */ true); - index.removeEmpty(); + contentIndex.load(); + if (fileIndex != null) { + Map fileMetadata = fileIndex.getAll(); + loadDirectory(cacheDir, /* isRoot= */ true, fileMetadata); + fileIndex.removeAll(fileMetadata.keySet()); + } else { + loadDirectory(cacheDir, /* isRoot= */ true, /* fileMetadata= */ null); + } + contentIndex.removeEmpty(); try { - index.store(); + contentIndex.store(); } catch (CacheException e) { Log.e(TAG, "Storing index file failed", e); } } - private void loadDirectory(File directory, boolean isRootDirectory) { + /** + * Loads a cache directory. If the root directory is passed, also loads any subdirectories. + * + * @param directory The directory to load. + * @param isRoot Whether the directory is the root directory. + * @param fileMetadata A mutable map containing cache file metadata, keyed by file name. The map + * is modified by removing entries for all loaded files. When the method call returns, the map + * will contain only metadata that was unused. May be null if no file metadata is available. + */ + private void loadDirectory( + File directory, boolean isRoot, @Nullable Map fileMetadata) { File[] files = directory.listFiles(); if (files == null) { // Not a directory. return; } - if (!isRootDirectory && files.length == 0) { + if (!isRoot && files.length == 0) { // Empty non-root directory. directory.delete(); return; } for (File file : files) { String fileName = file.getName(); - if (isRootDirectory && fileName.indexOf('.') == -1) { - loadDirectory(file, /* isRootDirectory= */ false); + if (isRoot && fileName.indexOf('.') == -1) { + loadDirectory(file, /* isRoot= */ false, fileMetadata); } else { - if (isRootDirectory && CachedContentIndex.isIndexFile(fileName)) { + if (isRoot && CachedContentIndex.isIndexFile(fileName)) { // Skip the (expected) index files in the root directory. continue; } - long fileLength = file.length(); + CacheFileMetadata metadata = + fileMetadata != null ? fileMetadata.remove(file.getName()) : null; + long length = C.LENGTH_UNSET; + long lastAccessTimestamp = C.TIME_UNSET; + if (metadata != null) { + length = metadata.length; + lastAccessTimestamp = metadata.lastAccessTimestamp; + } SimpleCacheSpan span = - fileLength > 0 ? SimpleCacheSpan.createCacheEntry(file, fileLength, index) : null; + SimpleCacheSpan.createCacheEntry(file, length, lastAccessTimestamp, contentIndex); if (span != null) { addSpan(span); } else { @@ -447,18 +485,21 @@ public final class SimpleCache implements Cache { * @param span The span to be added. */ private void addSpan(SimpleCacheSpan span) { - index.getOrAdd(span.key).addSpan(span); + contentIndex.getOrAdd(span.key).addSpan(span); totalSpace += span.length; notifySpanAdded(span); } private void removeSpanInternal(CacheSpan span) { - CachedContent cachedContent = index.get(span.key); + CachedContent cachedContent = contentIndex.get(span.key); if (cachedContent == null || !cachedContent.removeSpan(span)) { return; } totalSpace -= span.length; - index.maybeRemove(cachedContent.key); + if (fileIndex != null) { + fileIndex.remove(span.file.getName()); + } + contentIndex.maybeRemove(cachedContent.key); notifySpanRemoved(span); } @@ -468,7 +509,7 @@ public final class SimpleCache implements Cache { */ private void removeStaleSpans() { ArrayList spansToBeRemoved = new ArrayList<>(); - for (CachedContent cachedContent : index.getAll()) { + for (CachedContent cachedContent : contentIndex.getAll()) { for (CacheSpan span : cachedContent.getSpans()) { if (!span.file.exists()) { spansToBeRemoved.add(span); diff --git a/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/SimpleCacheSpan.java b/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/SimpleCacheSpan.java index decbe80c84..82563af01c 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/SimpleCacheSpan.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/SimpleCacheSpan.java @@ -38,16 +38,16 @@ import java.util.regex.Pattern; /** * Returns a new {@link File} instance from {@code cacheDir}, {@code id}, {@code position}, {@code - * lastAccessTimestamp}. + * timestamp}. * * @param cacheDir The parent abstract pathname. * @param id The cache file id. * @param position The position of the stored data in the original stream. - * @param lastAccessTimestamp The last access timestamp. + * @param timestamp The file timestamp. * @return The cache file. */ - public static File getCacheFile(File cacheDir, int id, long position, long lastAccessTimestamp) { - return new File(cacheDir, id + "." + position + "." + lastAccessTimestamp + SUFFIX); + public static File getCacheFile(File cacheDir, int id, long position, long timestamp) { + return new File(cacheDir, id + "." + position + "." + timestamp + SUFFIX); } /** @@ -84,22 +84,36 @@ import java.util.regex.Pattern; return new SimpleCacheSpan(key, position, length, C.TIME_UNSET, null); } - /* - * Note: {@code fileLength} is equivalent to {@code file.length()}, but passing it as an explicit - * argument can reduce the number of calls to this method if the calling code already knows the - * file length. This is preferable because calling {@code file.length()} can be expensive. See: - * https://github.com/google/ExoPlayer/issues/4253#issuecomment-451593889. - */ /** * Creates a cache span from an underlying cache file. Upgrades the file if necessary. * * @param file The cache file. - * @param length The length of the cache file in bytes. + * @param length The length of the cache file in bytes, or {@link C#LENGTH_UNSET} to query the + * underlying file system. Querying the underlying file system can be expensive, so callers + * that already know the length of the file should pass it explicitly. * @return The span, or null if the file name is not correctly formatted, or if the id is not - * present in the content index. + * present in the content index, or if the length is 0. */ @Nullable public static SimpleCacheSpan createCacheEntry(File file, long length, CachedContentIndex index) { + return createCacheEntry(file, length, /* lastAccessTimestamp= */ C.TIME_UNSET, index); + } + + /** + * Creates a cache span from an underlying cache file. Upgrades the file if necessary. + * + * @param file The cache file. + * @param length The length of the cache file in bytes, or {@link C#LENGTH_UNSET} to query the + * underlying file system. Querying the underlying file system can be expensive, so callers + * that already know the length of the file should pass it explicitly. + * @param lastAccessTimestamp The last access timestamp, or {@link C#TIME_UNSET} to use the file + * timestamp. + * @return The span, or null if the file name is not correctly formatted, or if the id is not + * present in the content index, or if the length is 0. + */ + @Nullable + public static SimpleCacheSpan createCacheEntry( + File file, long length, long lastAccessTimestamp, CachedContentIndex index) { String name = file.getName(); if (!name.endsWith(SUFFIX)) { file = upgradeFile(file, index); @@ -120,9 +134,18 @@ import java.util.regex.Pattern; return null; } + if (length == C.LENGTH_UNSET) { + length = file.length(); + } + if (length == 0) { + return null; + } + long position = Long.parseLong(matcher.group(2)); - long lastAccessTime = Long.parseLong(matcher.group(3)); - return new SimpleCacheSpan(key, position, length, lastAccessTime, file); + if (lastAccessTimestamp == C.TIME_UNSET) { + lastAccessTimestamp = Long.parseLong(matcher.group(3)); + } + return new SimpleCacheSpan(key, position, length, lastAccessTimestamp, file); } /** @@ -174,18 +197,16 @@ import java.util.regex.Pattern; } /** - * Returns a copy of this CacheSpan whose last access time stamp is set to current time. This - * doesn't copy or change the underlying cache file. + * Returns a copy of this CacheSpan with a new file and last access timestamp. * - * @param id The cache file id. - * @return A {@link SimpleCacheSpan} with updated last access time stamp. + * @param file The new file. + * @param lastAccessTimestamp The new last access time. + * @return A copy with the new file and last access timestamp. * @throws IllegalStateException If called on a non-cached span (i.e. {@link #isCached} is false). */ - public SimpleCacheSpan copyWithUpdatedLastAccessTime(int id) { + public SimpleCacheSpan copyWithFileAndLastAccessTimestamp(File file, long lastAccessTimestamp) { Assertions.checkState(isCached); - long now = System.currentTimeMillis(); - File newCacheFile = getCacheFile(file.getParentFile(), id, position, now); - return new SimpleCacheSpan(key, position, length, now, newCacheFile); + return new SimpleCacheSpan(key, position, length, lastAccessTimestamp, file); } }