Add VideoFrameMetadataListener

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=209193233

extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java

@@ -39,8 +39,10 @@ import com.google.android.exoplayer2.drm.DrmSessionManager;
 import com.google.android.exoplayer2.drm.ExoMediaCrypto;
 import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.util.MimeTypes;
+import com.google.android.exoplayer2.util.TimedValueQueue;
 import com.google.android.exoplayer2.util.TraceUtil;
 import com.google.android.exoplayer2.util.Util;
+import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
 import com.google.android.exoplayer2.video.VideoRendererEventListener;
 import com.google.android.exoplayer2.video.VideoRendererEventListener.EventDispatcher;
 import java.lang.annotation.Retention;
@@ -109,11 +111,14 @@ public class LibvpxVideoRenderer extends BaseRenderer {
   private final boolean playClearSamplesWithoutKeys;
   private final EventDispatcher eventDispatcher;
   private final FormatHolder formatHolder;
+  private final TimedValueQueue<Format> formatQueue;
   private final DecoderInputBuffer flagsOnlyBuffer;
   private final DrmSessionManager<ExoMediaCrypto> drmSessionManager;
   private final boolean useSurfaceYuvOutput;
 
   private Format format;
+  private Format pendingFormat;
+  private Format outputFormat;
   private VpxDecoder decoder;
   private VpxInputBuffer inputBuffer;
   private VpxOutputBuffer outputBuffer;
@@ -142,6 +147,8 @@ public class LibvpxVideoRenderer extends BaseRenderer {
   private int consecutiveDroppedFrameCount;
   private int buffersInCodecCount;
   private long lastRenderTimeUs;
+  private long outputStreamOffsetUs;
+  private VideoFrameMetadataListener frameMetadataListener;
 
   protected DecoderCounters decoderCounters;
 
@@ -219,6 +226,7 @@ public class LibvpxVideoRenderer extends BaseRenderer {
     joiningDeadlineMs = C.TIME_UNSET;
     clearReportedVideoSize();
     formatHolder = new FormatHolder();
+    formatQueue = new TimedValueQueue<>();
     flagsOnlyBuffer = DecoderInputBuffer.newFlagsOnlyInstance();
     eventDispatcher = new EventDispatcher(eventHandler, eventListener);
     outputMode = VpxDecoder.OUTPUT_MODE_NONE;
@@ -328,6 +336,7 @@ public class LibvpxVideoRenderer extends BaseRenderer {
     } else {
       joiningDeadlineMs = C.TIME_UNSET;
     }
+    formatQueue.clear();
   }
 
   @Override
@@ -371,6 +380,12 @@ public class LibvpxVideoRenderer extends BaseRenderer {
     }
   }
 
+  @Override
+  protected void onStreamChanged(Format[] formats, long offsetUs) throws ExoPlaybackException {
+    outputStreamOffsetUs = offsetUs;
+    super.onStreamChanged(formats, offsetUs);
+  }
+
   /**
    * Called when a decoder has been created and configured.
    *
@@ -437,6 +452,7 @@ public class LibvpxVideoRenderer extends BaseRenderer {
   protected void onInputFormatChanged(Format newFormat) throws ExoPlaybackException {
     Format oldFormat = format;
     format = newFormat;
+    pendingFormat = newFormat;
 
     boolean drmInitDataChanged = !Util.areEqual(format.drmInitData, oldFormat == null ? null
         : oldFormat.drmInitData);
@@ -629,6 +645,8 @@ public class LibvpxVideoRenderer extends BaseRenderer {
       setOutput((Surface) message, null);
     } else if (messageType == MSG_SET_OUTPUT_BUFFER_RENDERER) {
       setOutput(null, (VpxOutputBufferRenderer) message);
+    } else if (messageType == C.MSG_SET_VIDEO_FRAME_METADATA_LISTENER) {
+      frameMetadataListener = (VideoFrameMetadataListener) message;
     } else {
       super.handleMessage(messageType, message);
     }
@@ -772,6 +790,10 @@ public class LibvpxVideoRenderer extends BaseRenderer {
     if (waitingForKeys) {
       return false;
     }
+    if (pendingFormat != null) {
+      formatQueue.add(inputBuffer.timeUs, pendingFormat);
+      pendingFormat = null;
+    }
     inputBuffer.flip();
     inputBuffer.colorInfo = formatHolder.format.colorInfo;
     onQueueInputBuffer(inputBuffer);
@@ -851,11 +873,21 @@ public class LibvpxVideoRenderer extends BaseRenderer {
       return false;
     }
 
+    long presentationTimeUs = outputBuffer.timeUs - outputStreamOffsetUs;
+    Format format = formatQueue.pollFloor(presentationTimeUs);
+    if (format != null) {
+      outputFormat = format;
+    }
+
     long elapsedRealtimeNowUs = SystemClock.elapsedRealtime() * 1000;
     boolean isStarted = getState() == STATE_STARTED;
     if (!renderedFirstFrame
         || (isStarted
             && shouldForceRenderOutputBuffer(earlyUs, elapsedRealtimeNowUs - lastRenderTimeUs))) {
+      if (frameMetadataListener != null) {
+        frameMetadataListener.onVideoFrameAboutToBeRendered(
+            presentationTimeUs, System.nanoTime(), outputFormat);
+      }
       renderOutputBuffer(outputBuffer);
       return true;
     }
@@ -873,6 +905,10 @@ public class LibvpxVideoRenderer extends BaseRenderer {
     }
 
     if (earlyUs < 30000) {
+      if (frameMetadataListener != null) {
+        frameMetadataListener.onVideoFrameAboutToBeRendered(
+            presentationTimeUs, System.nanoTime(), outputFormat);
+      }
       renderOutputBuffer(outputBuffer);
       return true;
     }

library/core/src/main/java/com/google/android/exoplayer2/C.java

@@ -26,6 +26,7 @@ import android.view.Surface;
 import com.google.android.exoplayer2.PlayerMessage.Target;
 import com.google.android.exoplayer2.audio.AuxEffectInfo;
 import com.google.android.exoplayer2.util.Util;
+import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.util.UUID;
@@ -733,6 +734,13 @@ public final class C {
    */
   public static final int MSG_SET_AUX_EFFECT_INFO = 5;
 
+  /**
+   * The type of a message that can be passed to a video {@link Renderer} via {@link
+   * ExoPlayer#createMessage(Target)}. The message payload should be a {@link
+   * VideoFrameMetadataListener} instance, or null.
+   */
+  public static final int MSG_SET_VIDEO_FRAME_METADATA_LISTENER = 6;
+
   /**
    * Applications or extensions may define custom {@code MSG_*} constants that can be passed to
    * {@link Renderer}s. These custom constants must be greater than or equal to this value.

library/core/src/main/java/com/google/android/exoplayer2/Player.java

@@ -29,6 +29,7 @@ import com.google.android.exoplayer2.source.TrackGroupArray;
 import com.google.android.exoplayer2.text.TextOutput;
 import com.google.android.exoplayer2.trackselection.TrackSelectionArray;
 import com.google.android.exoplayer2.util.Util;
+import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
 import com.google.android.exoplayer2.video.VideoListener;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
@@ -165,6 +166,25 @@ public interface Player {
      */
     void removeVideoListener(VideoListener listener);
 
+    /**
+     * Sets a listener to receive video frame metadata events.
+     *
+     * <p>This method is intended to be called by the same component that sets the {@link Surface}
+     * onto which video will be rendered. If using ExoPlayer's standard UI components, this method
+     * should not be called directly from application code.
+     *
+     * @param listener The listener.
+     */
+    void setVideoFrameMetadataListener(VideoFrameMetadataListener listener);
+
+    /**
+     * Clears the listener which receives video frame metadata events if it matches the one passed.
+     * Else does nothing.
+     *
+     * @param listener The listener to clear.
+     */
+    void clearVideoFrameMetadataListener(VideoFrameMetadataListener listener);
+
     /**
      * Clears any {@link Surface}, {@link SurfaceHolder}, {@link SurfaceView} or {@link TextureView}
      * currently set on the player.

library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java

@@ -51,6 +51,7 @@ import com.google.android.exoplayer2.trackselection.TrackSelector;
 import com.google.android.exoplayer2.upstream.BandwidthMeter;
 import com.google.android.exoplayer2.util.Clock;
 import com.google.android.exoplayer2.util.Util;
+import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
 import com.google.android.exoplayer2.video.VideoRendererEventListener;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -105,6 +106,7 @@ public class SimpleExoPlayer
   private float audioVolume;
   private MediaSource mediaSource;
   private List<Cue> currentCues;
+  private VideoFrameMetadataListener videoFrameMetadataListener;
 
   /**
    * @param renderersFactory A factory for creating {@link Renderer}s to be used by the instance.
@@ -598,6 +600,36 @@ public class SimpleExoPlayer
     videoListeners.remove(listener);
   }
 
+  @Override
+  public void setVideoFrameMetadataListener(VideoFrameMetadataListener listener) {
+    videoFrameMetadataListener = listener;
+    for (Renderer renderer : renderers) {
+      if (renderer.getTrackType() == C.TRACK_TYPE_VIDEO) {
+        player
+            .createMessage(renderer)
+            .setType(C.MSG_SET_VIDEO_FRAME_METADATA_LISTENER)
+            .setPayload(listener)
+            .send();
+      }
+    }
+  }
+
+  @Override
+  public void clearVideoFrameMetadataListener(VideoFrameMetadataListener listener) {
+    if (videoFrameMetadataListener != listener) {
+      return;
+    }
+    for (Renderer renderer : renderers) {
+      if (renderer.getTrackType() == C.TRACK_TYPE_VIDEO) {
+        player
+            .createMessage(renderer)
+            .setType(C.MSG_SET_VIDEO_FRAME_METADATA_LISTENER)
+            .setPayload(null)
+            .send();
+      }
+    }
+  }
+
   /**
    * Sets a listener to receive video events, removing all existing listeners.
    *

library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java

@@ -547,9 +547,17 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
   }
 
   @Override
-  protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec,
+  protected boolean processOutputBuffer(
-      ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs,
+      long positionUs,
-      boolean shouldSkip) throws ExoPlaybackException {
+      long elapsedRealtimeUs,
+      MediaCodec codec,
+      ByteBuffer buffer,
+      int bufferIndex,
+      int bufferFlags,
+      long bufferPresentationTimeUs,
+      boolean shouldSkip,
+      Format format)
+      throws ExoPlaybackException {
     if (passthroughEnabled && (bufferFlags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
       // Discard output buffers from the passthrough (raw) decoder containing codec specific data.
       codec.releaseOutputBuffer(bufferIndex, false);

library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java

@@ -43,6 +43,7 @@ import com.google.android.exoplayer2.mediacodec.MediaCodecUtil.DecoderQueryExcep
 import com.google.android.exoplayer2.source.MediaPeriod;
 import com.google.android.exoplayer2.util.Assertions;
 import com.google.android.exoplayer2.util.NalUnitUtil;
+import com.google.android.exoplayer2.util.TimedValueQueue;
 import com.google.android.exoplayer2.util.TraceUtil;
 import com.google.android.exoplayer2.util.Util;
 import java.lang.annotation.Retention;
@@ -272,10 +273,13 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   private final DecoderInputBuffer buffer;
   private final DecoderInputBuffer flagsOnlyBuffer;
   private final FormatHolder formatHolder;
+  private final TimedValueQueue<Format> formatQueue;
   private final List<Long> decodeOnlyPresentationTimestamps;
   private final MediaCodec.BufferInfo outputBufferInfo;
 
   private Format format;
+  private Format pendingFormat;
+  private Format outputFormat;
   private DrmSession<FrameworkMediaCrypto> drmSession;
   private DrmSession<FrameworkMediaCrypto> pendingDrmSession;
   private MediaCodec codec;
@@ -344,6 +348,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
     flagsOnlyBuffer = DecoderInputBuffer.newFlagsOnlyInstance();
     formatHolder = new FormatHolder();
+    formatQueue = new TimedValueQueue<>();
     decodeOnlyPresentationTimestamps = new ArrayList<>();
     outputBufferInfo = new MediaCodec.BufferInfo();
     codecReconfigurationState = RECONFIGURATION_STATE_NONE;
@@ -501,6 +506,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     if (codec != null) {
       flushCodec();
     }
+    formatQueue.clear();
   }
 
   @Override
@@ -956,6 +962,10 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
       if (buffer.isDecodeOnly()) {
         decodeOnlyPresentationTimestamps.add(presentationTimeUs);
       }
+      if (pendingFormat != null) {
+        formatQueue.add(presentationTimeUs, pendingFormat);
+        pendingFormat = null;
+      }
 
       buffer.flip();
       onQueueInputBuffer(buffer);
@@ -1012,6 +1022,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   protected void onInputFormatChanged(Format newFormat) throws ExoPlaybackException {
     Format oldFormat = format;
     format = newFormat;
+    pendingFormat = newFormat;
 
     boolean drmInitDataChanged =
         !Util.areEqual(format.drmInitData, oldFormat == null ? null : oldFormat.drmInitData);
@@ -1234,35 +1245,15 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
             codec.dequeueOutputBuffer(outputBufferInfo, getDequeueOutputBufferTimeoutUs());
       }
 
-      if (outputIndex >= 0) {
+      if (outputIndex < 0) {
-        // We've dequeued a buffer.
+        if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED /* (-2) */) {
-        if (shouldSkipAdaptationWorkaroundOutputBuffer) {
+          processOutputFormat();
-          shouldSkipAdaptationWorkaroundOutputBuffer = false;
+          return true;
-          codec.releaseOutputBuffer(outputIndex, false);
+        } else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED /* (-3) */) {
+          processOutputBuffersChanged();
           return true;
-        } else if (outputBufferInfo.size == 0
-            && (outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
-          // The dequeued buffer indicates the end of the stream. Process it immediately.
-          processEndOfStream();
-          return false;
-        } else {
-          this.outputIndex = outputIndex;
-          outputBuffer = getOutputBuffer(outputIndex);
-          // The dequeued buffer is a media buffer. Do some initial setup.
-          // It will be processed by calling processOutputBuffer (possibly multiple times).
-          if (outputBuffer != null) {
-            outputBuffer.position(outputBufferInfo.offset);
-            outputBuffer.limit(outputBufferInfo.offset + outputBufferInfo.size);
-          }
-          shouldSkipOutputBuffer = shouldSkipOutputBuffer(outputBufferInfo.presentationTimeUs);
         }
-      } else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED /* (-2) */) {
+        /* MediaCodec.INFO_TRY_AGAIN_LATER (-1) or unknown negative return value */
-        processOutputFormat();
-        return true;
-      } else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED /* (-3) */) {
-        processOutputBuffersChanged();
-        return true;
-      } else /* MediaCodec.INFO_TRY_AGAIN_LATER (-1) or unknown negative return value */ {
         if (codecNeedsEosPropagationWorkaround
             && (inputStreamEnded
                 || codecReinitializationState == REINITIALIZATION_STATE_WAIT_END_OF_STREAM)) {
@@ -1270,6 +1261,32 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
         }
         return false;
       }
+
+      // We've dequeued a buffer.
+      if (shouldSkipAdaptationWorkaroundOutputBuffer) {
+        shouldSkipAdaptationWorkaroundOutputBuffer = false;
+        codec.releaseOutputBuffer(outputIndex, false);
+        return true;
+      } else if (outputBufferInfo.size == 0
+          && (outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
+        // The dequeued buffer indicates the end of the stream. Process it immediately.
+        processEndOfStream();
+        return false;
+      }
+
+      this.outputIndex = outputIndex;
+      outputBuffer = getOutputBuffer(outputIndex);
+      // The dequeued buffer is a media buffer. Do some initial setup.
+      // It will be processed by calling processOutputBuffer (possibly multiple times).
+      if (outputBuffer != null) {
+        outputBuffer.position(outputBufferInfo.offset);
+        outputBuffer.limit(outputBufferInfo.offset + outputBufferInfo.size);
+      }
+      shouldSkipOutputBuffer = shouldSkipOutputBuffer(outputBufferInfo.presentationTimeUs);
+      Format format = formatQueue.pollFloor(outputBufferInfo.presentationTimeUs);
+      if (format != null) {
+        outputFormat = format;
+      }
     }
 
     boolean processedOutputBuffer;
@@ -1284,7 +1301,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
                 outputIndex,
                 outputBufferInfo.flags,
                 outputBufferInfo.presentationTimeUs,
-                shouldSkipOutputBuffer);
+                shouldSkipOutputBuffer,
+                outputFormat);
       } catch (IllegalStateException e) {
         processEndOfStream();
         if (outputStreamEnded) {
@@ -1303,7 +1321,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
               outputIndex,
               outputBufferInfo.flags,
               outputBufferInfo.presentationTimeUs,
-              shouldSkipOutputBuffer);
+              shouldSkipOutputBuffer,
+              outputFormat);
     }
 
     if (processedOutputBuffer) {
@@ -1348,36 +1367,43 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
 
   /**
    * Processes an output media buffer.
-   * <p>
+   *
-   * When a new {@link ByteBuffer} is passed to this method its position and limit delineate the
+   * <p>When a new {@link ByteBuffer} is passed to this method its position and limit delineate the
    * data to be processed. The return value indicates whether the buffer was processed in full. If
    * true is returned then the next call to this method will receive a new buffer to be processed.
    * If false is returned then the same buffer will be passed to the next call. An implementation of
    * this method is free to modify the buffer and can assume that the buffer will not be externally
    * modified between successive calls. Hence an implementation can, for example, modify the
    * buffer's position to keep track of how much of the data it has processed.
-   * <p>
-   * Note that the first call to this method following a call to
-   * {@link #onPositionReset(long, boolean)} will always receive a new {@link ByteBuffer} to be
-   * processed.
    *
-   * @param positionUs The current media time in microseconds, measured at the start of the
+   * <p>Note that the first call to this method following a call to {@link #onPositionReset(long,
-   *     current iteration of the rendering loop.
+   * boolean)} will always receive a new {@link ByteBuffer} to be processed.
-   * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds,
+   *
-   *     measured at the start of the current iteration of the rendering loop.
+   * @param positionUs The current media time in microseconds, measured at the start of the current
+   *     iteration of the rendering loop.
+   * @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, measured at the
+   *     start of the current iteration of the rendering loop.
    * @param codec The {@link MediaCodec} instance.
    * @param buffer The output buffer to process.
    * @param bufferIndex The index of the output buffer.
    * @param bufferFlags The flags attached to the output buffer.
    * @param bufferPresentationTimeUs The presentation time of the output buffer in microseconds.
    * @param shouldSkip Whether the buffer should be skipped (i.e. not rendered).
-   *
+   * @param format The format associated with the buffer.
    * @return Whether the output buffer was fully processed (e.g. rendered or skipped).
    * @throws ExoPlaybackException If an error occurs processing the output buffer.
    */
-  protected abstract boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs,
+  protected abstract boolean processOutputBuffer(
-      MediaCodec codec, ByteBuffer buffer, int bufferIndex, int bufferFlags,
+      long positionUs,
-      long bufferPresentationTimeUs, boolean shouldSkip) throws ExoPlaybackException;
+      long elapsedRealtimeUs,
+      MediaCodec codec,
+      ByteBuffer buffer,
+      int bufferIndex,
+      int bufferFlags,
+      long bufferPresentationTimeUs,
+      boolean shouldSkip,
+      Format format)
+      throws ExoPlaybackException;
 
   /**
    * Incrementally renders any remaining output.

library/core/src/main/java/com/google/android/exoplayer2/util/TimedValueQueue.java

@@ -0,0 +1,160 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.util;
+
+import android.support.annotation.Nullable;
+import java.util.Arrays;
+import org.checkerframework.checker.nullness.compatqual.NullableType;
+
+/** A utility class to keep a queue of values with timestamps. This class is thread safe. */
+public final class TimedValueQueue<V> {
+  private static final int INITIAL_BUFFER_SIZE = 10;
+
+  // Looping buffer for timestamps and values
+  private long[] timestamps;
+  private @NullableType V[] values;
+  private int first;
+  private int size;
+
+  public TimedValueQueue() {
+    this(INITIAL_BUFFER_SIZE);
+  }
+
+  /** Creates a TimedValueBuffer with the given initial buffer size. */
+  public TimedValueQueue(int initialBufferSize) {
+    timestamps = new long[initialBufferSize];
+    values = newArray(initialBufferSize);
+  }
+
+  /**
+   * Associates the specified value with the specified timestamp. All new values should have a
+   * greater timestamp than the previously added values. Otherwise all values are removed before
+   * adding the new one.
+   */
+  public synchronized void add(long timestamp, V value) {
+    clearBufferOnTimeDiscontinuity(timestamp);
+    doubleCapacityIfFull();
+    addUnchecked(timestamp, value);
+  }
+
+  /** Removes all of the values. */
+  public synchronized void clear() {
+    first = 0;
+    size = 0;
+    Arrays.fill(values, null);
+  }
+
+  /** Returns number of the values buffered. */
+  public synchronized int size() {
+    return size;
+  }
+
+  /**
+   * Returns the value with the greatest timestamp which is less than or equal to the given
+   * timestamp. Removes all older values including the returned one from the buffer.
+   *
+   * @param timestamp The timestamp value.
+   * @return The value with the greatest timestamp which is less than or equal to the given
+   *     timestamp or null if there is no such value.
+   * @see #poll(long)
+   */
+  public synchronized @Nullable V pollFloor(long timestamp) {
+    return poll(timestamp, /* onlyOlder= */ true);
+  }
+
+  /**
+   * Returns the value with the closest timestamp to the given timestamp. Removes all older values
+   * including the returned one from the buffer.
+   *
+   * @param timestamp The timestamp value.
+   * @return The value with the closest timestamp or null if the buffer is empty.
+   * @see #pollFloor(long)
+   */
+  public synchronized @Nullable V poll(long timestamp) {
+    return poll(timestamp, /* onlyOlder= */ false);
+  }
+
+  /**
+   * Returns the value with the closest timestamp to the given timestamp. Removes all older values
+   * including the returned one from the buffer.
+   *
+   * @param timestamp The timestamp value.
+   * @param onlyOlder Whether this method can return a new value in case its timestamp value is
+   *     closest to {@code timestamp}.
+   * @return The value with the closest timestamp or null if the buffer is empty or there is no
+   *     older value and {@code onlyOlder} is true.
+   */
+  private @Nullable V poll(long timestamp, boolean onlyOlder) {
+    V value = null;
+    long previousTimeDiff = Long.MAX_VALUE;
+    while (size > 0) {
+      long timeDiff = timestamp - timestamps[first];
+      if (timeDiff < 0 && (onlyOlder || -timeDiff >= previousTimeDiff)) {
+        break;
+      }
+      previousTimeDiff = timeDiff;
+      value = values[first];
+      values[first] = null;
+      first = (first + 1) % values.length;
+      size--;
+    }
+    return value;
+  }
+
+  private void clearBufferOnTimeDiscontinuity(long timestamp) {
+    if (size > 0) {
+      int last = (first + size - 1) % values.length;
+      if (timestamp <= timestamps[last]) {
+        clear();
+      }
+    }
+  }
+
+  private void doubleCapacityIfFull() {
+    int capacity = values.length;
+    if (size < capacity) {
+      return;
+    }
+    int newCapacity = capacity * 2;
+    long[] newTimestamps = new long[newCapacity];
+    V[] newValues = newArray(newCapacity);
+    // Reset the loop starting index to 0 while coping to the new buffer.
+    // First copy the values from 'first' index to the end of original array.
+    int length = capacity - first;
+    System.arraycopy(timestamps, first, newTimestamps, 0, length);
+    System.arraycopy(values, first, newValues, 0, length);
+    // Then the values from index 0 to 'first' index.
+    if (first > 0) {
+      System.arraycopy(timestamps, 0, newTimestamps, length, first);
+      System.arraycopy(values, 0, newValues, length, first);
+    }
+    timestamps = newTimestamps;
+    values = newValues;
+    first = 0;
+  }
+
+  private void addUnchecked(long timestamp, V value) {
+    int next = (first + size) % values.length;
+    timestamps[next] = timestamp;
+    values[next] = value;
+    size++;
+  }
+
+  @SuppressWarnings("unchecked")
+  private static <V> V[] newArray(int length) {
+    return (V[]) new Object[length];
+  }
+}

library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java

@@ -136,6 +136,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   private long lastInputTimeUs;
   private long outputStreamOffsetUs;
   private int pendingOutputStreamOffsetCount;
+  private @Nullable VideoFrameMetadataListener frameMetadataListener;
 
   /**
    * @param context A context.
@@ -386,6 +387,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
       if (codec != null) {
         codec.setVideoScalingMode(scalingMode);
       }
+    } else if (messageType == C.MSG_SET_VIDEO_FRAME_METADATA_LISTENER) {
+      frameMetadataListener = (VideoFrameMetadataListener) message;
     } else {
       super.handleMessage(messageType, message);
     }
@@ -587,9 +590,17 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   }
 
   @Override
-  protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec,
+  protected boolean processOutputBuffer(
-      ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs,
+      long positionUs,
-      boolean shouldSkip) throws ExoPlaybackException {
+      long elapsedRealtimeUs,
+      MediaCodec codec,
+      ByteBuffer buffer,
+      int bufferIndex,
+      int bufferFlags,
+      long bufferPresentationTimeUs,
+      boolean shouldSkip,
+      Format format)
+      throws ExoPlaybackException {
     if (initialPositionUs == C.TIME_UNSET) {
       initialPositionUs = positionUs;
     }
@@ -616,8 +627,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     if (!renderedFirstFrame
         || (isStarted
             && shouldForceRenderOutputBuffer(earlyUs, elapsedRealtimeNowUs - lastRenderTimeUs))) {
+      long releaseTimeNs = System.nanoTime();
+      notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format);
       if (Util.SDK_INT >= 21) {
-        renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, System.nanoTime());
+        renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, releaseTimeNs);
       } else {
         renderOutputBuffer(codec, bufferIndex, presentationTimeUs);
       }
@@ -653,6 +666,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     if (Util.SDK_INT >= 21) {
       // Let the underlying framework time the release.
       if (earlyUs < 50000) {
+        notifyFrameMetadataListener(presentationTimeUs, adjustedReleaseTimeNs, format);
         renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, adjustedReleaseTimeNs);
         return true;
       }
@@ -670,6 +684,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
             return false;
           }
         }
+        notifyFrameMetadataListener(presentationTimeUs, adjustedReleaseTimeNs, format);
         renderOutputBuffer(codec, bufferIndex, presentationTimeUs);
         return true;
       }
@@ -679,10 +694,18 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     return false;
   }
 
+  private void notifyFrameMetadataListener(
+      long presentationTimeUs, long releaseTimeNs, Format format) {
+    if (frameMetadataListener != null) {
+      frameMetadataListener.onVideoFrameAboutToBeRendered(
+          presentationTimeUs, releaseTimeNs, format);
+    }
+  }
+
   /**
    * Returns the offset that should be subtracted from {@code bufferPresentationTimeUs} in {@link
-   * #processOutputBuffer(long, long, MediaCodec, ByteBuffer, int, int, long, boolean)} to get the
+   * #processOutputBuffer(long, long, MediaCodec, ByteBuffer, int, int, long, boolean, Format)} to
-   * playback position with respect to the media.
+   * get the playback position with respect to the media.
    */
   protected long getOutputStreamOffsetUs() {
     return outputStreamOffsetUs;

library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameMetadataListener.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.video;
+
+import com.google.android.exoplayer2.Format;
+
+/** A listener for metadata corresponding to video frame being rendered. */
+public interface VideoFrameMetadataListener {
+  /**
+   * Called when the video frame about to be rendered. This method is called on the playback thread.
+   *
+   * @param presentationTimeUs The presentation time of the output buffer, in microseconds.
+   * @param releaseTimeNs The wallclock time at which the frame should be displayed, in nanoseconds.
+   *     If the platform API version of the device is less than 21, then this is the best effort.
+   * @param format The format associated with the frame.
+   */
+  void onVideoFrameAboutToBeRendered(long presentationTimeUs, long releaseTimeNs, Format format);
+}

library/core/src/test/java/com/google/android/exoplayer2/util/TimedValueQueueTest.java

@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.util;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricTestRunner;
+
+/** Unit test for {@link TimedValueQueue}. */
+@RunWith(RobolectricTestRunner.class)
+public class TimedValueQueueTest {
+
+  private TimedValueQueue<String> queue;
+
+  @Before
+  public void setUp() throws Exception {
+    queue = new TimedValueQueue<>();
+  }
+
+  @Test
+  public void testAddAndPollValues() {
+    queue.add(0, "a");
+    queue.add(1, "b");
+    queue.add(2, "c");
+    assertThat(queue.poll(0)).isEqualTo("a");
+    assertThat(queue.poll(1)).isEqualTo("b");
+    assertThat(queue.poll(2)).isEqualTo("c");
+  }
+
+  @Test
+  public void testBufferCapacityIncreasesAutomatically() {
+    queue = new TimedValueQueue<>(1);
+    for (int i = 0; i < 20; i++) {
+      queue.add(i, "" + i);
+      if ((i & 1) == 1) {
+        assertThat(queue.poll(0)).isEqualTo("" + (i / 2));
+      }
+    }
+    assertThat(queue.size()).isEqualTo(10);
+  }
+
+  @Test
+  public void testTimeDiscontinuityClearsValues() {
+    queue.add(1, "b");
+    queue.add(2, "c");
+    queue.add(0, "a");
+    assertThat(queue.size()).isEqualTo(1);
+    assertThat(queue.poll(0)).isEqualTo("a");
+  }
+
+  @Test
+  public void testTimeDiscontinuityOnFullBufferClearsValues() {
+    queue = new TimedValueQueue<>(2);
+    queue.add(1, "b");
+    queue.add(3, "c");
+    queue.add(2, "a");
+    assertThat(queue.size()).isEqualTo(1);
+    assertThat(queue.poll(2)).isEqualTo("a");
+  }
+
+  @Test
+  public void testPollReturnsClosestValue() {
+    queue.add(0, "a");
+    queue.add(3, "b");
+    assertThat(queue.poll(2)).isEqualTo("b");
+    assertThat(queue.size()).isEqualTo(0);
+  }
+
+  @Test
+  public void testPollRemovesPreviousValues() {
+    queue.add(0, "a");
+    queue.add(1, "b");
+    queue.add(2, "c");
+    assertThat(queue.poll(1)).isEqualTo("b");
+    assertThat(queue.size()).isEqualTo(1);
+  }
+
+  @Test
+  public void testPollFloorReturnsClosestPreviousValue() {
+    queue.add(0, "a");
+    queue.add(3, "b");
+    assertThat(queue.pollFloor(2)).isEqualTo("a");
+    assertThat(queue.pollFloor(2)).isEqualTo(null);
+    assertThat(queue.pollFloor(3)).isEqualTo("b");
+    assertThat(queue.size()).isEqualTo(0);
+  }
+
+  @Test
+  public void testPollFloorRemovesPreviousValues() {
+    queue.add(0, "a");
+    queue.add(1, "b");
+    queue.add(2, "c");
+    assertThat(queue.pollFloor(1)).isEqualTo("b");
+    assertThat(queue.size()).isEqualTo(1);
+  }
+}

testutils/src/main/java/com/google/android/exoplayer2/testutil/DebugRenderersFactory.java

@@ -126,17 +126,33 @@ public class DebugRenderersFactory extends DefaultRenderersFactory {
     }
 
     @Override
-    protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec,
+    protected boolean processOutputBuffer(
-        ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs,
+        long positionUs,
-        boolean shouldSkip) throws ExoPlaybackException {
+        long elapsedRealtimeUs,
+        MediaCodec codec,
+        ByteBuffer buffer,
+        int bufferIndex,
+        int bufferFlags,
+        long bufferPresentationTimeUs,
+        boolean shouldSkip,
+        Format format)
+        throws ExoPlaybackException {
       if (skipToPositionBeforeRenderingFirstFrame && bufferPresentationTimeUs < positionUs) {
         // After the codec has been initialized, don't render the first frame until we've caught up
         // to the playback position. Else test runs on devices that do not support dummy surface
         // will drop frames between rendering the first one and catching up [Internal: b/66494991].
         shouldSkip = true;
       }
-      return super.processOutputBuffer(positionUs, elapsedRealtimeUs, codec, buffer, bufferIndex,
+      return super.processOutputBuffer(
-          bufferFlags, bufferPresentationTimeUs, shouldSkip);
+          positionUs,
+          elapsedRealtimeUs,
+          codec,
+          buffer,
+          bufferIndex,
+          bufferFlags,
+          bufferPresentationTimeUs,
+          shouldSkip,
+          format);
     }
 
     @Override