From b58f6940ebdfd966d6679beb2d9c985ce473fd7d Mon Sep 17 00:00:00 2001 From: eguven Date: Fri, 17 Aug 2018 12:57:54 -0700 Subject: [PATCH] Add VideoFrameMetadataListener ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=209193233 --- .../ext/vp9/LibvpxVideoRenderer.java | 36 ++++ .../java/com/google/android/exoplayer2/C.java | 8 + .../com/google/android/exoplayer2/Player.java | 20 +++ .../android/exoplayer2/SimpleExoPlayer.java | 32 ++++ .../audio/MediaCodecAudioRenderer.java | 14 +- .../mediacodec/MediaCodecRenderer.java | 112 +++++++----- .../exoplayer2/util/TimedValueQueue.java | 160 ++++++++++++++++++ .../video/MediaCodecVideoRenderer.java | 35 +++- .../video/VideoFrameMetadataListener.java | 31 ++++ .../exoplayer2/util/TimedValueQueueTest.java | 112 ++++++++++++ .../testutil/DebugRenderersFactory.java | 26 ++- 11 files changed, 529 insertions(+), 57 deletions(-) create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/util/TimedValueQueue.java create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameMetadataListener.java create mode 100644 library/core/src/test/java/com/google/android/exoplayer2/util/TimedValueQueueTest.java diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java b/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java index 08c413aba7..f0986d08be 100644 --- a/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java +++ b/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java @@ -39,8 +39,10 @@ import com.google.android.exoplayer2.drm.DrmSessionManager; import com.google.android.exoplayer2.drm.ExoMediaCrypto; import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.MimeTypes; +import com.google.android.exoplayer2.util.TimedValueQueue; import com.google.android.exoplayer2.util.TraceUtil; import com.google.android.exoplayer2.util.Util; +import com.google.android.exoplayer2.video.VideoFrameMetadataListener; import com.google.android.exoplayer2.video.VideoRendererEventListener; import com.google.android.exoplayer2.video.VideoRendererEventListener.EventDispatcher; import java.lang.annotation.Retention; @@ -109,11 +111,14 @@ public class LibvpxVideoRenderer extends BaseRenderer { private final boolean playClearSamplesWithoutKeys; private final EventDispatcher eventDispatcher; private final FormatHolder formatHolder; + private final TimedValueQueue formatQueue; private final DecoderInputBuffer flagsOnlyBuffer; private final DrmSessionManager drmSessionManager; private final boolean useSurfaceYuvOutput; private Format format; + private Format pendingFormat; + private Format outputFormat; private VpxDecoder decoder; private VpxInputBuffer inputBuffer; private VpxOutputBuffer outputBuffer; @@ -142,6 +147,8 @@ public class LibvpxVideoRenderer extends BaseRenderer { private int consecutiveDroppedFrameCount; private int buffersInCodecCount; private long lastRenderTimeUs; + private long outputStreamOffsetUs; + private VideoFrameMetadataListener frameMetadataListener; protected DecoderCounters decoderCounters; @@ -219,6 +226,7 @@ public class LibvpxVideoRenderer extends BaseRenderer { joiningDeadlineMs = C.TIME_UNSET; clearReportedVideoSize(); formatHolder = new FormatHolder(); + formatQueue = new TimedValueQueue<>(); flagsOnlyBuffer = DecoderInputBuffer.newFlagsOnlyInstance(); eventDispatcher = new EventDispatcher(eventHandler, eventListener); outputMode = VpxDecoder.OUTPUT_MODE_NONE; @@ -328,6 +336,7 @@ public class LibvpxVideoRenderer extends BaseRenderer { } else { joiningDeadlineMs = C.TIME_UNSET; } + formatQueue.clear(); } @Override @@ -371,6 +380,12 @@ public class LibvpxVideoRenderer extends BaseRenderer { } } + @Override + protected void onStreamChanged(Format[] formats, long offsetUs) throws ExoPlaybackException { + outputStreamOffsetUs = offsetUs; + super.onStreamChanged(formats, offsetUs); + } + /** * Called when a decoder has been created and configured. * @@ -437,6 +452,7 @@ public class LibvpxVideoRenderer extends BaseRenderer { protected void onInputFormatChanged(Format newFormat) throws ExoPlaybackException { Format oldFormat = format; format = newFormat; + pendingFormat = newFormat; boolean drmInitDataChanged = !Util.areEqual(format.drmInitData, oldFormat == null ? null : oldFormat.drmInitData); @@ -629,6 +645,8 @@ public class LibvpxVideoRenderer extends BaseRenderer { setOutput((Surface) message, null); } else if (messageType == MSG_SET_OUTPUT_BUFFER_RENDERER) { setOutput(null, (VpxOutputBufferRenderer) message); + } else if (messageType == C.MSG_SET_VIDEO_FRAME_METADATA_LISTENER) { + frameMetadataListener = (VideoFrameMetadataListener) message; } else { super.handleMessage(messageType, message); } @@ -772,6 +790,10 @@ public class LibvpxVideoRenderer extends BaseRenderer { if (waitingForKeys) { return false; } + if (pendingFormat != null) { + formatQueue.add(inputBuffer.timeUs, pendingFormat); + pendingFormat = null; + } inputBuffer.flip(); inputBuffer.colorInfo = formatHolder.format.colorInfo; onQueueInputBuffer(inputBuffer); @@ -851,11 +873,21 @@ public class LibvpxVideoRenderer extends BaseRenderer { return false; } + long presentationTimeUs = outputBuffer.timeUs - outputStreamOffsetUs; + Format format = formatQueue.pollFloor(presentationTimeUs); + if (format != null) { + outputFormat = format; + } + long elapsedRealtimeNowUs = SystemClock.elapsedRealtime() * 1000; boolean isStarted = getState() == STATE_STARTED; if (!renderedFirstFrame || (isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedRealtimeNowUs - lastRenderTimeUs))) { + if (frameMetadataListener != null) { + frameMetadataListener.onVideoFrameAboutToBeRendered( + presentationTimeUs, System.nanoTime(), outputFormat); + } renderOutputBuffer(outputBuffer); return true; } @@ -873,6 +905,10 @@ public class LibvpxVideoRenderer extends BaseRenderer { } if (earlyUs < 30000) { + if (frameMetadataListener != null) { + frameMetadataListener.onVideoFrameAboutToBeRendered( + presentationTimeUs, System.nanoTime(), outputFormat); + } renderOutputBuffer(outputBuffer); return true; } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/C.java b/library/core/src/main/java/com/google/android/exoplayer2/C.java index 87499a9cb1..6930975fca 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/C.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/C.java @@ -26,6 +26,7 @@ import android.view.Surface; import com.google.android.exoplayer2.PlayerMessage.Target; import com.google.android.exoplayer2.audio.AuxEffectInfo; import com.google.android.exoplayer2.util.Util; +import com.google.android.exoplayer2.video.VideoFrameMetadataListener; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.util.UUID; @@ -733,6 +734,13 @@ public final class C { */ public static final int MSG_SET_AUX_EFFECT_INFO = 5; + /** + * The type of a message that can be passed to a video {@link Renderer} via {@link + * ExoPlayer#createMessage(Target)}. The message payload should be a {@link + * VideoFrameMetadataListener} instance, or null. + */ + public static final int MSG_SET_VIDEO_FRAME_METADATA_LISTENER = 6; + /** * Applications or extensions may define custom {@code MSG_*} constants that can be passed to * {@link Renderer}s. These custom constants must be greater than or equal to this value. diff --git a/library/core/src/main/java/com/google/android/exoplayer2/Player.java b/library/core/src/main/java/com/google/android/exoplayer2/Player.java index 87aec0c253..4711933f17 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/Player.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/Player.java @@ -29,6 +29,7 @@ import com.google.android.exoplayer2.source.TrackGroupArray; import com.google.android.exoplayer2.text.TextOutput; import com.google.android.exoplayer2.trackselection.TrackSelectionArray; import com.google.android.exoplayer2.util.Util; +import com.google.android.exoplayer2.video.VideoFrameMetadataListener; import com.google.android.exoplayer2.video.VideoListener; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -165,6 +166,25 @@ public interface Player { */ void removeVideoListener(VideoListener listener); + /** + * Sets a listener to receive video frame metadata events. + * + *

This method is intended to be called by the same component that sets the {@link Surface} + * onto which video will be rendered. If using ExoPlayer's standard UI components, this method + * should not be called directly from application code. + * + * @param listener The listener. + */ + void setVideoFrameMetadataListener(VideoFrameMetadataListener listener); + + /** + * Clears the listener which receives video frame metadata events if it matches the one passed. + * Else does nothing. + * + * @param listener The listener to clear. + */ + void clearVideoFrameMetadataListener(VideoFrameMetadataListener listener); + /** * Clears any {@link Surface}, {@link SurfaceHolder}, {@link SurfaceView} or {@link TextureView} * currently set on the player. diff --git a/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java b/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java index 055cf1de17..65d1113be6 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java @@ -51,6 +51,7 @@ import com.google.android.exoplayer2.trackselection.TrackSelector; import com.google.android.exoplayer2.upstream.BandwidthMeter; import com.google.android.exoplayer2.util.Clock; import com.google.android.exoplayer2.util.Util; +import com.google.android.exoplayer2.video.VideoFrameMetadataListener; import com.google.android.exoplayer2.video.VideoRendererEventListener; import java.util.ArrayList; import java.util.Collections; @@ -105,6 +106,7 @@ public class SimpleExoPlayer private float audioVolume; private MediaSource mediaSource; private List currentCues; + private VideoFrameMetadataListener videoFrameMetadataListener; /** * @param renderersFactory A factory for creating {@link Renderer}s to be used by the instance. @@ -598,6 +600,36 @@ public class SimpleExoPlayer videoListeners.remove(listener); } + @Override + public void setVideoFrameMetadataListener(VideoFrameMetadataListener listener) { + videoFrameMetadataListener = listener; + for (Renderer renderer : renderers) { + if (renderer.getTrackType() == C.TRACK_TYPE_VIDEO) { + player + .createMessage(renderer) + .setType(C.MSG_SET_VIDEO_FRAME_METADATA_LISTENER) + .setPayload(listener) + .send(); + } + } + } + + @Override + public void clearVideoFrameMetadataListener(VideoFrameMetadataListener listener) { + if (videoFrameMetadataListener != listener) { + return; + } + for (Renderer renderer : renderers) { + if (renderer.getTrackType() == C.TRACK_TYPE_VIDEO) { + player + .createMessage(renderer) + .setType(C.MSG_SET_VIDEO_FRAME_METADATA_LISTENER) + .setPayload(null) + .send(); + } + } + } + /** * Sets a listener to receive video events, removing all existing listeners. * diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java index 1197cb5a71..1d3e65f7ff 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java @@ -547,9 +547,17 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media } @Override - protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec, - ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs, - boolean shouldSkip) throws ExoPlaybackException { + protected boolean processOutputBuffer( + long positionUs, + long elapsedRealtimeUs, + MediaCodec codec, + ByteBuffer buffer, + int bufferIndex, + int bufferFlags, + long bufferPresentationTimeUs, + boolean shouldSkip, + Format format) + throws ExoPlaybackException { if (passthroughEnabled && (bufferFlags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { // Discard output buffers from the passthrough (raw) decoder containing codec specific data. codec.releaseOutputBuffer(bufferIndex, false); diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java index 3630977fca..0a80780148 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java @@ -43,6 +43,7 @@ import com.google.android.exoplayer2.mediacodec.MediaCodecUtil.DecoderQueryExcep import com.google.android.exoplayer2.source.MediaPeriod; import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.NalUnitUtil; +import com.google.android.exoplayer2.util.TimedValueQueue; import com.google.android.exoplayer2.util.TraceUtil; import com.google.android.exoplayer2.util.Util; import java.lang.annotation.Retention; @@ -272,10 +273,13 @@ public abstract class MediaCodecRenderer extends BaseRenderer { private final DecoderInputBuffer buffer; private final DecoderInputBuffer flagsOnlyBuffer; private final FormatHolder formatHolder; + private final TimedValueQueue formatQueue; private final List decodeOnlyPresentationTimestamps; private final MediaCodec.BufferInfo outputBufferInfo; private Format format; + private Format pendingFormat; + private Format outputFormat; private DrmSession drmSession; private DrmSession pendingDrmSession; private MediaCodec codec; @@ -344,6 +348,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); flagsOnlyBuffer = DecoderInputBuffer.newFlagsOnlyInstance(); formatHolder = new FormatHolder(); + formatQueue = new TimedValueQueue<>(); decodeOnlyPresentationTimestamps = new ArrayList<>(); outputBufferInfo = new MediaCodec.BufferInfo(); codecReconfigurationState = RECONFIGURATION_STATE_NONE; @@ -501,6 +506,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { if (codec != null) { flushCodec(); } + formatQueue.clear(); } @Override @@ -956,6 +962,10 @@ public abstract class MediaCodecRenderer extends BaseRenderer { if (buffer.isDecodeOnly()) { decodeOnlyPresentationTimestamps.add(presentationTimeUs); } + if (pendingFormat != null) { + formatQueue.add(presentationTimeUs, pendingFormat); + pendingFormat = null; + } buffer.flip(); onQueueInputBuffer(buffer); @@ -1012,6 +1022,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { protected void onInputFormatChanged(Format newFormat) throws ExoPlaybackException { Format oldFormat = format; format = newFormat; + pendingFormat = newFormat; boolean drmInitDataChanged = !Util.areEqual(format.drmInitData, oldFormat == null ? null : oldFormat.drmInitData); @@ -1234,35 +1245,15 @@ public abstract class MediaCodecRenderer extends BaseRenderer { codec.dequeueOutputBuffer(outputBufferInfo, getDequeueOutputBufferTimeoutUs()); } - if (outputIndex >= 0) { - // We've dequeued a buffer. - if (shouldSkipAdaptationWorkaroundOutputBuffer) { - shouldSkipAdaptationWorkaroundOutputBuffer = false; - codec.releaseOutputBuffer(outputIndex, false); + if (outputIndex < 0) { + if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED /* (-2) */) { + processOutputFormat(); + return true; + } else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED /* (-3) */) { + processOutputBuffersChanged(); return true; - } else if (outputBufferInfo.size == 0 - && (outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { - // The dequeued buffer indicates the end of the stream. Process it immediately. - processEndOfStream(); - return false; - } else { - this.outputIndex = outputIndex; - outputBuffer = getOutputBuffer(outputIndex); - // The dequeued buffer is a media buffer. Do some initial setup. - // It will be processed by calling processOutputBuffer (possibly multiple times). - if (outputBuffer != null) { - outputBuffer.position(outputBufferInfo.offset); - outputBuffer.limit(outputBufferInfo.offset + outputBufferInfo.size); - } - shouldSkipOutputBuffer = shouldSkipOutputBuffer(outputBufferInfo.presentationTimeUs); } - } else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED /* (-2) */) { - processOutputFormat(); - return true; - } else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED /* (-3) */) { - processOutputBuffersChanged(); - return true; - } else /* MediaCodec.INFO_TRY_AGAIN_LATER (-1) or unknown negative return value */ { + /* MediaCodec.INFO_TRY_AGAIN_LATER (-1) or unknown negative return value */ if (codecNeedsEosPropagationWorkaround && (inputStreamEnded || codecReinitializationState == REINITIALIZATION_STATE_WAIT_END_OF_STREAM)) { @@ -1270,6 +1261,32 @@ public abstract class MediaCodecRenderer extends BaseRenderer { } return false; } + + // We've dequeued a buffer. + if (shouldSkipAdaptationWorkaroundOutputBuffer) { + shouldSkipAdaptationWorkaroundOutputBuffer = false; + codec.releaseOutputBuffer(outputIndex, false); + return true; + } else if (outputBufferInfo.size == 0 + && (outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { + // The dequeued buffer indicates the end of the stream. Process it immediately. + processEndOfStream(); + return false; + } + + this.outputIndex = outputIndex; + outputBuffer = getOutputBuffer(outputIndex); + // The dequeued buffer is a media buffer. Do some initial setup. + // It will be processed by calling processOutputBuffer (possibly multiple times). + if (outputBuffer != null) { + outputBuffer.position(outputBufferInfo.offset); + outputBuffer.limit(outputBufferInfo.offset + outputBufferInfo.size); + } + shouldSkipOutputBuffer = shouldSkipOutputBuffer(outputBufferInfo.presentationTimeUs); + Format format = formatQueue.pollFloor(outputBufferInfo.presentationTimeUs); + if (format != null) { + outputFormat = format; + } } boolean processedOutputBuffer; @@ -1284,7 +1301,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer { outputIndex, outputBufferInfo.flags, outputBufferInfo.presentationTimeUs, - shouldSkipOutputBuffer); + shouldSkipOutputBuffer, + outputFormat); } catch (IllegalStateException e) { processEndOfStream(); if (outputStreamEnded) { @@ -1303,7 +1321,8 @@ public abstract class MediaCodecRenderer extends BaseRenderer { outputIndex, outputBufferInfo.flags, outputBufferInfo.presentationTimeUs, - shouldSkipOutputBuffer); + shouldSkipOutputBuffer, + outputFormat); } if (processedOutputBuffer) { @@ -1348,36 +1367,43 @@ public abstract class MediaCodecRenderer extends BaseRenderer { /** * Processes an output media buffer. - *

- * When a new {@link ByteBuffer} is passed to this method its position and limit delineate the + * + *

When a new {@link ByteBuffer} is passed to this method its position and limit delineate the * data to be processed. The return value indicates whether the buffer was processed in full. If * true is returned then the next call to this method will receive a new buffer to be processed. * If false is returned then the same buffer will be passed to the next call. An implementation of * this method is free to modify the buffer and can assume that the buffer will not be externally * modified between successive calls. Hence an implementation can, for example, modify the * buffer's position to keep track of how much of the data it has processed. - *

- * Note that the first call to this method following a call to - * {@link #onPositionReset(long, boolean)} will always receive a new {@link ByteBuffer} to be - * processed. * - * @param positionUs The current media time in microseconds, measured at the start of the - * current iteration of the rendering loop. - * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, - * measured at the start of the current iteration of the rendering loop. + *

Note that the first call to this method following a call to {@link #onPositionReset(long, + * boolean)} will always receive a new {@link ByteBuffer} to be processed. + * + * @param positionUs The current media time in microseconds, measured at the start of the current + * iteration of the rendering loop. + * @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, measured at the + * start of the current iteration of the rendering loop. * @param codec The {@link MediaCodec} instance. * @param buffer The output buffer to process. * @param bufferIndex The index of the output buffer. * @param bufferFlags The flags attached to the output buffer. * @param bufferPresentationTimeUs The presentation time of the output buffer in microseconds. * @param shouldSkip Whether the buffer should be skipped (i.e. not rendered). - * + * @param format The format associated with the buffer. * @return Whether the output buffer was fully processed (e.g. rendered or skipped). * @throws ExoPlaybackException If an error occurs processing the output buffer. */ - protected abstract boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, - MediaCodec codec, ByteBuffer buffer, int bufferIndex, int bufferFlags, - long bufferPresentationTimeUs, boolean shouldSkip) throws ExoPlaybackException; + protected abstract boolean processOutputBuffer( + long positionUs, + long elapsedRealtimeUs, + MediaCodec codec, + ByteBuffer buffer, + int bufferIndex, + int bufferFlags, + long bufferPresentationTimeUs, + boolean shouldSkip, + Format format) + throws ExoPlaybackException; /** * Incrementally renders any remaining output. diff --git a/library/core/src/main/java/com/google/android/exoplayer2/util/TimedValueQueue.java b/library/core/src/main/java/com/google/android/exoplayer2/util/TimedValueQueue.java new file mode 100644 index 0000000000..160db74eda --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/util/TimedValueQueue.java @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.util; + +import android.support.annotation.Nullable; +import java.util.Arrays; +import org.checkerframework.checker.nullness.compatqual.NullableType; + +/** A utility class to keep a queue of values with timestamps. This class is thread safe. */ +public final class TimedValueQueue { + private static final int INITIAL_BUFFER_SIZE = 10; + + // Looping buffer for timestamps and values + private long[] timestamps; + private @NullableType V[] values; + private int first; + private int size; + + public TimedValueQueue() { + this(INITIAL_BUFFER_SIZE); + } + + /** Creates a TimedValueBuffer with the given initial buffer size. */ + public TimedValueQueue(int initialBufferSize) { + timestamps = new long[initialBufferSize]; + values = newArray(initialBufferSize); + } + + /** + * Associates the specified value with the specified timestamp. All new values should have a + * greater timestamp than the previously added values. Otherwise all values are removed before + * adding the new one. + */ + public synchronized void add(long timestamp, V value) { + clearBufferOnTimeDiscontinuity(timestamp); + doubleCapacityIfFull(); + addUnchecked(timestamp, value); + } + + /** Removes all of the values. */ + public synchronized void clear() { + first = 0; + size = 0; + Arrays.fill(values, null); + } + + /** Returns number of the values buffered. */ + public synchronized int size() { + return size; + } + + /** + * Returns the value with the greatest timestamp which is less than or equal to the given + * timestamp. Removes all older values including the returned one from the buffer. + * + * @param timestamp The timestamp value. + * @return The value with the greatest timestamp which is less than or equal to the given + * timestamp or null if there is no such value. + * @see #poll(long) + */ + public synchronized @Nullable V pollFloor(long timestamp) { + return poll(timestamp, /* onlyOlder= */ true); + } + + /** + * Returns the value with the closest timestamp to the given timestamp. Removes all older values + * including the returned one from the buffer. + * + * @param timestamp The timestamp value. + * @return The value with the closest timestamp or null if the buffer is empty. + * @see #pollFloor(long) + */ + public synchronized @Nullable V poll(long timestamp) { + return poll(timestamp, /* onlyOlder= */ false); + } + + /** + * Returns the value with the closest timestamp to the given timestamp. Removes all older values + * including the returned one from the buffer. + * + * @param timestamp The timestamp value. + * @param onlyOlder Whether this method can return a new value in case its timestamp value is + * closest to {@code timestamp}. + * @return The value with the closest timestamp or null if the buffer is empty or there is no + * older value and {@code onlyOlder} is true. + */ + private @Nullable V poll(long timestamp, boolean onlyOlder) { + V value = null; + long previousTimeDiff = Long.MAX_VALUE; + while (size > 0) { + long timeDiff = timestamp - timestamps[first]; + if (timeDiff < 0 && (onlyOlder || -timeDiff >= previousTimeDiff)) { + break; + } + previousTimeDiff = timeDiff; + value = values[first]; + values[first] = null; + first = (first + 1) % values.length; + size--; + } + return value; + } + + private void clearBufferOnTimeDiscontinuity(long timestamp) { + if (size > 0) { + int last = (first + size - 1) % values.length; + if (timestamp <= timestamps[last]) { + clear(); + } + } + } + + private void doubleCapacityIfFull() { + int capacity = values.length; + if (size < capacity) { + return; + } + int newCapacity = capacity * 2; + long[] newTimestamps = new long[newCapacity]; + V[] newValues = newArray(newCapacity); + // Reset the loop starting index to 0 while coping to the new buffer. + // First copy the values from 'first' index to the end of original array. + int length = capacity - first; + System.arraycopy(timestamps, first, newTimestamps, 0, length); + System.arraycopy(values, first, newValues, 0, length); + // Then the values from index 0 to 'first' index. + if (first > 0) { + System.arraycopy(timestamps, 0, newTimestamps, length, first); + System.arraycopy(values, 0, newValues, length, first); + } + timestamps = newTimestamps; + values = newValues; + first = 0; + } + + private void addUnchecked(long timestamp, V value) { + int next = (first + size) % values.length; + timestamps[next] = timestamp; + values[next] = value; + size++; + } + + @SuppressWarnings("unchecked") + private static V[] newArray(int length) { + return (V[]) new Object[length]; + } +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index 181232b7b2..0c3cd74b74 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -136,6 +136,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { private long lastInputTimeUs; private long outputStreamOffsetUs; private int pendingOutputStreamOffsetCount; + private @Nullable VideoFrameMetadataListener frameMetadataListener; /** * @param context A context. @@ -386,6 +387,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { if (codec != null) { codec.setVideoScalingMode(scalingMode); } + } else if (messageType == C.MSG_SET_VIDEO_FRAME_METADATA_LISTENER) { + frameMetadataListener = (VideoFrameMetadataListener) message; } else { super.handleMessage(messageType, message); } @@ -587,9 +590,17 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } @Override - protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec, - ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs, - boolean shouldSkip) throws ExoPlaybackException { + protected boolean processOutputBuffer( + long positionUs, + long elapsedRealtimeUs, + MediaCodec codec, + ByteBuffer buffer, + int bufferIndex, + int bufferFlags, + long bufferPresentationTimeUs, + boolean shouldSkip, + Format format) + throws ExoPlaybackException { if (initialPositionUs == C.TIME_UNSET) { initialPositionUs = positionUs; } @@ -616,8 +627,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { if (!renderedFirstFrame || (isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedRealtimeNowUs - lastRenderTimeUs))) { + long releaseTimeNs = System.nanoTime(); + notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); if (Util.SDK_INT >= 21) { - renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, System.nanoTime()); + renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, releaseTimeNs); } else { renderOutputBuffer(codec, bufferIndex, presentationTimeUs); } @@ -653,6 +666,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { if (Util.SDK_INT >= 21) { // Let the underlying framework time the release. if (earlyUs < 50000) { + notifyFrameMetadataListener(presentationTimeUs, adjustedReleaseTimeNs, format); renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, adjustedReleaseTimeNs); return true; } @@ -670,6 +684,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return false; } } + notifyFrameMetadataListener(presentationTimeUs, adjustedReleaseTimeNs, format); renderOutputBuffer(codec, bufferIndex, presentationTimeUs); return true; } @@ -679,10 +694,18 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return false; } + private void notifyFrameMetadataListener( + long presentationTimeUs, long releaseTimeNs, Format format) { + if (frameMetadataListener != null) { + frameMetadataListener.onVideoFrameAboutToBeRendered( + presentationTimeUs, releaseTimeNs, format); + } + } + /** * Returns the offset that should be subtracted from {@code bufferPresentationTimeUs} in {@link - * #processOutputBuffer(long, long, MediaCodec, ByteBuffer, int, int, long, boolean)} to get the - * playback position with respect to the media. + * #processOutputBuffer(long, long, MediaCodec, ByteBuffer, int, int, long, boolean, Format)} to + * get the playback position with respect to the media. */ protected long getOutputStreamOffsetUs() { return outputStreamOffsetUs; diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameMetadataListener.java b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameMetadataListener.java new file mode 100644 index 0000000000..b467d0f421 --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameMetadataListener.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.video; + +import com.google.android.exoplayer2.Format; + +/** A listener for metadata corresponding to video frame being rendered. */ +public interface VideoFrameMetadataListener { + /** + * Called when the video frame about to be rendered. This method is called on the playback thread. + * + * @param presentationTimeUs The presentation time of the output buffer, in microseconds. + * @param releaseTimeNs The wallclock time at which the frame should be displayed, in nanoseconds. + * If the platform API version of the device is less than 21, then this is the best effort. + * @param format The format associated with the frame. + */ + void onVideoFrameAboutToBeRendered(long presentationTimeUs, long releaseTimeNs, Format format); +} diff --git a/library/core/src/test/java/com/google/android/exoplayer2/util/TimedValueQueueTest.java b/library/core/src/test/java/com/google/android/exoplayer2/util/TimedValueQueueTest.java new file mode 100644 index 0000000000..ca34bc3216 --- /dev/null +++ b/library/core/src/test/java/com/google/android/exoplayer2/util/TimedValueQueueTest.java @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.util; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; + +/** Unit test for {@link TimedValueQueue}. */ +@RunWith(RobolectricTestRunner.class) +public class TimedValueQueueTest { + + private TimedValueQueue queue; + + @Before + public void setUp() throws Exception { + queue = new TimedValueQueue<>(); + } + + @Test + public void testAddAndPollValues() { + queue.add(0, "a"); + queue.add(1, "b"); + queue.add(2, "c"); + assertThat(queue.poll(0)).isEqualTo("a"); + assertThat(queue.poll(1)).isEqualTo("b"); + assertThat(queue.poll(2)).isEqualTo("c"); + } + + @Test + public void testBufferCapacityIncreasesAutomatically() { + queue = new TimedValueQueue<>(1); + for (int i = 0; i < 20; i++) { + queue.add(i, "" + i); + if ((i & 1) == 1) { + assertThat(queue.poll(0)).isEqualTo("" + (i / 2)); + } + } + assertThat(queue.size()).isEqualTo(10); + } + + @Test + public void testTimeDiscontinuityClearsValues() { + queue.add(1, "b"); + queue.add(2, "c"); + queue.add(0, "a"); + assertThat(queue.size()).isEqualTo(1); + assertThat(queue.poll(0)).isEqualTo("a"); + } + + @Test + public void testTimeDiscontinuityOnFullBufferClearsValues() { + queue = new TimedValueQueue<>(2); + queue.add(1, "b"); + queue.add(3, "c"); + queue.add(2, "a"); + assertThat(queue.size()).isEqualTo(1); + assertThat(queue.poll(2)).isEqualTo("a"); + } + + @Test + public void testPollReturnsClosestValue() { + queue.add(0, "a"); + queue.add(3, "b"); + assertThat(queue.poll(2)).isEqualTo("b"); + assertThat(queue.size()).isEqualTo(0); + } + + @Test + public void testPollRemovesPreviousValues() { + queue.add(0, "a"); + queue.add(1, "b"); + queue.add(2, "c"); + assertThat(queue.poll(1)).isEqualTo("b"); + assertThat(queue.size()).isEqualTo(1); + } + + @Test + public void testPollFloorReturnsClosestPreviousValue() { + queue.add(0, "a"); + queue.add(3, "b"); + assertThat(queue.pollFloor(2)).isEqualTo("a"); + assertThat(queue.pollFloor(2)).isEqualTo(null); + assertThat(queue.pollFloor(3)).isEqualTo("b"); + assertThat(queue.size()).isEqualTo(0); + } + + @Test + public void testPollFloorRemovesPreviousValues() { + queue.add(0, "a"); + queue.add(1, "b"); + queue.add(2, "c"); + assertThat(queue.pollFloor(1)).isEqualTo("b"); + assertThat(queue.size()).isEqualTo(1); + } +} diff --git a/testutils/src/main/java/com/google/android/exoplayer2/testutil/DebugRenderersFactory.java b/testutils/src/main/java/com/google/android/exoplayer2/testutil/DebugRenderersFactory.java index 02a8a0597d..627b5b72f3 100644 --- a/testutils/src/main/java/com/google/android/exoplayer2/testutil/DebugRenderersFactory.java +++ b/testutils/src/main/java/com/google/android/exoplayer2/testutil/DebugRenderersFactory.java @@ -126,17 +126,33 @@ public class DebugRenderersFactory extends DefaultRenderersFactory { } @Override - protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec, - ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs, - boolean shouldSkip) throws ExoPlaybackException { + protected boolean processOutputBuffer( + long positionUs, + long elapsedRealtimeUs, + MediaCodec codec, + ByteBuffer buffer, + int bufferIndex, + int bufferFlags, + long bufferPresentationTimeUs, + boolean shouldSkip, + Format format) + throws ExoPlaybackException { if (skipToPositionBeforeRenderingFirstFrame && bufferPresentationTimeUs < positionUs) { // After the codec has been initialized, don't render the first frame until we've caught up // to the playback position. Else test runs on devices that do not support dummy surface // will drop frames between rendering the first one and catching up [Internal: b/66494991]. shouldSkip = true; } - return super.processOutputBuffer(positionUs, elapsedRealtimeUs, codec, buffer, bufferIndex, - bufferFlags, bufferPresentationTimeUs, shouldSkip); + return super.processOutputBuffer( + positionUs, + elapsedRealtimeUs, + codec, + buffer, + bufferIndex, + bufferFlags, + bufferPresentationTimeUs, + shouldSkip, + format); } @Override