From 88fddb42eca2da65297cfacd1a068aadc47b2614 Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Tue, 23 May 2017 01:43:24 -0700 Subject: [PATCH] Provide video frame timestamps to subclasses Expose the stream offset to BaseRenderer subclasses. Issue: #2267 ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=156837514 --- .../android/exoplayer2/BaseRenderer.java | 11 ++- .../exoplayer2/metadata/MetadataRenderer.java | 2 +- .../android/exoplayer2/text/TextRenderer.java | 2 +- .../video/MediaCodecVideoRenderer.java | 99 ++++++++++++++++--- 4 files changed, 92 insertions(+), 22 deletions(-) diff --git a/library/core/src/main/java/com/google/android/exoplayer2/BaseRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/BaseRenderer.java index 396584a39e..a88a1dd615 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/BaseRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/BaseRenderer.java @@ -96,7 +96,7 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities { this.stream = stream; readEndOfStream = false; streamOffsetUs = offsetUs; - onStreamChanged(formats); + onStreamChanged(formats, offsetUs); } @Override @@ -183,16 +183,19 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities { * The default implementation is a no-op. * * @param formats The enabled formats. + * @param offsetUs The offset that will be added to the timestamps of buffers read via + * {@link #readSource(FormatHolder, DecoderInputBuffer, boolean)} so that decoder input + * buffers have monotonically increasing timestamps. * @throws ExoPlaybackException If an error occurs. */ - protected void onStreamChanged(Format[] formats) throws ExoPlaybackException { + protected void onStreamChanged(Format[] formats, long offsetUs) throws ExoPlaybackException { // Do nothing. } /** * Called when the position is reset. This occurs when the renderer is enabled after - * {@link #onStreamChanged(Format[])} has been called, and also when a position discontinuity - * is encountered. + * {@link #onStreamChanged(Format[], long)} has been called, and also when a position + * discontinuity is encountered. *

* After a position reset, the renderer's {@link SampleStream} is guaranteed to provide samples * starting from a key frame. diff --git a/library/core/src/main/java/com/google/android/exoplayer2/metadata/MetadataRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/metadata/MetadataRenderer.java index 70b2d8aab9..7ff426e2df 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/metadata/MetadataRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/metadata/MetadataRenderer.java @@ -104,7 +104,7 @@ public final class MetadataRenderer extends BaseRenderer implements Callback { } @Override - protected void onStreamChanged(Format[] formats) throws ExoPlaybackException { + protected void onStreamChanged(Format[] formats, long offsetUs) throws ExoPlaybackException { decoder = decoderFactory.createDecoder(formats[0]); } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/text/TextRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/text/TextRenderer.java index 4950549b19..1820d43e75 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/text/TextRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/text/TextRenderer.java @@ -130,7 +130,7 @@ public final class TextRenderer extends BaseRenderer implements Callback { } @Override - protected void onStreamChanged(Format[] formats) throws ExoPlaybackException { + protected void onStreamChanged(Format[] formats, long offsetUs) throws ExoPlaybackException { streamFormat = formats[0]; if (decoder != null) { decoderReplacementState = REPLACEMENT_STATE_SIGNAL_END_OF_STREAM; diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index dd0c5356ea..aabec0eaa7 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -62,11 +62,16 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { private static final int[] STANDARD_LONG_EDGE_VIDEO_PX = new int[] { 1920, 1600, 1440, 1280, 960, 854, 640, 540, 480}; + // Generally there is zero or one pending output stream offset. We track more offsets to allow for + // pending output streams that have fewer frames than the codec latency. + private static final int MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT = 10; + private final VideoFrameReleaseTimeHelper frameReleaseTimeHelper; private final EventDispatcher eventDispatcher; private final long allowedJoiningTimeMs; private final int maxDroppedFramesToNotify; private final boolean deviceNeedsAutoFrcWorkaround; + private final long[] pendingOutputStreamOffsetsUs; private Format[] streamFormats; private CodecMaxValues codecMaxValues; @@ -95,6 +100,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { private int tunnelingAudioSessionId; /* package */ OnFrameRenderedListenerV23 tunnelingOnFrameRenderedListener; + private long outputStreamOffsetUs; + private int pendingOutputStreamOffsetCount; + /** * @param context A context. * @param mediaCodecSelector A decoder selector. @@ -160,6 +168,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { frameReleaseTimeHelper = new VideoFrameReleaseTimeHelper(context); eventDispatcher = new EventDispatcher(eventHandler, eventListener); deviceNeedsAutoFrcWorkaround = deviceNeedsAutoFrcWorkaround(); + pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT]; + outputStreamOffsetUs = C.TIME_UNSET; joiningDeadlineMs = C.TIME_UNSET; currentWidth = Format.NO_VALUE; currentHeight = Format.NO_VALUE; @@ -219,9 +229,20 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } @Override - protected void onStreamChanged(Format[] formats) throws ExoPlaybackException { + protected void onStreamChanged(Format[] formats, long offsetUs) throws ExoPlaybackException { streamFormats = formats; - super.onStreamChanged(formats); + if (outputStreamOffsetUs == C.TIME_UNSET) { + outputStreamOffsetUs = offsetUs; + } else { + if (pendingOutputStreamOffsetCount == pendingOutputStreamOffsetsUs.length) { + Log.w(TAG, "Too many stream changes, so dropping offset: " + + pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1]); + } else { + pendingOutputStreamOffsetCount++; + } + pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1] = offsetUs; + } + super.onStreamChanged(formats, offsetUs); } @Override @@ -229,6 +250,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { super.onPositionReset(positionUs, joining); clearRenderedFirstFrame(); consecutiveDroppedFrameCount = 0; + if (pendingOutputStreamOffsetCount != 0) { + outputStreamOffsetUs = pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1]; + pendingOutputStreamOffsetCount = 0; + } if (joining) { setJoiningDeadlineMs(); } else { @@ -275,6 +300,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { currentHeight = Format.NO_VALUE; currentPixelWidthHeightRatio = Format.NO_VALUE; pendingPixelWidthHeightRatio = Format.NO_VALUE; + outputStreamOffsetUs = C.TIME_UNSET; + pendingOutputStreamOffsetCount = 0; clearReportedVideoSize(); clearRenderedFirstFrame(); frameReleaseTimeHelper.disable(); @@ -417,16 +444,24 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { protected boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec, ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs, boolean shouldSkip) { + while (pendingOutputStreamOffsetCount != 0 + && bufferPresentationTimeUs >= pendingOutputStreamOffsetsUs[0]) { + outputStreamOffsetUs = pendingOutputStreamOffsetsUs[0]; + pendingOutputStreamOffsetCount--; + System.arraycopy(pendingOutputStreamOffsetsUs, 1, pendingOutputStreamOffsetsUs, 0, + pendingOutputStreamOffsetCount); + } + long presentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs; if (shouldSkip) { - skipOutputBuffer(codec, bufferIndex); + skipOutputBuffer(codec, bufferIndex, presentationTimeUs); return true; } if (!renderedFirstFrame) { if (Util.SDK_INT >= 21) { - renderOutputBufferV21(codec, bufferIndex, System.nanoTime()); + renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, System.nanoTime()); } else { - renderOutputBuffer(codec, bufferIndex); + renderOutputBuffer(codec, bufferIndex, presentationTimeUs); } return true; } @@ -450,14 +485,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { if (shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs)) { // We're more than 30ms late rendering the frame. - dropOutputBuffer(codec, bufferIndex); + dropOutputBuffer(codec, bufferIndex, presentationTimeUs); return true; } if (Util.SDK_INT >= 21) { // Let the underlying framework time the release. if (earlyUs < 50000) { - renderOutputBufferV21(codec, bufferIndex, adjustedReleaseTimeNs); + renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, adjustedReleaseTimeNs); return true; } } else { @@ -473,7 +508,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { Thread.currentThread().interrupt(); } } - renderOutputBuffer(codec, bufferIndex); + renderOutputBuffer(codec, bufferIndex, presentationTimeUs); return true; } } @@ -495,16 +530,30 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return earlyUs < -30000; } - private void skipOutputBuffer(MediaCodec codec, int bufferIndex) { + /** + * Skips the output buffer with the specified index. + * + * @param codec The codec that owns the output buffer. + * @param index The index of the output buffer to skip. + * @param presentationTimeUs The presentation time of the output buffer, in microseconds. + */ + protected void skipOutputBuffer(MediaCodec codec, int index, long presentationTimeUs) { TraceUtil.beginSection("skipVideoBuffer"); - codec.releaseOutputBuffer(bufferIndex, false); + codec.releaseOutputBuffer(index, false); TraceUtil.endSection(); decoderCounters.skippedOutputBufferCount++; } - private void dropOutputBuffer(MediaCodec codec, int bufferIndex) { + /** + * Drops the output buffer with the specified index. + * + * @param codec The codec that owns the output buffer. + * @param index The index of the output buffer to drop. + * @param presentationTimeUs The presentation time of the output buffer, in microseconds. + */ + protected void dropOutputBuffer(MediaCodec codec, int index, long presentationTimeUs) { TraceUtil.beginSection("dropVideoBuffer"); - codec.releaseOutputBuffer(bufferIndex, false); + codec.releaseOutputBuffer(index, false); TraceUtil.endSection(); decoderCounters.droppedOutputBufferCount++; droppedFrames++; @@ -516,21 +565,39 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } } - private void renderOutputBuffer(MediaCodec codec, int bufferIndex) { + /** + * Renders the output buffer with the specified index. This method is only called if the platform + * API version of the device is less than 21. + * + * @param codec The codec that owns the output buffer. + * @param index The index of the output buffer to drop. + * @param presentationTimeUs The presentation time of the output buffer, in microseconds. + */ + protected void renderOutputBuffer(MediaCodec codec, int index, long presentationTimeUs) { maybeNotifyVideoSizeChanged(); TraceUtil.beginSection("releaseOutputBuffer"); - codec.releaseOutputBuffer(bufferIndex, true); + codec.releaseOutputBuffer(index, true); TraceUtil.endSection(); decoderCounters.renderedOutputBufferCount++; consecutiveDroppedFrameCount = 0; maybeNotifyRenderedFirstFrame(); } + /** + * Renders the output buffer with the specified index. This method is only called if the platform + * API version of the device is 21 or later. + * + * @param codec The codec that owns the output buffer. + * @param index The index of the output buffer to drop. + * @param presentationTimeUs The presentation time of the output buffer, in microseconds. + * @param releaseTimeNs The wallclock time at which the frame should be displayed, in nanoseconds. + */ @TargetApi(21) - private void renderOutputBufferV21(MediaCodec codec, int bufferIndex, long releaseTimeNs) { + protected void renderOutputBufferV21(MediaCodec codec, int index, long presentationTimeUs, + long releaseTimeNs) { maybeNotifyVideoSizeChanged(); TraceUtil.beginSection("releaseOutputBuffer"); - codec.releaseOutputBuffer(bufferIndex, releaseTimeNs); + codec.releaseOutputBuffer(index, releaseTimeNs); TraceUtil.endSection(); decoderCounters.renderedOutputBufferCount++; consecutiveDroppedFrameCount = 0;