From df85a996ba9b8b08cc4256e89a0d845690ddb779 Mon Sep 17 00:00:00 2001 From: tonihei Date: Wed, 4 Oct 2023 10:35:15 -0700 Subject: [PATCH] Add Decoder.setOutputStartTimeUs and use it in extension decoders This gets rid of the reliance on the decode only flag that is still set on input buffers to the decoder if they are less than the start time. We still need to set and check the decode-only flag in SimpleDecoder to ensure compatbility with custom decoders that use the flag while it's not fully removed. PiperOrigin-RevId: 570736692 (cherry picked from commit a03e20fe6c423389d54eb08d3b1f1d19499a0d9a) --- RELEASENOTES.md | 9 ++++-- .../java/androidx/media3/decoder/Decoder.java | 13 ++++++++ .../media3/decoder/SimpleDecoder.java | 32 +++++++++++++++++-- .../media3/decoder/av1/Gav1Decoder.java | 2 +- .../media3/decoder/midi/MidiDecoder.java | 5 +-- .../media3/decoder/vp9/VpxDecoder.java | 2 +- .../exoplayer/audio/DecoderAudioRenderer.java | 4 +++ .../exoplayer/text/ExoplayerCuesDecoder.java | 5 +++ .../exoplayer/video/DecoderVideoRenderer.java | 5 ++- .../media3/extractor/text/cea/CeaDecoder.java | 5 +++ 10 files changed, 72 insertions(+), 10 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 2afe9d5d58..50cb07500a 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -33,11 +33,14 @@ * Allow multiple of the same DASH identifier in segment template url. * Smooth Streaming Extension: * RTSP Extension: -* Decoder Extensions (FFmpeg, VP9, AV1, etc.): +* Decoder Extensions (FFmpeg, VP9, AV1, MIDI, etc.): * Add `DecoderOutputBuffer.shouldBeSkipped` to directly mark output buffers that don't need to be presented. This is preferred over - `C.BUFFER_FLAG_DECODE_ONLY`. -* MIDI extension: + `C.BUFFER_FLAG_DECODE_ONLY` that will be deprecated. + * Add `Decoder.setOutputStartTimeUs` and + `SimpleDecoder.isAtLeastOutputStartTimeUs` to allow decoders to drop + decode-only samples before the start time. This should be preferred to + `Buffer.isDecodeOnly` that will be deprecated. * Leanback extension: * Cast Extension: * Test Utilities: diff --git a/libraries/decoder/src/main/java/androidx/media3/decoder/Decoder.java b/libraries/decoder/src/main/java/androidx/media3/decoder/Decoder.java index c6f9968992..1cd1a6c6f5 100644 --- a/libraries/decoder/src/main/java/androidx/media3/decoder/Decoder.java +++ b/libraries/decoder/src/main/java/androidx/media3/decoder/Decoder.java @@ -35,6 +35,19 @@ public interface Decoder { */ String getName(); + /** + * Sets the timestamp from which output buffers should be produced, in microseconds. + * + *

Any decoded buffer with a timestamp less than {@code outputStartTimeUs} should be skipped by + * the implementation and not made available via {@link #dequeueOutputBuffer}. + * + *

This method must only be called before {@linkplain #queueInputBuffer queuing the first input + * buffer} initially or after {@link #flush()}. + * + * @param outputStartTimeUs The time from which output buffer should be produced, in microseconds. + */ + void setOutputStartTimeUs(long outputStartTimeUs); + /** * Dequeues the next input buffer to be filled and queued to the decoder. * diff --git a/libraries/decoder/src/main/java/androidx/media3/decoder/SimpleDecoder.java b/libraries/decoder/src/main/java/androidx/media3/decoder/SimpleDecoder.java index d4084a904e..557c66cd8f 100644 --- a/libraries/decoder/src/main/java/androidx/media3/decoder/SimpleDecoder.java +++ b/libraries/decoder/src/main/java/androidx/media3/decoder/SimpleDecoder.java @@ -48,6 +48,7 @@ public abstract class SimpleDecoder< private boolean flushed; private boolean released; private int skippedOutputBufferCount; + private long outputStartTimeUs; /** * @param inputBuffers An array of nulls that will be used to store references to input buffers. @@ -56,6 +57,7 @@ public abstract class SimpleDecoder< @SuppressWarnings("nullness:method.invocation") protected SimpleDecoder(I[] inputBuffers, O[] outputBuffers) { lock = new Object(); + outputStartTimeUs = C.TIME_UNSET; queuedInputBuffers = new ArrayDeque<>(); queuedOutputBuffers = new ArrayDeque<>(); availableInputBuffers = inputBuffers; @@ -93,6 +95,30 @@ public abstract class SimpleDecoder< } } + /** + * Returns whether a sample time is greater or equal to the {@link #setOutputStartTimeUs output + * start time}, if set. + * + *

If this method returns false, the buffer will not be made available as an output buffer. + * + * @param timeUs The buffer time, in microseconds. + * @return Whether the buffer time is greater or equal to the output start time, or {@code true} + * if the output start time is not set. + */ + protected final boolean isAtLeastOutputStartTimeUs(long timeUs) { + synchronized (lock) { + return outputStartTimeUs == C.TIME_UNSET || timeUs >= outputStartTimeUs; + } + } + + @Override + public final void setOutputStartTimeUs(long outputStartTimeUs) { + synchronized (lock) { + Assertions.checkState(availableInputBufferCount == availableInputBuffers.length || flushed); + this.outputStartTimeUs = outputStartTimeUs; + } + } + @Override @Nullable public final I dequeueInputBuffer() throws E { @@ -233,7 +259,7 @@ public abstract class SimpleDecoder< outputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM); } else { outputBuffer.timeUs = inputBuffer.timeUs; - if (inputBuffer.isDecodeOnly()) { + if (!isAtLeastOutputStartTimeUs(inputBuffer.timeUs) || inputBuffer.isDecodeOnly()) { outputBuffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY); } if (inputBuffer.isFirstSample()) { @@ -263,7 +289,9 @@ public abstract class SimpleDecoder< synchronized (lock) { if (flushed) { outputBuffer.release(); - } else if (outputBuffer.isDecodeOnly() || outputBuffer.shouldBeSkipped) { + } else if ((!outputBuffer.isEndOfStream() && !isAtLeastOutputStartTimeUs(outputBuffer.timeUs)) + || outputBuffer.isDecodeOnly() + || outputBuffer.shouldBeSkipped) { skippedOutputBufferCount++; outputBuffer.release(); } else { diff --git a/libraries/decoder_av1/src/main/java/androidx/media3/decoder/av1/Gav1Decoder.java b/libraries/decoder_av1/src/main/java/androidx/media3/decoder/av1/Gav1Decoder.java index 49feb4e68c..9b9de0e700 100644 --- a/libraries/decoder_av1/src/main/java/androidx/media3/decoder/av1/Gav1Decoder.java +++ b/libraries/decoder_av1/src/main/java/androidx/media3/decoder/av1/Gav1Decoder.java @@ -105,7 +105,7 @@ public final class Gav1Decoder "gav1Decode error: " + gav1GetErrorMessage(gav1DecoderContext)); } - boolean decodeOnly = inputBuffer.isDecodeOnly(); + boolean decodeOnly = !isAtLeastOutputStartTimeUs(inputBuffer.timeUs); if (!decodeOnly) { outputBuffer.init(inputBuffer.timeUs, outputMode, /* supplementalData= */ null); } diff --git a/libraries/decoder_midi/src/main/java/androidx/media3/decoder/midi/MidiDecoder.java b/libraries/decoder_midi/src/main/java/androidx/media3/decoder/midi/MidiDecoder.java index c7b886e0f4..cb7d11957e 100644 --- a/libraries/decoder_midi/src/main/java/androidx/media3/decoder/midi/MidiDecoder.java +++ b/libraries/decoder_midi/src/main/java/androidx/media3/decoder/midi/MidiDecoder.java @@ -143,8 +143,9 @@ import org.checkerframework.checker.nullness.qual.EnsuresNonNull; if (lastReceivedTimestampUs == C.TIME_UNSET) { outputTimeUs = inputBuffer.timeUs; } + boolean isDecodeOnly = !isAtLeastOutputStartTimeUs(inputBuffer.timeUs); try { - if (!inputBuffer.isDecodeOnly()) { + if (!isDecodeOnly) { // Yield the thread to the Synthesizer to produce PCM samples up to this buffer's timestamp. if (lastReceivedTimestampUs != C.TIME_UNSET) { double timeToSleepSecs = (inputBuffer.timeUs - lastReceivedTimestampUs) * 0.000001D; @@ -167,7 +168,7 @@ import org.checkerframework.checker.nullness.qual.EnsuresNonNull; int availableSamples = reader.available(); // Ensure there are no remaining bytes if the input buffer is decode only. - checkState(!inputBuffer.isDecodeOnly() || reader.available() == 0); + checkState(!isDecodeOnly || availableSamples == 0); if (availableSamples > audioStreamOutputBuffer.length) { // Increase the size of the buffer by 25% of the availableSamples (arbitrary number). diff --git a/libraries/decoder_vp9/src/main/java/androidx/media3/decoder/vp9/VpxDecoder.java b/libraries/decoder_vp9/src/main/java/androidx/media3/decoder/vp9/VpxDecoder.java index db85283ec6..25326c071a 100644 --- a/libraries/decoder_vp9/src/main/java/androidx/media3/decoder/vp9/VpxDecoder.java +++ b/libraries/decoder_vp9/src/main/java/androidx/media3/decoder/vp9/VpxDecoder.java @@ -165,7 +165,7 @@ public final class VpxDecoder } } - if (!inputBuffer.isDecodeOnly()) { + if (isAtLeastOutputStartTimeUs(inputBuffer.timeUs)) { outputBuffer.init(inputBuffer.timeUs, outputMode, lastSupplementalData); int getFrameResult = vpxGetFrame(vpxDecContext, outputBuffer); if (getFrameResult == 1) { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java index 699c1ee8b2..3f4faa6ee7 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java @@ -15,6 +15,7 @@ */ package androidx.media3.exoplayer.audio; +import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_DRM_SESSION_CHANGED; import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_REUSE_NOT_IMPLEMENTED; import static androidx.media3.exoplayer.DecoderReuseEvaluation.REUSE_RESULT_NO; @@ -557,7 +558,9 @@ public abstract class DecoderAudioRenderer< outputBuffer.release(); outputBuffer = null; } + Decoder decoder = checkNotNull(this.decoder); decoder.flush(); + decoder.setOutputStartTimeUs(getLastResetPositionUs()); decoderReceivedBuffers = false; } } @@ -735,6 +738,7 @@ public abstract class DecoderAudioRenderer< long codecInitializingTimestamp = SystemClock.elapsedRealtime(); TraceUtil.beginSection("createAudioDecoder"); decoder = createDecoder(inputFormat, cryptoConfig); + decoder.setOutputStartTimeUs(getLastResetPositionUs()); TraceUtil.endSection(); long codecInitializedTimestamp = SystemClock.elapsedRealtime(); eventDispatcher.decoderInitialized( diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/ExoplayerCuesDecoder.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/ExoplayerCuesDecoder.java index 49c0b925f4..ac7c63baf3 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/ExoplayerCuesDecoder.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/ExoplayerCuesDecoder.java @@ -86,6 +86,11 @@ public final class ExoplayerCuesDecoder implements SubtitleDecoder { return "ExoplayerCuesDecoder"; } + @Override + public void setOutputStartTimeUs(long outputStartTimeUs) { + // Do nothing. + } + @Nullable @Override public SubtitleInputBuffer dequeueInputBuffer() throws SubtitleDecoderException { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DecoderVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DecoderVideoRenderer.java index c0f7e74723..29b1c2889c 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DecoderVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/DecoderVideoRenderer.java @@ -366,7 +366,9 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { outputBuffer.release(); outputBuffer = null; } - checkNotNull(decoder).flush(); + Decoder decoder = checkNotNull(this.decoder); + decoder.flush(); + decoder.setOutputStartTimeUs(getLastResetPositionUs()); decoderReceivedBuffers = false; } } @@ -718,6 +720,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { try { long decoderInitializingTimestamp = SystemClock.elapsedRealtime(); decoder = createDecoder(checkNotNull(inputFormat), cryptoConfig); + decoder.setOutputStartTimeUs(getLastResetPositionUs()); setDecoderOutputMode(outputMode); long decoderInitializedTimestamp = SystemClock.elapsedRealtime(); eventDispatcher.decoderInitialized( diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/text/cea/CeaDecoder.java b/libraries/extractor/src/main/java/androidx/media3/extractor/text/cea/CeaDecoder.java index 3d591858db..feac7fd1ed 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/text/cea/CeaDecoder.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/text/cea/CeaDecoder.java @@ -58,6 +58,11 @@ import java.util.PriorityQueue; @Override public abstract String getName(); + @Override + public final void setOutputStartTimeUs(long outputStartTimeUs) { + // Do nothing. + } + @Override public void setPositionUs(long positionUs) { playbackPositionUs = positionUs;