diff --git a/RELEASENOTES.md b/RELEASENOTES.md index acce6295ca..605c1c20f6 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,5 +1,14 @@ # Release notes +### Unreleased changes + +* Core library: + * Add `Buffer.isLastSample()` that denotes if `Buffer` contains flag + `C.BUFFER_FLAG_LAST_SAMPLE`. + * Fix issue where last frame may not be rendered if the last sample with + frames is dequeued without reading the 'end of stream' sample. + ([#11079](https://github.com/google/ExoPlayer/issues/11079)). + ### 1.0.1 (2023-04-18) This release corresponds to the diff --git a/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java b/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java index cf2be91c65..8d1c636da5 100644 --- a/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java +++ b/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java @@ -49,6 +49,11 @@ public abstract class Buffer { return getFlag(C.BUFFER_FLAG_KEY_FRAME); } + /** Returns whether the {@link C#BUFFER_FLAG_LAST_SAMPLE} flag is set. */ + public final boolean isLastSample() { + return getFlag(C.BUFFER_FLAG_LAST_SAMPLE); + } + /** Returns whether the {@link C#BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA} flag is set. */ public final boolean hasSupplementalData() { return getFlag(C.BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index fddfea4e2a..c944762e0a 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -1244,7 +1244,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { return true; } - if (hasReadStreamToEnd()) { + if (hasReadStreamToEnd() || buffer.isLastSample()) { // Notify output queue of the last buffer's timestamp. lastBufferInStreamPresentationTimeUs = largestQueuedPresentationTimeUs; } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SampleQueue.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SampleQueue.java index 203c5f8b9b..a9d2c57caa 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SampleQueue.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SampleQueue.java @@ -716,6 +716,9 @@ public class SampleQueue implements TrackOutput { } buffer.setFlags(flags[relativeReadIndex]); + if (readPosition == (length - 1) && (loadingFinished || isLastSampleQueued)) { + buffer.addFlag(C.BUFFER_FLAG_LAST_SAMPLE); + } buffer.timeUs = timesUs[relativeReadIndex]; if (buffer.timeUs < startTimeUs) { buffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY); diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/SampleQueueTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/SampleQueueTest.java index b7042b0457..a7f935ef17 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/SampleQueueTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/SampleQueueTest.java @@ -354,6 +354,32 @@ public final class SampleQueueTest { assertAllocationCount(0); } + @Test + public void readSingleSampleWithLoadingFinished() { + sampleQueue.sampleData(new ParsableByteArray(DATA), ALLOCATION_SIZE); + sampleQueue.format(FORMAT_1); + sampleQueue.sampleMetadata(1000, C.BUFFER_FLAG_KEY_FRAME, ALLOCATION_SIZE, 0, null); + + assertAllocationCount(1); + // If formatRequired, should read the format rather than the sample. + assertReadFormat(true, FORMAT_1); + // Otherwise should read the sample with loading finished. + assertReadLastSample( + 1000, + /* isKeyFrame= */ true, + /* isDecodeOnly= */ false, + /* isEncrypted= */ false, + DATA, + /* offset= */ 0, + ALLOCATION_SIZE); + // Allocation should still be held. + assertAllocationCount(1); + + sampleQueue.discardToRead(); + // The allocation should have been released. + assertAllocationCount(0); + } + @Test public void readMultiSamples() { writeTestData(); @@ -1642,13 +1668,27 @@ public final class SampleQueueTest { FLAG_OMIT_SAMPLE_DATA | FLAG_PEEK, /* loadingFinished= */ false); assertSampleBufferReadResult( - flagsOnlyBuffer, result, timeUs, isKeyFrame, isDecodeOnly, isEncrypted); + flagsOnlyBuffer, + result, + timeUs, + isKeyFrame, + isDecodeOnly, + isEncrypted, + /* isLastSample= */ false); // Check that peek yields the expected values. clearFormatHolderAndInputBuffer(); result = sampleQueue.read(formatHolder, inputBuffer, FLAG_PEEK, /* loadingFinished= */ false); assertSampleBufferReadResult( - result, timeUs, isKeyFrame, isDecodeOnly, isEncrypted, sampleData, offset, length); + result, + timeUs, + isKeyFrame, + isDecodeOnly, + isEncrypted, + /* isLastSample= */ false, + sampleData, + offset, + length); // Check that read yields the expected values. clearFormatHolderAndInputBuffer(); @@ -1656,7 +1696,85 @@ public final class SampleQueueTest { sampleQueue.read( formatHolder, inputBuffer, /* readFlags= */ 0, /* loadingFinished= */ false); assertSampleBufferReadResult( - result, timeUs, isKeyFrame, isDecodeOnly, isEncrypted, sampleData, offset, length); + result, + timeUs, + isKeyFrame, + isDecodeOnly, + isEncrypted, + /* isLastSample= */ false, + sampleData, + offset, + length); + } + + /** + * Asserts {@link SampleQueue#read} returns {@link C#RESULT_BUFFER_READ} and that the buffer is + * filled with the specified sample data. Also asserts that being the last sample and loading is + * finished, that the {@link C#BUFFER_FLAG_LAST_SAMPLE} flag is set. + * + * @param timeUs The expected buffer timestamp. + * @param isKeyFrame The expected keyframe flag. + * @param isDecodeOnly The expected decodeOnly flag. + * @param isEncrypted The expected encrypted flag. + * @param sampleData An array containing the expected sample data. + * @param offset The offset in {@code sampleData} of the expected sample data. + * @param length The length of the expected sample data. + */ + private void assertReadLastSample( + long timeUs, + boolean isKeyFrame, + boolean isDecodeOnly, + boolean isEncrypted, + byte[] sampleData, + int offset, + int length) { + // Check that peek whilst omitting data yields the expected values. + formatHolder.format = null; + DecoderInputBuffer flagsOnlyBuffer = DecoderInputBuffer.newNoDataInstance(); + int result = + sampleQueue.read( + formatHolder, + flagsOnlyBuffer, + FLAG_OMIT_SAMPLE_DATA | FLAG_PEEK, + /* loadingFinished= */ true); + assertSampleBufferReadResult( + flagsOnlyBuffer, + result, + timeUs, + isKeyFrame, + isDecodeOnly, + isEncrypted, + /* isLastSample= */ true); + + // Check that peek yields the expected values. + clearFormatHolderAndInputBuffer(); + result = sampleQueue.read(formatHolder, inputBuffer, FLAG_PEEK, /* loadingFinished= */ true); + assertSampleBufferReadResult( + result, + timeUs, + isKeyFrame, + isDecodeOnly, + isEncrypted, + /* isLastSample= */ true, + sampleData, + offset, + length); + + // Check that read yields the expected values. + clearFormatHolderAndInputBuffer(); + result = + sampleQueue.read( + formatHolder, inputBuffer, /* readFlags= */ 0, /* loadingFinished= */ true); + assertSampleBufferReadResult( + result, + timeUs, + isKeyFrame, + isDecodeOnly, + isEncrypted, + /* isLastSample= */ true, + sampleData, + offset, + length); } private void assertSampleBufferReadResult( @@ -1665,7 +1783,8 @@ public final class SampleQueueTest { long timeUs, boolean isKeyFrame, boolean isDecodeOnly, - boolean isEncrypted) { + boolean isEncrypted, + boolean isLastSample) { assertThat(result).isEqualTo(RESULT_BUFFER_READ); // formatHolder should not be populated. assertThat(formatHolder.format).isNull(); @@ -1674,6 +1793,7 @@ public final class SampleQueueTest { assertThat(inputBuffer.isKeyFrame()).isEqualTo(isKeyFrame); assertThat(inputBuffer.isDecodeOnly()).isEqualTo(isDecodeOnly); assertThat(inputBuffer.isEncrypted()).isEqualTo(isEncrypted); + assertThat(inputBuffer.isLastSample()).isEqualTo(isLastSample); } private void assertSampleBufferReadResult( @@ -1682,11 +1802,12 @@ public final class SampleQueueTest { boolean isKeyFrame, boolean isDecodeOnly, boolean isEncrypted, + boolean isLastSample, byte[] sampleData, int offset, int length) { assertSampleBufferReadResult( - inputBuffer, result, timeUs, isKeyFrame, isDecodeOnly, isEncrypted); + inputBuffer, result, timeUs, isKeyFrame, isDecodeOnly, isEncrypted, isLastSample); // inputBuffer should be populated with data. inputBuffer.flip(); assertThat(inputBuffer.data.limit()).isEqualTo(length); diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index b7ad8dc6c2..5d6dac43d8 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -31,11 +31,14 @@ import static org.robolectric.Shadows.shadowOf; import android.content.Context; import android.graphics.SurfaceTexture; import android.hardware.display.DisplayManager; +import android.media.MediaCodec; import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaCodecInfo.CodecProfileLevel; import android.media.MediaFormat; +import android.os.Bundle; import android.os.Handler; import android.os.Looper; +import android.os.PersistableBundle; import android.os.SystemClock; import android.view.Display; import android.view.Surface; @@ -44,6 +47,8 @@ import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.VideoSize; +import androidx.media3.decoder.CryptoInfo; +import androidx.media3.exoplayer.DecoderCounters; import androidx.media3.exoplayer.Renderer; import androidx.media3.exoplayer.RendererCapabilities; import androidx.media3.exoplayer.RendererCapabilities.Capabilities; @@ -51,13 +56,17 @@ import androidx.media3.exoplayer.RendererConfiguration; import androidx.media3.exoplayer.analytics.PlayerId; import androidx.media3.exoplayer.drm.DrmSessionEventListener; import androidx.media3.exoplayer.drm.DrmSessionManager; +import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter; import androidx.media3.exoplayer.mediacodec.MediaCodecInfo; import androidx.media3.exoplayer.mediacodec.MediaCodecSelector; +import androidx.media3.exoplayer.mediacodec.SynchronousMediaCodecAdapter; import androidx.media3.exoplayer.upstream.DefaultAllocator; import androidx.media3.test.utils.FakeSampleStream; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @@ -117,6 +126,7 @@ public class MediaCodecVideoRendererTest { private Looper testMainLooper; private Surface surface; private MediaCodecVideoRenderer mediaCodecVideoRenderer; + private MediaCodecSelector mediaCodecSelector; @Nullable private Format currentOutputFormat; @Mock private VideoRendererEventListener eventListener; @@ -124,7 +134,7 @@ public class MediaCodecVideoRendererTest { @Before public void setUp() throws Exception { testMainLooper = Looper.getMainLooper(); - MediaCodecSelector mediaCodecSelector = + mediaCodecSelector = (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) -> Collections.singletonList( MediaCodecInfo.newInstance( @@ -207,6 +217,65 @@ public class MediaCodecVideoRendererTest { verify(eventListener).onDroppedFrames(eq(1), anyLong()); } + @Test + public void render_withBufferLimitEqualToNumberOfSamples_rendersLastFrameAfterEndOfStream() + throws Exception { + ArgumentCaptor argumentDecoderCounters = + ArgumentCaptor.forClass(DecoderCounters.class); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ VIDEO_H264, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer. + oneByteSample(/* timeUs= */ 10_000), + oneByteSample(/* timeUs= */ 20_000), // Last buffer. + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + // Seek to time after samples. + fakeSampleStream.seekToUs(30_000, /* allowTimeBeyondBuffer= */ true); + mediaCodecVideoRenderer = + new MediaCodecVideoRenderer( + ApplicationProvider.getApplicationContext(), + new ForwardingSynchronousMediaCodecAdapterWithBufferLimit.Factory(/* bufferLimit= */ 3), + mediaCodecSelector, + /* allowedJoiningTimeMs= */ 0, + /* enableDecoderFallback= */ false, + /* eventHandler= */ new Handler(testMainLooper), + /* eventListener= */ eventListener, + /* maxDroppedFramesToNotify= */ 1); + mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {VIDEO_H264}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.setCurrentStreamFinal(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + // Call to render should have read all samples up to but not including the END_OF_STREAM_ITEM. + assertThat(mediaCodecVideoRenderer.hasReadStreamToEnd()).isFalse(); + int posUs = 30_000; + while (!mediaCodecVideoRenderer.isEnded()) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + posUs += 40_000; + } + shadowOf(testMainLooper).idle(); + + verify(eventListener).onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong()); + verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture()); + assertThat(argumentDecoderCounters.getValue().renderedOutputBufferCount).isEqualTo(1); + assertThat(argumentDecoderCounters.getValue().skippedOutputBufferCount).isEqualTo(2); + } + @Test public void render_sendsVideoSizeChangeWithCurrentFormatValues() throws Exception { FakeSampleStream fakeSampleStream = @@ -1194,4 +1263,146 @@ public class MediaCodecVideoRendererTest { .setHeight(height) .build(); } + + private static final class ForwardingSynchronousMediaCodecAdapterWithBufferLimit + extends ForwardingSynchronousMediaCodecAdapter { + /** A factory for {@link ForwardingSynchronousMediaCodecAdapterWithBufferLimit} instances. */ + public static final class Factory implements MediaCodecAdapter.Factory { + private final int bufferLimit; + + Factory(int bufferLimit) { + this.bufferLimit = bufferLimit; + } + + @Override + public MediaCodecAdapter createAdapter(Configuration configuration) throws IOException { + return new ForwardingSynchronousMediaCodecAdapterWithBufferLimit( + bufferLimit, new SynchronousMediaCodecAdapter.Factory().createAdapter(configuration)); + } + } + + private int bufferCounter; + + ForwardingSynchronousMediaCodecAdapterWithBufferLimit( + int bufferCounter, MediaCodecAdapter adapter) { + super(adapter); + this.bufferCounter = bufferCounter; + } + + @Override + public int dequeueInputBufferIndex() { + if (bufferCounter > 0) { + bufferCounter--; + return super.dequeueInputBufferIndex(); + } + return -1; + } + + @Override + public int dequeueOutputBufferIndex(MediaCodec.BufferInfo bufferInfo) { + int outputIndex = super.dequeueOutputBufferIndex(bufferInfo); + if (outputIndex > 0) { + bufferCounter++; + } + return outputIndex; + } + } + + private abstract static class ForwardingSynchronousMediaCodecAdapter + implements MediaCodecAdapter { + private final MediaCodecAdapter adapter; + + ForwardingSynchronousMediaCodecAdapter(MediaCodecAdapter adapter) { + this.adapter = adapter; + } + + @Override + public int dequeueInputBufferIndex() { + return adapter.dequeueInputBufferIndex(); + } + + @Override + public int dequeueOutputBufferIndex(MediaCodec.BufferInfo bufferInfo) { + return adapter.dequeueOutputBufferIndex(bufferInfo); + } + + @Override + public MediaFormat getOutputFormat() { + return adapter.getOutputFormat(); + } + + @Nullable + @Override + public ByteBuffer getInputBuffer(int index) { + return adapter.getInputBuffer(index); + } + + @Nullable + @Override + public ByteBuffer getOutputBuffer(int index) { + return adapter.getOutputBuffer(index); + } + + @Override + public void queueInputBuffer( + int index, int offset, int size, long presentationTimeUs, int flags) { + adapter.queueInputBuffer(index, offset, size, presentationTimeUs, flags); + } + + @Override + public void queueSecureInputBuffer( + int index, int offset, CryptoInfo info, long presentationTimeUs, int flags) { + adapter.queueSecureInputBuffer(index, offset, info, presentationTimeUs, flags); + } + + @Override + public void releaseOutputBuffer(int index, boolean render) { + adapter.releaseOutputBuffer(index, render); + } + + @Override + public void releaseOutputBuffer(int index, long renderTimeStampNs) { + adapter.releaseOutputBuffer(index, renderTimeStampNs); + } + + @Override + public void flush() { + adapter.flush(); + } + + @Override + public void release() { + adapter.release(); + } + + @Override + public void setOnFrameRenderedListener(OnFrameRenderedListener listener, Handler handler) { + adapter.setOnFrameRenderedListener(listener, handler); + } + + @Override + public void setOutputSurface(Surface surface) { + adapter.setOutputSurface(surface); + } + + @Override + public void setParameters(Bundle params) { + adapter.setParameters(params); + } + + @Override + public void setVideoScalingMode(int scalingMode) { + adapter.setVideoScalingMode(scalingMode); + } + + @Override + public boolean needsReconfiguration() { + return adapter.needsReconfiguration(); + } + + @Override + public PersistableBundle getMetrics() { + return adapter.getMetrics(); + } + } } diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeMediaPeriod.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeMediaPeriod.java index 7ec904028e..aa1cc64bff 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeMediaPeriod.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeMediaPeriod.java @@ -338,7 +338,8 @@ public class FakeMediaPeriod implements MediaPeriod { lastSeekPositionUs = seekPositionUs; boolean seekedInsideStreams = true; for (FakeSampleStream sampleStream : sampleStreams) { - seekedInsideStreams &= sampleStream.seekToUs(seekPositionUs); + seekedInsideStreams &= + sampleStream.seekToUs(seekPositionUs, /* allowTimeBeyondBuffer= */ false); } if (!seekedInsideStreams) { for (FakeSampleStream sampleStream : sampleStreams) { diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeSampleStream.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeSampleStream.java index d779b21109..b5a53a40c9 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeSampleStream.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeSampleStream.java @@ -204,10 +204,12 @@ public class FakeSampleStream implements SampleStream { * Seeks the stream to a new position using already available data in the queue. * * @param positionUs The new position, in microseconds. + * @param allowTimeBeyondBuffer Whether the operation can succeed if timeUs is beyond the end of + * the queue, by seeking to the last sample (or keyframe). * @return Whether seeking inside the available data was possible. */ - public boolean seekToUs(long positionUs) { - return sampleQueue.seekTo(positionUs, /* allowTimeBeyondBuffer= */ false); + public boolean seekToUs(long positionUs, boolean allowTimeBeyondBuffer) { + return sampleQueue.seekTo(positionUs, allowTimeBeyondBuffer); } /**