loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Render last frame even if have not read BUFFER_FLAG_END_OF_STREAM

Browse Source 
If the limited number of input buffers causes reading of all samples except the last one conveying end of stream, then the last frame will not be rendered.

PiperOrigin-RevId: 525974445
This commit is contained in:
michaelkatz 2023-04-21 10:09:45 +01:00 committed by Rohit Singh
parent 353523bb07
commit affbb7c57e
8 changed files with 358 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
RELEASENOTES.md
View File

@ -31,6 +31,11 @@
* Deprecate `Player.COMMAND_GET_MEDIA_ITEMS_METADATA` and
`COMMAND_SET_MEDIA_ITEMS_METADATA`. Use `COMMAND_GET_METADATA` and
`COMMAND_SET_PLAYLIST_METADATA` instead.
* Add `Buffer.isLastSample()` that denotes if `Buffer` contains flag
`C.BUFFER_FLAG_LAST_SAMPLE`.
* Fix issue where last frame may not be rendered if the last sample with
frames is dequeued without reading the 'end of stream' sample.
([#11079](https://github.com/google/ExoPlayer/issues/11079)).
* Session:
* Deprecate 4 volume-controlling methods in `Player` and add overloaded
methods which allow users to specify volume flags:

5
libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java
View File

@ -49,6 +49,11 @@ public abstract class Buffer {
return getFlag(C.BUFFER_FLAG_KEY_FRAME);
}
/** Returns whether the {@link C#BUFFER_FLAG_LAST_SAMPLE} flag is set. */
public final boolean isLastSample() {
return getFlag(C.BUFFER_FLAG_LAST_SAMPLE);
}
/** Returns whether the {@link C#BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA} flag is set. */
public final boolean hasSupplementalData() {
return getFlag(C.BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA);

2
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java
View File

@ -1249,7 +1249,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
return true;
}
if (hasReadStreamToEnd()) {
if (hasReadStreamToEnd() || buffer.isLastSample()) {
// Notify output queue of the last buffer's timestamp.
lastBufferInStreamPresentationTimeUs = largestQueuedPresentationTimeUs;
}

3
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SampleQueue.java
View File

@ -716,6 +716,9 @@ public class SampleQueue implements TrackOutput {
}
buffer.setFlags(flags[relativeReadIndex]);
if (readPosition == (length - 1) && (loadingFinished || isLastSampleQueued)) {
buffer.addFlag(C.BUFFER_FLAG_LAST_SAMPLE);
}
buffer.timeUs = timesUs[relativeReadIndex];
if (buffer.timeUs < startTimeUs) {
buffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY);

131
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/SampleQueueTest.java
View File

@ -354,6 +354,32 @@ public final class SampleQueueTest {
assertAllocationCount(0);
}
@Test
public void readSingleSampleWithLoadingFinished() {
sampleQueue.sampleData(new ParsableByteArray(DATA), ALLOCATION_SIZE);
sampleQueue.format(FORMAT_1);
sampleQueue.sampleMetadata(1000, C.BUFFER_FLAG_KEY_FRAME, ALLOCATION_SIZE, 0, null);
assertAllocationCount(1);
// If formatRequired, should read the format rather than the sample.
assertReadFormat(true, FORMAT_1);
// Otherwise should read the sample with loading finished.
assertReadLastSample(
1000,
/* isKeyFrame= */ true,
/* isDecodeOnly= */ false,
/* isEncrypted= */ false,
DATA,
/* offset= */ 0,
ALLOCATION_SIZE);
// Allocation should still be held.
assertAllocationCount(1);
sampleQueue.discardToRead();
// The allocation should have been released.
assertAllocationCount(0);
}
@Test
public void readMultiSamples() {
writeTestData();
@ -1642,13 +1668,27 @@ public final class SampleQueueTest {
FLAG_OMIT_SAMPLE_DATA | FLAG_PEEK,
/* loadingFinished= */ false);
assertSampleBufferReadResult(
flagsOnlyBuffer, result, timeUs, isKeyFrame, isDecodeOnly, isEncrypted);
flagsOnlyBuffer,
result,
timeUs,
isKeyFrame,
isDecodeOnly,
isEncrypted,
/* isLastSample= */ false);
// Check that peek yields the expected values.
clearFormatHolderAndInputBuffer();
result = sampleQueue.read(formatHolder, inputBuffer, FLAG_PEEK, /* loadingFinished= */ false);
assertSampleBufferReadResult(
result, timeUs, isKeyFrame, isDecodeOnly, isEncrypted, sampleData, offset, length);
result,
timeUs,
isKeyFrame,
isDecodeOnly,
isEncrypted,
/* isLastSample= */ false,
sampleData,
offset,
length);
// Check that read yields the expected values.
clearFormatHolderAndInputBuffer();
@ -1656,7 +1696,85 @@ public final class SampleQueueTest {
sampleQueue.read(
formatHolder, inputBuffer, /* readFlags= */ 0, /* loadingFinished= */ false);
assertSampleBufferReadResult(
result, timeUs, isKeyFrame, isDecodeOnly, isEncrypted, sampleData, offset, length);
result,
timeUs,
isKeyFrame,
isDecodeOnly,
isEncrypted,
/* isLastSample= */ false,
sampleData,
offset,
length);
}
/**
* Asserts {@link SampleQueue#read} returns {@link C#RESULT_BUFFER_READ} and that the buffer is
* filled with the specified sample data. Also asserts that being the last sample and loading is
* finished, that the {@link C#BUFFER_FLAG_LAST_SAMPLE} flag is set.
*
* @param timeUs The expected buffer timestamp.
* @param isKeyFrame The expected keyframe flag.
* @param isDecodeOnly The expected decodeOnly flag.
* @param isEncrypted The expected encrypted flag.
* @param sampleData An array containing the expected sample data.
* @param offset The offset in {@code sampleData} of the expected sample data.
* @param length The length of the expected sample data.
*/
private void assertReadLastSample(
long timeUs,
boolean isKeyFrame,
boolean isDecodeOnly,
boolean isEncrypted,
byte[] sampleData,
int offset,
int length) {
// Check that peek whilst omitting data yields the expected values.
formatHolder.format = null;
DecoderInputBuffer flagsOnlyBuffer = DecoderInputBuffer.newNoDataInstance();
int result =
sampleQueue.read(
formatHolder,
flagsOnlyBuffer,
FLAG_OMIT_SAMPLE_DATA | FLAG_PEEK,
/* loadingFinished= */ true);
assertSampleBufferReadResult(
flagsOnlyBuffer,
result,
timeUs,
isKeyFrame,
isDecodeOnly,
isEncrypted,
/* isLastSample= */ true);
// Check that peek yields the expected values.
clearFormatHolderAndInputBuffer();
result = sampleQueue.read(formatHolder, inputBuffer, FLAG_PEEK, /* loadingFinished= */ true);
assertSampleBufferReadResult(
result,
timeUs,
isKeyFrame,
isDecodeOnly,
isEncrypted,
/* isLastSample= */ true,
sampleData,
offset,
length);
// Check that read yields the expected values.
clearFormatHolderAndInputBuffer();
result =
sampleQueue.read(
formatHolder, inputBuffer, /* readFlags= */ 0, /* loadingFinished= */ true);
assertSampleBufferReadResult(
result,
timeUs,
isKeyFrame,
isDecodeOnly,
isEncrypted,
/* isLastSample= */ true,
sampleData,
offset,
length);
}
private void assertSampleBufferReadResult(
@ -1665,7 +1783,8 @@ public final class SampleQueueTest {
long timeUs,
boolean isKeyFrame,
boolean isDecodeOnly,
boolean isEncrypted) {
boolean isEncrypted,
boolean isLastSample) {
assertThat(result).isEqualTo(RESULT_BUFFER_READ);
// formatHolder should not be populated.
assertThat(formatHolder.format).isNull();
@ -1674,6 +1793,7 @@ public final class SampleQueueTest {
assertThat(inputBuffer.isKeyFrame()).isEqualTo(isKeyFrame);
assertThat(inputBuffer.isDecodeOnly()).isEqualTo(isDecodeOnly);
assertThat(inputBuffer.isEncrypted()).isEqualTo(isEncrypted);
assertThat(inputBuffer.isLastSample()).isEqualTo(isLastSample);
}
private void assertSampleBufferReadResult(
@ -1682,11 +1802,12 @@ public final class SampleQueueTest {
boolean isKeyFrame,
boolean isDecodeOnly,
boolean isEncrypted,
boolean isLastSample,
byte[] sampleData,
int offset,
int length) {
assertSampleBufferReadResult(
inputBuffer, result, timeUs, isKeyFrame, isDecodeOnly, isEncrypted);
inputBuffer, result, timeUs, isKeyFrame, isDecodeOnly, isEncrypted, isLastSample);
// inputBuffer should be populated with data.
inputBuffer.flip();
assertThat(inputBuffer.data.limit()).isEqualTo(length);

213
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java
View File

@ -31,11 +31,14 @@ import static org.robolectric.Shadows.shadowOf;
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.hardware.display.DisplayManager;
import android.media.MediaCodec;
import android.media.MediaCodecInfo.CodecCapabilities;
import android.media.MediaCodecInfo.CodecProfileLevel;
import android.media.MediaFormat;
import android.os.Bundle;
import android.os.Handler;
import android.os.Looper;
import android.os.PersistableBundle;
import android.os.SystemClock;
import android.view.Display;
import android.view.Surface;
@ -44,6 +47,8 @@ import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.VideoSize;
import androidx.media3.decoder.CryptoInfo;
import androidx.media3.exoplayer.DecoderCounters;
import androidx.media3.exoplayer.Renderer;
import androidx.media3.exoplayer.RendererCapabilities;
import androidx.media3.exoplayer.RendererCapabilities.Capabilities;
@ -51,13 +56,17 @@ import androidx.media3.exoplayer.RendererConfiguration;
import androidx.media3.exoplayer.analytics.PlayerId;
import androidx.media3.exoplayer.drm.DrmSessionEventListener;
import androidx.media3.exoplayer.drm.DrmSessionManager;
import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter;
import androidx.media3.exoplayer.mediacodec.MediaCodecInfo;
import androidx.media3.exoplayer.mediacodec.MediaCodecSelector;
import androidx.media3.exoplayer.mediacodec.SynchronousMediaCodecAdapter;
import androidx.media3.exoplayer.upstream.DefaultAllocator;
import androidx.media3.test.utils.FakeSampleStream;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
@ -117,6 +126,7 @@ public class MediaCodecVideoRendererTest {
private Looper testMainLooper;
private Surface surface;
private MediaCodecVideoRenderer mediaCodecVideoRenderer;
private MediaCodecSelector mediaCodecSelector;
@Nullable private Format currentOutputFormat;
@Mock private VideoRendererEventListener eventListener;
@ -124,7 +134,7 @@ public class MediaCodecVideoRendererTest {
@Before
public void setUp() throws Exception {
testMainLooper = Looper.getMainLooper();
MediaCodecSelector mediaCodecSelector =
mediaCodecSelector =
(mimeType, requiresSecureDecoder, requiresTunnelingDecoder) ->
Collections.singletonList(
MediaCodecInfo.newInstance(
@ -207,6 +217,65 @@ public class MediaCodecVideoRendererTest {
verify(eventListener).onDroppedFrames(eq(1), anyLong());
}
@Test
public void render_withBufferLimitEqualToNumberOfSamples_rendersLastFrameAfterEndOfStream()
throws Exception {
ArgumentCaptor<DecoderCounters> argumentDecoderCounters =
ArgumentCaptor.forClass(DecoderCounters.class);
FakeSampleStream fakeSampleStream =
new FakeSampleStream(
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
/* mediaSourceEventDispatcher= */ null,
DrmSessionManager.DRM_UNSUPPORTED,
new DrmSessionEventListener.EventDispatcher(),
/* initialFormat= */ VIDEO_H264,
ImmutableList.of(
oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer.
oneByteSample(/* timeUs= */ 10_000),
oneByteSample(/* timeUs= */ 20_000), // Last buffer.
END_OF_STREAM_ITEM));
fakeSampleStream.writeData(/* startPositionUs= */ 0);
// Seek to time after samples.
fakeSampleStream.seekToUs(30_000, /* allowTimeBeyondBuffer= */ true);
mediaCodecVideoRenderer =
new MediaCodecVideoRenderer(
ApplicationProvider.getApplicationContext(),
new ForwardingSynchronousMediaCodecAdapterWithBufferLimit.Factory(/* bufferLimit= */ 3),
mediaCodecSelector,
/* allowedJoiningTimeMs= */ 0,
/* enableDecoderFallback= */ false,
/* eventHandler= */ new Handler(testMainLooper),
/* eventListener= */ eventListener,
/* maxDroppedFramesToNotify= */ 1);
mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
mediaCodecVideoRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {VIDEO_H264},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 0,
/* offsetUs= */ 0);
mediaCodecVideoRenderer.start();
mediaCodecVideoRenderer.setCurrentStreamFinal();
mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000);
// Call to render should have read all samples up to but not including the END_OF_STREAM_ITEM.
assertThat(mediaCodecVideoRenderer.hasReadStreamToEnd()).isFalse();
int posUs = 30_000;
while (!mediaCodecVideoRenderer.isEnded()) {
mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000);
posUs += 40_000;
}
shadowOf(testMainLooper).idle();
verify(eventListener).onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong());
verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
assertThat(argumentDecoderCounters.getValue().renderedOutputBufferCount).isEqualTo(1);
assertThat(argumentDecoderCounters.getValue().skippedOutputBufferCount).isEqualTo(2);
}
@Test
public void render_sendsVideoSizeChangeWithCurrentFormatValues() throws Exception {
FakeSampleStream fakeSampleStream =
@ -1194,4 +1263,146 @@ public class MediaCodecVideoRendererTest {
.setHeight(height)
.build();
}
private static final class ForwardingSynchronousMediaCodecAdapterWithBufferLimit
extends ForwardingSynchronousMediaCodecAdapter {
/** A factory for {@link ForwardingSynchronousMediaCodecAdapterWithBufferLimit} instances. */
public static final class Factory implements MediaCodecAdapter.Factory {
private final int bufferLimit;
Factory(int bufferLimit) {
this.bufferLimit = bufferLimit;
}
@Override
public MediaCodecAdapter createAdapter(Configuration configuration) throws IOException {
return new ForwardingSynchronousMediaCodecAdapterWithBufferLimit(
bufferLimit, new SynchronousMediaCodecAdapter.Factory().createAdapter(configuration));
}
}
private int bufferCounter;
ForwardingSynchronousMediaCodecAdapterWithBufferLimit(
int bufferCounter, MediaCodecAdapter adapter) {
super(adapter);
this.bufferCounter = bufferCounter;
}
@Override
public int dequeueInputBufferIndex() {
if (bufferCounter > 0) {
bufferCounter--;
return super.dequeueInputBufferIndex();
}
return -1;
}
@Override
public int dequeueOutputBufferIndex(MediaCodec.BufferInfo bufferInfo) {
int outputIndex = super.dequeueOutputBufferIndex(bufferInfo);
if (outputIndex > 0) {
bufferCounter++;
}
return outputIndex;
}
}
private abstract static class ForwardingSynchronousMediaCodecAdapter
implements MediaCodecAdapter {
private final MediaCodecAdapter adapter;
ForwardingSynchronousMediaCodecAdapter(MediaCodecAdapter adapter) {
this.adapter = adapter;
}
@Override
public int dequeueInputBufferIndex() {
return adapter.dequeueInputBufferIndex();
}
@Override
public int dequeueOutputBufferIndex(MediaCodec.BufferInfo bufferInfo) {
return adapter.dequeueOutputBufferIndex(bufferInfo);
}
@Override
public MediaFormat getOutputFormat() {
return adapter.getOutputFormat();
}
@Nullable
@Override
public ByteBuffer getInputBuffer(int index) {
return adapter.getInputBuffer(index);
}
@Nullable
@Override
public ByteBuffer getOutputBuffer(int index) {
return adapter.getOutputBuffer(index);
}
@Override
public void queueInputBuffer(
int index, int offset, int size, long presentationTimeUs, int flags) {
adapter.queueInputBuffer(index, offset, size, presentationTimeUs, flags);
}
@Override
public void queueSecureInputBuffer(
int index, int offset, CryptoInfo info, long presentationTimeUs, int flags) {
adapter.queueSecureInputBuffer(index, offset, info, presentationTimeUs, flags);
}
@Override
public void releaseOutputBuffer(int index, boolean render) {
adapter.releaseOutputBuffer(index, render);
}
@Override
public void releaseOutputBuffer(int index, long renderTimeStampNs) {
adapter.releaseOutputBuffer(index, renderTimeStampNs);
}
@Override
public void flush() {
adapter.flush();
}
@Override
public void release() {
adapter.release();
}
@Override
public void setOnFrameRenderedListener(OnFrameRenderedListener listener, Handler handler) {
adapter.setOnFrameRenderedListener(listener, handler);
}
@Override
public void setOutputSurface(Surface surface) {
adapter.setOutputSurface(surface);
}
@Override
public void setParameters(Bundle params) {
adapter.setParameters(params);
}
@Override
public void setVideoScalingMode(int scalingMode) {
adapter.setVideoScalingMode(scalingMode);
}
@Override
public boolean needsReconfiguration() {
return adapter.needsReconfiguration();
}
@Override
public PersistableBundle getMetrics() {
return adapter.getMetrics();
}
}
}

3
libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeMediaPeriod.java
View File

@ -338,7 +338,8 @@ public class FakeMediaPeriod implements MediaPeriod {
lastSeekPositionUs = seekPositionUs;
boolean seekedInsideStreams = true;
for (FakeSampleStream sampleStream : sampleStreams) {
seekedInsideStreams &= sampleStream.seekToUs(seekPositionUs);
seekedInsideStreams &=
sampleStream.seekToUs(seekPositionUs, /* allowTimeBeyondBuffer= */ false);
}
if (!seekedInsideStreams) {
for (FakeSampleStream sampleStream : sampleStreams) {

6
libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeSampleStream.java
View File

@ -204,10 +204,12 @@ public class FakeSampleStream implements SampleStream {
* Seeks the stream to a new position using already available data in the queue.
*
* @param positionUs The new position, in microseconds.
* @param allowTimeBeyondBuffer Whether the operation can succeed if timeUs is beyond the end of
* the queue, by seeking to the last sample (or keyframe).
* @return Whether seeking inside the available data was possible.
*/
public boolean seekToUs(long positionUs) {
return sampleQueue.seekTo(positionUs, /* allowTimeBeyondBuffer= */ false);
public boolean seekToUs(long positionUs, boolean allowTimeBeyondBuffer) {
return sampleQueue.seekTo(positionUs, allowTimeBeyondBuffer);
}
/**