diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 9d84efea26..1389fb1c1f 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -162,6 +162,10 @@ This release includes the following changes since the * Catch `OutOfMemoryError` when parsing very large ID3 frames, meaning playback can continue without the tag info instead of playback failing completely. +* Image: + * Add support for DASH thumbnails. Grid images are cropped and individual + thumbnails are provided to `ImageOutput` close to their presentation + times. * DRM: * Extend workaround for spurious ClearKey `https://default.url` license URL to API 33+ (previously the workaround only applied on API 33 diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java index f62cf582e0..55b082fd5f 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java @@ -46,6 +46,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.Nullable; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -83,6 +84,11 @@ public class ImageRenderer extends BaseRenderer { */ private static final int REINITIALIZATION_STATE_WAIT_END_OF_STREAM = 3; + /** + * A time threshold, in microseconds, for the window during which an image should be presented. + */ + private static final long IMAGE_PRESENTATION_WINDOW_THRESHOLD_US = 30_000; + private final ImageDecoder.Factory decoderFactory; private final DecoderInputBuffer flagsOnlyBuffer; private final LongArrayQueue offsetQueue; @@ -94,8 +100,12 @@ public class ImageRenderer extends BaseRenderer { private @Nullable Format inputFormat; private @Nullable ImageDecoder decoder; private @Nullable DecoderInputBuffer inputBuffer; - private @Nullable ImageOutputBuffer outputBuffer; private ImageOutput imageOutput; + private @Nullable Bitmap outputBitmap; + private boolean readyToOutputTiles; + private @Nullable TileInfo tileInfo; + private @Nullable TileInfo nextTileInfo; + private int currentTileIndex; /** * Creates an instance. @@ -156,8 +166,8 @@ public class ImageRenderer extends BaseRenderer { try { // Rendering loop. TraceUtil.beginSection("drainAndFeedDecoder"); - while (drainOutputBuffer(positionUs, elapsedRealtimeUs)) {} - while (feedInputBuffer()) {} + while (drainOutput(positionUs, elapsedRealtimeUs)) {} + while (feedInputBuffer(positionUs)) {} TraceUtil.endSection(); } catch (ImageDecoderException e) { throw createRendererException(e, null, PlaybackException.ERROR_CODE_DECODING_FAILED); @@ -168,7 +178,7 @@ public class ImageRenderer extends BaseRenderer { public boolean isReady() { return firstFrameState == FIRST_FRAME_RENDERED || (firstFrameState == FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED - && outputBuffer != null); + && readyToOutputTiles); } @Override @@ -198,8 +208,18 @@ public class ImageRenderer extends BaseRenderer { } @Override - protected void onPositionReset(long positionUs, boolean joining) { + protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException { lowerFirstFrameState(FIRST_FRAME_NOT_RENDERED); + outputStreamEnded = false; + inputStreamEnded = false; + outputBitmap = null; + tileInfo = null; + nextTileInfo = null; + readyToOutputTiles = false; + inputBuffer = null; + if (decoder != null) { + decoder.flush(); + } } @Override @@ -238,63 +258,108 @@ public class ImageRenderer extends BaseRenderer { } /** - * Attempts to dequeue an output buffer from the decoder and, if successful and permitted to, - * renders it. + * Checks if there is data to output. If there is no data to output, it attempts dequeuing the + * output buffer from the decoder. If there is data to output, it attempts to render it. * * @param positionUs The player's current position. * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, * measured at the start of the current iteration of the rendering loop. - * @return Whether it may be possible to drain more output data. + * @return Whether it may be possible to output more data. * @throws ImageDecoderException If an error occurs draining the output buffer. */ - private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs) + private boolean drainOutput(long positionUs, long elapsedRealtimeUs) throws ImageDecoderException, ExoPlaybackException { - if (outputBuffer == null) { - checkStateNotNull(decoder); - outputBuffer = decoder.dequeueOutputBuffer(); - if (outputBuffer == null) { - return false; - } + // If tileInfo and outputBitmap are both null, we must not return early. The EOS may have been + // queued to the decoder, and we must stay in this method to deque it further down. + if (outputBitmap != null && tileInfo == null) { + return false; } if (firstFrameState == FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED && getState() != STATE_STARTED) { return false; } - if (checkStateNotNull(outputBuffer).isEndOfStream()) { - offsetQueue.remove(); - if (decoderReinitializationState == REINITIALIZATION_STATE_WAIT_END_OF_STREAM) { - // We're waiting to re-initialize the decoder, and have now processed all final buffers. - releaseDecoderResources(); - checkStateNotNull(inputFormat); - initDecoder(); - } else { - checkStateNotNull(outputBuffer).release(); - outputBuffer = null; - if (offsetQueue.isEmpty()) { - outputStreamEnded = true; - } + if (outputBitmap == null) { + checkStateNotNull(decoder); + ImageOutputBuffer outputBuffer = decoder.dequeueOutputBuffer(); + if (outputBuffer == null) { + return false; } - return false; + if (checkStateNotNull(outputBuffer).isEndOfStream()) { + offsetQueue.remove(); + if (decoderReinitializationState == REINITIALIZATION_STATE_WAIT_END_OF_STREAM) { + // We're waiting to re-initialize the decoder, and have now processed all final buffers. + releaseDecoderResources(); + checkStateNotNull(inputFormat); + initDecoder(); + } else { + checkStateNotNull(outputBuffer).release(); + if (offsetQueue.isEmpty()) { + outputStreamEnded = true; + } + } + return false; + } + checkStateNotNull( + outputBuffer.bitmap, "Non-EOS buffer came back from the decoder without bitmap."); + outputBitmap = outputBuffer.bitmap; + checkStateNotNull(outputBuffer).release(); } - ImageOutputBuffer imageOutputBuffer = checkStateNotNull(outputBuffer); - checkStateNotNull( - imageOutputBuffer.bitmap, "Non-EOS buffer came back from the decoder without bitmap."); - if (!processOutputBuffer( - positionUs, elapsedRealtimeUs, imageOutputBuffer.bitmap, imageOutputBuffer.timeUs)) { - return false; + if (readyToOutputTiles && outputBitmap != null && tileInfo != null) { + checkStateNotNull(inputFormat); + boolean isThumbnailGrid = + (inputFormat.tileCountHorizontal != 1 || inputFormat.tileCountVertical != 1) + && inputFormat.tileCountHorizontal != Format.NO_VALUE + && inputFormat.tileCountVertical != Format.NO_VALUE; + // Lazily crop and store the bitmap to ensure we only have one tile in memory rather than + // proactively storing a tile whenever creating TileInfos. + if (!tileInfo.hasTileBitmap()) { + tileInfo.setTileBitmap( + isThumbnailGrid + ? cropTileFromImageGrid(tileInfo.getTileIndex()) + : checkStateNotNull(outputBitmap)); + } + if (!processOutputBuffer( + positionUs, + elapsedRealtimeUs, + checkStateNotNull(tileInfo.getTileBitmap()), + tileInfo.getPresentationTimeUs())) { + return false; + } + firstFrameState = FIRST_FRAME_RENDERED; + if (!isThumbnailGrid + || checkStateNotNull(tileInfo).getTileIndex() + == checkStateNotNull(inputFormat).tileCountVertical + * checkStateNotNull(inputFormat).tileCountHorizontal + - 1) { + outputBitmap = null; + } + tileInfo = nextTileInfo; + nextTileInfo = null; + return true; + } + return false; + } + + private boolean shouldForceRender() { + boolean isStarted = getState() == STATE_STARTED; + switch (firstFrameState) { + case C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED: + return isStarted; + case C.FIRST_FRAME_NOT_RENDERED: + return true; + case C.FIRST_FRAME_RENDERED: + return false; + default: + throw new IllegalStateException(); } - checkStateNotNull(outputBuffer).release(); - outputBuffer = null; - firstFrameState = FIRST_FRAME_RENDERED; - return true; } /** * Processes an output image. * - * @param positionUs The current media time in microseconds, measured at the start of the current - * iteration of the rendering loop. + * @param positionUs The current playback position in microseconds, measured at the start of the + * current iteration of the rendering loop. * @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, measured at the * start of the current iteration of the rendering loop. * @param outputBitmap The {@link Bitmap}. @@ -305,18 +370,25 @@ public class ImageRenderer extends BaseRenderer { protected boolean processOutputBuffer( long positionUs, long elapsedRealtimeUs, Bitmap outputBitmap, long bufferPresentationTimeUs) throws ExoPlaybackException { - if (positionUs < bufferPresentationTimeUs) { - // It's too early to render the buffer. - return false; + // TODO: b/319484746 - ImageRenderer should consider startPositionUs when choosing to output an + // image. + long earlyUs = bufferPresentationTimeUs - positionUs; + if (shouldForceRender() || earlyUs < IMAGE_PRESENTATION_WINDOW_THRESHOLD_US) { + imageOutput.onImageAvailable(bufferPresentationTimeUs - offsetQueue.element(), outputBitmap); + return true; } - imageOutput.onImageAvailable(bufferPresentationTimeUs - offsetQueue.element(), outputBitmap); - return true; + return false; } /** + * @param positionUs The current playback position in microseconds, measured at the start of the + * current iteration of the rendering loop. * @return Whether we can feed more input data to the decoder. */ - private boolean feedInputBuffer() throws ImageDecoderException { + private boolean feedInputBuffer(long positionUs) throws ImageDecoderException { + if (readyToOutputTiles && tileInfo != null) { + return false; + } FormatHolder formatHolder = getFormatHolder(); if (decoder == null || decoderReinitializationState == REINITIALIZATION_STATE_WAIT_END_OF_STREAM @@ -349,8 +421,12 @@ public class ImageRenderer extends BaseRenderer { checkStateNotNull(inputBuffer.data).remaining() > 0 || checkStateNotNull(inputBuffer).isEndOfStream(); if (shouldQueueBuffer) { + // TODO: b/318696449 - Don't use the deprecated BUFFER_FLAG_DECODE_ONLY with image chunks. + checkStateNotNull(inputBuffer).clearFlag(C.BUFFER_FLAG_DECODE_ONLY); checkStateNotNull(decoder).queueInputBuffer(checkStateNotNull(inputBuffer)); + currentTileIndex = 0; } + maybeAdvanceTileInfo(positionUs, checkStateNotNull(inputBuffer)); if (checkStateNotNull(inputBuffer).isEndOfStream()) { inputStreamEnded = true; inputBuffer = null; @@ -363,7 +439,7 @@ public class ImageRenderer extends BaseRenderer { } else { checkStateNotNull(inputBuffer).clear(); } - return true; + return !readyToOutputTiles; case C.RESULT_FORMAT_READ: inputFormat = checkStateNotNull(formatHolder.format); decoderReinitializationState = REINITIALIZATION_STATE_SIGNAL_END_OF_STREAM_THEN_WAIT; @@ -401,10 +477,6 @@ public class ImageRenderer extends BaseRenderer { private void releaseDecoderResources() { inputBuffer = null; - if (outputBuffer != null) { - outputBuffer.release(); - } - outputBuffer = null; decoderReinitializationState = REINITIALIZATION_STATE_NONE; if (decoder != null) { decoder.release(); @@ -416,7 +488,72 @@ public class ImageRenderer extends BaseRenderer { this.imageOutput = getImageOutput(imageOutput); } + private void maybeAdvanceTileInfo(long positionUs, DecoderInputBuffer inputBuffer) { + if (inputBuffer.isEndOfStream()) { + readyToOutputTiles = true; + return; + } + nextTileInfo = new TileInfo(currentTileIndex, inputBuffer.timeUs); + currentTileIndex++; + // TODO: b/319484746 - ImageRenderer should consider startPositionUs when choosing to output an + // image. + if (nextTileInfo.getPresentationTimeUs() - IMAGE_PRESENTATION_WINDOW_THRESHOLD_US <= positionUs + && positionUs + <= nextTileInfo.getPresentationTimeUs() + IMAGE_PRESENTATION_WINDOW_THRESHOLD_US) { + readyToOutputTiles = true; + } else if (tileInfo != null + && nextTileInfo != null + && tileInfo.getPresentationTimeUs() <= positionUs + && positionUs < checkStateNotNull(nextTileInfo).getPresentationTimeUs()) { + readyToOutputTiles = true; + return; + } + tileInfo = nextTileInfo; + nextTileInfo = null; + } + + private Bitmap cropTileFromImageGrid(int tileIndex) { + checkStateNotNull(outputBitmap); + int tileWidth = outputBitmap.getWidth() / checkStateNotNull(inputFormat).tileCountHorizontal; + int tileHeight = outputBitmap.getHeight() / checkStateNotNull(inputFormat).tileCountVertical; + int tileStartXCoordinate = tileWidth * (tileIndex % inputFormat.tileCountVertical); + int tileStartYCoordinate = tileHeight * (tileIndex / inputFormat.tileCountHorizontal); + return Bitmap.createBitmap( + outputBitmap, tileStartXCoordinate, tileStartYCoordinate, tileWidth, tileHeight); + } + private static ImageOutput getImageOutput(@Nullable ImageOutput imageOutput) { return imageOutput == null ? ImageOutput.NO_OP : imageOutput; } + + private static class TileInfo { + private final int tileIndex; + private final long presentationTimeUs; + private @MonotonicNonNull Bitmap tileBitmap; + + public TileInfo(int tileIndex, long presentationTimeUs) { + this.tileIndex = tileIndex; + this.presentationTimeUs = presentationTimeUs; + } + + public int getTileIndex() { + return this.tileIndex; + } + + public long getPresentationTimeUs() { + return presentationTimeUs; + } + + public @Nullable Bitmap getTileBitmap() { + return tileBitmap; + } + + public void setTileBitmap(Bitmap tileBitmap) { + this.tileBitmap = tileBitmap; + } + + public boolean hasTileBitmap() { + return tileBitmap != null; + } + } } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/ContainerMediaChunk.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/ContainerMediaChunk.java index 11d358e852..b2dff4d01e 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/ContainerMediaChunk.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/ContainerMediaChunk.java @@ -15,8 +15,6 @@ */ package androidx.media3.exoplayer.source.chunk; -import static androidx.media3.common.C.BUFFER_FLAG_KEY_FRAME; - import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; @@ -171,11 +169,7 @@ public class ContainerMediaChunk extends BaseMediaChunk { long tileStartTimeUs = i * tileDurationUs; trackOutput.sampleData(new ParsableByteArray(), /* length= */ 0); trackOutput.sampleMetadata( - tileStartTimeUs, - /* flags= */ BUFFER_FLAG_KEY_FRAME, - /* size= */ 0, - /* offset= */ 0, - /* cryptoData= */ null); + tileStartTimeUs, /* flags= */ 0, /* size= */ 0, /* offset= */ 0, /* cryptoData= */ null); } } } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/ImageRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/ImageRendererTest.java index c598c67fcd..59555f692f 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/ImageRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/ImageRendererTest.java @@ -17,6 +17,7 @@ package androidx.media3.exoplayer.image; import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM; import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.oneByteSample; +import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.sample; import static com.google.common.truth.Truth.assertThat; import android.graphics.Bitmap; @@ -65,12 +66,18 @@ public class ImageRendererTest { .setTileCountVertical(1) .setTileCountHorizontal(1) .build(); + private static final Format JPEG_FORMAT_WITH_FOUR_TILES = + new Format.Builder() + .setContainerMimeType(MimeTypes.IMAGE_JPEG) + .setTileCountVertical(2) + .setTileCountHorizontal(2) + .build(); private final List> renderedBitmaps = new ArrayList<>(); private final Bitmap fakeDecodedBitmap1 = - Bitmap.createBitmap(/* width= */ 1, /* height= */ 1, Bitmap.Config.ARGB_8888); - private final Bitmap fakeDecodedBitmap2 = Bitmap.createBitmap(/* width= */ 2, /* height= */ 2, Bitmap.Config.ARGB_8888); + private final Bitmap fakeDecodedBitmap2 = + Bitmap.createBitmap(/* width= */ 4, /* height= */ 4, Bitmap.Config.ARGB_8888); private ImageRenderer renderer; private int decodeCallCount; @@ -256,6 +263,285 @@ public class ImageRendererTest { assertThat(renderedBitmaps.get(1).second).isSameInstanceAs(fakeDecodedBitmap2); } + @Test + public void render_tiledImage_cropsAndRendersToImageOutput() throws Exception { + FakeSampleStream fakeSampleStream = + createSampleStream( + JPEG_FORMAT_WITH_FOUR_TILES, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0L, /* flags= */ C.BUFFER_FLAG_KEY_FRAME), + emptySample(/* timeUs= */ 100_000L, /* flags= */ 0), + emptySample(/* timeUs= */ 200_000L, /* flags= */ 0), + emptySample(/* timeUs= */ 300_000L, /* flags= */ 0), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {JPEG_FORMAT_WITH_FOUR_TILES}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.setCurrentStreamFinal(); + + StopWatch isReadyStopWatch = new StopWatch(IS_READY_TIMEOUT_MESSAGE); + while (!renderer.isReady() && isReadyStopWatch.ensureNotExpired()) { + renderer.render( + /* positionUs= */ 0, + /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000); + } + StopWatch isEndedStopWatch = new StopWatch(IS_ENDED_TIMEOUT_MESSAGE); + long positionUs = 0; + while (!renderer.isEnded() && isEndedStopWatch.ensureNotExpired()) { + renderer.render( + positionUs, /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000); + positionUs += 100_000; + } + + assertThat(renderedBitmaps).hasSize(4); + assertThat(renderedBitmaps.get(0).first).isEqualTo(0L); + assertThat(renderedBitmaps.get(0).second.getHeight()).isEqualTo(1); + assertThat(renderedBitmaps.get(0).second.getWidth()).isEqualTo(1); + assertThat(renderedBitmaps.get(1).first).isEqualTo(100_000L); + assertThat(renderedBitmaps.get(1).second.getHeight()).isEqualTo(1); + assertThat(renderedBitmaps.get(1).second.getWidth()).isEqualTo(1); + assertThat(renderedBitmaps.get(2).first).isEqualTo(200_000L); + assertThat(renderedBitmaps.get(2).second.getHeight()).isEqualTo(1); + assertThat(renderedBitmaps.get(2).second.getWidth()).isEqualTo(1); + assertThat(renderedBitmaps.get(3).first).isEqualTo(300_000L); + assertThat(renderedBitmaps.get(3).second.getHeight()).isEqualTo(1); + assertThat(renderedBitmaps.get(3).second.getWidth()).isEqualTo(1); + } + + @Test + public void render_tiledImageWithNonZeroStartPosition_rendersToImageOutput() throws Exception { + FakeSampleStream fakeSampleStream = + createSampleStream( + JPEG_FORMAT_WITH_FOUR_TILES, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0L, /* flags= */ C.BUFFER_FLAG_KEY_FRAME), + emptySample(/* timeUs= */ 100_000L, /* flags= */ 0), + emptySample(/* timeUs= */ 200_000L, /* flags= */ 0), + emptySample(/* timeUs= */ 300_000L, /* flags= */ 0), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {JPEG_FORMAT_WITH_FOUR_TILES}, + fakeSampleStream, + /* positionUs= */ 200_000, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.setCurrentStreamFinal(); + + StopWatch isReadyStopWatch = new StopWatch(IS_READY_TIMEOUT_MESSAGE); + while (!renderer.isReady() && isReadyStopWatch.ensureNotExpired()) { + renderer.render( + /* positionUs= */ 200_000, + /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000); + } + StopWatch isEndedStopWatch = new StopWatch(IS_ENDED_TIMEOUT_MESSAGE); + long positionUs = 200_000; + while (!renderer.isEnded() && isEndedStopWatch.ensureNotExpired()) { + renderer.render( + positionUs, /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000); + positionUs += 100_000; + } + + assertThat(renderedBitmaps).hasSize(2); + assertThat(renderedBitmaps.get(0).first).isEqualTo(200_000L); + assertThat(renderedBitmaps.get(1).first).isEqualTo(300_000L); + } + + @Test + public void render_tiledImageStartPositionIsAfterLastTile_rendersToImageOutput() + throws Exception { + FakeSampleStream fakeSampleStream = + createSampleStream( + JPEG_FORMAT_WITH_FOUR_TILES, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0L, /* flags= */ C.BUFFER_FLAG_KEY_FRAME), + emptySample(/* timeUs= */ 100_000L, /* flags= */ 0), + emptySample(/* timeUs= */ 200_000L, /* flags= */ 0), + emptySample(/* timeUs= */ 300_000L, /* flags= */ 0), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {JPEG_FORMAT_WITH_FOUR_TILES}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.setCurrentStreamFinal(); + + StopWatch isReadyStopWatch = new StopWatch(IS_READY_TIMEOUT_MESSAGE); + while (!renderer.isReady() && isReadyStopWatch.ensureNotExpired()) { + renderer.render( + /* positionUs= */ 350_000, + /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000); + } + StopWatch isEndedStopWatch = new StopWatch(IS_ENDED_TIMEOUT_MESSAGE); + long positionUs = 350_000; + while (!renderer.isEnded() && isEndedStopWatch.ensureNotExpired()) { + renderer.render( + positionUs, /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000); + positionUs += 100_000; + } + + assertThat(renderedBitmaps).hasSize(1); + assertThat(renderedBitmaps.get(0).first).isEqualTo(300_000L); + } + + @Test + public void + render_tiledImageStartPositionBeforePresentationTimeAndWithinThreshold_rendersIncomingTile() + throws Exception { + FakeSampleStream fakeSampleStream = + createSampleStream( + JPEG_FORMAT_WITH_FOUR_TILES, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0L, /* flags= */ C.BUFFER_FLAG_KEY_FRAME), + emptySample(/* timeUs= */ 100_000L, /* flags= */ 0), + emptySample(/* timeUs= */ 200_000L, /* flags= */ 0), + emptySample(/* timeUs= */ 300_000L, /* flags= */ 0), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {JPEG_FORMAT_WITH_FOUR_TILES}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.setCurrentStreamFinal(); + + StopWatch isReadyStopWatch = new StopWatch(IS_READY_TIMEOUT_MESSAGE); + while (!renderer.isReady() && isReadyStopWatch.ensureNotExpired()) { + renderer.render( + /* positionUs= */ 70_000, + /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000); + } + StopWatch isEndedStopWatch = new StopWatch(IS_ENDED_TIMEOUT_MESSAGE); + long positionUs = 70_000; + while (!renderer.isEnded() && isEndedStopWatch.ensureNotExpired()) { + renderer.render( + positionUs, /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000); + positionUs += 100_000; + } + + assertThat(renderedBitmaps).hasSize(3); + assertThat(renderedBitmaps.get(0).first).isEqualTo(100_000L); + assertThat(renderedBitmaps.get(1).first).isEqualTo(200_000L); + assertThat(renderedBitmaps.get(2).first).isEqualTo(300_000L); + } + + @Test + public void + render_tiledImageStartPositionAfterPresentationTimeAndWithinThreshold_rendersLastReadTile() + throws Exception { + FakeSampleStream fakeSampleStream = + createSampleStream( + JPEG_FORMAT_WITH_FOUR_TILES, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0L, /* flags= */ C.BUFFER_FLAG_KEY_FRAME), + emptySample(/* timeUs= */ 100_000L, /* flags= */ 0), + emptySample(/* timeUs= */ 200_000L, /* flags= */ 0), + emptySample(/* timeUs= */ 300_000L, /* flags= */ 0), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {JPEG_FORMAT_WITH_FOUR_TILES}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.setCurrentStreamFinal(); + + StopWatch isReadyStopWatch = new StopWatch(IS_READY_TIMEOUT_MESSAGE); + while (!renderer.isReady() && isReadyStopWatch.ensureNotExpired()) { + renderer.render( + /* positionUs= */ 130_000, + /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000); + } + StopWatch isEndedStopWatch = new StopWatch(IS_ENDED_TIMEOUT_MESSAGE); + long positionUs = 130_000; + while (!renderer.isEnded() && isEndedStopWatch.ensureNotExpired()) { + renderer.render( + positionUs, /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000); + positionUs += 100_000; + } + + assertThat(renderedBitmaps).hasSize(3); + assertThat(renderedBitmaps.get(0).first).isEqualTo(100_000L); + assertThat(renderedBitmaps.get(1).first).isEqualTo(200_000L); + assertThat(renderedBitmaps.get(2).first).isEqualTo(300_000L); + } + + @Test + public void render_tiledImageStartPositionRightBeforeEOSAndWithinThreshold_rendersLastTileInGrid() + throws Exception { + FakeSampleStream fakeSampleStream = + createSampleStream( + JPEG_FORMAT_WITH_FOUR_TILES, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0L, /* flags= */ C.BUFFER_FLAG_KEY_FRAME), + emptySample(/* timeUs= */ 100_000L, /* flags= */ 0), + emptySample(/* timeUs= */ 200_000L, /* flags= */ 0), + emptySample(/* timeUs= */ 300_000L, /* flags= */ 0), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {JPEG_FORMAT_WITH_FOUR_TILES}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.setCurrentStreamFinal(); + + StopWatch isReadyStopWatch = new StopWatch(IS_READY_TIMEOUT_MESSAGE); + while (!renderer.isReady() && isReadyStopWatch.ensureNotExpired()) { + renderer.render( + /* positionUs= */ 330_000, + /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000); + } + StopWatch isEndedStopWatch = new StopWatch(IS_ENDED_TIMEOUT_MESSAGE); + long positionUs = 330_000; + while (!renderer.isEnded() && isEndedStopWatch.ensureNotExpired()) { + renderer.render( + positionUs, /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000); + positionUs += 100_000; + } + + assertThat(renderedBitmaps).hasSize(1); + assertThat(renderedBitmaps.get(0).first).isEqualTo(300_000L); + } + + private static FakeSampleStream.FakeSampleStreamItem emptySample( + long timeUs, @C.BufferFlags int flags) { + return sample(timeUs, flags, new byte[] {}); + } + private static FakeSampleStream createSampleStream(long timeUs) { return new FakeSampleStream( new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), @@ -266,6 +552,17 @@ public class ImageRendererTest { ImmutableList.of(oneByteSample(timeUs, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM)); } + private static FakeSampleStream createSampleStream( + Format format, List fakeSampleStreamItems) { + return new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + format, + fakeSampleStreamItems); + } + private static final class StopWatch { private final long startTimeMs; private final long timeOutMs; diff --git a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java index a6fdeac5fe..cc70a47693 100644 --- a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java +++ b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java @@ -322,6 +322,11 @@ public final class DashPlaybackTest { applicationContext, playbackOutput, "playbackdumps/dash/metadata_from_early_output.dump"); } + /** + * This test might be flaky. The {@link ExoPlayer} instantiated in this test uses a {@link + * FakeClock} that runs much faster than real time. This might cause the {@link ExoPlayer} to skip + * and not present some images. That will cause the test to fail. + */ @Test public void playThumbnailGrid() throws Exception { Context applicationContext = ApplicationProvider.getApplicationContext(); diff --git a/libraries/test_data/src/test/assets/playbackdumps/dash/loadimage.dump b/libraries/test_data/src/test/assets/playbackdumps/dash/loadimage.dump index 631f9b1219..7bf760e51b 100644 --- a/libraries/test_data/src/test/assets/playbackdumps/dash/loadimage.dump +++ b/libraries/test_data/src/test/assets/playbackdumps/dash/loadimage.dump @@ -1,5 +1,194 @@ ImageOutput: - rendered image count = 1 + rendered image count = 64 image output #1: presentationTimeUs = 0 - bitmap hash = 90169190 + bitmap hash = 1662699756 + image output #2: + presentationTimeUs = 937500 + bitmap hash = -1574796305 + image output #3: + presentationTimeUs = 1875000 + bitmap hash = 113939771 + image output #4: + presentationTimeUs = 2812500 + bitmap hash = -1947261277 + image output #5: + presentationTimeUs = 3750000 + bitmap hash = 516207300 + image output #6: + presentationTimeUs = 4687500 + bitmap hash = -1362626860 + image output #7: + presentationTimeUs = 5625000 + bitmap hash = 1681263366 + image output #8: + presentationTimeUs = 6562500 + bitmap hash = -2107026381 + image output #9: + presentationTimeUs = 7500000 + bitmap hash = -1582997950 + image output #10: + presentationTimeUs = 8437500 + bitmap hash = -1077263789 + image output #11: + presentationTimeUs = 9375000 + bitmap hash = 1900793242 + image output #12: + presentationTimeUs = 10312500 + bitmap hash = -310799297 + image output #13: + presentationTimeUs = 11250000 + bitmap hash = 1499050092 + image output #14: + presentationTimeUs = 12187500 + bitmap hash = -889397582 + image output #15: + presentationTimeUs = 13125000 + bitmap hash = -522250446 + image output #16: + presentationTimeUs = 14062500 + bitmap hash = -581506812 + image output #17: + presentationTimeUs = 15000000 + bitmap hash = -767045995 + image output #18: + presentationTimeUs = 15937500 + bitmap hash = -1205030348 + image output #19: + presentationTimeUs = 16875000 + bitmap hash = 1338041318 + image output #20: + presentationTimeUs = 17812500 + bitmap hash = 1352738203 + image output #21: + presentationTimeUs = 18750000 + bitmap hash = 883432223 + image output #22: + presentationTimeUs = 19687500 + bitmap hash = -535832768 + image output #23: + presentationTimeUs = 20625000 + bitmap hash = -1338903762 + image output #24: + presentationTimeUs = 21562500 + bitmap hash = -125811797 + image output #25: + presentationTimeUs = 22500000 + bitmap hash = 549453620 + image output #26: + presentationTimeUs = 23437500 + bitmap hash = 1677558048 + image output #27: + presentationTimeUs = 24375000 + bitmap hash = -256549269 + image output #28: + presentationTimeUs = 25312500 + bitmap hash = -630960808 + image output #29: + presentationTimeUs = 26250000 + bitmap hash = 1015145670 + image output #30: + presentationTimeUs = 27187500 + bitmap hash = -1795307136 + image output #31: + presentationTimeUs = 28125000 + bitmap hash = 1272159394 + image output #32: + presentationTimeUs = 29062500 + bitmap hash = -93678600 + image output #33: + presentationTimeUs = 30000000 + bitmap hash = -600076145 + image output #34: + presentationTimeUs = 30937500 + bitmap hash = -97251290 + image output #35: + presentationTimeUs = 31875000 + bitmap hash = 1281484249 + image output #36: + presentationTimeUs = 32812500 + bitmap hash = -1728867849 + image output #37: + presentationTimeUs = 33750000 + bitmap hash = 380034424 + image output #38: + presentationTimeUs = 34687500 + bitmap hash = 1913328953 + image output #39: + presentationTimeUs = 35625000 + bitmap hash = 1616828465 + image output #40: + presentationTimeUs = 36562500 + bitmap hash = 1579225474 + image output #41: + presentationTimeUs = 37500000 + bitmap hash = -1263537508 + image output #42: + presentationTimeUs = 38437500 + bitmap hash = 1469560805 + image output #43: + presentationTimeUs = 39375000 + bitmap hash = -1949117971 + image output #44: + presentationTimeUs = 40312500 + bitmap hash = 1890332461 + image output #45: + presentationTimeUs = 41250000 + bitmap hash = 381486112 + image output #46: + presentationTimeUs = 42187500 + bitmap hash = 943544370 + image output #47: + presentationTimeUs = 43125000 + bitmap hash = -449507486 + image output #48: + presentationTimeUs = 44062500 + bitmap hash = -1456959112 + image output #49: + presentationTimeUs = 45000000 + bitmap hash = -919717716 + image output #50: + presentationTimeUs = 45937500 + bitmap hash = -1852787702 + image output #51: + presentationTimeUs = 46875000 + bitmap hash = 2000481270 + image output #52: + presentationTimeUs = 47812500 + bitmap hash = 1399518428 + image output #53: + presentationTimeUs = 48750000 + bitmap hash = -158658633 + image output #54: + presentationTimeUs = 49687500 + bitmap hash = 587265344 + image output #55: + presentationTimeUs = 50625000 + bitmap hash = -1857190760 + image output #56: + presentationTimeUs = 51562500 + bitmap hash = -392855012 + image output #57: + presentationTimeUs = 52500000 + bitmap hash = -1222466861 + image output #58: + presentationTimeUs = 53437500 + bitmap hash = 2060648653 + image output #59: + presentationTimeUs = 54375000 + bitmap hash = 1407821609 + image output #60: + presentationTimeUs = 55312500 + bitmap hash = -1744072926 + image output #61: + presentationTimeUs = 56250000 + bitmap hash = -1355216794 + image output #62: + presentationTimeUs = 57187500 + bitmap hash = -7610058 + image output #63: + presentationTimeUs = 58125000 + bitmap hash = 1362483058 + image output #64: + presentationTimeUs = 59062500 + bitmap hash = 442567684