diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java index a37c40f000..1edc42b4cb 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java @@ -541,21 +541,34 @@ public class ImageRenderer extends BaseRenderer { currentTileIndex++; // TODO: b/319484746 - ImageRenderer should consider startPositionUs when choosing to output an // image. - if (nextTileInfo.getPresentationTimeUs() - IMAGE_PRESENTATION_WINDOW_THRESHOLD_US <= positionUs - && positionUs - <= nextTileInfo.getPresentationTimeUs() + IMAGE_PRESENTATION_WINDOW_THRESHOLD_US) { - readyToOutputTiles = true; - } else if (tileInfo != null - && nextTileInfo != null - && tileInfo.getPresentationTimeUs() <= positionUs - && positionUs < checkStateNotNull(nextTileInfo).getPresentationTimeUs()) { - readyToOutputTiles = true; - return; + if (!readyToOutputTiles) { + long tilePresentationTimeUs = nextTileInfo.getPresentationTimeUs(); + boolean isNextTileWithinPresentationThreshold = + tilePresentationTimeUs - IMAGE_PRESENTATION_WINDOW_THRESHOLD_US <= positionUs + && positionUs <= tilePresentationTimeUs + IMAGE_PRESENTATION_WINDOW_THRESHOLD_US; + boolean isPositionBetweenTiles = + tileInfo != null + && tileInfo.getPresentationTimeUs() <= positionUs + && positionUs < tilePresentationTimeUs; + boolean isNextTileLastInGrid = isTileLastInGrid(checkStateNotNull(nextTileInfo)); + readyToOutputTiles = + isNextTileWithinPresentationThreshold || isPositionBetweenTiles || isNextTileLastInGrid; + if (isPositionBetweenTiles && !isNextTileWithinPresentationThreshold) { + return; + } } tileInfo = nextTileInfo; nextTileInfo = null; } + private boolean isTileLastInGrid(TileInfo tileInfo) { + return checkStateNotNull(inputFormat).tileCountHorizontal == Format.NO_VALUE + || inputFormat.tileCountVertical == Format.NO_VALUE + || (tileInfo.getTileIndex() + == checkStateNotNull(inputFormat).tileCountVertical * inputFormat.tileCountHorizontal + - 1); + } + private Bitmap cropTileFromImageGrid(int tileIndex) { checkStateNotNull(outputBitmap); int tileWidth = outputBitmap.getWidth() / checkStateNotNull(inputFormat).tileCountHorizontal; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ImagePlaybackTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ImagePlaybackTest.java index 34cbdc74ff..048ffe5375 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ImagePlaybackTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ImagePlaybackTest.java @@ -15,14 +15,11 @@ */ package androidx.media3.exoplayer.e2etest; -import static com.google.common.truth.Truth.assertThat; import static org.robolectric.annotation.GraphicsMode.Mode.NATIVE; import android.content.Context; -import androidx.media3.common.C; import androidx.media3.common.MediaItem; import androidx.media3.common.Player; -import androidx.media3.common.util.Clock; import androidx.media3.exoplayer.ExoPlayer; import androidx.media3.test.utils.CapturingRenderersFactory; import androidx.media3.test.utils.DumpFileAsserts; @@ -30,88 +27,45 @@ import androidx.media3.test.utils.FakeClock; import androidx.media3.test.utils.robolectric.PlaybackOutput; import androidx.media3.test.utils.robolectric.TestPlayerRunHelper; import androidx.test.core.app.ApplicationProvider; -import com.google.common.collect.Collections2; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; import org.junit.Test; import org.junit.runner.RunWith; -import org.robolectric.ParameterizedRobolectricTestRunner; -import org.robolectric.ParameterizedRobolectricTestRunner.Parameter; -import org.robolectric.ParameterizedRobolectricTestRunner.Parameters; import org.robolectric.annotation.GraphicsMode; /** End-to-end tests using image samples. */ -@RunWith(ParameterizedRobolectricTestRunner.class) +@RunWith(AndroidJUnit4.class) @GraphicsMode(value = NATIVE) public class ImagePlaybackTest { - @Parameter public Set inputFiles; - - @Parameters(name = "{0}") - public static List> mediaSamples() { - // Robolectric's ShadowNativeBitmapFactory doesn't support decoding HEIF format, so we don't - // test that here. - // TODO b/300457060 - Find out why jpegs cause flaky failures in this test and then add jpegs to - // this list if possible. - return new ArrayList<>( - Collections2.filter( - Sets.powerSet( - ImmutableSet.of( - "bitmap/input_images/media3test.png", - "bmp/non-motion-photo-shortened-cropped.bmp", - "png/non-motion-photo-shortened.png", - "webp/ic_launcher_round.webp")), - /* predicate= */ input -> !input.isEmpty())); - } @Test - public void test() throws Exception { + public void playImagePlaylist_withSeek_rendersExpectedImages() throws Exception { Context applicationContext = ApplicationProvider.getApplicationContext(); CapturingRenderersFactory renderersFactory = new CapturingRenderersFactory(applicationContext); - Clock clock = new FakeClock(/* isAutoAdvancing= */ true); ExoPlayer player = - new ExoPlayer.Builder(applicationContext, renderersFactory).setClock(clock).build(); + new ExoPlayer.Builder(applicationContext, renderersFactory) + .setClock(new FakeClock(/* isAutoAdvancing= */ true)) + .build(); PlaybackOutput playbackOutput = PlaybackOutput.register(player, renderersFactory); - List sortedInputFiles = new ArrayList<>(inputFiles); - Collections.sort(sortedInputFiles); - List mediaItems = new ArrayList<>(inputFiles.size()); - long totalDurationMs = 0; - long currentDurationMs = 3 * C.MILLIS_PER_SECOND; - for (String inputFile : sortedInputFiles) { - mediaItems.add( - new MediaItem.Builder() - .setUri("asset:///media/" + inputFile) - .setImageDurationMs(currentDurationMs) - .build()); - totalDurationMs += currentDurationMs; - if (currentDurationMs < 5 * C.MILLIS_PER_SECOND) { - currentDurationMs += C.MILLIS_PER_SECOND; - } - } - player.setMediaItems(mediaItems); + MediaItem mediaItem1 = + new MediaItem.Builder() + .setUri("asset:///media/bitmap/input_images/media3test.png") + .setImageDurationMs(3000L) + .build(); + MediaItem mediaItem2 = + new MediaItem.Builder() + .setUri("asset:///media/png/non-motion-photo-shortened.png") + .setImageDurationMs(3000L) + .build(); + player.setMediaItems(ImmutableList.of(mediaItem1, mediaItem2)); player.prepare(); - TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_READY); - long playerStartedMs = clock.elapsedRealtime(); - player.play(); - TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED); - long playbackDurationMs = clock.elapsedRealtime() - playerStartedMs; - player.release(); - assertThat(playbackDurationMs).isAtLeast(totalDurationMs); - DumpFileAsserts.assertOutput( - applicationContext, - playbackOutput, - "playbackdumps/image/" + generateName(sortedInputFiles) + ".dump"); - } - private static String generateName(List sortedInputFiles) { - StringBuilder name = new StringBuilder(); - for (String inputFile : sortedInputFiles) { - name.append(inputFile, inputFile.lastIndexOf("/") + 1, inputFile.length()).append("+"); - } - name.setLength(name.length() - 1); - return name.toString(); + TestPlayerRunHelper.playUntilPosition(player, /* mediaItemIndex= */ 0, /* positionMs= */ 1000L); + player.seekTo(/* mediaItemIndex= */ 0, /* positionMs= */ 2000L); + TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED); + player.release(); + + DumpFileAsserts.assertOutput( + applicationContext, playbackOutput, "playbackdumps/image/image_playlist_with_seek.dump"); } } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ParameterizedImagePlaybackTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ParameterizedImagePlaybackTest.java new file mode 100644 index 0000000000..70a0ca6f18 --- /dev/null +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ParameterizedImagePlaybackTest.java @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.e2etest; + +import static com.google.common.truth.Truth.assertThat; +import static org.robolectric.annotation.GraphicsMode.Mode.NATIVE; + +import android.content.Context; +import androidx.media3.common.C; +import androidx.media3.common.MediaItem; +import androidx.media3.common.Player; +import androidx.media3.common.util.Clock; +import androidx.media3.exoplayer.ExoPlayer; +import androidx.media3.test.utils.CapturingRenderersFactory; +import androidx.media3.test.utils.DumpFileAsserts; +import androidx.media3.test.utils.FakeClock; +import androidx.media3.test.utils.robolectric.PlaybackOutput; +import androidx.media3.test.utils.robolectric.TestPlayerRunHelper; +import androidx.test.core.app.ApplicationProvider; +import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.ParameterizedRobolectricTestRunner; +import org.robolectric.ParameterizedRobolectricTestRunner.Parameter; +import org.robolectric.ParameterizedRobolectricTestRunner.Parameters; +import org.robolectric.annotation.GraphicsMode; + +/** Parameterized end-to-end tests using image samples. */ +@RunWith(ParameterizedRobolectricTestRunner.class) +@GraphicsMode(value = NATIVE) +public class ParameterizedImagePlaybackTest { + @Parameter public Set inputFiles; + + @Parameters(name = "{0}") + public static List> mediaSamples() { + // Robolectric's ShadowNativeBitmapFactory doesn't support decoding HEIF format, so we don't + // test that here. + // TODO b/300457060 - Find out why jpegs cause flaky failures in this test and then add jpegs to + // this list if possible. + return new ArrayList<>( + Collections2.filter( + Sets.powerSet( + ImmutableSet.of( + "bitmap/input_images/media3test.png", + "bmp/non-motion-photo-shortened-cropped.bmp", + "png/non-motion-photo-shortened.png", + "webp/ic_launcher_round.webp")), + /* predicate= */ input -> !input.isEmpty())); + } + + @Test + public void test() throws Exception { + Context applicationContext = ApplicationProvider.getApplicationContext(); + CapturingRenderersFactory renderersFactory = new CapturingRenderersFactory(applicationContext); + Clock clock = new FakeClock(/* isAutoAdvancing= */ true); + ExoPlayer player = + new ExoPlayer.Builder(applicationContext, renderersFactory).setClock(clock).build(); + PlaybackOutput playbackOutput = PlaybackOutput.register(player, renderersFactory); + List sortedInputFiles = new ArrayList<>(inputFiles); + Collections.sort(sortedInputFiles); + List mediaItems = new ArrayList<>(inputFiles.size()); + long totalDurationMs = 0; + long currentDurationMs = 3 * C.MILLIS_PER_SECOND; + for (String inputFile : sortedInputFiles) { + mediaItems.add( + new MediaItem.Builder() + .setUri("asset:///media/" + inputFile) + .setImageDurationMs(currentDurationMs) + .build()); + totalDurationMs += currentDurationMs; + if (currentDurationMs < 5 * C.MILLIS_PER_SECOND) { + currentDurationMs += C.MILLIS_PER_SECOND; + } + } + player.setMediaItems(mediaItems); + player.prepare(); + TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_READY); + long playerStartedMs = clock.elapsedRealtime(); + player.play(); + TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED); + long playbackDurationMs = clock.elapsedRealtime() - playerStartedMs; + player.release(); + assertThat(playbackDurationMs).isAtLeast(totalDurationMs); + DumpFileAsserts.assertOutput( + applicationContext, + playbackOutput, + "playbackdumps/image/" + generateName(sortedInputFiles) + ".dump"); + } + + private static String generateName(List sortedInputFiles) { + StringBuilder name = new StringBuilder(); + for (String inputFile : sortedInputFiles) { + name.append(inputFile, inputFile.lastIndexOf("/") + 1, inputFile.length()).append("+"); + } + name.setLength(name.length() - 1); + return name.toString(); + } +} diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/ImageRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/ImageRendererTest.java index 636e74fd9f..ffc161498e 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/ImageRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/ImageRendererTest.java @@ -532,6 +532,50 @@ public class ImageRendererTest { assertThat(renderedBitmaps.get(0).first).isEqualTo(300_000L); } + @Test + public void render_tiledImageStartPositionIsBeforeLastTileAndNotWithinThreshold_rendersPriorTile() + throws Exception { + FakeSampleStream fakeSampleStream = + createSampleStream( + JPEG_FORMAT_WITH_FOUR_TILES, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0L, /* flags= */ C.BUFFER_FLAG_KEY_FRAME), + emptySample(/* timeUs= */ 100_000L, /* flags= */ 0), + emptySample(/* timeUs= */ 200_000L, /* flags= */ 0), + emptySample(/* timeUs= */ 300_000L, /* flags= */ 0), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {JPEG_FORMAT_WITH_FOUR_TILES}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.setCurrentStreamFinal(); + + StopWatch isReadyStopWatch = new StopWatch(IS_READY_TIMEOUT_MESSAGE); + while (!renderer.isReady() && isReadyStopWatch.ensureNotExpired()) { + renderer.render( + /* positionUs= */ 250_000L, + /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000); + } + StopWatch isEndedStopWatch = new StopWatch(IS_ENDED_TIMEOUT_MESSAGE); + long positionUs = 250_000L; + while (!renderer.isEnded() && isEndedStopWatch.ensureNotExpired()) { + renderer.render( + positionUs, /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000); + positionUs += 100_000L; + } + + assertThat(renderedBitmaps).hasSize(2); + assertThat(renderedBitmaps.get(0).first).isEqualTo(200_000L); + assertThat(renderedBitmaps.get(1).first).isEqualTo(300_000L); + } + @Test public void render_tiledImageStartPositionBeforePresentationTimeAndWithinThreshold_rendersIncomingTile() diff --git a/libraries/test_data/src/test/assets/playbackdumps/image/image_playlist_with_seek.dump b/libraries/test_data/src/test/assets/playbackdumps/image/image_playlist_with_seek.dump new file mode 100644 index 0000000000..151cfe4f9c --- /dev/null +++ b/libraries/test_data/src/test/assets/playbackdumps/image/image_playlist_with_seek.dump @@ -0,0 +1,11 @@ +ImageOutput: + rendered image count = 3 + image output #1: + presentationTimeUs = 0 + bitmap hash = -389047680 + image output #2: + presentationTimeUs = 0 + bitmap hash = -389047680 + image output #3: + presentationTimeUs = 0 + bitmap hash = 1367007828