Render last image despite not receiving EoS

If seeking between last image sample and end of the file where the current stream is not final, then EoS sample will not be provided to `ImageRenderer`. ImageRenderer must still produce the last image sample.

PiperOrigin-RevId: 603312090

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java

@@ -541,21 +541,34 @@ public class ImageRenderer extends BaseRenderer {
     currentTileIndex++;
     // TODO: b/319484746 - ImageRenderer should consider startPositionUs when choosing to output an
     // image.
-    if (nextTileInfo.getPresentationTimeUs() - IMAGE_PRESENTATION_WINDOW_THRESHOLD_US <= positionUs
+    if (!readyToOutputTiles) {
-        && positionUs
+      long tilePresentationTimeUs = nextTileInfo.getPresentationTimeUs();
-            <= nextTileInfo.getPresentationTimeUs() + IMAGE_PRESENTATION_WINDOW_THRESHOLD_US) {
+      boolean isNextTileWithinPresentationThreshold =
-      readyToOutputTiles = true;
+          tilePresentationTimeUs - IMAGE_PRESENTATION_WINDOW_THRESHOLD_US <= positionUs
-    } else if (tileInfo != null
+              && positionUs <= tilePresentationTimeUs + IMAGE_PRESENTATION_WINDOW_THRESHOLD_US;
-        && nextTileInfo != null
+      boolean isPositionBetweenTiles =
-        && tileInfo.getPresentationTimeUs() <= positionUs
+          tileInfo != null
-        && positionUs < checkStateNotNull(nextTileInfo).getPresentationTimeUs()) {
+              && tileInfo.getPresentationTimeUs() <= positionUs
-      readyToOutputTiles = true;
+              && positionUs < tilePresentationTimeUs;
-      return;
+      boolean isNextTileLastInGrid = isTileLastInGrid(checkStateNotNull(nextTileInfo));
+      readyToOutputTiles =
+          isNextTileWithinPresentationThreshold || isPositionBetweenTiles || isNextTileLastInGrid;
+      if (isPositionBetweenTiles && !isNextTileWithinPresentationThreshold) {
+        return;
+      }
     }
     tileInfo = nextTileInfo;
     nextTileInfo = null;
   }
 
+  private boolean isTileLastInGrid(TileInfo tileInfo) {
+    return checkStateNotNull(inputFormat).tileCountHorizontal == Format.NO_VALUE
+        || inputFormat.tileCountVertical == Format.NO_VALUE
+        || (tileInfo.getTileIndex()
+            == checkStateNotNull(inputFormat).tileCountVertical * inputFormat.tileCountHorizontal
+                - 1);
+  }
+
   private Bitmap cropTileFromImageGrid(int tileIndex) {
     checkStateNotNull(outputBitmap);
     int tileWidth = outputBitmap.getWidth() / checkStateNotNull(inputFormat).tileCountHorizontal;

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ImagePlaybackTest.java

@@ -15,14 +15,11 @@
  */
 package androidx.media3.exoplayer.e2etest;
 
-import static com.google.common.truth.Truth.assertThat;
 import static org.robolectric.annotation.GraphicsMode.Mode.NATIVE;
 
 import android.content.Context;
-import androidx.media3.common.C;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.Player;
-import androidx.media3.common.util.Clock;
 import androidx.media3.exoplayer.ExoPlayer;
 import androidx.media3.test.utils.CapturingRenderersFactory;
 import androidx.media3.test.utils.DumpFileAsserts;
@@ -30,88 +27,45 @@ import androidx.media3.test.utils.FakeClock;
 import androidx.media3.test.utils.robolectric.PlaybackOutput;
 import androidx.media3.test.utils.robolectric.TestPlayerRunHelper;
 import androidx.test.core.app.ApplicationProvider;
-import com.google.common.collect.Collections2;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
-import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Sets;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
 import org.junit.Test;
 import org.junit.runner.RunWith;
-import org.robolectric.ParameterizedRobolectricTestRunner;
-import org.robolectric.ParameterizedRobolectricTestRunner.Parameter;
-import org.robolectric.ParameterizedRobolectricTestRunner.Parameters;
 import org.robolectric.annotation.GraphicsMode;
 
 /** End-to-end tests using image samples. */
-@RunWith(ParameterizedRobolectricTestRunner.class)
+@RunWith(AndroidJUnit4.class)
 @GraphicsMode(value = NATIVE)
 public class ImagePlaybackTest {
-  @Parameter public Set<String> inputFiles;
-
-  @Parameters(name = "{0}")
-  public static List<Set<String>> mediaSamples() {
-    // Robolectric's ShadowNativeBitmapFactory doesn't support decoding HEIF format, so we don't
-    // test that here.
-    // TODO b/300457060 - Find out why jpegs cause flaky failures in this test and then add jpegs to
-    // this list if possible.
-    return new ArrayList<>(
-        Collections2.filter(
-            Sets.powerSet(
-                ImmutableSet.of(
-                    "bitmap/input_images/media3test.png",
-                    "bmp/non-motion-photo-shortened-cropped.bmp",
-                    "png/non-motion-photo-shortened.png",
-                    "webp/ic_launcher_round.webp")),
-            /* predicate= */ input -> !input.isEmpty()));
-  }
 
   @Test
-  public void test() throws Exception {
+  public void playImagePlaylist_withSeek_rendersExpectedImages() throws Exception {
     Context applicationContext = ApplicationProvider.getApplicationContext();
     CapturingRenderersFactory renderersFactory = new CapturingRenderersFactory(applicationContext);
-    Clock clock = new FakeClock(/* isAutoAdvancing= */ true);
     ExoPlayer player =
-        new ExoPlayer.Builder(applicationContext, renderersFactory).setClock(clock).build();
+        new ExoPlayer.Builder(applicationContext, renderersFactory)
+            .setClock(new FakeClock(/* isAutoAdvancing= */ true))
+            .build();
     PlaybackOutput playbackOutput = PlaybackOutput.register(player, renderersFactory);
-    List<String> sortedInputFiles = new ArrayList<>(inputFiles);
+    MediaItem mediaItem1 =
-    Collections.sort(sortedInputFiles);
+        new MediaItem.Builder()
-    List<MediaItem> mediaItems = new ArrayList<>(inputFiles.size());
+            .setUri("asset:///media/bitmap/input_images/media3test.png")
-    long totalDurationMs = 0;
+            .setImageDurationMs(3000L)
-    long currentDurationMs = 3 * C.MILLIS_PER_SECOND;
+            .build();
-    for (String inputFile : sortedInputFiles) {
+    MediaItem mediaItem2 =
-      mediaItems.add(
+        new MediaItem.Builder()
-          new MediaItem.Builder()
+            .setUri("asset:///media/png/non-motion-photo-shortened.png")
-              .setUri("asset:///media/" + inputFile)
+            .setImageDurationMs(3000L)
-              .setImageDurationMs(currentDurationMs)
+            .build();
-              .build());
+    player.setMediaItems(ImmutableList.of(mediaItem1, mediaItem2));
-      totalDurationMs += currentDurationMs;
-      if (currentDurationMs < 5 * C.MILLIS_PER_SECOND) {
-        currentDurationMs += C.MILLIS_PER_SECOND;
-      }
-    }
-    player.setMediaItems(mediaItems);
     player.prepare();
-    TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_READY);
-    long playerStartedMs = clock.elapsedRealtime();
-    player.play();
-    TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED);
-    long playbackDurationMs = clock.elapsedRealtime() - playerStartedMs;
-    player.release();
-    assertThat(playbackDurationMs).isAtLeast(totalDurationMs);
-    DumpFileAsserts.assertOutput(
-        applicationContext,
-        playbackOutput,
-        "playbackdumps/image/" + generateName(sortedInputFiles) + ".dump");
-  }
 
-  private static String generateName(List<String> sortedInputFiles) {
+    TestPlayerRunHelper.playUntilPosition(player, /* mediaItemIndex= */ 0, /* positionMs= */ 1000L);
-    StringBuilder name = new StringBuilder();
+    player.seekTo(/* mediaItemIndex= */ 0, /* positionMs= */ 2000L);
-    for (String inputFile : sortedInputFiles) {
+    TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED);
-      name.append(inputFile, inputFile.lastIndexOf("/") + 1, inputFile.length()).append("+");
+    player.release();
-    }
+
-    name.setLength(name.length() - 1);
+    DumpFileAsserts.assertOutput(
-    return name.toString();
+        applicationContext, playbackOutput, "playbackdumps/image/image_playlist_with_seek.dump");
   }
 }

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/ParameterizedImagePlaybackTest.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.e2etest;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.robolectric.annotation.GraphicsMode.Mode.NATIVE;
+
+import android.content.Context;
+import androidx.media3.common.C;
+import androidx.media3.common.MediaItem;
+import androidx.media3.common.Player;
+import androidx.media3.common.util.Clock;
+import androidx.media3.exoplayer.ExoPlayer;
+import androidx.media3.test.utils.CapturingRenderersFactory;
+import androidx.media3.test.utils.DumpFileAsserts;
+import androidx.media3.test.utils.FakeClock;
+import androidx.media3.test.utils.robolectric.PlaybackOutput;
+import androidx.media3.test.utils.robolectric.TestPlayerRunHelper;
+import androidx.test.core.app.ApplicationProvider;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.ParameterizedRobolectricTestRunner;
+import org.robolectric.ParameterizedRobolectricTestRunner.Parameter;
+import org.robolectric.ParameterizedRobolectricTestRunner.Parameters;
+import org.robolectric.annotation.GraphicsMode;
+
+/** Parameterized end-to-end tests using image samples. */
+@RunWith(ParameterizedRobolectricTestRunner.class)
+@GraphicsMode(value = NATIVE)
+public class ParameterizedImagePlaybackTest {
+  @Parameter public Set<String> inputFiles;
+
+  @Parameters(name = "{0}")
+  public static List<Set<String>> mediaSamples() {
+    // Robolectric's ShadowNativeBitmapFactory doesn't support decoding HEIF format, so we don't
+    // test that here.
+    // TODO b/300457060 - Find out why jpegs cause flaky failures in this test and then add jpegs to
+    // this list if possible.
+    return new ArrayList<>(
+        Collections2.filter(
+            Sets.powerSet(
+                ImmutableSet.of(
+                    "bitmap/input_images/media3test.png",
+                    "bmp/non-motion-photo-shortened-cropped.bmp",
+                    "png/non-motion-photo-shortened.png",
+                    "webp/ic_launcher_round.webp")),
+            /* predicate= */ input -> !input.isEmpty()));
+  }
+
+  @Test
+  public void test() throws Exception {
+    Context applicationContext = ApplicationProvider.getApplicationContext();
+    CapturingRenderersFactory renderersFactory = new CapturingRenderersFactory(applicationContext);
+    Clock clock = new FakeClock(/* isAutoAdvancing= */ true);
+    ExoPlayer player =
+        new ExoPlayer.Builder(applicationContext, renderersFactory).setClock(clock).build();
+    PlaybackOutput playbackOutput = PlaybackOutput.register(player, renderersFactory);
+    List<String> sortedInputFiles = new ArrayList<>(inputFiles);
+    Collections.sort(sortedInputFiles);
+    List<MediaItem> mediaItems = new ArrayList<>(inputFiles.size());
+    long totalDurationMs = 0;
+    long currentDurationMs = 3 * C.MILLIS_PER_SECOND;
+    for (String inputFile : sortedInputFiles) {
+      mediaItems.add(
+          new MediaItem.Builder()
+              .setUri("asset:///media/" + inputFile)
+              .setImageDurationMs(currentDurationMs)
+              .build());
+      totalDurationMs += currentDurationMs;
+      if (currentDurationMs < 5 * C.MILLIS_PER_SECOND) {
+        currentDurationMs += C.MILLIS_PER_SECOND;
+      }
+    }
+    player.setMediaItems(mediaItems);
+    player.prepare();
+    TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_READY);
+    long playerStartedMs = clock.elapsedRealtime();
+    player.play();
+    TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED);
+    long playbackDurationMs = clock.elapsedRealtime() - playerStartedMs;
+    player.release();
+    assertThat(playbackDurationMs).isAtLeast(totalDurationMs);
+    DumpFileAsserts.assertOutput(
+        applicationContext,
+        playbackOutput,
+        "playbackdumps/image/" + generateName(sortedInputFiles) + ".dump");
+  }
+
+  private static String generateName(List<String> sortedInputFiles) {
+    StringBuilder name = new StringBuilder();
+    for (String inputFile : sortedInputFiles) {
+      name.append(inputFile, inputFile.lastIndexOf("/") + 1, inputFile.length()).append("+");
+    }
+    name.setLength(name.length() - 1);
+    return name.toString();
+  }
+}

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/ImageRendererTest.java

@@ -532,6 +532,50 @@ public class ImageRendererTest {
     assertThat(renderedBitmaps.get(0).first).isEqualTo(300_000L);
   }
 
+  @Test
+  public void render_tiledImageStartPositionIsBeforeLastTileAndNotWithinThreshold_rendersPriorTile()
+      throws Exception {
+    FakeSampleStream fakeSampleStream =
+        createSampleStream(
+            JPEG_FORMAT_WITH_FOUR_TILES,
+            ImmutableList.of(
+                oneByteSample(/* timeUs= */ 0L, /* flags= */ C.BUFFER_FLAG_KEY_FRAME),
+                emptySample(/* timeUs= */ 100_000L, /* flags= */ 0),
+                emptySample(/* timeUs= */ 200_000L, /* flags= */ 0),
+                emptySample(/* timeUs= */ 300_000L, /* flags= */ 0),
+                END_OF_STREAM_ITEM));
+    fakeSampleStream.writeData(/* startPositionUs= */ 0);
+    renderer.enable(
+        RendererConfiguration.DEFAULT,
+        new Format[] {JPEG_FORMAT_WITH_FOUR_TILES},
+        fakeSampleStream,
+        /* positionUs= */ 0,
+        /* joining= */ false,
+        /* mayRenderStartOfStream= */ true,
+        /* startPositionUs= */ 0,
+        /* offsetUs= */ 0,
+        new MediaSource.MediaPeriodId(new Object()));
+    renderer.setCurrentStreamFinal();
+
+    StopWatch isReadyStopWatch = new StopWatch(IS_READY_TIMEOUT_MESSAGE);
+    while (!renderer.isReady() && isReadyStopWatch.ensureNotExpired()) {
+      renderer.render(
+          /* positionUs= */ 250_000L,
+          /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000);
+    }
+    StopWatch isEndedStopWatch = new StopWatch(IS_ENDED_TIMEOUT_MESSAGE);
+    long positionUs = 250_000L;
+    while (!renderer.isEnded() && isEndedStopWatch.ensureNotExpired()) {
+      renderer.render(
+          positionUs, /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000);
+      positionUs += 100_000L;
+    }
+
+    assertThat(renderedBitmaps).hasSize(2);
+    assertThat(renderedBitmaps.get(0).first).isEqualTo(200_000L);
+    assertThat(renderedBitmaps.get(1).first).isEqualTo(300_000L);
+  }
+
   @Test
   public void
       render_tiledImageStartPositionBeforePresentationTimeAndWithinThreshold_rendersIncomingTile()

libraries/test_data/src/test/assets/playbackdumps/image/image_playlist_with_seek.dump

@@ -0,0 +1,11 @@
+ImageOutput:
+  rendered image count = 3
+  image output #1:
+    presentationTimeUs = 0
+    bitmap hash = -389047680
+  image output #2:
+    presentationTimeUs = 0
+    bitmap hash = -389047680
+  image output #3:
+    presentationTimeUs = 0
+    bitmap hash = 1367007828