Implement custom Frame Extractor renderer

Render only one frame per seek to reduce the amount of work done

PiperOrigin-RevId: 697946350

libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java

@@ -28,12 +28,10 @@ import static org.junit.Assert.assertThrows;
 import android.app.Instrumentation;
 import android.content.Context;
 import android.graphics.Bitmap;
-import androidx.annotation.Nullable;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.util.ConditionVariable;
 import androidx.media3.common.util.NullableType;
 import androidx.media3.effect.Presentation;
-import androidx.media3.exoplayer.DecoderCounters;
 import androidx.media3.exoplayer.ExoPlaybackException;
 import androidx.media3.transformer.ExperimentalFrameExtractor.Frame;
 import androidx.test.core.app.ApplicationProvider;
@@ -113,7 +111,7 @@ public class FrameExtractorTest {
                 .getDecoderCounters()
                 .get(TIMEOUT_SECONDS, SECONDS)
                 .renderedOutputBufferCount)
-        .isAtLeast(4);
+        .isEqualTo(2);
   }
 
   @Test
@@ -141,7 +139,7 @@ public class FrameExtractorTest {
                 .getDecoderCounters()
                 .get(TIMEOUT_SECONDS, SECONDS)
                 .renderedOutputBufferCount)
-        .isAtLeast(4);
+        .isEqualTo(2);
   }
 
   @Test
@@ -170,7 +168,7 @@ public class FrameExtractorTest {
                 .getDecoderCounters()
                 .get(TIMEOUT_SECONDS, SECONDS)
                 .renderedOutputBufferCount)
-        .isAtLeast(3);
+        .isEqualTo(2);
   }
 
   @Test
@@ -202,14 +200,12 @@ public class FrameExtractorTest {
       assertBitmapsAreSimilar(expectedBitmap, frame.bitmap, PSNR_THRESHOLD);
       assertThat(frame.presentationTimeMs).isEqualTo(expectedFramePositionsMs.get(i));
     }
-    // TODO: b/350498258 - some decoders break right after extracting all the frames for this test.
+    assertThat(
-    // Fix and remove this hack.
+            frameExtractor
-    @Nullable
+                .getDecoderCounters()
-    DecoderCounters decoderCounters =
+                .get(TIMEOUT_SECONDS, SECONDS)
-        frameExtractor.getDecoderCounters().get(TIMEOUT_SECONDS, SECONDS);
+                .renderedOutputBufferCount)
-    if (decoderCounters != null) {
+        .isEqualTo(3);
-      assertThat(decoderCounters.renderedOutputBufferCount).isAtLeast(7);
-    }
   }
 
   @Test
@@ -237,7 +233,7 @@ public class FrameExtractorTest {
                 .getDecoderCounters()
                 .get(TIMEOUT_SECONDS, SECONDS)
                 .renderedOutputBufferCount)
-        .isAtLeast(10);
+        .isEqualTo(6);
   }
 
   @Test
@@ -269,7 +265,7 @@ public class FrameExtractorTest {
                 .getDecoderCounters()
                 .get(TIMEOUT_SECONDS, SECONDS)
                 .renderedOutputBufferCount)
-        .isAtLeast(8);
+        .isEqualTo(6);
   }
 
   @Test
@@ -329,7 +325,7 @@ public class FrameExtractorTest {
                 .getDecoderCounters()
                 .get(TIMEOUT_SECONDS, SECONDS)
                 .renderedOutputBufferCount)
-        .isAtLeast(1);
+        .isEqualTo(1);
   }
 
   @Test

libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java

@@ -21,6 +21,7 @@ import static androidx.media3.common.Player.DISCONTINUITY_REASON_SEEK;
 import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.common.util.Assertions.checkState;
 import static androidx.media3.common.util.Util.usToMs;
+import static androidx.media3.exoplayer.mediacodec.MediaCodecSelector.DEFAULT;
 import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
 
 import android.content.Context;
@@ -32,6 +33,7 @@ import android.os.Looper;
 import androidx.annotation.Nullable;
 import androidx.annotation.VisibleForTesting;
 import androidx.media3.common.Effect;
+import androidx.media3.common.Format;
 import androidx.media3.common.GlObjectsProvider;
 import androidx.media3.common.GlTextureInfo;
 import androidx.media3.common.MediaItem;
@@ -40,14 +42,20 @@ import androidx.media3.common.Player;
 import androidx.media3.common.util.ConditionVariable;
 import androidx.media3.common.util.GlUtil;
 import androidx.media3.common.util.NullableType;
+import androidx.media3.common.util.Util;
 import androidx.media3.effect.GlEffect;
 import androidx.media3.effect.GlShaderProgram;
 import androidx.media3.effect.MatrixTransformation;
 import androidx.media3.effect.PassthroughShaderProgram;
 import androidx.media3.exoplayer.DecoderCounters;
+import androidx.media3.exoplayer.ExoPlaybackException;
 import androidx.media3.exoplayer.ExoPlayer;
+import androidx.media3.exoplayer.Renderer;
 import androidx.media3.exoplayer.SeekParameters;
 import androidx.media3.exoplayer.analytics.AnalyticsListener;
+import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter;
+import androidx.media3.exoplayer.video.MediaCodecVideoRenderer;
+import androidx.media3.exoplayer.video.VideoRendererEventListener;
 import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.FutureCallback;
 import com.google.common.util.concurrent.Futures;
@@ -159,7 +167,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   // TODO: b/350498258 - Support changing the MediaItem.
   public ExperimentalFrameExtractor(
       Context context, Configuration configuration, MediaItem mediaItem, List<Effect> effects) {
-    player = new ExoPlayer.Builder(context).setSeekParameters(configuration.seekParameters).build();
+    player =
+        new ExoPlayer.Builder(
+                context,
+                /* renderersFactory= */ (eventHandler,
+                    videoRendererEventListener,
+                    audioRendererEventListener,
+                    textRendererOutput,
+                    metadataRendererOutput) ->
+                    new Renderer[] {
+                      new FrameExtractorRenderer(context, videoRendererEventListener)
+                    })
+            .setSeekParameters(configuration.seekParameters)
+            .build();
     playerApplicationThreadHandler = new Handler(player.getApplicationLooper());
     lastRequestedFrameFuture = SettableFuture.create();
     // TODO: b/350498258 - Extracting the first frame is a workaround for ExoPlayer.setVideoEffects
@@ -354,4 +374,76 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       getInputListener().onInputFrameProcessed(inputTexture);
     }
   }
+
+  /** A custom MediaCodecVideoRenderer that renders only one frame per position reset. */
+  private static final class FrameExtractorRenderer extends MediaCodecVideoRenderer {
+
+    private boolean frameRenderedSinceLastReset;
+
+    public FrameExtractorRenderer(
+        Context context, VideoRendererEventListener videoRendererEventListener) {
+      super(
+          context,
+          /* mediaCodecSelector= */ DEFAULT,
+          /* allowedJoiningTimeMs= */ 0,
+          Util.createHandlerForCurrentOrMainLooper(),
+          videoRendererEventListener,
+          /* maxDroppedFramesToNotify= */ 0);
+    }
+
+    @Override
+    public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
+      if (!frameRenderedSinceLastReset) {
+        super.render(positionUs, elapsedRealtimeUs);
+      }
+    }
+
+    @Override
+    protected boolean processOutputBuffer(
+        long positionUs,
+        long elapsedRealtimeUs,
+        @Nullable MediaCodecAdapter codec,
+        @Nullable ByteBuffer buffer,
+        int bufferIndex,
+        int bufferFlags,
+        int sampleCount,
+        long bufferPresentationTimeUs,
+        boolean isDecodeOnlyBuffer,
+        boolean isLastBuffer,
+        Format format)
+        throws ExoPlaybackException {
+      if (frameRenderedSinceLastReset) {
+        return false;
+      }
+      return super.processOutputBuffer(
+          positionUs,
+          elapsedRealtimeUs,
+          codec,
+          buffer,
+          bufferIndex,
+          bufferFlags,
+          sampleCount,
+          bufferPresentationTimeUs,
+          isDecodeOnlyBuffer,
+          isLastBuffer,
+          format);
+    }
+
+    @Override
+    protected void renderOutputBufferV21(
+        MediaCodecAdapter codec, int index, long presentationTimeUs, long releaseTimeNs) {
+      if (frameRenderedSinceLastReset) {
+        // Do not skip this buffer to prevent the decoder from making more progress.
+        return;
+      }
+      frameRenderedSinceLastReset = true;
+      super.renderOutputBufferV21(codec, index, presentationTimeUs, releaseTimeNs);
+    }
+
+    @Override
+    protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException {
+      frameRenderedSinceLastReset = false;
+      super.onPositionReset(positionUs, joining);
+    }
+  }
 }