diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/FrameExtractorTest/internal_emulator_transformer_output_180_rotated_0.000.png b/libraries/test_data/src/test/assets/test-generated-goldens/FrameExtractorTest/internal_emulator_transformer_output_180_rotated_0.000.png new file mode 100644 index 0000000000..ce5c8c235d Binary files /dev/null and b/libraries/test_data/src/test/assets/test-generated-goldens/FrameExtractorTest/internal_emulator_transformer_output_180_rotated_0.000.png differ diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java index f19ac30391..2c0007fc32 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameExtractorTest.java @@ -20,6 +20,7 @@ import static androidx.media3.exoplayer.SeekParameters.CLOSEST_SYNC; import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; import static androidx.media3.test.utils.TestUtil.assertBitmapsAreSimilar; +import static androidx.media3.transformer.AndroidTestUtil.MP4_TRIM_OPTIMIZATION_270; import static com.google.common.truth.Truth.assertThat; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static java.util.concurrent.TimeUnit.SECONDS; @@ -341,4 +342,32 @@ public class FrameExtractorTest { instrumentation.runOnMainSync(frameExtractor::release); frameExtractor = null; } + + @Test + public void extractFrame_oneFrameRotated_returnsFrameInCorrectOrientation() throws Exception { + frameExtractor = + new ExperimentalFrameExtractor( + context, + new ExperimentalFrameExtractor.Configuration.Builder().build(), + MediaItem.fromUri(MP4_TRIM_OPTIMIZATION_270.uri), + /* effects= */ ImmutableList.of()); + + ListenableFuture frameFuture = frameExtractor.getFrame(/* positionMs= */ 0); + Frame frame = frameFuture.get(TIMEOUT_SECONDS, SECONDS); + Bitmap actualBitmap = frame.bitmap; + Bitmap expectedBitmap = + readBitmap( + /* assetString= */ GOLDEN_ASSET_FOLDER_PATH + + "internal_emulator_transformer_output_180_rotated_0.000.png"); + maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual", actualBitmap, /* path= */ null); + + assertThat(frame.presentationTimeMs).isEqualTo(0); + assertBitmapsAreSimilar(expectedBitmap, actualBitmap, PSNR_THRESHOLD); + assertThat( + frameExtractor + .getDecoderCounters() + .get(TIMEOUT_SECONDS, SECONDS) + .renderedOutputBufferCount) + .isEqualTo(1); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java index 9976b047e2..f6e3bf969b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractor.java @@ -47,9 +47,12 @@ import androidx.media3.effect.GlEffect; import androidx.media3.effect.GlShaderProgram; import androidx.media3.effect.MatrixTransformation; import androidx.media3.effect.PassthroughShaderProgram; +import androidx.media3.effect.ScaleAndRotateTransformation; import androidx.media3.exoplayer.DecoderCounters; +import androidx.media3.exoplayer.DecoderReuseEvaluation; import androidx.media3.exoplayer.ExoPlaybackException; import androidx.media3.exoplayer.ExoPlayer; +import androidx.media3.exoplayer.FormatHolder; import androidx.media3.exoplayer.Renderer; import androidx.media3.exoplayer.SeekParameters; import androidx.media3.exoplayer.analytics.AnalyticsListener; @@ -379,6 +382,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final class FrameExtractorRenderer extends MediaCodecVideoRenderer { private boolean frameRenderedSinceLastReset; + private List effectsFromPlayer; + private @MonotonicNonNull Effect rotation; public FrameExtractorRenderer( Context context, VideoRendererEventListener videoRendererEventListener) { @@ -389,6 +394,43 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Util.createHandlerForCurrentOrMainLooper(), videoRendererEventListener, /* maxDroppedFramesToNotify= */ 0); + effectsFromPlayer = ImmutableList.of(); + } + + @Override + public void setVideoEffects(List effects) { + effectsFromPlayer = effects; + setEffectsWithRotation(); + } + + @Override + @Nullable + protected DecoderReuseEvaluation onInputFormatChanged(FormatHolder formatHolder) + throws ExoPlaybackException { + if (formatHolder.format != null) { + Format format = formatHolder.format; + if (format.rotationDegrees != 0) { + // Some decoders do not apply rotation. It's no extra cost to rotate with a GL matrix + // transformation effect instead. + // https://developer.android.com/reference/android/media/MediaCodec#transformations-when-rendering-onto-surface + rotation = + new ScaleAndRotateTransformation.Builder() + .setRotationDegrees(360 - format.rotationDegrees) + .build(); + setEffectsWithRotation(); + formatHolder.format = format.buildUpon().setRotationDegrees(0).build(); + } + } + return super.onInputFormatChanged(formatHolder); + } + + private void setEffectsWithRotation() { + ImmutableList.Builder effectBuilder = new ImmutableList.Builder<>(); + if (rotation != null) { + effectBuilder.add(rotation); + } + effectBuilder.addAll(effectsFromPlayer); + super.setVideoEffects(effectBuilder.build()); } @Override