diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/scale_wide.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/scale_wide.png new file mode 100644 index 0000000000..67ac8a7583 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/scale_wide.png differ diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java index 9caf06c59f..72de79d5ec 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java @@ -53,6 +53,8 @@ import org.junit.runner.RunWith; public final class FrameProcessorChainPixelTest { public static final String ORIGINAL_PNG_ASSET_PATH = "media/bitmap/sample_mp4_first_frame/original.png"; + public static final String SCALE_WIDE_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/scale_wide.png"; public static final String TRANSLATE_RIGHT_PNG_ASSET_PATH = "media/bitmap/sample_mp4_first_frame/translate_right.png"; public static final String ROTATE_THEN_TRANSLATE_PNG_ASSET_PATH = @@ -74,7 +76,7 @@ public final class FrameProcessorChainPixelTest { */ private static final int FRAME_PROCESSING_WAIT_MS = 5000; /** The ratio of width over height, for each pixel in a frame. */ - private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1; + private static final float DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO = 1; private @MonotonicNonNull FrameProcessorChain frameProcessorChain; private @MonotonicNonNull ImageReader outputImageReader; @@ -90,7 +92,7 @@ public final class FrameProcessorChainPixelTest { @Test public void processData_noEdits_producesExpectedOutput() throws Exception { String testId = "processData_noEdits"; - setUpAndPrepareFirstFrame(); + setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); Bitmap actualBitmap = processFirstFrameAndEnd(); @@ -104,6 +106,23 @@ public final class FrameProcessorChainPixelTest { assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } + @Test + public void processData_withPixelWidthHeightRatio_producesExpectedOutput() throws Exception { + String testId = "processData_withPixelWidthHeightRatio"; + setUpAndPrepareFirstFrame(/* pixelWidthHeightRatio= */ 2f); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_WIDE_PNG_ASSET_PATH); + + Bitmap actualBitmap = processFirstFrameAndEnd(); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap); + // TODO(b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + @Test public void processData_withAdvancedFrameProcessor_translateRight_producesExpectedOutput() throws Exception { @@ -111,7 +130,7 @@ public final class FrameProcessorChainPixelTest { Matrix translateRightMatrix = new Matrix(); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); GlFrameProcessor glFrameProcessor = new AdvancedFrameProcessor(translateRightMatrix); - setUpAndPrepareFirstFrame(glFrameProcessor); + setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, glFrameProcessor); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH); Bitmap actualBitmap = processFirstFrameAndEnd(); @@ -135,7 +154,8 @@ public final class FrameProcessorChainPixelTest { new AdvancedFrameProcessor(translateRightMatrix); GlFrameProcessor rotate45FrameProcessor = new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build(); - setUpAndPrepareFirstFrame(translateRightFrameProcessor, rotate45FrameProcessor); + setUpAndPrepareFirstFrame( + DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, translateRightFrameProcessor, rotate45FrameProcessor); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_THEN_ROTATE_PNG_ASSET_PATH); Bitmap actualBitmap = processFirstFrameAndEnd(); @@ -159,7 +179,8 @@ public final class FrameProcessorChainPixelTest { translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); GlFrameProcessor translateRightFrameProcessor = new AdvancedFrameProcessor(translateRightMatrix); - setUpAndPrepareFirstFrame(rotate45FrameProcessor, translateRightFrameProcessor); + setUpAndPrepareFirstFrame( + DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, rotate45FrameProcessor, translateRightFrameProcessor); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_THEN_TRANSLATE_PNG_ASSET_PATH); Bitmap actualBitmap = processFirstFrameAndEnd(); @@ -179,7 +200,7 @@ public final class FrameProcessorChainPixelTest { String testId = "processData_withPresentationFrameProcessor_setResolution"; GlFrameProcessor glFrameProcessor = new PresentationFrameProcessor.Builder().setResolution(480).build(); - setUpAndPrepareFirstFrame(glFrameProcessor); + setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, glFrameProcessor); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(REQUEST_OUTPUT_HEIGHT_PNG_ASSET_PATH); Bitmap actualBitmap = processFirstFrameAndEnd(); @@ -199,7 +220,7 @@ public final class FrameProcessorChainPixelTest { String testId = "processData_withScaleToFitFrameProcessor_rotate45"; GlFrameProcessor glFrameProcessor = new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build(); - setUpAndPrepareFirstFrame(glFrameProcessor); + setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, glFrameProcessor); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE45_SCALE_TO_FIT_PNG_ASSET_PATH); Bitmap actualBitmap = processFirstFrameAndEnd(); @@ -218,10 +239,12 @@ public final class FrameProcessorChainPixelTest { * infrastructure. The frame will be sent towards the {@link FrameProcessorChain}, and may be * accessed on the {@link FrameProcessorChain}'s output {@code outputImageReader}. * + * @param pixelWidthHeightRatio The ratio of width over height for each pixel. * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} that will apply changes * to the input frame. */ - private void setUpAndPrepareFirstFrame(GlFrameProcessor... frameProcessors) throws Exception { + private void setUpAndPrepareFirstFrame( + float pixelWidthHeightRatio, GlFrameProcessor... frameProcessors) throws Exception { // Set up the extractor to read the first video frame and get its format. MediaExtractor mediaExtractor = new MediaExtractor(); @Nullable MediaCodec mediaCodec = null; @@ -241,7 +264,7 @@ public final class FrameProcessorChainPixelTest { frameProcessorChain = FrameProcessorChain.create( context, - PIXEL_WIDTH_HEIGHT_RATIO, + pixelWidthHeightRatio, inputWidth, inputHeight, asList(frameProcessors), diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java index 0807c3e13a..976b300aef 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java @@ -17,7 +17,6 @@ package androidx.media3.transformer; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertThrows; import android.content.Context; import android.util.Size; @@ -36,46 +35,13 @@ import org.junit.runner.RunWith; public final class FrameProcessorChainTest { @Test - public void create_withSupportedPixelWidthHeightRatio_completesSuccessfully() - throws TransformationException { - Context context = getApplicationContext(); - - FrameProcessorChain.create( - context, - /* pixelWidthHeightRatio= */ 1, - /* inputWidth= */ 200, - /* inputHeight= */ 100, - /* frameProcessors= */ ImmutableList.of(), - /* enableExperimentalHdrEditing= */ false); - } - - @Test - public void create_withUnsupportedPixelWidthHeightRatio_throwsException() { - Context context = getApplicationContext(); - - TransformationException exception = - assertThrows( - TransformationException.class, - () -> - FrameProcessorChain.create( - context, - /* pixelWidthHeightRatio= */ 2, - /* inputWidth= */ 200, - /* inputHeight= */ 100, - /* frameProcessors= */ ImmutableList.of(), - /* enableExperimentalHdrEditing= */ false)); - - assertThat(exception).hasCauseThat().isInstanceOf(UnsupportedOperationException.class); - assertThat(exception).hasCauseThat().hasMessageThat().contains("pixelWidthHeightRatio"); - } - - @Test - public void getOutputSize_withoutFrameProcessors_returnsInputSize() - throws TransformationException { + public void getOutputSize_noOperation_returnsInputSize() throws Exception { Size inputSize = new Size(200, 100); FrameProcessorChain frameProcessorChain = createFrameProcessorChainWithFakeFrameProcessors( - inputSize, /* frameProcessorOutputSizes= */ ImmutableList.of()); + /* pixelWidthHeightRatio= */ 1f, + inputSize, + /* frameProcessorOutputSizes= */ ImmutableList.of()); Size outputSize = frameProcessorChain.getOutputSize(); @@ -83,13 +49,42 @@ public final class FrameProcessorChainTest { } @Test - public void getOutputSize_withOneFrameProcessor_returnsItsOutputSize() - throws TransformationException { + public void getOutputSize_withWidePixels_returnsWiderOutputSize() throws Exception { + Size inputSize = new Size(200, 100); + FrameProcessorChain frameProcessorChain = + createFrameProcessorChainWithFakeFrameProcessors( + /* pixelWidthHeightRatio= */ 2f, + inputSize, + /* frameProcessorOutputSizes= */ ImmutableList.of()); + + Size outputSize = frameProcessorChain.getOutputSize(); + + assertThat(outputSize).isEqualTo(new Size(400, 100)); + } + + @Test + public void getOutputSize_withTallPixels_returnsTallerOutputSize() throws Exception { + Size inputSize = new Size(200, 100); + FrameProcessorChain frameProcessorChain = + createFrameProcessorChainWithFakeFrameProcessors( + /* pixelWidthHeightRatio= */ .5f, + inputSize, + /* frameProcessorOutputSizes= */ ImmutableList.of()); + + Size outputSize = frameProcessorChain.getOutputSize(); + + assertThat(outputSize).isEqualTo(new Size(200, 200)); + } + + @Test + public void getOutputSize_withOneFrameProcessor_returnsItsOutputSize() throws Exception { Size inputSize = new Size(200, 100); Size frameProcessorOutputSize = new Size(300, 250); FrameProcessorChain frameProcessorChain = createFrameProcessorChainWithFakeFrameProcessors( - inputSize, /* frameProcessorOutputSizes= */ ImmutableList.of(frameProcessorOutputSize)); + /* pixelWidthHeightRatio= */ 1f, + inputSize, + /* frameProcessorOutputSizes= */ ImmutableList.of(frameProcessorOutputSize)); Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize(); @@ -97,14 +92,14 @@ public final class FrameProcessorChainTest { } @Test - public void getOutputSize_withThreeFrameProcessors_returnsLastOutputSize() - throws TransformationException { + public void getOutputSize_withThreeFrameProcessors_returnsLastOutputSize() throws Exception { Size inputSize = new Size(200, 100); Size outputSize1 = new Size(300, 250); Size outputSize2 = new Size(400, 244); Size outputSize3 = new Size(150, 160); FrameProcessorChain frameProcessorChain = createFrameProcessorChainWithFakeFrameProcessors( + /* pixelWidthHeightRatio= */ 1f, inputSize, /* frameProcessorOutputSizes= */ ImmutableList.of( outputSize1, outputSize2, outputSize3)); @@ -115,14 +110,15 @@ public final class FrameProcessorChainTest { } private static FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors( - Size inputSize, List frameProcessorOutputSizes) throws TransformationException { + float pixelWidthHeightRatio, Size inputSize, List frameProcessorOutputSizes) + throws TransformationException { ImmutableList.Builder frameProcessors = new ImmutableList.Builder<>(); for (Size element : frameProcessorOutputSizes) { frameProcessors.add(new FakeFrameProcessor(element)); } return FrameProcessorChain.create( getApplicationContext(), - /* pixelWidthHeightRatio= */ 1, + pixelWidthHeightRatio, inputSize.getWidth(), inputSize.getHeight(), frameProcessors.build(), diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java index 2f01fed83f..1bf814eec4 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java @@ -71,15 +71,15 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * Creates a new instance. * * @param context A {@link Context}. - * @param pixelWidthHeightRatio The ratio of width over height, for each pixel. + * @param pixelWidthHeightRatio The ratio of width over height for each pixel. Pixels are expanded + * by this ratio so that the output frame's pixels have a ratio of 1. * @param inputWidth The input frame width, in pixels. * @param inputHeight The input frame height, in pixels. * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame. * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. * @return A new instance. - * @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1, reading shader - * files fails, or an OpenGL error occurs while creating and configuring the OpenGL - * components. + * @throws TransformationException If reading shader files fails, or an OpenGL error occurs while + * creating and configuring the OpenGL components. */ public static FrameProcessorChain create( Context context, @@ -92,19 +92,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; checkArgument(inputWidth > 0, "inputWidth must be positive"); checkArgument(inputHeight > 0, "inputHeight must be positive"); - if (pixelWidthHeightRatio != 1.0f) { - // TODO(b/211782176): Consider implementing support for non-square pixels. - throw TransformationException.createForFrameProcessorChain( - new UnsupportedOperationException( - "Transformer's FrameProcessorChain currently does not support frame edits on" - + " non-square pixels. The pixelWidthHeightRatio is: " - + pixelWidthHeightRatio), - TransformationException.ERROR_CODE_GL_INIT_FAILED); - } - ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); - ExternalCopyFrameProcessor externalCopyFrameProcessor = - new ExternalCopyFrameProcessor(enableExperimentalHdrEditing); try { return singleThreadExecutorService @@ -112,12 +100,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; () -> createOpenGlObjectsAndFrameProcessorChain( context, + pixelWidthHeightRatio, inputWidth, inputHeight, frameProcessors, enableExperimentalHdrEditing, - singleThreadExecutorService, - externalCopyFrameProcessor)) + singleThreadExecutorService)) .get(); } catch (ExecutionException e) { throw TransformationException.createForFrameProcessorChain( @@ -138,16 +126,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @WorkerThread private static FrameProcessorChain createOpenGlObjectsAndFrameProcessorChain( Context context, + float pixelWidthHeightRatio, int inputWidth, int inputHeight, List frameProcessors, boolean enableExperimentalHdrEditing, - ExecutorService singleThreadExecutorService, - ExternalCopyFrameProcessor externalCopyFrameProcessor) + ExecutorService singleThreadExecutorService) throws IOException { checkState(Thread.currentThread().getName().equals(THREAD_NAME)); EGLDisplay eglDisplay = GlUtil.createEglDisplay(); + ExternalCopyFrameProcessor externalCopyFrameProcessor = + new ExternalCopyFrameProcessor(enableExperimentalHdrEditing); EGLContext eglContext = enableExperimentalHdrEditing ? GlUtil.createEglContextEs3Rgba1010102(eglDisplay) @@ -163,18 +153,22 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay); } + ImmutableList expandedFrameProcessors = + getExpandedFrameProcessors( + externalCopyFrameProcessor, pixelWidthHeightRatio, frameProcessors); + + // Initialize frame processors. int inputExternalTexId = GlUtil.createExternalTexture(); externalCopyFrameProcessor.initialize(context, inputExternalTexId, inputWidth, inputHeight); - int[] framebuffers = new int[frameProcessors.size()]; + int[] framebuffers = new int[expandedFrameProcessors.size() - 1]; Size inputSize = externalCopyFrameProcessor.getOutputSize(); - for (int i = 0; i < frameProcessors.size(); i++) { + for (int i = 1; i < expandedFrameProcessors.size(); i++) { int inputTexId = GlUtil.createTexture(inputSize.getWidth(), inputSize.getHeight()); - framebuffers[i] = GlUtil.createFboForTexture(inputTexId); - frameProcessors - .get(i) - .initialize(context, inputTexId, inputSize.getWidth(), inputSize.getHeight()); - inputSize = frameProcessors.get(i).getOutputSize(); + framebuffers[i - 1] = GlUtil.createFboForTexture(inputTexId); + GlFrameProcessor frameProcessor = expandedFrameProcessors.get(i); + frameProcessor.initialize(context, inputTexId, inputSize.getWidth(), inputSize.getHeight()); + inputSize = frameProcessor.getOutputSize(); } return new FrameProcessorChain( eglDisplay, @@ -182,13 +176,33 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; singleThreadExecutorService, inputExternalTexId, framebuffers, - new ImmutableList.Builder() - .add(externalCopyFrameProcessor) - .addAll(frameProcessors) - .build(), + expandedFrameProcessors, enableExperimentalHdrEditing); } + private static ImmutableList getExpandedFrameProcessors( + ExternalCopyFrameProcessor externalCopyFrameProcessor, + float pixelWidthHeightRatio, + List frameProcessors) { + ImmutableList.Builder frameProcessorListBuilder = + new ImmutableList.Builder().add(externalCopyFrameProcessor); + + // Scale to expand the frame to apply the pixelWidthHeightRatio. + if (pixelWidthHeightRatio > 1f) { + frameProcessorListBuilder.add( + new ScaleToFitFrameProcessor.Builder() + .setScale(/* scaleX= */ pixelWidthHeightRatio, /* scaleY= */ 1f) + .build()); + } else if (pixelWidthHeightRatio < 1f) { + frameProcessorListBuilder.add( + new ScaleToFitFrameProcessor.Builder() + .setScale(/* scaleX= */ 1f, /* scaleY= */ 1f / pixelWidthHeightRatio) + .build()); + } + frameProcessorListBuilder.addAll(frameProcessors); + return frameProcessorListBuilder.build(); + } + private static final String TAG = "FrameProcessorChain"; private static final String THREAD_NAME = "Transformer:FrameProcessorChain"; private static final long RELEASE_WAIT_TIME_MS = 100; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java index d8789fc382..b32bd877c5 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java @@ -121,6 +121,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) { return false; } + if (inputFormat.pixelWidthHeightRatio != 1f) { + return false; + } if (transformationRequest.rotationDegrees != 0f) { return false; }