Encoders commonly support higher maximum widths than maximum heights. This may rotate the + * decoded frame before encoding, so the encoded frame's width >= height, and set {@link + * Format#rotationDegrees} to ensure the frame is displayed in the correct orientation. + */ +/* package */ class EncoderCompatibilityFrameProcessor implements GlFrameProcessor { + // TODO(b/218488308): Allow reconfiguration of the output size, as encoders may not support the + // requested output resolution. + + static { + GlUtil.glAssertionsEnabled = true; + } + + private int outputRotationDegrees; + private @MonotonicNonNull ScaleToFitFrameProcessor rotateFrameProcessor; + + /** Creates a new instance. */ + /* package */ EncoderCompatibilityFrameProcessor() { + + outputRotationDegrees = C.LENGTH_UNSET; + } + + @Override + public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) + throws IOException { + configureOutputSizeAndRotation(inputWidth, inputHeight); + rotateFrameProcessor = + new ScaleToFitFrameProcessor.Builder().setRotationDegrees(outputRotationDegrees).build(); + rotateFrameProcessor.initialize(context, inputTexId, inputWidth, inputHeight); + } + + @Override + public Size getOutputSize() { + return checkStateNotNull(rotateFrameProcessor).getOutputSize(); + } + + /** + * Returns {@link Format#rotationDegrees} for the output frame. + * + *
Return values may be {@code 0} or {@code 90} degrees. + * + *
The frame processor must be {@linkplain GlFrameProcessor#initialize(Context, int, int, int) + * initialized}. + */ + public int getOutputRotationDegrees() { + checkState( + outputRotationDegrees != C.LENGTH_UNSET, + "configureOutputSizeAndTransformationMatrix must be called before" + + " getOutputRotationDegrees"); + return outputRotationDegrees; + } + + @Override + public void drawFrame(long presentationTimeUs) { + checkStateNotNull(rotateFrameProcessor).drawFrame(presentationTimeUs); + } + + @Override + public void release() { + if (rotateFrameProcessor != null) { + rotateFrameProcessor.release(); + } + } + + @VisibleForTesting // Allows robolectric testing of output size calculation without OpenGL. + /* package */ void configureOutputSizeAndRotation(int inputWidth, int inputHeight) { + checkArgument(inputWidth > 0, "inputWidth must be positive"); + checkArgument(inputHeight > 0, "inputHeight must be positive"); + + outputRotationDegrees = (inputHeight > inputWidth) ? 90 : 0; + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java index 16dd161972..58e669371d 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java @@ -27,7 +27,6 @@ import android.util.Size; import androidx.annotation.IntDef; import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; -import androidx.media3.common.Format; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.UnstableApi; import java.io.IOException; @@ -234,7 +233,6 @@ public final class PresentationFrameProcessor implements GlFrameProcessor { private final float requestedAspectRatio; private final @Layout int layout; - private int outputRotationDegrees; private int outputWidth; private int outputHeight; private @MonotonicNonNull Matrix transformationMatrix; @@ -259,7 +257,6 @@ public final class PresentationFrameProcessor implements GlFrameProcessor { outputWidth = C.LENGTH_UNSET; outputHeight = C.LENGTH_UNSET; - outputRotationDegrees = C.LENGTH_UNSET; transformationMatrix = new Matrix(); } @@ -279,22 +276,6 @@ public final class PresentationFrameProcessor implements GlFrameProcessor { return new Size(outputWidth, outputHeight); } - /** - * Returns {@link Format#rotationDegrees} for the output frame. - * - *
Return values may be {@code 0} or {@code 90} degrees. - * - *
The frame processor must be {@linkplain GlFrameProcessor#initialize(Context, int, int, int) - * initialized}. - */ - public int getOutputRotationDegrees() { - checkState( - outputRotationDegrees != C.LENGTH_UNSET, - "configureOutputSizeAndTransformationMatrix must be called before" - + " getOutputRotationDegrees"); - return outputRotationDegrees; - } - @Override public void drawFrame(long presentationTimeUs) { checkStateNotNull(advancedFrameProcessor).drawFrame(presentationTimeUs); @@ -331,20 +312,6 @@ public final class PresentationFrameProcessor implements GlFrameProcessor { outputWidth = Math.round((float) requestedHeightPixels * outputWidth / outputHeight); outputHeight = requestedHeightPixels; } - - // Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded - // frame before encoding, so the encoded frame's width >= height, and set - // outputRotationDegrees to ensure the frame is displayed in the correct orientation. - if (outputHeight > outputWidth) { - outputRotationDegrees = 90; - // TODO(b/201293185): Put postRotate in a later GlFrameProcessor. - transformationMatrix.postRotate(outputRotationDegrees); - int swap = outputWidth; - outputWidth = outputHeight; - outputHeight = swap; - } else { - outputRotationDegrees = 0; - } } @RequiresNonNull("transformationMatrix") diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 3f2b492a47..6789d9e164 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -70,7 +70,8 @@ import org.checkerframework.dataflow.qual.Pure; int decodedHeight = (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; - // TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset. + // TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset, + // and don't create a PresentationFrameProcessor if resolution is unset. ScaleToFitFrameProcessor scaleToFitFrameProcessor = new ScaleToFitFrameProcessor.Builder() .setScale(transformationRequest.scaleX, transformationRequest.scaleY) @@ -80,6 +81,8 @@ import org.checkerframework.dataflow.qual.Pure; new PresentationFrameProcessor.Builder() .setResolution(transformationRequest.outputHeight) .build(); + EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor = + new EncoderCompatibilityFrameProcessor(); frameProcessorChain = FrameProcessorChain.create( context, @@ -90,10 +93,11 @@ import org.checkerframework.dataflow.qual.Pure; .addAll(frameProcessors) .add(scaleToFitFrameProcessor) .add(presentationFrameProcessor) + .add(encoderCompatibilityFrameProcessor) .build(), transformationRequest.enableHdrEditing); Size requestedEncoderSize = frameProcessorChain.getOutputSize(); - outputRotationDegrees = presentationFrameProcessor.getOutputRotationDegrees(); + outputRotationDegrees = encoderCompatibilityFrameProcessor.getOutputRotationDegrees(); Format requestedEncoderFormat = new Format.Builder() diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/EncoderCompatibilityFrameProcessorTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/EncoderCompatibilityFrameProcessorTest.java new file mode 100644 index 0000000000..4fb73c9024 --- /dev/null +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/EncoderCompatibilityFrameProcessorTest.java @@ -0,0 +1,73 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link EncoderCompatibilityFrameProcessor}. */ +@RunWith(AndroidJUnit4.class) +public final class EncoderCompatibilityFrameProcessorTest { + @Test + public void getOutputSize_noEditsLandscape_leavesOrientationUnchanged() { + int inputWidth = 200; + int inputHeight = 150; + EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor = + new EncoderCompatibilityFrameProcessor(); + + encoderCompatibilityFrameProcessor.configureOutputSizeAndRotation(inputWidth, inputHeight); + + assertThat(encoderCompatibilityFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); + } + + @Test + public void getOutputSize_noEditsSquare_leavesOrientationUnchanged() { + int inputWidth = 150; + int inputHeight = 150; + EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor = + new EncoderCompatibilityFrameProcessor(); + + encoderCompatibilityFrameProcessor.configureOutputSizeAndRotation(inputWidth, inputHeight); + + assertThat(encoderCompatibilityFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); + } + + @Test + public void getOutputSize_noEditsPortrait_flipsOrientation() { + int inputWidth = 150; + int inputHeight = 200; + EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor = + new EncoderCompatibilityFrameProcessor(); + + encoderCompatibilityFrameProcessor.configureOutputSizeAndRotation(inputWidth, inputHeight); + + assertThat(encoderCompatibilityFrameProcessor.getOutputRotationDegrees()).isEqualTo(90); + } + + @Test + public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() { + EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor = + new EncoderCompatibilityFrameProcessor(); + + // configureOutputSize not called before getOutputRotationDegrees. + assertThrows( + IllegalStateException.class, encoderCompatibilityFrameProcessor::getOutputRotationDegrees); + } +} diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java index e311820253..09b999241e 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java @@ -27,13 +27,13 @@ import org.junit.runner.RunWith; /** * Unit tests for {@link PresentationFrameProcessor}. * - *
See {@code AdvancedFrameProcessorPixelTest} for pixel tests testing {@link - * AdvancedFrameProcessor} given a transformation matrix. + *
See {@code PresentationFrameProcessorPixelTest} for pixel tests testing {@link + * PresentationFrameProcessor}. */ @RunWith(AndroidJUnit4.class) public final class PresentationFrameProcessorTest { @Test - public void getOutputSize_noEditsLandscape_leavesFramesUnchanged() { + public void getOutputSize_noEdits_leavesFramesUnchanged() { int inputWidth = 200; int inputHeight = 150; PresentationFrameProcessor presentationFrameProcessor = @@ -42,41 +42,10 @@ public final class PresentationFrameProcessorTest { presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight); Size outputSize = presentationFrameProcessor.getOutputSize(); - assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); assertThat(outputSize.getWidth()).isEqualTo(inputWidth); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } - @Test - public void getOutputSize_noEditsSquare_leavesFramesUnchanged() { - int inputWidth = 150; - int inputHeight = 150; - PresentationFrameProcessor presentationFrameProcessor = - new PresentationFrameProcessor.Builder().build(); - - presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight); - Size outputSize = presentationFrameProcessor.getOutputSize(); - - assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); - assertThat(outputSize.getWidth()).isEqualTo(inputWidth); - assertThat(outputSize.getHeight()).isEqualTo(inputHeight); - } - - @Test - public void getOutputSize_noEditsPortrait_flipsOrientation() { - int inputWidth = 150; - int inputHeight = 200; - PresentationFrameProcessor presentationFrameProcessor = - new PresentationFrameProcessor.Builder().build(); - - presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight); - Size outputSize = presentationFrameProcessor.getOutputSize(); - - assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(90); - assertThat(outputSize.getWidth()).isEqualTo(inputHeight); - assertThat(outputSize.getHeight()).isEqualTo(inputWidth); - } - @Test public void getOutputSize_setResolution_changesDimensions() { int inputWidth = 200; @@ -88,7 +57,6 @@ public final class PresentationFrameProcessorTest { presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight); Size outputSize = presentationFrameProcessor.getOutputSize(); - assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); assertThat(outputSize.getWidth()).isEqualTo(requestedHeight * inputWidth / inputHeight); assertThat(outputSize.getHeight()).isEqualTo(requestedHeight); } @@ -107,7 +75,6 @@ public final class PresentationFrameProcessorTest { presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight); Size outputSize = presentationFrameProcessor.getOutputSize(); - assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC); int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC); assertThat(outputSize.getWidth()).isEqualTo(expectedPostCropWidth); @@ -132,7 +99,6 @@ public final class PresentationFrameProcessorTest { presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight); Size outputSize = presentationFrameProcessor.getOutputSize(); - assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC); int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC); assertThat(outputSize.getWidth()) @@ -159,7 +125,6 @@ public final class PresentationFrameProcessorTest { presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight); Size outputSize = presentationFrameProcessor.getOutputSize(); - assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC); int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC); assertThat(outputSize.getWidth()) @@ -181,7 +146,6 @@ public final class PresentationFrameProcessorTest { presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight); Size outputSize = presentationFrameProcessor.getOutputSize(); - assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); assertThat(outputSize.getWidth()).isEqualTo(Math.round(aspectRatio * inputHeight)); assertThat(outputSize.getHeight()).isEqualTo(inputHeight); } @@ -201,7 +165,6 @@ public final class PresentationFrameProcessorTest { presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight); Size outputSize = presentationFrameProcessor.getOutputSize(); - assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); assertThat(outputSize.getWidth()).isEqualTo(Math.round(aspectRatio * requestedHeight)); assertThat(outputSize.getHeight()).isEqualTo(requestedHeight); } @@ -231,13 +194,4 @@ public final class PresentationFrameProcessorTest { presentationFrameProcessor.setAspectRatio( /* aspectRatio= */ 2f, PresentationFrameProcessor.LAYOUT_SCALE_TO_FIT)); } - - @Test - public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() { - PresentationFrameProcessor presentationFrameProcessor = - new PresentationFrameProcessor.Builder().build(); - - // configureOutputSize not called before getOutputRotationDegrees. - assertThrows(IllegalStateException.class, presentationFrameProcessor::getOutputRotationDegrees); - } }