loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FrameProcessor: Create EncoderCompatibilityFrameProcessor.

Browse Source 
Split rotationDegrees changes to EncoderCompatibilityFrameProcessor, a new
FrameProcessor.

This removes automatic rotationDegrees adjustments from Presentation, which
allows Presentation to be used for changes before the end of a
FrameProcessorChain pipeline.

PiperOrigin-RevId: 443387226
This commit is contained in:
huangdarwin 2022-04-21 16:38:53 +01:00 committed by Ian Baker
parent 16b0cee0b6
commit 3cfdfb41c6
5 changed files with 187 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
libraries/transformer/src/main/java/androidx/media3/transformer/EncoderCompatibilityFrameProcessor.java Normal file
View File

@ -0,0 +1,105 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.util.Size;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.util.GlUtil;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* Copies frames from a texture and applies {@link Format#rotationDegrees} for encoder
* compatibility, if needed.
*
* <p>Encoders commonly support higher maximum widths than maximum heights. This may rotate the
* decoded frame before encoding, so the encoded frame's width >= height, and set {@link
* Format#rotationDegrees} to ensure the frame is displayed in the correct orientation.
*/
/* package */ class EncoderCompatibilityFrameProcessor implements GlFrameProcessor {
// TODO(b/218488308): Allow reconfiguration of the output size, as encoders may not support the
// requested output resolution.
static {
GlUtil.glAssertionsEnabled = true;
}
private int outputRotationDegrees;
private @MonotonicNonNull ScaleToFitFrameProcessor rotateFrameProcessor;
/** Creates a new instance. */
/* package */ EncoderCompatibilityFrameProcessor() {
outputRotationDegrees = C.LENGTH_UNSET;
}
@Override
public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight)
throws IOException {
configureOutputSizeAndRotation(inputWidth, inputHeight);
rotateFrameProcessor =
new ScaleToFitFrameProcessor.Builder().setRotationDegrees(outputRotationDegrees).build();
rotateFrameProcessor.initialize(context, inputTexId, inputWidth, inputHeight);
}
@Override
public Size getOutputSize() {
return checkStateNotNull(rotateFrameProcessor).getOutputSize();
}
/**
* Returns {@link Format#rotationDegrees} for the output frame.
*
* <p>Return values may be {@code 0} or {@code 90} degrees.
*
* <p>The frame processor must be {@linkplain GlFrameProcessor#initialize(Context, int, int, int)
* initialized}.
*/
public int getOutputRotationDegrees() {
checkState(
outputRotationDegrees != C.LENGTH_UNSET,
"configureOutputSizeAndTransformationMatrix must be called before"
+ " getOutputRotationDegrees");
return outputRotationDegrees;
}
@Override
public void drawFrame(long presentationTimeUs) {
checkStateNotNull(rotateFrameProcessor).drawFrame(presentationTimeUs);
}
@Override
public void release() {
if (rotateFrameProcessor != null) {
rotateFrameProcessor.release();
}
}
@VisibleForTesting // Allows robolectric testing of output size calculation without OpenGL.
/* package */ void configureOutputSizeAndRotation(int inputWidth, int inputHeight) {
checkArgument(inputWidth > 0, "inputWidth must be positive");
checkArgument(inputHeight > 0, "inputHeight must be positive");
outputRotationDegrees = (inputHeight > inputWidth) ? 90 : 0;
}
}

33
libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java
View File

@ -27,7 +27,6 @@ import android.util.Size;
import androidx.annotation.IntDef;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
import java.io.IOException;
@ -234,7 +233,6 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
private final float requestedAspectRatio;
private final @Layout int layout;
private int outputRotationDegrees;
private int outputWidth;
private int outputHeight;
private @MonotonicNonNull Matrix transformationMatrix;
@ -259,7 +257,6 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
outputWidth = C.LENGTH_UNSET;
outputHeight = C.LENGTH_UNSET;
outputRotationDegrees = C.LENGTH_UNSET;
transformationMatrix = new Matrix();
}
@ -279,22 +276,6 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
return new Size(outputWidth, outputHeight);
}
/**
* Returns {@link Format#rotationDegrees} for the output frame.
*
* <p>Return values may be {@code 0} or {@code 90} degrees.
*
* <p>The frame processor must be {@linkplain GlFrameProcessor#initialize(Context, int, int, int)
* initialized}.
*/
public int getOutputRotationDegrees() {
checkState(
outputRotationDegrees != C.LENGTH_UNSET,
"configureOutputSizeAndTransformationMatrix must be called before"
+ " getOutputRotationDegrees");
return outputRotationDegrees;
}
@Override
public void drawFrame(long presentationTimeUs) {
checkStateNotNull(advancedFrameProcessor).drawFrame(presentationTimeUs);
@ -331,20 +312,6 @@ public final class PresentationFrameProcessor implements GlFrameProcessor {
outputWidth = Math.round((float) requestedHeightPixels * outputWidth / outputHeight);
outputHeight = requestedHeightPixels;
}
// Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded
// frame before encoding, so the encoded frame's width >= height, and set
// outputRotationDegrees to ensure the frame is displayed in the correct orientation.
if (outputHeight > outputWidth) {
outputRotationDegrees = 90;
// TODO(b/201293185): Put postRotate in a later GlFrameProcessor.
transformationMatrix.postRotate(outputRotationDegrees);
int swap = outputWidth;
outputWidth = outputHeight;
outputHeight = swap;
} else {
outputRotationDegrees = 0;
}
}
@RequiresNonNull("transformationMatrix")

8
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -70,7 +70,8 @@ import org.checkerframework.dataflow.qual.Pure;
int decodedHeight =
(inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
// TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset.
// TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset,
// and don't create a PresentationFrameProcessor if resolution is unset.
ScaleToFitFrameProcessor scaleToFitFrameProcessor =
new ScaleToFitFrameProcessor.Builder()
.setScale(transformationRequest.scaleX, transformationRequest.scaleY)
@ -80,6 +81,8 @@ import org.checkerframework.dataflow.qual.Pure;
new PresentationFrameProcessor.Builder()
.setResolution(transformationRequest.outputHeight)
.build();
EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor =
new EncoderCompatibilityFrameProcessor();
frameProcessorChain =
FrameProcessorChain.create(
context,
@ -90,10 +93,11 @@ import org.checkerframework.dataflow.qual.Pure;
.addAll(frameProcessors)
.add(scaleToFitFrameProcessor)
.add(presentationFrameProcessor)
.add(encoderCompatibilityFrameProcessor)
.build(),
transformationRequest.enableHdrEditing);
Size requestedEncoderSize = frameProcessorChain.getOutputSize();
outputRotationDegrees = presentationFrameProcessor.getOutputRotationDegrees();
outputRotationDegrees = encoderCompatibilityFrameProcessor.getOutputRotationDegrees();
Format requestedEncoderFormat =
new Format.Builder()

73
libraries/transformer/src/test/java/androidx/media3/transformer/EncoderCompatibilityFrameProcessorTest.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Test;
import org.junit.runner.RunWith;
/** Unit tests for {@link EncoderCompatibilityFrameProcessor}. */
@RunWith(AndroidJUnit4.class)
public final class EncoderCompatibilityFrameProcessorTest {
@Test
public void getOutputSize_noEditsLandscape_leavesOrientationUnchanged() {
int inputWidth = 200;
int inputHeight = 150;
EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor =
new EncoderCompatibilityFrameProcessor();
encoderCompatibilityFrameProcessor.configureOutputSizeAndRotation(inputWidth, inputHeight);
assertThat(encoderCompatibilityFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
}
@Test
public void getOutputSize_noEditsSquare_leavesOrientationUnchanged() {
int inputWidth = 150;
int inputHeight = 150;
EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor =
new EncoderCompatibilityFrameProcessor();
encoderCompatibilityFrameProcessor.configureOutputSizeAndRotation(inputWidth, inputHeight);
assertThat(encoderCompatibilityFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
}
@Test
public void getOutputSize_noEditsPortrait_flipsOrientation() {
int inputWidth = 150;
int inputHeight = 200;
EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor =
new EncoderCompatibilityFrameProcessor();
encoderCompatibilityFrameProcessor.configureOutputSizeAndRotation(inputWidth, inputHeight);
assertThat(encoderCompatibilityFrameProcessor.getOutputRotationDegrees()).isEqualTo(90);
}
@Test
public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() {
EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor =
new EncoderCompatibilityFrameProcessor();
// configureOutputSize not called before getOutputRotationDegrees.
assertThrows(
IllegalStateException.class, encoderCompatibilityFrameProcessor::getOutputRotationDegrees);
}
}

52
libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java
View File

@ -27,13 +27,13 @@ import org.junit.runner.RunWith;
/**
* Unit tests for {@link PresentationFrameProcessor}.
*
* <p>See {@code AdvancedFrameProcessorPixelTest} for pixel tests testing {@link
* AdvancedFrameProcessor} given a transformation matrix.
* <p>See {@code PresentationFrameProcessorPixelTest} for pixel tests testing {@link
* PresentationFrameProcessor}.
*/
@RunWith(AndroidJUnit4.class)
public final class PresentationFrameProcessorTest {
@Test
public void getOutputSize_noEditsLandscape_leavesFramesUnchanged() {
public void getOutputSize_noEdits_leavesFramesUnchanged() {
int inputWidth = 200;
int inputHeight = 150;
PresentationFrameProcessor presentationFrameProcessor =
@ -42,41 +42,10 @@ public final class PresentationFrameProcessorTest {
presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
}
@Test
public void getOutputSize_noEditsSquare_leavesFramesUnchanged() {
int inputWidth = 150;
int inputHeight = 150;
PresentationFrameProcessor presentationFrameProcessor =
new PresentationFrameProcessor.Builder().build();
presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(outputSize.getWidth()).isEqualTo(inputWidth);
assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
}
@Test
public void getOutputSize_noEditsPortrait_flipsOrientation() {
int inputWidth = 150;
int inputHeight = 200;
PresentationFrameProcessor presentationFrameProcessor =
new PresentationFrameProcessor.Builder().build();
presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(90);
assertThat(outputSize.getWidth()).isEqualTo(inputHeight);
assertThat(outputSize.getHeight()).isEqualTo(inputWidth);
}
@Test
public void getOutputSize_setResolution_changesDimensions() {
int inputWidth = 200;
@ -88,7 +57,6 @@ public final class PresentationFrameProcessorTest {
presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(outputSize.getWidth()).isEqualTo(requestedHeight * inputWidth / inputHeight);
assertThat(outputSize.getHeight()).isEqualTo(requestedHeight);
}
@ -107,7 +75,6 @@ public final class PresentationFrameProcessorTest {
presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC);
int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC);
assertThat(outputSize.getWidth()).isEqualTo(expectedPostCropWidth);
@ -132,7 +99,6 @@ public final class PresentationFrameProcessorTest {
presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC);
int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC);
assertThat(outputSize.getWidth())
@ -159,7 +125,6 @@ public final class PresentationFrameProcessorTest {
presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
int expectedPostCropWidth = Math.round(inputWidth * (right - left) / GlUtil.LENGTH_NDC);
int expectedPostCropHeight = Math.round(inputHeight * (top - bottom) / GlUtil.LENGTH_NDC);
assertThat(outputSize.getWidth())
@ -181,7 +146,6 @@ public final class PresentationFrameProcessorTest {
presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(outputSize.getWidth()).isEqualTo(Math.round(aspectRatio * inputHeight));
assertThat(outputSize.getHeight()).isEqualTo(inputHeight);
}
@ -201,7 +165,6 @@ public final class PresentationFrameProcessorTest {
presentationFrameProcessor.configureOutputSizeAndTransformationMatrix(inputWidth, inputHeight);
Size outputSize = presentationFrameProcessor.getOutputSize();
assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(outputSize.getWidth()).isEqualTo(Math.round(aspectRatio * requestedHeight));
assertThat(outputSize.getHeight()).isEqualTo(requestedHeight);
}
@ -231,13 +194,4 @@ public final class PresentationFrameProcessorTest {
presentationFrameProcessor.setAspectRatio(
/* aspectRatio= */ 2f, PresentationFrameProcessor.LAYOUT_SCALE_TO_FIT));
}
@Test
public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() {
PresentationFrameProcessor presentationFrameProcessor =
new PresentationFrameProcessor.Builder().build();
// configureOutputSize not called before getOutputRotationDegrees.
assertThrows(IllegalStateException.class, presentationFrameProcessor::getOutputRotationDegrees);
}
}