loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-05-07 15:40:37 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Transformer GL: Split out ScaleToFit and Advanced GlFrameProcessors

Browse Source 
* Move auto-adjustments for transformation matrices from the
  VideoTranscodingSamplePipeline constructor to the new
  ScaleToFitFrameProcessor.
* Add GlFrameProcessor#getOutputDimensions() to allow for GlFrameProcessors with
  different input and output dimensions. This is a prerequisite for
  Presentation.
* Tested with unit tests (and manually just in case).
* A follow up CL will implement change the FrameProcessor input to be scale and
  rotate values as requested by the user. This was kept out of this CL to
  reduce CL review size. Presentation will also be implemented in a follow up
  CL.

PiperOrigin-RevId: 434774854
This commit is contained in:
huangdarwin 2022-03-15 16:36:52 +00:00 committed by Ian Baker
parent 8763843c84
commit 371c5c1b2e
15 changed files with 652 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java
View File

@ -221,6 +221,30 @@ public final class GlUtil {
}
}
/**
* Asserts that dimensions are valid for a texture.
*
* @param width The width for a texture.
* @param height The height for a texture.
* @throws GlException If the texture width or height is invalid.
*/
public static void assertValidTextureDimensions(int width, int height) {
// TODO(b/201293185): Consider handling adjustments for resolutions > GL_MAX_TEXTURE_SIZE
// (ex. downscaling appropriately) in a FrameProcessor instead of asserting incorrect values.
// For valid GL resolutions, see:
// https://www.khronos.org/registry/OpenGL-Refpages/es2.0/xhtml/glTexImage2D.xml
int[] maxTextureSizeBuffer = new int[1];
GLES20.glGetIntegerv(GLES20.GL_MAX_TEXTURE_SIZE, maxTextureSizeBuffer, 0);
int maxTextureSize = maxTextureSizeBuffer[0];
if (width < 0 || height < 0) {
throwGlException("width or height is less than 0");
}
if (width > maxTextureSize || height > maxTextureSize) {
throwGlException("width or height is greater than GL_MAX_TEXTURE_SIZE " + maxTextureSize);
}
}
/**
* Makes the specified {@code eglSurface} the render target, using a viewport of {@code width} by
* {@code height} pixels.

42
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformationFrameProcessorTest.java → library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessorPixelTest.java
View File

@ -39,7 +39,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
/**
* Pixel test for frame processing via {@link TransformationFrameProcessor}.
* Pixel test for frame processing via {@link AdvancedFrameProcessor}.
*
* <p>Expected images are taken from an emulator, so tests on different emulators or physical
* devices may fail. To test on other devices, please increase the {@link
@ -47,7 +47,7 @@ import org.junit.runner.RunWith;
* as recommended in {@link FrameEditorDataProcessingTest}.
*/
@RunWith(AndroidJUnit4.class)
public final class TransformationFrameProcessorTest {
public final class AdvancedFrameProcessorPixelTest {
static {
GlUtil.glAssertionsEnabled = true;
@ -55,7 +55,7 @@ public final class TransformationFrameProcessorTest {
private final EGLDisplay eglDisplay = GlUtil.createEglDisplay();
private final EGLContext eglContext = GlUtil.createEglContext(eglDisplay);
private @MonotonicNonNull GlFrameProcessor transformationFrameProcessor;
private @MonotonicNonNull GlFrameProcessor advancedFrameProcessor;
private int inputTexId;
private int outputTexId;
// TODO(b/214975934): Once the frame processors are allowed to have different input and output
@ -82,8 +82,8 @@ public final class TransformationFrameProcessorTest {
@After
public void release() {
if (transformationFrameProcessor != null) {
transformationFrameProcessor.release();
if (advancedFrameProcessor != null) {
advancedFrameProcessor.release();
}
GlUtil.destroyEglContext(eglDisplay, eglContext);
}
@ -92,12 +92,11 @@ public final class TransformationFrameProcessorTest {
public void updateProgramAndDraw_noEdits_producesExpectedOutput() throws Exception {
final String testId = "updateProgramAndDraw_noEdits";
Matrix identityMatrix = new Matrix();
transformationFrameProcessor =
new TransformationFrameProcessor(getApplicationContext(), identityMatrix);
transformationFrameProcessor.initialize(inputTexId);
advancedFrameProcessor = new AdvancedFrameProcessor(getApplicationContext(), identityMatrix);
advancedFrameProcessor.initialize(inputTexId);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING);
transformationFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
@ -115,13 +114,13 @@ public final class TransformationFrameProcessorTest {
final String testId = "updateProgramAndDraw_translateRight";
Matrix translateRightMatrix = new Matrix();
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
transformationFrameProcessor =
new TransformationFrameProcessor(getApplicationContext(), translateRightMatrix);
transformationFrameProcessor.initialize(inputTexId);
advancedFrameProcessor =
new AdvancedFrameProcessor(getApplicationContext(), translateRightMatrix);
advancedFrameProcessor.initialize(inputTexId);
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING);
transformationFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
@ -139,13 +138,12 @@ public final class TransformationFrameProcessorTest {
final String testId = "updateProgramAndDraw_scaleNarrow";
Matrix scaleNarrowMatrix = new Matrix();
scaleNarrowMatrix.postScale(.5f, 1.2f);
transformationFrameProcessor =
new TransformationFrameProcessor(getApplicationContext(), scaleNarrowMatrix);
transformationFrameProcessor.initialize(inputTexId);
advancedFrameProcessor = new AdvancedFrameProcessor(getApplicationContext(), scaleNarrowMatrix);
advancedFrameProcessor.initialize(inputTexId);
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING);
transformationFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);
@ -161,17 +159,13 @@ public final class TransformationFrameProcessorTest {
@Test
public void updateProgramAndDraw_rotate90_producesExpectedOutput() throws Exception {
final String testId = "updateProgramAndDraw_rotate90";
// TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline
// resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can
// test that rotation doesn't distort the image.
Matrix rotate90Matrix = new Matrix();
rotate90Matrix.postRotate(/* degrees= */ 90);
transformationFrameProcessor =
new TransformationFrameProcessor(getApplicationContext(), rotate90Matrix);
transformationFrameProcessor.initialize(inputTexId);
advancedFrameProcessor = new AdvancedFrameProcessor(getApplicationContext(), rotate90Matrix);
advancedFrameProcessor.initialize(inputTexId);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING);
transformationFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0);
Bitmap actualBitmap =
BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height);

4
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/BitmapTestUtil.java
View File

@ -55,6 +55,10 @@ public class BitmapTestUtil {
"media/bitmap/sample_mp4_first_frame_scale_narrow.png";
public static final String ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_rotate90.png";
public static final String REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_request_output_height.png";
public static final String ROTATE45_SCALE_TO_FIT_EXPECTED_OUTPUT_PNG_ASSET_STRING =
"media/bitmap/sample_mp4_first_frame_rotate_45_scale_to_fit.png";
/**
* Maximum allowed average pixel difference between the expected and actual edited images for the
* test to pass. The value is chosen so that differences in decoder behavior across emulator

107
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java
View File

@ -18,6 +18,8 @@ package com.google.android.exoplayer2.transformer;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.android.exoplayer2.transformer.BitmapTestUtil.FIRST_FRAME_PNG_ASSET_STRING;
import static com.google.android.exoplayer2.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
import static com.google.android.exoplayer2.transformer.BitmapTestUtil.REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static com.google.android.exoplayer2.transformer.BitmapTestUtil.ROTATE45_SCALE_TO_FIT_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static com.google.android.exoplayer2.transformer.BitmapTestUtil.ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static com.google.android.exoplayer2.transformer.BitmapTestUtil.SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING;
import static com.google.android.exoplayer2.transformer.BitmapTestUtil.TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING;
@ -34,8 +36,10 @@ import android.media.ImageReader;
import android.media.MediaCodec;
import android.media.MediaExtractor;
import android.media.MediaFormat;
import android.util.Pair;
import androidx.annotation.Nullable;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.util.MimeTypes;
import java.nio.ByteBuffer;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@ -84,7 +88,9 @@ public final class FrameEditorDataProcessingTest {
public void processData_noEdits_producesExpectedOutput() throws Exception {
final String testId = "processData_noEdits";
Matrix identityMatrix = new Matrix();
setUpAndPrepareFirstFrame(identityMatrix);
GlFrameProcessor glFrameProcessor =
new AdvancedFrameProcessor(getApplicationContext(), identityMatrix);
setUpAndPrepareFirstFrame(glFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING);
Bitmap actualBitmap = processFirstFrameAndEnd();
@ -103,7 +109,9 @@ public final class FrameEditorDataProcessingTest {
final String testId = "processData_translateRight";
Matrix translateRightMatrix = new Matrix();
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
setUpAndPrepareFirstFrame(translateRightMatrix);
GlFrameProcessor glFrameProcessor =
new AdvancedFrameProcessor(getApplicationContext(), translateRightMatrix);
setUpAndPrepareFirstFrame(glFrameProcessor);
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING);
@ -123,7 +131,9 @@ public final class FrameEditorDataProcessingTest {
final String testId = "processData_scaleNarrow";
Matrix scaleNarrowMatrix = new Matrix();
scaleNarrowMatrix.postScale(.5f, 1.2f);
setUpAndPrepareFirstFrame(scaleNarrowMatrix);
GlFrameProcessor glFrameProcessor =
new AdvancedFrameProcessor(getApplicationContext(), scaleNarrowMatrix);
setUpAndPrepareFirstFrame(glFrameProcessor);
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING);
@ -141,12 +151,11 @@ public final class FrameEditorDataProcessingTest {
@Test
public void processData_rotate90_producesExpectedOutput() throws Exception {
final String testId = "processData_rotate90";
// TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline
// resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can
// test that rotation doesn't distort the image.
Matrix rotate90Matrix = new Matrix();
rotate90Matrix.postRotate(/* degrees= */ 90);
setUpAndPrepareFirstFrame(rotate90Matrix);
GlFrameProcessor glFrameProcessor =
new AdvancedFrameProcessor(getApplicationContext(), rotate90Matrix);
setUpAndPrepareFirstFrame(glFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING);
Bitmap actualBitmap = processFirstFrameAndEnd();
@ -160,12 +169,70 @@ public final class FrameEditorDataProcessingTest {
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
private void setUpAndPrepareFirstFrame(Matrix transformationMatrix) throws Exception {
@Test
public void processData_requestOutputHeight_producesExpectedOutput() throws Exception {
final String testId = "processData_requestOutputHeight";
// TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline
// resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can
// test that rotation doesn't distort the image.
Matrix identityMatrix = new Matrix();
GlFrameProcessor glFrameProcessor =
new ScaleToFitFrameProcessor(
getApplicationContext(), identityMatrix, /* requestedHeight= */ 480);
setUpAndPrepareFirstFrame(glFrameProcessor);
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING);
Bitmap actualBitmap = processFirstFrameAndEnd();
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
expectedBitmap, actualBitmap, testId);
BitmapTestUtil.saveTestBitmapToCacheDirectory(
testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void processData_rotate45_scaleToFit_producesExpectedOutput() throws Exception {
final String testId = "processData_rotate45_scaleToFit";
// TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline
// resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can
// test that rotation doesn't distort the image.
Matrix rotate45Matrix = new Matrix();
rotate45Matrix.postRotate(/* degrees= */ 45);
GlFrameProcessor glFrameProcessor =
new ScaleToFitFrameProcessor(
getApplicationContext(), rotate45Matrix, /* requestedHeight= */ C.LENGTH_UNSET);
setUpAndPrepareFirstFrame(glFrameProcessor);
Bitmap expectedBitmap =
BitmapTestUtil.readBitmap(ROTATE45_SCALE_TO_FIT_EXPECTED_OUTPUT_PNG_ASSET_STRING);
Bitmap actualBitmap = processFirstFrameAndEnd();
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
expectedBitmap, actualBitmap, testId);
BitmapTestUtil.saveTestBitmapToCacheDirectory(
testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
/**
* Set up and prepare the first frame from an input video, as well as relevant test
* infrastructure. The frame will be sent towards the {@link FrameEditor}, and may be accessed on
* the {@link FrameEditor}'s output {@code frameEditorOutputImageReader}.
*
* @param glFrameProcessor The frame processor that will apply changes to the input frame.
*/
private void setUpAndPrepareFirstFrame(GlFrameProcessor glFrameProcessor) throws Exception {
// Set up the extractor to read the first video frame and get its format.
MediaExtractor mediaExtractor = new MediaExtractor();
@Nullable MediaCodec mediaCodec = null;
try (AssetFileDescriptor afd =
getApplicationContext().getAssets().openFd(INPUT_MP4_ASSET_STRING)) {
Context context = getApplicationContext();
try (AssetFileDescriptor afd = context.getAssets().openFd(INPUT_MP4_ASSET_STRING)) {
mediaExtractor.setDataSource(afd.getFileDescriptor(), afd.getStartOffset(), afd.getLength());
for (int i = 0; i < mediaExtractor.getTrackCount(); i++) {
if (MimeTypes.isVideo(mediaExtractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME))) {
@ -175,18 +242,24 @@ public final class FrameEditorDataProcessingTest {
}
}
int width = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH);
int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
int inputWidth = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH);
int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
Pair<Integer, Integer> outputDimensions =
glFrameProcessor.configureOutputDimensions(inputWidth, inputHeight);
int outputWidth = outputDimensions.first;
int outputHeight = outputDimensions.second;
frameEditorOutputImageReader =
ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
Context context = getApplicationContext();
ImageReader.newInstance(
outputWidth, outputHeight, PixelFormat.RGBA_8888, /* maxImages= */ 1);
frameEditor =
FrameEditor.create(
context,
width,
height,
inputWidth,
inputHeight,
outputWidth,
outputHeight,
PIXEL_WIDTH_HEIGHT_RATIO,
new TransformationFrameProcessor(context, transformationMatrix),
glFrameProcessor,
frameEditorOutputImageReader.getSurface(),
/* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE);

12
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorTest.java
View File

@ -28,8 +28,8 @@ import org.junit.Test;
import org.junit.runner.RunWith;
/**
* Test for {@link FrameEditor#create(Context, int, int, float, GlFrameProcessor, Surface, boolean,
* Transformer.DebugViewProvider) creating} a {@link FrameEditor}.
* Test for {@link FrameEditor#create(Context, int, int, int, int, float, GlFrameProcessor, Surface,
* boolean, Transformer.DebugViewProvider) creating} a {@link FrameEditor}.
*/
@RunWith(AndroidJUnit4.class)
public final class FrameEditorTest {
@ -43,10 +43,12 @@ public final class FrameEditorTest {
FrameEditor.create(
context,
/* inputWidth= */ 200,
/* inputHeight= */ 100,
/* outputWidth= */ 200,
/* outputHeight= */ 100,
/* pixelWidthHeightRatio= */ 1,
new TransformationFrameProcessor(context, new Matrix()),
new AdvancedFrameProcessor(context, new Matrix()),
new Surface(new SurfaceTexture(false)),
/* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE);
@ -62,10 +64,12 @@ public final class FrameEditorTest {
() ->
FrameEditor.create(
context,
/* inputWidth= */ 200,
/* inputHeight= */ 100,
/* outputWidth= */ 200,
/* outputHeight= */ 100,
/* pixelWidthHeightRatio= */ 2,
new TransformationFrameProcessor(context, new Matrix()),
new AdvancedFrameProcessor(context, new Matrix()),
new Surface(new SurfaceTexture(false)),
/* enableExperimentalHdrEditing= */ false,
Transformer.DebugViewProvider.NONE));

21
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationFrameProcessor.java → library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessor.java
View File

@ -20,13 +20,19 @@ import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.graphics.Matrix;
import android.opengl.GLES20;
import android.util.Pair;
import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Applies a transformation matrix in the vertex shader. */
/* package */ class TransformationFrameProcessor implements GlFrameProcessor {
/**
* Applies a transformation matrix in the vertex shader. Operations are done on normalized device
* coordinates (-1 to 1 on x and y axes). No automatic adjustments (like done in {@link
* ScaleToFitFrameProcessor}) are applied on the transformation. Width and height are not modified.
* The background color will default to black.
*/
/* package */ final class AdvancedFrameProcessor implements GlFrameProcessor {
static {
GlUtil.glAssertionsEnabled = true;
@ -85,13 +91,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* Creates a new instance.
*
* @param context The {@link Context}.
* @param transformationMatrix The transformation matrix to apply to each frame.
* @param transformationMatrix The transformation matrix to apply to each frame. Operations are
* done on normalized device coordinates (-1 to 1 on x and y), and no automatic adjustments
* are applied on the transformation matrix.
*/
public TransformationFrameProcessor(Context context, Matrix transformationMatrix) {
public AdvancedFrameProcessor(Context context, Matrix transformationMatrix) {
this.context = context;
this.transformationMatrix = transformationMatrix;
}
@Override
public Pair<Integer, Integer> configureOutputDimensions(int inputWidth, int inputHeight) {
return new Pair<>(inputWidth, inputHeight);
}
@Override
public void initialize(int inputTexId) throws IOException {
// TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms

6
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExternalCopyFrameProcessor.java
View File

@ -19,6 +19,7 @@ import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.opengl.GLES20;
import android.util.Pair;
import com.google.android.exoplayer2.util.GlProgram;
import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException;
@ -57,6 +58,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
}
@Override
public Pair<Integer, Integer> configureOutputDimensions(int inputWidth, int inputHeight) {
return new Pair<>(inputWidth, inputHeight);
}
@Override
public void initialize(int inputTexId) throws IOException {
// TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms

21
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java
View File

@ -49,7 +49,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* is processed on a background thread as it becomes available. All input frames should be {@link
* #registerInputFrame() registered} before they are rendered to the input surface. {@link
* #hasPendingFrames()} can be used to check whether there are frames that have not been fully
* processed yet. Output is written to its {@link #create(Context, int, int, float,
* processed yet. Output is written to its {@link #create(Context, int, int, int, int, float,
* GlFrameProcessor, Surface, boolean, Transformer.DebugViewProvider) output surface}.
*/
/* package */ final class FrameEditor {
@ -62,6 +62,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* Returns a new {@code FrameEditor} for applying changes to individual frames.
*
* @param context A {@link Context}.
* @param inputWidth The input width in pixels.
* @param inputHeight The input height in pixels.
* @param outputWidth The output width in pixels.
* @param outputHeight The output height in pixels.
* @param pixelWidthHeightRatio The ratio of width over height, for each pixel.
@ -76,8 +78,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
*/
// TODO(b/214975934): Take a List<GlFrameProcessor> as input and rename FrameEditor to
// FrameProcessorChain.
// TODO(b/218488308): Remove the need to input outputWidth and outputHeight into FrameEditor, that
// stems from encoder fallback resolution. This could maybe be input into the last
// GlFrameProcessor in the FrameEditor instead of being input directly into the FrameEditor.
public static FrameEditor create(
Context context,
int inputWidth,
int inputHeight,
int outputWidth,
int outputHeight,
float pixelWidthHeightRatio,
@ -121,6 +128,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
externalCopyFrameProcessor,
transformationFrameProcessor,
outputSurface,
inputWidth,
inputHeight,
outputWidth,
outputHeight,
enableExperimentalHdrEditing,
@ -152,6 +161,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
ExternalCopyFrameProcessor externalCopyFrameProcessor,
GlFrameProcessor transformationFrameProcessor,
Surface outputSurface,
int inputWidth,
int inputHeight,
int outputWidth,
int outputHeight,
boolean enableExperimentalHdrEditing,
@ -182,7 +193,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
}
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
GlUtil.assertValidTextureDimensions(outputWidth, outputHeight);
int inputExternalTexId = GlUtil.createExternalTexture();
externalCopyFrameProcessor.configureOutputDimensions(inputWidth, inputHeight);
externalCopyFrameProcessor.initialize(inputExternalTexId);
int intermediateTexId = GlUtil.createTexture(outputWidth, outputHeight);
int frameBuffer = GlUtil.createFboForTexture(intermediateTexId);
@ -227,9 +240,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private final float[] textureTransformMatrix;
/**
* Identifier of a framebuffer object associated with the intermediate texture that the output of
* the {@link ExternalCopyFrameProcessor} is written to and the {@link
* TransformationFrameProcessor} reads its input from.
* Identifier of a framebuffer object associated with the intermediate texture that receives
* output from the prior {@link ExternalCopyFrameProcessor}, and provides input for the following
* {@link GlFrameProcessor}.
*/
private final int frameBuffer;

25
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/GlFrameProcessor.java
View File

@ -15,21 +15,36 @@
*/
package com.google.android.exoplayer2.transformer;
import android.util.Pair;
import java.io.IOException;
/** Manages a GLSL shader program for processing a frame. */
/**
* Manages a GLSL shader program for processing a frame.
*
* <p>Methods must be called in the following order:
*
* <ol>
* <li>The constructor, for implementation-specific arguments.
* <li>{@link #configureOutputDimensions(int, int)}, to configure based on input dimensions.
* <li>{@link #initialize(int)}, to set up graphics initialization.
* <li>{@link #updateProgramAndDraw(long)}, to process one frame.
* <li>{@link #release()}, upon conclusion of processing.
* </ol>
*/
/* package */ interface GlFrameProcessor {
// TODO(b/214975934): Add getOutputDimensions(inputWidth, inputHeight) and move output dimension
// calculations out of the VideoTranscodingSamplePipeline into the frame processors.
/**
* Returns the output dimensions of frames processed through {@link #updateProgramAndDraw(long)}.
*
* <p>This method must be called before {@link #initialize(int)} and does not use OpenGL.
*/
Pair<Integer, Integer> configureOutputDimensions(int inputWidth, int inputHeight);
/**
* Does any initialization necessary such as loading and compiling a GLSL shader programs.
*
* <p>This method may only be called after creating the OpenGL context and focusing a render
* target.
*
* @param inputTexId The identifier of an OpenGL texture that the fragment shader can sample from.
*/
void initialize(int inputTexId) throws IOException;

197
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessor.java Normal file
View File

@ -0,0 +1,197 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import static java.lang.Math.max;
import static java.lang.Math.min;
import android.content.Context;
import android.graphics.Matrix;
import android.util.Pair;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/**
* Applies a simple rotation and/or scale in the vertex shader. All input frames' pixels will be
* preserved, potentially changing the width and height of the video by scaling dimensions to fit.
* The background color will default to black.
*/
/* package */ final class ScaleToFitFrameProcessor implements GlFrameProcessor {
static {
GlUtil.glAssertionsEnabled = true;
}
private final Context context;
private final Matrix transformationMatrix;
private final int requestedHeight;
private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor;
private int inputWidth;
private int inputHeight;
private int outputWidth;
private int outputHeight;
private int outputRotationDegrees;
private @MonotonicNonNull Matrix adjustedTransformationMatrix;
/**
* Creates a new instance.
*
* @param context The {@link Context}.
* @param transformationMatrix The transformation matrix to apply to each frame.
* @param requestedHeight The height of the output frame, in pixels.
*/
public ScaleToFitFrameProcessor(
Context context, Matrix transformationMatrix, int requestedHeight) {
// TODO(b/201293185): Replace transformationMatrix parameter with scale and rotation.
this.context = context;
this.transformationMatrix = new Matrix(transformationMatrix);
this.requestedHeight = requestedHeight;
inputWidth = C.LENGTH_UNSET;
inputHeight = C.LENGTH_UNSET;
outputWidth = C.LENGTH_UNSET;
outputHeight = C.LENGTH_UNSET;
outputRotationDegrees = C.LENGTH_UNSET;
}
/**
* Returns {@link Format#rotationDegrees} for the output frame.
*
* <p>Return values may be {@code 0} or {@code 90} degrees.
*
* <p>This method can only be called after {@link #configureOutputDimensions(int, int)}.
*/
public int getOutputRotationDegrees() {
checkState(outputRotationDegrees != C.LENGTH_UNSET);
return outputRotationDegrees;
}
/**
* Returns whether this ScaleToFitFrameProcessor will apply any changes on a frame.
*
* <p>The ScaleToFitFrameProcessor should only be used if this returns true.
*
* <p>This method can only be called after {@link #configureOutputDimensions(int, int)}.
*/
@RequiresNonNull("adjustedTransformationMatrix")
public boolean shouldProcess() {
return inputWidth != outputWidth
|| inputHeight != outputHeight
|| !adjustedTransformationMatrix.isIdentity();
}
@Override
@EnsuresNonNull("adjustedTransformationMatrix")
public Pair<Integer, Integer> configureOutputDimensions(int inputWidth, int inputHeight) {
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
adjustedTransformationMatrix = new Matrix(transformationMatrix);
int displayWidth = inputWidth;
int displayHeight = inputHeight;
if (!transformationMatrix.isIdentity()) {
float inputAspectRatio = (float) inputWidth / inputHeight;
// Scale frames by inputAspectRatio, to account for FrameEditor's normalized device
// coordinates (NDC) (a square from -1 to 1 for both x and y) and preserve rectangular
// display of input pixels during transformations (ex. rotations). With scaling,
// transformationMatrix operations operate on a rectangle for x from -inputAspectRatio to
// inputAspectRatio, and y from -1 to 1.
adjustedTransformationMatrix.preScale(/* sx= */ inputAspectRatio, /* sy= */ 1f);
adjustedTransformationMatrix.postScale(/* sx= */ 1f / inputAspectRatio, /* sy= */ 1f);
// Modify transformationMatrix to keep input pixels.
float[][] transformOnNdcPoints = {{-1, -1, 0, 1}, {-1, 1, 0, 1}, {1, -1, 0, 1}, {1, 1, 0, 1}};
float xMin = Float.MAX_VALUE;
float xMax = Float.MIN_VALUE;
float yMin = Float.MAX_VALUE;
float yMax = Float.MIN_VALUE;
for (float[] transformOnNdcPoint : transformOnNdcPoints) {
adjustedTransformationMatrix.mapPoints(transformOnNdcPoint);
xMin = min(xMin, transformOnNdcPoint[0]);
xMax = max(xMax, transformOnNdcPoint[0]);
yMin = min(yMin, transformOnNdcPoint[1]);
yMax = max(yMax, transformOnNdcPoint[1]);
}
float xCenter = (xMax + xMin) / 2f;
float yCenter = (yMax + yMin) / 2f;
adjustedTransformationMatrix.postTranslate(-xCenter, -yCenter);
float ndcWidthAndHeight = 2f; // Length from -1 to 1.
float xScale = (xMax - xMin) / ndcWidthAndHeight;
float yScale = (yMax - yMin) / ndcWidthAndHeight;
adjustedTransformationMatrix.postScale(1f / xScale, 1f / yScale);
displayWidth = Math.round(inputWidth * xScale);
displayHeight = Math.round(inputHeight * yScale);
}
// TODO(b/214975934): Move following requestedHeight and outputRotationDegrees logic into
// separate GlFrameProcessors (ex. Presentation).
// Scale width and height to desired requestedHeight, preserving aspect ratio.
if (requestedHeight != C.LENGTH_UNSET && requestedHeight != displayHeight) {
displayWidth = Math.round((float) requestedHeight * displayWidth / displayHeight);
displayHeight = requestedHeight;
}
// Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded
// video before encoding, so the encoded video's width >= height, and set
// outputRotationDegrees to ensure the video is displayed in the correct orientation.
if (displayHeight > displayWidth) {
outputRotationDegrees = 90;
outputWidth = displayHeight;
outputHeight = displayWidth;
// TODO(b/201293185): After fragment shader transformations are implemented, put
// postRotate in a later GlFrameProcessor.
adjustedTransformationMatrix.postRotate(outputRotationDegrees);
} else {
outputRotationDegrees = 0;
outputWidth = displayWidth;
outputHeight = displayHeight;
}
return new Pair<>(outputWidth, outputHeight);
}
@Override
public void initialize(int inputTexId) throws IOException {
checkStateNotNull(adjustedTransformationMatrix);
advancedFrameProcessor = new AdvancedFrameProcessor(context, adjustedTransformationMatrix);
advancedFrameProcessor.configureOutputDimensions(inputWidth, inputHeight);
advancedFrameProcessor.initialize(inputTexId);
}
@Override
public void updateProgramAndDraw(long presentationTimeNs) {
checkStateNotNull(advancedFrameProcessor).updateProgramAndDraw(presentationTimeNs);
}
@Override
public void release() {
if (advancedFrameProcessor != null) {
advancedFrameProcessor.release();
}
}
}

94
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
View File

@ -18,16 +18,13 @@ package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Util.SDK_INT;
import static java.lang.Math.max;
import static java.lang.Math.min;
import android.content.Context;
import android.graphics.Matrix;
import android.media.MediaCodec;
import android.media.MediaFormat;
import android.util.Pair;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.util.Util;
@ -70,77 +67,20 @@ import org.checkerframework.dataflow.qual.Pure;
(inputFormat.rotationDegrees % 180 == 0) ? inputFormat.width : inputFormat.height;
int decodedHeight =
(inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
float decodedAspectRatio = (float) decodedWidth / decodedHeight;
Matrix transformationMatrix = new Matrix(transformationRequest.transformationMatrix);
int outputWidth = decodedWidth;
int outputHeight = decodedHeight;
if (!transformationMatrix.isIdentity()) {
// Scale frames by decodedAspectRatio, to account for FrameEditor's normalized device
// coordinates (NDC) (a square from -1 to 1 for both x and y) and preserve rectangular display
// of input pixels during transformations (ex. rotations). With scaling, transformationMatrix
// operations operate on a rectangle for x from -decodedAspectRatio to decodedAspectRatio, and
// y from -1 to 1.
transformationMatrix.preScale(/* sx= */ decodedAspectRatio, /* sy= */ 1f);
transformationMatrix.postScale(/* sx= */ 1f / decodedAspectRatio, /* sy= */ 1f);
float[][] transformOnNdcPoints = {{-1, -1, 0, 1}, {-1, 1, 0, 1}, {1, -1, 0, 1}, {1, 1, 0, 1}};
float xMin = Float.MAX_VALUE;
float xMax = Float.MIN_VALUE;
float yMin = Float.MAX_VALUE;
float yMax = Float.MIN_VALUE;
for (float[] transformOnNdcPoint : transformOnNdcPoints) {
transformationMatrix.mapPoints(transformOnNdcPoint);
xMin = min(xMin, transformOnNdcPoint[0]);
xMax = max(xMax, transformOnNdcPoint[0]);
yMin = min(yMin, transformOnNdcPoint[1]);
yMax = max(yMax, transformOnNdcPoint[1]);
}
float xCenter = (xMax + xMin) / 2f;
float yCenter = (yMax + yMin) / 2f;
transformationMatrix.postTranslate(-xCenter, -yCenter);
float ndcWidthAndHeight = 2f; // Length from -1 to 1.
float xScale = (xMax - xMin) / ndcWidthAndHeight;
float yScale = (yMax - yMin) / ndcWidthAndHeight;
transformationMatrix.postScale(1f / xScale, 1f / yScale);
outputWidth = Math.round(decodedWidth * xScale);
outputHeight = Math.round(decodedHeight * yScale);
}
// Scale width and height to desired transformationRequest.outputHeight, preserving
// aspect ratio.
if (transformationRequest.outputHeight != C.LENGTH_UNSET
&& transformationRequest.outputHeight != outputHeight) {
outputWidth =
Math.round((float) transformationRequest.outputHeight * outputWidth / outputHeight);
outputHeight = transformationRequest.outputHeight;
}
// Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded
// video before encoding, so the encoded video's width >= height, and set outputRotationDegrees
// to ensure the video is displayed in the correct orientation.
int requestedEncoderWidth;
int requestedEncoderHeight;
boolean swapEncodingDimensions = outputHeight > outputWidth;
if (swapEncodingDimensions) {
outputRotationDegrees = 90;
requestedEncoderWidth = outputHeight;
requestedEncoderHeight = outputWidth;
// TODO(b/201293185): After fragment shader transformations are implemented, put
// postRotate in a later vertex shader.
transformationMatrix.postRotate(outputRotationDegrees);
} else {
outputRotationDegrees = 0;
requestedEncoderWidth = outputWidth;
requestedEncoderHeight = outputHeight;
}
ScaleToFitFrameProcessor scaleToFitFrameProcessor =
new ScaleToFitFrameProcessor(
context,
transformationRequest.transformationMatrix,
transformationRequest.outputHeight);
Pair<Integer, Integer> requestedEncoderDimensions =
scaleToFitFrameProcessor.configureOutputDimensions(decodedWidth, decodedHeight);
outputRotationDegrees = scaleToFitFrameProcessor.getOutputRotationDegrees();
Format requestedEncoderFormat =
new Format.Builder()
.setWidth(requestedEncoderWidth)
.setHeight(requestedEncoderHeight)
.setWidth(requestedEncoderDimensions.first)
.setHeight(requestedEncoderDimensions.second)
.setRotationDegrees(0)
.setSampleMimeType(
transformationRequest.videoMimeType != null
@ -152,21 +92,23 @@ import org.checkerframework.dataflow.qual.Pure;
fallbackListener.onTransformationRequestFinalized(
createFallbackTransformationRequest(
transformationRequest,
/* hasOutputFormatRotation= */ swapEncodingDimensions,
/* hasOutputFormatRotation= */ outputRotationDegrees == 0,
requestedEncoderFormat,
encoderSupportedFormat));
if (transformationRequest.enableHdrEditing
|| inputFormat.height != encoderSupportedFormat.height
|| inputFormat.width != encoderSupportedFormat.width
|| !transformationMatrix.isIdentity()) {
|| scaleToFitFrameProcessor.shouldProcess()) {
frameEditor =
FrameEditor.create(
context,
encoderSupportedFormat.width,
encoderSupportedFormat.height,
/* inputWidth= */ decodedWidth,
/* inputHeight= */ decodedHeight,
/* outputWidth= */ encoderSupportedFormat.width,
/* outputHeight= */ encoderSupportedFormat.height,
inputFormat.pixelWidthHeightRatio,
new TransformationFrameProcessor(context, transformationMatrix),
scaleToFitFrameProcessor,
/* outputSurface= */ encoder.getInputSurface(),
transformationRequest.enableHdrEditing,
debugViewProvider);

66
library/transformer/src/test/java/com/google/android/exoplayer2/transformer/AdvancedFrameProcessorTest.java Normal file
View File

@ -0,0 +1,66 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.transformer;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat;
import android.graphics.Matrix;
import android.util.Pair;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Test;
import org.junit.runner.RunWith;
/**
* Unit tests for {@link AdvancedFrameProcessor}.
*
* <p>See {@link AdvancedFrameProcessorPixelTest} for pixel tests testing {@link
* AdvancedFrameProcessor} given a transformation matrix.
*/
@RunWith(AndroidJUnit4.class)
public final class AdvancedFrameProcessorTest {
@Test
public void getOutputDimensions_withIdentityMatrix_leavesDimensionsUnchanged() {
Matrix identityMatrix = new Matrix();
int inputWidth = 200;
int inputHeight = 150;
AdvancedFrameProcessor advancedFrameProcessor =
new AdvancedFrameProcessor(getApplicationContext(), identityMatrix);
Pair<Integer, Integer> outputDimensions =
advancedFrameProcessor.configureOutputDimensions(inputWidth, inputHeight);
assertThat(outputDimensions.first).isEqualTo(inputWidth);
assertThat(outputDimensions.second).isEqualTo(inputHeight);
}
@Test
public void getOutputDimensions_withTransformationMatrix_leavesDimensionsUnchanged() {
Matrix transformationMatrix = new Matrix();
transformationMatrix.postRotate(/* degrees= */ 90);
transformationMatrix.postScale(/* sx= */ .5f, /* sy= */ 1.2f);
int inputWidth = 200;
int inputHeight = 150;
AdvancedFrameProcessor advancedFrameProcessor =
new AdvancedFrameProcessor(getApplicationContext(), transformationMatrix);
Pair<Integer, Integer> outputDimensions =
advancedFrameProcessor.configureOutputDimensions(inputWidth, inputHeight);
assertThat(outputDimensions.first).isEqualTo(inputWidth);
assertThat(outputDimensions.second).isEqualTo(inputHeight);
}
}

167
library/transformer/src/test/java/com/google/android/exoplayer2/transformer/ScaleToFitFrameProcessorTest.java Normal file
View File

@ -0,0 +1,167 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.transformer;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import android.graphics.Matrix;
import android.util.Pair;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.C;
import org.junit.Test;
import org.junit.runner.RunWith;
/**
* Unit tests for {@link ScaleToFitFrameProcessor}.
*
* <p>See {@code AdvancedFrameProcessorPixelTest} for pixel tests testing {@link
* AdvancedFrameProcessor} given a transformation matrix.
*/
@RunWith(AndroidJUnit4.class)
public final class ScaleToFitFrameProcessorTest {
@Test
public void configureOutputDimensions_noEdits_producesExpectedOutput() {
Matrix identityMatrix = new Matrix();
int inputWidth = 200;
int inputHeight = 150;
ScaleToFitFrameProcessor scaleToFitFrameProcessor =
new ScaleToFitFrameProcessor(getApplicationContext(), identityMatrix, C.LENGTH_UNSET);
Pair<Integer, Integer> outputDimensions =
scaleToFitFrameProcessor.configureOutputDimensions(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isFalse();
assertThat(outputDimensions.first).isEqualTo(inputWidth);
assertThat(outputDimensions.second).isEqualTo(inputHeight);
}
@Test
public void initializeBeforeConfigure_throwsIllegalStateException() {
Matrix identityMatrix = new Matrix();
ScaleToFitFrameProcessor scaleToFitFrameProcessor =
new ScaleToFitFrameProcessor(getApplicationContext(), identityMatrix, C.LENGTH_UNSET);
// configureOutputDimensions not called before initialize.
assertThrows(
IllegalStateException.class,
() -> scaleToFitFrameProcessor.initialize(/* inputTexId= */ 0));
}
@Test
public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() {
Matrix identityMatrix = new Matrix();
ScaleToFitFrameProcessor scaleToFitFrameProcessor =
new ScaleToFitFrameProcessor(getApplicationContext(), identityMatrix, C.LENGTH_UNSET);
// configureOutputDimensions not called before initialize.
assertThrows(IllegalStateException.class, scaleToFitFrameProcessor::getOutputRotationDegrees);
}
@Test
public void configureOutputDimensions_scaleNarrow_producesExpectedOutput() {
Matrix scaleNarrowMatrix = new Matrix();
scaleNarrowMatrix.postScale(/* sx= */ .5f, /* sy= */ 1.0f);
int inputWidth = 200;
int inputHeight = 150;
ScaleToFitFrameProcessor scaleToFitFrameProcessor =
new ScaleToFitFrameProcessor(getApplicationContext(), scaleNarrowMatrix, C.LENGTH_UNSET);
Pair<Integer, Integer> outputDimensions =
scaleToFitFrameProcessor.configureOutputDimensions(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(90);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
assertThat(outputDimensions.first).isEqualTo(inputHeight);
assertThat(outputDimensions.second).isEqualTo(Math.round(inputWidth * .5f));
}
@Test
public void configureOutputDimensions_scaleWide_producesExpectedOutput() {
Matrix scaleNarrowMatrix = new Matrix();
scaleNarrowMatrix.postScale(/* sx= */ 2f, /* sy= */ 1.0f);
int inputWidth = 200;
int inputHeight = 150;
ScaleToFitFrameProcessor scaleToFitFrameProcessor =
new ScaleToFitFrameProcessor(getApplicationContext(), scaleNarrowMatrix, C.LENGTH_UNSET);
Pair<Integer, Integer> outputDimensions =
scaleToFitFrameProcessor.configureOutputDimensions(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
assertThat(outputDimensions.first).isEqualTo(inputWidth * 2);
assertThat(outputDimensions.second).isEqualTo(inputHeight);
}
@Test
public void configureOutputDimensions_rotate90_producesExpectedOutput() {
Matrix rotate90Matrix = new Matrix();
rotate90Matrix.postRotate(/* degrees= */ 90);
int inputWidth = 200;
int inputHeight = 150;
ScaleToFitFrameProcessor scaleToFitFrameProcessor =
new ScaleToFitFrameProcessor(getApplicationContext(), rotate90Matrix, C.LENGTH_UNSET);
Pair<Integer, Integer> outputDimensions =
scaleToFitFrameProcessor.configureOutputDimensions(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(90);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
assertThat(outputDimensions.first).isEqualTo(inputWidth);
assertThat(outputDimensions.second).isEqualTo(inputHeight);
}
@Test
public void configureOutputDimensions_rotate45_producesExpectedOutput() {
Matrix rotate45Matrix = new Matrix();
rotate45Matrix.postRotate(/* degrees= */ 45);
int inputWidth = 200;
int inputHeight = 150;
ScaleToFitFrameProcessor scaleToFitFrameProcessor =
new ScaleToFitFrameProcessor(getApplicationContext(), rotate45Matrix, C.LENGTH_UNSET);
long expectedOutputWidthHeight = 247;
Pair<Integer, Integer> outputDimensions =
scaleToFitFrameProcessor.configureOutputDimensions(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
assertThat(outputDimensions.first).isEqualTo(expectedOutputWidthHeight);
assertThat(outputDimensions.second).isEqualTo(expectedOutputWidthHeight);
}
@Test
public void configureOutputDimensions_setResolution_producesExpectedOutput() {
Matrix identityMatrix = new Matrix();
int inputWidth = 200;
int inputHeight = 150;
int requestedHeight = 300;
ScaleToFitFrameProcessor scaleToFitFrameProcessor =
new ScaleToFitFrameProcessor(getApplicationContext(), identityMatrix, requestedHeight);
Pair<Integer, Integer> outputDimensions =
scaleToFitFrameProcessor.configureOutputDimensions(inputWidth, inputHeight);
assertThat(scaleToFitFrameProcessor.getOutputRotationDegrees()).isEqualTo(0);
assertThat(scaleToFitFrameProcessor.shouldProcess()).isTrue();
assertThat(outputDimensions.first).isEqualTo(requestedHeight * inputWidth / inputHeight);
assertThat(outputDimensions.second).isEqualTo(requestedHeight);
}
}

BIN
testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_request_output_height.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 310 KiB

BIN
testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_rotate_45_scale_to_fit.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 728 KiB