loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Transformer GL: Add support for pixelWidthHeightRatio.

Browse Source 
To ensure frame processor operations operate on square pixels,
make the frame taller or wider for non-square input pixels.

In addition to automated tests, this was tested by changing the
inputFormat.pixelWidthHeightRatio in the TransformerVideoRenderer.

PiperOrigin-RevId: 444553517
This commit is contained in:
huangdarwin 2022-04-26 15:27:10 +01:00 committed by Ian Baker
parent de871ea273
commit f269963e89
5 changed files with 121 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/scale_wide.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 811 KiB

41
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java
View File

@ -53,6 +53,8 @@ import org.junit.runner.RunWith;
public final class FrameProcessorChainPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/original.png";
public static final String SCALE_WIDE_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/scale_wide.png";
public static final String TRANSLATE_RIGHT_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/translate_right.png";
public static final String ROTATE_THEN_TRANSLATE_PNG_ASSET_PATH =
@ -74,7 +76,7 @@ public final class FrameProcessorChainPixelTest {
*/
private static final int FRAME_PROCESSING_WAIT_MS = 5000;
/** The ratio of width over height, for each pixel in a frame. */
private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1;
private static final float DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO = 1;
private @MonotonicNonNull FrameProcessorChain frameProcessorChain;
private @MonotonicNonNull ImageReader outputImageReader;
@ -90,7 +92,7 @@ public final class FrameProcessorChainPixelTest {
@Test
public void processData_noEdits_producesExpectedOutput() throws Exception {
String testId = "processData_noEdits";
setUpAndPrepareFirstFrame();
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
@ -104,6 +106,23 @@ public final class FrameProcessorChainPixelTest {
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void processData_withPixelWidthHeightRatio_producesExpectedOutput() throws Exception {
String testId = "processData_withPixelWidthHeightRatio";
setUpAndPrepareFirstFrame(/* pixelWidthHeightRatio= */ 2f);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_WIDE_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory(
testId, /* bitmapLabel= */ "actual", actualBitmap);
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888(
expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void processData_withAdvancedFrameProcessor_translateRight_producesExpectedOutput()
throws Exception {
@ -111,7 +130,7 @@ public final class FrameProcessorChainPixelTest {
Matrix translateRightMatrix = new Matrix();
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
GlFrameProcessor glFrameProcessor = new AdvancedFrameProcessor(translateRightMatrix);
setUpAndPrepareFirstFrame(glFrameProcessor);
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, glFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
@ -135,7 +154,8 @@ public final class FrameProcessorChainPixelTest {
new AdvancedFrameProcessor(translateRightMatrix);
GlFrameProcessor rotate45FrameProcessor =
new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build();
setUpAndPrepareFirstFrame(translateRightFrameProcessor, rotate45FrameProcessor);
setUpAndPrepareFirstFrame(
DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, translateRightFrameProcessor, rotate45FrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_THEN_ROTATE_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
@ -159,7 +179,8 @@ public final class FrameProcessorChainPixelTest {
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
GlFrameProcessor translateRightFrameProcessor =
new AdvancedFrameProcessor(translateRightMatrix);
setUpAndPrepareFirstFrame(rotate45FrameProcessor, translateRightFrameProcessor);
setUpAndPrepareFirstFrame(
DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, rotate45FrameProcessor, translateRightFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_THEN_TRANSLATE_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
@ -179,7 +200,7 @@ public final class FrameProcessorChainPixelTest {
String testId = "processData_withPresentationFrameProcessor_setResolution";
GlFrameProcessor glFrameProcessor =
new PresentationFrameProcessor.Builder().setResolution(480).build();
setUpAndPrepareFirstFrame(glFrameProcessor);
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, glFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(REQUEST_OUTPUT_HEIGHT_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
@ -199,7 +220,7 @@ public final class FrameProcessorChainPixelTest {
String testId = "processData_withScaleToFitFrameProcessor_rotate45";
GlFrameProcessor glFrameProcessor =
new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build();
setUpAndPrepareFirstFrame(glFrameProcessor);
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, glFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE45_SCALE_TO_FIT_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
@ -218,10 +239,12 @@ public final class FrameProcessorChainPixelTest {
* infrastructure. The frame will be sent towards the {@link FrameProcessorChain}, and may be
* accessed on the {@link FrameProcessorChain}'s output {@code outputImageReader}.
*
* @param pixelWidthHeightRatio The ratio of width over height for each pixel.
* @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} that will apply changes
* to the input frame.
*/
private void setUpAndPrepareFirstFrame(GlFrameProcessor... frameProcessors) throws Exception {
private void setUpAndPrepareFirstFrame(
float pixelWidthHeightRatio, GlFrameProcessor... frameProcessors) throws Exception {
// Set up the extractor to read the first video frame and get its format.
MediaExtractor mediaExtractor = new MediaExtractor();
@Nullable MediaCodec mediaCodec = null;
@ -241,7 +264,7 @@ public final class FrameProcessorChainPixelTest {
frameProcessorChain =
FrameProcessorChain.create(
context,
PIXEL_WIDTH_HEIGHT_RATIO,
pixelWidthHeightRatio,
inputWidth,
inputHeight,
asList(frameProcessors),

86
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java
View File

@ -17,7 +17,6 @@ package androidx.media3.transformer;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import android.content.Context;
import android.util.Size;
@ -36,46 +35,13 @@ import org.junit.runner.RunWith;
public final class FrameProcessorChainTest {
@Test
public void create_withSupportedPixelWidthHeightRatio_completesSuccessfully()
throws TransformationException {
Context context = getApplicationContext();
FrameProcessorChain.create(
context,
/* pixelWidthHeightRatio= */ 1,
/* inputWidth= */ 200,
/* inputHeight= */ 100,
/* frameProcessors= */ ImmutableList.of(),
/* enableExperimentalHdrEditing= */ false);
}
@Test
public void create_withUnsupportedPixelWidthHeightRatio_throwsException() {
Context context = getApplicationContext();
TransformationException exception =
assertThrows(
TransformationException.class,
() ->
FrameProcessorChain.create(
context,
/* pixelWidthHeightRatio= */ 2,
/* inputWidth= */ 200,
/* inputHeight= */ 100,
/* frameProcessors= */ ImmutableList.of(),
/* enableExperimentalHdrEditing= */ false));
assertThat(exception).hasCauseThat().isInstanceOf(UnsupportedOperationException.class);
assertThat(exception).hasCauseThat().hasMessageThat().contains("pixelWidthHeightRatio");
}
@Test
public void getOutputSize_withoutFrameProcessors_returnsInputSize()
throws TransformationException {
public void getOutputSize_noOperation_returnsInputSize() throws Exception {
Size inputSize = new Size(200, 100);
FrameProcessorChain frameProcessorChain =
createFrameProcessorChainWithFakeFrameProcessors(
inputSize, /* frameProcessorOutputSizes= */ ImmutableList.of());
/* pixelWidthHeightRatio= */ 1f,
inputSize,
/* frameProcessorOutputSizes= */ ImmutableList.of());
Size outputSize = frameProcessorChain.getOutputSize();
@ -83,13 +49,42 @@ public final class FrameProcessorChainTest {
}
@Test
public void getOutputSize_withOneFrameProcessor_returnsItsOutputSize()
throws TransformationException {
public void getOutputSize_withWidePixels_returnsWiderOutputSize() throws Exception {
Size inputSize = new Size(200, 100);
FrameProcessorChain frameProcessorChain =
createFrameProcessorChainWithFakeFrameProcessors(
/* pixelWidthHeightRatio= */ 2f,
inputSize,
/* frameProcessorOutputSizes= */ ImmutableList.of());
Size outputSize = frameProcessorChain.getOutputSize();
assertThat(outputSize).isEqualTo(new Size(400, 100));
}
@Test
public void getOutputSize_withTallPixels_returnsTallerOutputSize() throws Exception {
Size inputSize = new Size(200, 100);
FrameProcessorChain frameProcessorChain =
createFrameProcessorChainWithFakeFrameProcessors(
/* pixelWidthHeightRatio= */ .5f,
inputSize,
/* frameProcessorOutputSizes= */ ImmutableList.of());
Size outputSize = frameProcessorChain.getOutputSize();
assertThat(outputSize).isEqualTo(new Size(200, 200));
}
@Test
public void getOutputSize_withOneFrameProcessor_returnsItsOutputSize() throws Exception {
Size inputSize = new Size(200, 100);
Size frameProcessorOutputSize = new Size(300, 250);
FrameProcessorChain frameProcessorChain =
createFrameProcessorChainWithFakeFrameProcessors(
inputSize, /* frameProcessorOutputSizes= */ ImmutableList.of(frameProcessorOutputSize));
/* pixelWidthHeightRatio= */ 1f,
inputSize,
/* frameProcessorOutputSizes= */ ImmutableList.of(frameProcessorOutputSize));
Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize();
@ -97,14 +92,14 @@ public final class FrameProcessorChainTest {
}
@Test
public void getOutputSize_withThreeFrameProcessors_returnsLastOutputSize()
throws TransformationException {
public void getOutputSize_withThreeFrameProcessors_returnsLastOutputSize() throws Exception {
Size inputSize = new Size(200, 100);
Size outputSize1 = new Size(300, 250);
Size outputSize2 = new Size(400, 244);
Size outputSize3 = new Size(150, 160);
FrameProcessorChain frameProcessorChain =
createFrameProcessorChainWithFakeFrameProcessors(
/* pixelWidthHeightRatio= */ 1f,
inputSize,
/* frameProcessorOutputSizes= */ ImmutableList.of(
outputSize1, outputSize2, outputSize3));
@ -115,14 +110,15 @@ public final class FrameProcessorChainTest {
}
private static FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors(
Size inputSize, List<Size> frameProcessorOutputSizes) throws TransformationException {
float pixelWidthHeightRatio, Size inputSize, List<Size> frameProcessorOutputSizes)
throws TransformationException {
ImmutableList.Builder<GlFrameProcessor> frameProcessors = new ImmutableList.Builder<>();
for (Size element : frameProcessorOutputSizes) {
frameProcessors.add(new FakeFrameProcessor(element));
}
return FrameProcessorChain.create(
getApplicationContext(),
/* pixelWidthHeightRatio= */ 1,
pixelWidthHeightRatio,
inputSize.getWidth(),
inputSize.getHeight(),
frameProcessors.build(),

76
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java
View File

@ -71,15 +71,15 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* Creates a new instance.
*
* @param context A {@link Context}.
* @param pixelWidthHeightRatio The ratio of width over height, for each pixel.
* @param pixelWidthHeightRatio The ratio of width over height for each pixel. Pixels are expanded
* by this ratio so that the output frame's pixels have a ratio of 1.
* @param inputWidth The input frame width, in pixels.
* @param inputHeight The input frame height, in pixels.
* @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @return A new instance.
* @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1, reading shader
* files fails, or an OpenGL error occurs while creating and configuring the OpenGL
* components.
* @throws TransformationException If reading shader files fails, or an OpenGL error occurs while
* creating and configuring the OpenGL components.
*/
public static FrameProcessorChain create(
Context context,
@ -92,19 +92,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
checkArgument(inputWidth > 0, "inputWidth must be positive");
checkArgument(inputHeight > 0, "inputHeight must be positive");
if (pixelWidthHeightRatio != 1.0f) {
// TODO(b/211782176): Consider implementing support for non-square pixels.
throw TransformationException.createForFrameProcessorChain(
new UnsupportedOperationException(
"Transformer's FrameProcessorChain currently does not support frame edits on"
+ " non-square pixels. The pixelWidthHeightRatio is: "
+ pixelWidthHeightRatio),
TransformationException.ERROR_CODE_GL_INIT_FAILED);
}
ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
ExternalCopyFrameProcessor externalCopyFrameProcessor =
new ExternalCopyFrameProcessor(enableExperimentalHdrEditing);
try {
return singleThreadExecutorService
@ -112,12 +100,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
() ->
createOpenGlObjectsAndFrameProcessorChain(
context,
pixelWidthHeightRatio,
inputWidth,
inputHeight,
frameProcessors,
enableExperimentalHdrEditing,
singleThreadExecutorService,
externalCopyFrameProcessor))
singleThreadExecutorService))
.get();
} catch (ExecutionException e) {
throw TransformationException.createForFrameProcessorChain(
@ -138,16 +126,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@WorkerThread
private static FrameProcessorChain createOpenGlObjectsAndFrameProcessorChain(
Context context,
float pixelWidthHeightRatio,
int inputWidth,
int inputHeight,
List<GlFrameProcessor> frameProcessors,
boolean enableExperimentalHdrEditing,
ExecutorService singleThreadExecutorService,
ExternalCopyFrameProcessor externalCopyFrameProcessor)
ExecutorService singleThreadExecutorService)
throws IOException {
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
EGLDisplay eglDisplay = GlUtil.createEglDisplay();
ExternalCopyFrameProcessor externalCopyFrameProcessor =
new ExternalCopyFrameProcessor(enableExperimentalHdrEditing);
EGLContext eglContext =
enableExperimentalHdrEditing
? GlUtil.createEglContextEs3Rgba1010102(eglDisplay)
@ -163,18 +153,22 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay);
}
ImmutableList<GlFrameProcessor> expandedFrameProcessors =
getExpandedFrameProcessors(
externalCopyFrameProcessor, pixelWidthHeightRatio, frameProcessors);
// Initialize frame processors.
int inputExternalTexId = GlUtil.createExternalTexture();
externalCopyFrameProcessor.initialize(context, inputExternalTexId, inputWidth, inputHeight);
int[] framebuffers = new int[frameProcessors.size()];
int[] framebuffers = new int[expandedFrameProcessors.size() - 1];
Size inputSize = externalCopyFrameProcessor.getOutputSize();
for (int i = 0; i < frameProcessors.size(); i++) {
for (int i = 1; i < expandedFrameProcessors.size(); i++) {
int inputTexId = GlUtil.createTexture(inputSize.getWidth(), inputSize.getHeight());
framebuffers[i] = GlUtil.createFboForTexture(inputTexId);
frameProcessors
.get(i)
.initialize(context, inputTexId, inputSize.getWidth(), inputSize.getHeight());
inputSize = frameProcessors.get(i).getOutputSize();
framebuffers[i - 1] = GlUtil.createFboForTexture(inputTexId);
GlFrameProcessor frameProcessor = expandedFrameProcessors.get(i);
frameProcessor.initialize(context, inputTexId, inputSize.getWidth(), inputSize.getHeight());
inputSize = frameProcessor.getOutputSize();
}
return new FrameProcessorChain(
eglDisplay,
@ -182,13 +176,33 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
singleThreadExecutorService,
inputExternalTexId,
framebuffers,
new ImmutableList.Builder<GlFrameProcessor>()
.add(externalCopyFrameProcessor)
.addAll(frameProcessors)
.build(),
expandedFrameProcessors,
enableExperimentalHdrEditing);
}
private static ImmutableList<GlFrameProcessor> getExpandedFrameProcessors(
ExternalCopyFrameProcessor externalCopyFrameProcessor,
float pixelWidthHeightRatio,
List<GlFrameProcessor> frameProcessors) {
ImmutableList.Builder<GlFrameProcessor> frameProcessorListBuilder =
new ImmutableList.Builder<GlFrameProcessor>().add(externalCopyFrameProcessor);
// Scale to expand the frame to apply the pixelWidthHeightRatio.
if (pixelWidthHeightRatio > 1f) {
frameProcessorListBuilder.add(
new ScaleToFitFrameProcessor.Builder()
.setScale(/* scaleX= */ pixelWidthHeightRatio, /* scaleY= */ 1f)
.build());
} else if (pixelWidthHeightRatio < 1f) {
frameProcessorListBuilder.add(
new ScaleToFitFrameProcessor.Builder()
.setScale(/* scaleX= */ 1f, /* scaleY= */ 1f / pixelWidthHeightRatio)
.build());
}
frameProcessorListBuilder.addAll(frameProcessors);
return frameProcessorListBuilder.build();
}
private static final String TAG = "FrameProcessorChain";
private static final String THREAD_NAME = "Transformer:FrameProcessorChain";
private static final long RELEASE_WAIT_TIME_MS = 100;

3
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java
View File

@ -121,6 +121,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
&& !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) {
return false;
}
if (inputFormat.pixelWidthHeightRatio != 1f) {
return false;
}
if (transformationRequest.rotationDegrees != 0f) {
return false;
}