loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Configure the frame sizes in FrameProcessorChain instead of caller.

Browse Source 
Configuring the frame sizes between frame processors is now the
FrameProcessorChain's rather than the caller's responsibility.
The caller can getOutputSize() and override it for encoder fallback
in configure().

PiperOrigin-RevId: 437048436
This commit is contained in:
hschlueter 2022-03-24 18:44:15 +00:00 committed by Ian Baker
parent 20daaa20ef
commit 37559deacf
4 changed files with 126 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java
View File

@ -41,9 +41,7 @@ import android.util.Size;
import androidx.annotation.Nullable;
import androidx.media3.common.MimeTypes;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.Iterables;
import java.nio.ByteBuffer;
import java.util.List;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.junit.After;
import org.junit.Test;
@ -247,24 +245,26 @@ public final class FrameProcessorChainPixelTest {
int inputWidth = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH);
int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
List<GlFrameProcessor> frameProcessorsList = asList(frameProcessors);
List<Size> sizes =
FrameProcessorChain.configureSizes(inputWidth, inputHeight, frameProcessorsList);
assertThat(sizes).isNotEmpty();
int outputWidth = Iterables.getLast(sizes).getWidth();
int outputHeight = Iterables.getLast(sizes).getHeight();
outputImageReader =
ImageReader.newInstance(
outputWidth, outputHeight, PixelFormat.RGBA_8888, /* maxImages= */ 1);
frameProcessorChain =
new FrameProcessorChain(
context,
PIXEL_WIDTH_HEIGHT_RATIO,
frameProcessorsList,
sizes,
inputWidth,
inputHeight,
asList(frameProcessors),
/* enableExperimentalHdrEditing= */ false);
Size outputSize = frameProcessorChain.getOutputSize();
outputImageReader =
ImageReader.newInstance(
outputSize.getWidth(),
outputSize.getHeight(),
PixelFormat.RGBA_8888,
/* maxImages= */ 1);
frameProcessorChain.configure(
outputImageReader.getSurface(), outputWidth, outputHeight, /* debugSurfaceView= */ null);
outputImageReader.getSurface(),
outputSize.getWidth(),
outputSize.getHeight(),
/* debugSurfaceView= */ null);
frameProcessorChain.registerInputFrame();
// Queue the first video frame from the extractor.

112
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java
View File

@ -15,7 +15,6 @@
*/
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
@ -29,6 +28,7 @@ import android.opengl.EGLDisplay;
import android.opengl.EGLExt;
import android.opengl.EGLSurface;
import android.opengl.GLES20;
import android.util.Pair;
import android.util.Size;
import android.view.Surface;
import android.view.SurfaceView;
@ -36,8 +36,8 @@ import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
@ -56,8 +56,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* processed on a background thread as it becomes available. All input frames should be {@link
* #registerInputFrame() registered} before they are rendered to the input surface. {@link
* #hasPendingFrames()} can be used to check whether there are frames that have not been fully
* processed yet. The {@code FrameProcessorChain} writes output to the surface passed to {@link
* #configure(Surface, int, int, SurfaceView)}.
* processed yet. Output is written to its {@link #configure(Surface, int, int, SurfaceView) output
* surface}.
*/
/* package */ final class FrameProcessorChain {
@ -65,32 +65,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
GlUtil.glAssertionsEnabled = true;
}
/**
* Configures the output {@link Size sizes} of a list of {@link GlFrameProcessor
* GlFrameProcessors}.
*
* @param inputWidth The width of frames passed to the first {@link GlFrameProcessor}.
* @param inputHeight The height of frames passed to the first {@link GlFrameProcessor}.
* @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors}.
* @return A mutable {@link List} containing the input {@link Size} as well as the output {@link
* Size} of each {@link GlFrameProcessor}.
*/
// TODO(b/218488308): Return an immutable list once VideoTranscodingSamplePipeline no longer needs
// to modify this list for encoder fallback.
public static List<Size> configureSizes(
int inputWidth, int inputHeight, List<GlFrameProcessor> frameProcessors) {
List<Size> sizes = new ArrayList<>(frameProcessors.size() + 1);
sizes.add(new Size(inputWidth, inputHeight));
for (int i = 0; i < frameProcessors.size(); i++) {
sizes.add(
frameProcessors
.get(i)
.configureOutputSize(getLast(sizes).getWidth(), getLast(sizes).getHeight()));
}
return sizes;
}
private static final String THREAD_NAME = "Transformer:FrameProcessorChain";
private final boolean enableExperimentalHdrEditing;
@ -129,12 +103,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* <p>The {@link ExternalCopyFrameProcessor} writes to the first framebuffer.
*/
private final int[] framebuffers;
/**
* The input {@link Size}, i.e., the output {@link Size} of the {@link
* ExternalCopyFrameProcessor}), as well as the output {@link Size} of each of the {@code
* frameProcessors}.
*/
private final List<Size> sizes;
/** The input {@link Size} of each of the {@code frameProcessors}. */
private final ImmutableList<Size> inputSizes;
private int outputWidth;
private int outputHeight;
@ -157,23 +127,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
*
* @param context A {@link Context}.
* @param pixelWidthHeightRatio The ratio of width over height, for each pixel.
* @param inputWidth The input frame width, in pixels.
* @param inputHeight The input frame height, in pixels.
* @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame.
* Their output sizes must be {@link GlFrameProcessor#configureOutputSize(int, int)}
* configured}.
* @param sizes The input {@link Size} as well as the output {@link Size} of each {@link
* GlFrameProcessor}.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1.
*/
public FrameProcessorChain(
Context context,
float pixelWidthHeightRatio,
int inputWidth,
int inputHeight,
List<GlFrameProcessor> frameProcessors,
List<Size> sizes,
boolean enableExperimentalHdrEditing)
throws TransformationException {
checkArgument(frameProcessors.size() + 1 == sizes.size());
if (pixelWidthHeightRatio != 1.0f) {
// TODO(b/211782176): Consider implementing support for non-square pixels.
throw TransformationException.createForFrameProcessorChain(
@ -186,7 +153,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
this.enableExperimentalHdrEditing = enableExperimentalHdrEditing;
this.frameProcessors = frameProcessors;
this.sizes = sizes;
try {
eglDisplay = GlUtil.createEglDisplay();
@ -205,12 +171,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
externalCopyFrameProcessor =
new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing);
framebuffers = new int[frameProcessors.size()];
outputWidth = getLast(sizes).getWidth();
outputHeight = getLast(sizes).getHeight();
Pair<ImmutableList<Size>, Size> sizes =
configureFrameProcessorSizes(inputWidth, inputHeight, frameProcessors);
inputSizes = sizes.first;
outputWidth = sizes.second.getWidth();
outputHeight = sizes.second.getHeight();
debugPreviewWidth = C.LENGTH_UNSET;
debugPreviewHeight = C.LENGTH_UNSET;
}
/** Returns the output {@link Size}. */
public Size getOutputSize() {
return new Size(outputWidth, outputHeight);
}
/**
* Configures the {@code FrameProcessorChain} to process frames to the specified output targets.
*
@ -399,12 +373,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
inputExternalTexId = GlUtil.createExternalTexture();
externalCopyFrameProcessor.configureOutputSize(
/* inputWidth= */ sizes.get(0).getWidth(), /* inputHeight= */ sizes.get(0).getHeight());
Size inputSize = inputSizes.get(0);
externalCopyFrameProcessor.configureOutputSize(inputSize.getWidth(), inputSize.getHeight());
externalCopyFrameProcessor.initialize(inputExternalTexId);
for (int i = 0; i < frameProcessors.size(); i++) {
int inputTexId = GlUtil.createTexture(sizes.get(i).getWidth(), sizes.get(i).getHeight());
inputSize = inputSizes.get(i);
int inputTexId = GlUtil.createTexture(inputSize.getWidth(), inputSize.getHeight());
framebuffers[i] = GlUtil.createFboForTexture(inputTexId);
frameProcessors.get(i).initialize(inputTexId);
}
@ -423,16 +398,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private void processFrame() {
checkState(Thread.currentThread().equals(glThread));
Size outputSize = inputSizes.get(0);
if (frameProcessors.isEmpty()) {
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
GlUtil.focusEglSurface(
eglDisplay, eglContext, eglSurface, outputSize.getWidth(), outputSize.getHeight());
} else {
GlUtil.focusFramebuffer(
eglDisplay,
eglContext,
eglSurface,
framebuffers[0],
sizes.get(0).getWidth(),
sizes.get(0).getHeight());
outputSize.getWidth(),
outputSize.getHeight());
}
inputSurfaceTexture.updateTexImage();
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
@ -441,13 +418,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
externalCopyFrameProcessor.updateProgramAndDraw(presentationTimeNs);
for (int i = 0; i < frameProcessors.size() - 1; i++) {
outputSize = inputSizes.get(i + 1);
GlUtil.focusFramebuffer(
eglDisplay,
eglContext,
eglSurface,
framebuffers[i + 1],
sizes.get(i + 1).getWidth(),
sizes.get(i + 1).getHeight());
outputSize.getWidth(),
outputSize.getHeight());
frameProcessors.get(i).updateProgramAndDraw(presentationTimeNs);
}
if (!frameProcessors.isEmpty()) {
@ -470,4 +448,30 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
checkState(pendingFrameCount.getAndDecrement() > 0);
}
/**
* Configures the input and output {@link Size sizes} of a list of {@link GlFrameProcessor
* GlFrameProcessors}.
*
* @param inputWidth The width of frames passed to the first {@link GlFrameProcessor}, in pixels.
* @param inputHeight The height of frames passed to the first {@link GlFrameProcessor}, in
* pixels.
* @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors}.
* @return The input {@link Size} of each {@link GlFrameProcessor} and the output {@link Size} of
* the final {@link GlFrameProcessor}.
*/
private static Pair<ImmutableList<Size>, Size> configureFrameProcessorSizes(
int inputWidth, int inputHeight, List<GlFrameProcessor> frameProcessors) {
Size size = new Size(inputWidth, inputHeight);
if (frameProcessors.isEmpty()) {
return Pair.create(ImmutableList.of(size), size);
}
ImmutableList.Builder<Size> inputSizes = new ImmutableList.Builder<>();
for (int i = 0; i < frameProcessors.size(); i++) {
inputSizes.add(size);
size = frameProcessors.get(i).configureOutputSize(size.getWidth(), size.getHeight());
}
return Pair.create(inputSizes.build(), size);
}
}

25
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -29,7 +29,6 @@ import androidx.media3.common.Format;
import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import java.util.List;
import org.checkerframework.dataflow.qual.Pure;
@ -70,6 +69,7 @@ import org.checkerframework.dataflow.qual.Pure;
int decodedHeight =
(inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
// TODO(b/214975934): Allow a list of frame processors to be passed into the sample pipeline.
// TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset.
ScaleToFitFrameProcessor scaleToFitFrameProcessor =
new ScaleToFitFrameProcessor.Builder(context)
@ -80,13 +80,15 @@ import org.checkerframework.dataflow.qual.Pure;
new PresentationFrameProcessor.Builder(context)
.setResolution(transformationRequest.outputHeight)
.build();
// TODO(b/214975934): Allow a list of frame processors to be passed into the sample pipeline.
ImmutableList<GlFrameProcessor> frameProcessors =
ImmutableList.of(scaleToFitFrameProcessor, presentationFrameProcessor);
List<Size> frameProcessorSizes =
FrameProcessorChain.configureSizes(decodedWidth, decodedHeight, frameProcessors);
Size requestedEncoderSize = Iterables.getLast(frameProcessorSizes);
// TODO(b/213190310): Move output rotation configuration to PresentationFrameProcessor.
frameProcessorChain =
new FrameProcessorChain(
context,
inputFormat.pixelWidthHeightRatio,
/* inputWidth= */ decodedWidth,
/* inputHeight= */ decodedHeight,
ImmutableList.of(scaleToFitFrameProcessor, presentationFrameProcessor),
transformationRequest.enableHdrEditing);
Size requestedEncoderSize = frameProcessorChain.getOutputSize();
outputRotationDegrees = presentationFrameProcessor.getOutputRotationDegrees();
Format requestedEncoderFormat =
@ -110,13 +112,6 @@ import org.checkerframework.dataflow.qual.Pure;
requestedEncoderFormat,
encoderSupportedFormat));
frameProcessorChain =
new FrameProcessorChain(
context,
inputFormat.pixelWidthHeightRatio,
frameProcessors,
frameProcessorSizes,
transformationRequest.enableHdrEditing);
frameProcessorChain.configure(
/* outputSurface= */ encoder.getInputSurface(),
/* outputWidth= */ encoderSupportedFormat.width,

68
libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java
View File

@ -35,6 +35,7 @@ import org.junit.runner.RunWith;
*/
@RunWith(AndroidJUnit4.class)
public final class FrameProcessorChainTest {
@Test
public void construct_withSupportedPixelWidthHeightRatio_completesSuccessfully()
throws TransformationException {
@ -43,8 +44,9 @@ public final class FrameProcessorChainTest {
new FrameProcessorChain(
context,
/* pixelWidthHeightRatio= */ 1,
/* inputWidth= */ 200,
/* inputHeight= */ 100,
/* frameProcessors= */ ImmutableList.of(),
/* sizes= */ ImmutableList.of(new Size(200, 100)),
/* enableExperimentalHdrEditing= */ false);
}
@ -59,8 +61,9 @@ public final class FrameProcessorChainTest {
new FrameProcessorChain(
context,
/* pixelWidthHeightRatio= */ 2,
/* inputWidth= */ 200,
/* inputHeight= */ 100,
/* frameProcessors= */ ImmutableList.of(),
/* sizes= */ ImmutableList.of(new Size(200, 100)),
/* enableExperimentalHdrEditing= */ false));
assertThat(exception).hasCauseThat().isInstanceOf(UnsupportedOperationException.class);
@ -68,46 +71,63 @@ public final class FrameProcessorChainTest {
}
@Test
public void configureOutputDimensions_withEmptyList_returnsInputSize() {
public void getOutputSize_withoutFrameProcessors_returnsInputSize()
throws TransformationException {
Size inputSize = new Size(200, 100);
FrameProcessorChain frameProcessorChain =
createFrameProcessorChainWithFakeFrameProcessors(
inputSize, /* frameProcessorOutputSizes= */ ImmutableList.of());
List<Size> sizes =
FrameProcessorChain.configureSizes(
inputSize.getWidth(), inputSize.getHeight(), /* frameProcessors= */ ImmutableList.of());
Size outputSize = frameProcessorChain.getOutputSize();
assertThat(sizes).containsExactly(inputSize);
assertThat(outputSize).isEqualTo(inputSize);
}
@Test
public void configureOutputDimensions_withOneFrameProcessor_returnsItsInputAndOutputDimensions() {
public void getOutputSize_withOneFrameProcessor_returnsItsOutputSize()
throws TransformationException {
Size inputSize = new Size(200, 100);
Size outputSize = new Size(300, 250);
GlFrameProcessor frameProcessor = new FakeFrameProcessor(outputSize);
Size frameProcessorOutputSize = new Size(300, 250);
FrameProcessorChain frameProcessorChain =
createFrameProcessorChainWithFakeFrameProcessors(
inputSize, /* frameProcessorOutputSizes= */ ImmutableList.of(frameProcessorOutputSize));
List<Size> sizes =
FrameProcessorChain.configureSizes(
inputSize.getWidth(), inputSize.getHeight(), ImmutableList.of(frameProcessor));
Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize();
assertThat(sizes).containsExactly(inputSize, outputSize).inOrder();
assertThat(frameProcessorChainOutputSize).isEqualTo(frameProcessorOutputSize);
}
@Test
public void configureOutputDimensions_withThreeFrameProcessors_propagatesOutputDimensions() {
public void getOutputSize_withThreeFrameProcessors_returnsLastOutputSize()
throws TransformationException {
Size inputSize = new Size(200, 100);
Size outputSize1 = new Size(300, 250);
Size outputSize2 = new Size(400, 244);
Size outputSize3 = new Size(150, 160);
GlFrameProcessor frameProcessor1 = new FakeFrameProcessor(outputSize1);
GlFrameProcessor frameProcessor2 = new FakeFrameProcessor(outputSize2);
GlFrameProcessor frameProcessor3 = new FakeFrameProcessor(outputSize3);
FrameProcessorChain frameProcessorChain =
createFrameProcessorChainWithFakeFrameProcessors(
inputSize,
/* frameProcessorOutputSizes= */ ImmutableList.of(
outputSize1, outputSize2, outputSize3));
List<Size> sizes =
FrameProcessorChain.configureSizes(
inputSize.getWidth(),
inputSize.getHeight(),
ImmutableList.of(frameProcessor1, frameProcessor2, frameProcessor3));
Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize();
assertThat(sizes).containsExactly(inputSize, outputSize1, outputSize2, outputSize3).inOrder();
assertThat(frameProcessorChainOutputSize).isEqualTo(outputSize3);
}
private static FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors(
Size inputSize, List<Size> frameProcessorOutputSizes) throws TransformationException {
ImmutableList.Builder<GlFrameProcessor> frameProcessors = new ImmutableList.Builder<>();
for (Size element : frameProcessorOutputSizes) {
frameProcessors.add(new FakeFrameProcessor(element));
}
return new FrameProcessorChain(
getApplicationContext(),
/* pixelWidthHeightRatio= */ 1,
inputSize.getWidth(),
inputSize.getHeight(),
frameProcessors.build(),
/* enableExperimentalHdrEditing= */ false);
}
private static class FakeFrameProcessor implements GlFrameProcessor {