Expected images are taken from an emulator, so tests on different emulators or physical * devices may fail. To test on other devices, please increase the {@link * BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps - * as recommended in {@link FrameEditorDataProcessingTest}. + * as recommended in {@link FrameProcessorChainPixelTest}. */ @RunWith(AndroidJUnit4.class) public final class AdvancedFrameProcessorPixelTest { diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/BitmapTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/BitmapTestUtil.java index 3285a07220..db2df0c07e 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/BitmapTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/BitmapTestUtil.java @@ -39,8 +39,8 @@ import java.io.InputStream; import java.nio.ByteBuffer; /** - * Utilities for instrumentation tests for the {@link FrameEditor} and {@link GlFrameProcessor - * GlFrameProcessors}. + * Utilities for instrumentation tests for the {@link FrameProcessorChain} and {@link + * GlFrameProcessor GlFrameProcessors}. */ public class BitmapTestUtil { @@ -53,6 +53,10 @@ public class BitmapTestUtil { "media/bitmap/sample_mp4_first_frame_translate_right.png"; public static final String SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING = "media/bitmap/sample_mp4_first_frame_scale_narrow.png"; + public static final String ROTATE_THEN_TRANSLATE_EXPECTED_OUTPUT_PNG_ASSET_STRING = + "media/bitmap/sample_mp4_first_frame_rotate_then_translate.png"; + public static final String TRANSLATE_THEN_ROTATE_EXPECTED_OUTPUT_PNG_ASSET_STRING = + "media/bitmap/sample_mp4_first_frame_translate_then_rotate.png"; public static final String ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING = "media/bitmap/sample_mp4_first_frame_rotate90.png"; public static final String REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING = @@ -63,13 +67,15 @@ public class BitmapTestUtil { * Maximum allowed average pixel difference between the expected and actual edited images for the * test to pass. The value is chosen so that differences in decoder behavior across emulator * versions don't affect whether the test passes for most emulators, but substantial distortions - * introduced by changes in the behavior of the frame editor will cause the test to fail. + * introduced by changes in the behavior of the {@link GlFrameProcessor GlFrameProcessors} will + * cause the test to fail. * *
To run this test on physical devices, please use a value of 5f, rather than 0.1f. This * higher value will ignore some very small errors, but will allow for some differences caused by * graphics implementations to be ignored. When the difference is close to the threshold, manually * inspect expected/actual bitmaps to confirm failure, as it's possible this is caused by a - * difference in the codec or graphics implementation as opposed to a FrameEditor issue. + * difference in the codec or graphics implementation as opposed to a {@link GlFrameProcessor} + * issue. */ public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 0.1f; diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java similarity index 70% rename from libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java rename to libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java index 78162591b6..80fdf71232 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java @@ -20,11 +20,12 @@ import static androidx.media3.transformer.BitmapTestUtil.FIRST_FRAME_PNG_ASSET_S import static androidx.media3.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; import static androidx.media3.transformer.BitmapTestUtil.REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING; import static androidx.media3.transformer.BitmapTestUtil.ROTATE45_SCALE_TO_FIT_EXPECTED_OUTPUT_PNG_ASSET_STRING; -import static androidx.media3.transformer.BitmapTestUtil.ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING; -import static androidx.media3.transformer.BitmapTestUtil.SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING; +import static androidx.media3.transformer.BitmapTestUtil.ROTATE_THEN_TRANSLATE_EXPECTED_OUTPUT_PNG_ASSET_STRING; import static androidx.media3.transformer.BitmapTestUtil.TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING; +import static androidx.media3.transformer.BitmapTestUtil.TRANSLATE_THEN_ROTATE_EXPECTED_OUTPUT_PNG_ASSET_STRING; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; +import static java.util.Arrays.asList; import android.content.Context; import android.content.res.AssetFileDescriptor; @@ -40,14 +41,16 @@ import android.util.Size; import androidx.annotation.Nullable; import androidx.media3.common.MimeTypes; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.Iterables; import java.nio.ByteBuffer; +import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.junit.After; import org.junit.Test; import org.junit.runner.RunWith; /** - * Pixel test for frame processing via {@link FrameEditor}. + * Pixel test for frame processing via {@link FrameProcessorChain}. * *
Expected images are taken from an emulator, so tests on different emulators or physical
* devices may fail. To test on other devices, please increase the {@link
@@ -55,41 +58,35 @@ import org.junit.runner.RunWith;
* bitmaps.
*/
@RunWith(AndroidJUnit4.class)
-public final class FrameEditorDataProcessingTest {
- // TODO(b/214975934): Once FrameEditor is converted to a FrameProcessorChain, replace these tests
- // with a test for a few example combinations of GlFrameProcessors rather than testing all use
- // cases of TransformationFrameProcessor.
+public final class FrameProcessorChainPixelTest {
/** Input video of which we only use the first frame. */
private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4";
/** Timeout for dequeueing buffers from the codec, in microseconds. */
private static final int DEQUEUE_TIMEOUT_US = 5_000_000;
/**
- * Time to wait for the decoded frame to populate the frame editor's input surface and the frame
- * editor to finish processing the frame, in milliseconds.
+ * Time to wait for the decoded frame to populate the {@link FrameProcessorChain}'s input surface
+ * and the {@link FrameProcessorChain} to finish processing the frame, in milliseconds.
*/
- private static final int FRAME_PROCESSING_WAIT_MS = 1000;
+ private static final int FRAME_PROCESSING_WAIT_MS = 2000;
/** The ratio of width over height, for each pixel in a frame. */
private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1;
- private @MonotonicNonNull FrameEditor frameEditor;
- private @MonotonicNonNull ImageReader frameEditorOutputImageReader;
+ private @MonotonicNonNull FrameProcessorChain frameProcessorChain;
+ private @MonotonicNonNull ImageReader outputImageReader;
private @MonotonicNonNull MediaFormat mediaFormat;
@After
public void release() {
- if (frameEditor != null) {
- frameEditor.release();
+ if (frameProcessorChain != null) {
+ frameProcessorChain.release();
}
}
@Test
public void processData_noEdits_producesExpectedOutput() throws Exception {
final String testId = "processData_noEdits";
- Matrix identityMatrix = new Matrix();
- GlFrameProcessor glFrameProcessor =
- new AdvancedFrameProcessor(getApplicationContext(), identityMatrix);
- setUpAndPrepareFirstFrame(glFrameProcessor);
+ setUpAndPrepareFirstFrame();
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING);
Bitmap actualBitmap = processFirstFrameAndEnd();
@@ -104,8 +101,9 @@ public final class FrameEditorDataProcessingTest {
}
@Test
- public void processData_translateRight_producesExpectedOutput() throws Exception {
- final String testId = "processData_translateRight";
+ public void processData_withAdvancedFrameProcessor_translateRight_producesExpectedOutput()
+ throws Exception {
+ final String testId = "processData_withAdvancedFrameProcessor_translateRight";
Matrix translateRightMatrix = new Matrix();
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
GlFrameProcessor glFrameProcessor =
@@ -126,15 +124,20 @@ public final class FrameEditorDataProcessingTest {
}
@Test
- public void processData_scaleNarrow_producesExpectedOutput() throws Exception {
- final String testId = "processData_scaleNarrow";
- Matrix scaleNarrowMatrix = new Matrix();
- scaleNarrowMatrix.postScale(.5f, 1.2f);
- GlFrameProcessor glFrameProcessor =
- new AdvancedFrameProcessor(getApplicationContext(), scaleNarrowMatrix);
- setUpAndPrepareFirstFrame(glFrameProcessor);
+ public void processData_withAdvancedAndScaleToFitFrameProcessors_producesExpectedOutput()
+ throws Exception {
+ final String testId = "processData_withAdvancedAndScaleToFitFrameProcessors";
+ Matrix translateRightMatrix = new Matrix();
+ translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
+ GlFrameProcessor translateRightFrameProcessor =
+ new AdvancedFrameProcessor(getApplicationContext(), translateRightMatrix);
+ GlFrameProcessor rotate45FrameProcessor =
+ new ScaleToFitFrameProcessor.Builder(getApplicationContext())
+ .setRotationDegrees(45)
+ .build();
+ setUpAndPrepareFirstFrame(translateRightFrameProcessor, rotate45FrameProcessor);
Bitmap expectedBitmap =
- BitmapTestUtil.readBitmap(SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING);
+ BitmapTestUtil.readBitmap(TRANSLATE_THEN_ROTATE_EXPECTED_OUTPUT_PNG_ASSET_STRING);
Bitmap actualBitmap = processFirstFrameAndEnd();
@@ -148,14 +151,20 @@ public final class FrameEditorDataProcessingTest {
}
@Test
- public void processData_rotate90_producesExpectedOutput() throws Exception {
- final String testId = "processData_rotate90";
- Matrix rotate90Matrix = new Matrix();
- rotate90Matrix.postRotate(/* degrees= */ 90);
- GlFrameProcessor glFrameProcessor =
- new AdvancedFrameProcessor(getApplicationContext(), rotate90Matrix);
- setUpAndPrepareFirstFrame(glFrameProcessor);
- Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING);
+ public void processData_withScaleToFitAndAdvancedFrameProcessors_producesExpectedOutput()
+ throws Exception {
+ final String testId = "processData_withScaleToFitAndAdvancedFrameProcessors";
+ GlFrameProcessor rotate45FrameProcessor =
+ new ScaleToFitFrameProcessor.Builder(getApplicationContext())
+ .setRotationDegrees(45)
+ .build();
+ Matrix translateRightMatrix = new Matrix();
+ translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
+ GlFrameProcessor translateRightFrameProcessor =
+ new AdvancedFrameProcessor(getApplicationContext(), translateRightMatrix);
+ setUpAndPrepareFirstFrame(rotate45FrameProcessor, translateRightFrameProcessor);
+ Bitmap expectedBitmap =
+ BitmapTestUtil.readBitmap(ROTATE_THEN_TRANSLATE_EXPECTED_OUTPUT_PNG_ASSET_STRING);
Bitmap actualBitmap = processFirstFrameAndEnd();
@@ -169,8 +178,9 @@ public final class FrameEditorDataProcessingTest {
}
@Test
- public void processData_requestOutputHeight_producesExpectedOutput() throws Exception {
- final String testId = "processData_requestOutputHeight";
+ public void processData_withScaleToFitFrameProcessor_requestOutputHeight_producesExpectedOutput()
+ throws Exception {
+ final String testId = "processData_withScaleToFitFrameProcessor_requestOutputHeight";
// TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline
// resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can
// test that rotation doesn't distort the image.
@@ -192,8 +202,9 @@ public final class FrameEditorDataProcessingTest {
}
@Test
- public void processData_rotate45_scaleToFit_producesExpectedOutput() throws Exception {
- final String testId = "processData_rotate45_scaleToFit";
+ public void processData_withScaleToFitFrameProcessor_rotate45_producesExpectedOutput()
+ throws Exception {
+ final String testId = "processData_withScaleToFitFrameProcessor_rotate45";
// TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline
// resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can
// test that rotation doesn't distort the image.
@@ -218,12 +229,13 @@ public final class FrameEditorDataProcessingTest {
/**
* Set up and prepare the first frame from an input video, as well as relevant test
- * infrastructure. The frame will be sent towards the {@link FrameEditor}, and may be accessed on
- * the {@link FrameEditor}'s output {@code frameEditorOutputImageReader}.
+ * infrastructure. The frame will be sent towards the {@link FrameProcessorChain}, and may be
+ * accessed on the {@link FrameProcessorChain}'s output {@code outputImageReader}.
*
- * @param glFrameProcessor The frame processor that will apply changes to the input frame.
+ * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} that will apply changes
+ * to the input frame.
*/
- private void setUpAndPrepareFirstFrame(GlFrameProcessor glFrameProcessor) throws Exception {
+ private void setUpAndPrepareFirstFrame(GlFrameProcessor... frameProcessors) throws Exception {
// Set up the extractor to read the first video frame and get its format.
MediaExtractor mediaExtractor = new MediaExtractor();
@Nullable MediaCodec mediaCodec = null;
@@ -240,31 +252,35 @@ public final class FrameEditorDataProcessingTest {
int inputWidth = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH);
int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
- Size outputSize = glFrameProcessor.configureOutputSize(inputWidth, inputHeight);
- int outputWidth = outputSize.getWidth();
- int outputHeight = outputSize.getHeight();
- frameEditorOutputImageReader =
+ List Input becomes available on its {@link #createInputSurface() input surface} asynchronously and
* is processed on a background thread as it becomes available. All input frames should be {@link
* #registerInputFrame() registered} before they are rendered to the input surface. {@link
* #hasPendingFrames()} can be used to check whether there are frames that have not been fully
- * processed yet. Output is written to its {@link #create(Context, int, int, int, int, float,
- * GlFrameProcessor, Surface, boolean, Transformer.DebugViewProvider) output surface}.
+ * processed yet. The {@code FrameProcessorChain} writes output to the surface passed to {@link
+ * #create(Context, float, List, List, Surface, boolean, Transformer.DebugViewProvider)}.
*/
-/* package */ final class FrameEditor {
+/* package */ final class FrameProcessorChain {
static {
GlUtil.glAssertionsEnabled = true;
}
/**
- * Returns a new {@code FrameEditor} for applying changes to individual frames.
+ * Configures the output {@link Size sizes} of a list of {@link GlFrameProcessor
+ * GlFrameProcessors}.
+ *
+ * @param inputWidth The width of frames passed to the first {@link GlFrameProcessor}.
+ * @param inputHeight The height of frames passed to the first {@link GlFrameProcessor}.
+ * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors}.
+ * @return A mutable {@link List} containing the input {@link Size} as well as the output {@link
+ * Size} of each {@link GlFrameProcessor}.
+ */
+ // TODO(b/218488308): Return an immutable list once VideoTranscodingSamplePipeline no longer needs
+ // to modify this list for encoder fallback.
+ public static List As the {@code FrameEditor} will call OpenGL commands on the {@code
+ * As the {@code FrameProcessorChain} will call OpenGL commands on the {@code
* singleThreadExecutorService}'s thread, the OpenGL context and objects also need to be created
* on that thread. So this method should only be called on the {@code
* singleThreadExecutorService}'s thread.
*/
- private static FrameEditor createOpenGlObjectsAndFrameEditor(
+ private static FrameProcessorChain createOpenGlObjectsAndFrameProcessorChain(
ExecutorService singleThreadExecutorService,
ExternalCopyFrameProcessor externalCopyFrameProcessor,
- GlFrameProcessor transformationFrameProcessor,
+ List The {@link ExternalCopyFrameProcessor} writes to the first framebuffer.
+ */
+ private final int[] framebuffers;
+ /**
+ * The input {@link Size}, i.e., the output {@link Size} of the {@link
+ * ExternalCopyFrameProcessor}), as well as the output {@link Size} of each of the {@code
+ * frameProcessors}.
+ */
+ private final List Should be called before rendering a frame to the frame editor's input surface.
+ * Should be called before rendering a frame to the frame processor chain's input surface.
*
* @throws IllegalStateException If called after {@link #signalEndOfInputStream()}.
*/
@@ -353,11 +372,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
try {
oldestGlProcessingFuture.get();
} catch (ExecutionException e) {
- throw TransformationException.createForFrameEditor(
+ throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
- throw TransformationException.createForFrameEditor(
+ throw TransformationException.createForFrameProcessorChain(
e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED);
}
oldestGlProcessingFuture = futures.peek();
@@ -377,7 +396,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return inputStreamEnded && !hasPendingFrames();
}
- /** Informs the {@code FrameEditor} that no further input frames should be accepted. */
+ /** Informs the {@code FrameProcessorChain} that no further input frames should be accepted. */
public void signalEndOfInputStream() {
inputStreamEnded = true;
}
@@ -385,9 +404,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/**
* Releases all resources.
*
- * If the frame editor is released before it has {@link #isEnded() ended}, it will attempt to
- * cancel processing any input frames that have already become available. Input frames that become
- * available after release are ignored.
+ * If the frame processor chain is released before it has {@link #isEnded() ended}, it will
+ * attempt to cancel processing any input frames that have already become available. Input frames
+ * that become available after release are ignored.
*/
public void release() {
releaseRequested = true;
@@ -398,7 +417,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
singleThreadExecutorService.submit(
() -> {
externalCopyFrameProcessor.release();
- transformationFrameProcessor.release();
+ for (int i = 0; i < frameProcessors.size(); i++) {
+ frameProcessors.get(i).release();
+ }
GlUtil.destroyEglContext(eglDisplay, eglContext);
}));
if (inputSurfaceTexture != null) {
@@ -417,13 +438,41 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
long presentationTimeNs = inputSurfaceTexture.getTimestamp();
- GlUtil.focusFramebuffer(
- eglDisplay, eglContext, eglSurface, frameBuffer, inputWidth, inputHeight);
+ if (frameProcessors.isEmpty()) {
+ GlUtil.focusEglSurface(
+ eglDisplay, eglContext, eglSurface, sizes.get(0).getWidth(), sizes.get(0).getHeight());
+ } else {
+ GlUtil.focusFramebuffer(
+ eglDisplay,
+ eglContext,
+ eglSurface,
+ framebuffers[0],
+ sizes.get(0).getWidth(),
+ sizes.get(0).getHeight());
+ }
externalCopyFrameProcessor.setTextureTransformMatrix(textureTransformMatrix);
externalCopyFrameProcessor.updateProgramAndDraw(presentationTimeNs);
- GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
- transformationFrameProcessor.updateProgramAndDraw(presentationTimeNs);
+ for (int i = 0; i < frameProcessors.size() - 1; i++) {
+ GlUtil.focusFramebuffer(
+ eglDisplay,
+ eglContext,
+ eglSurface,
+ framebuffers[i + 1],
+ sizes.get(i + 1).getWidth(),
+ sizes.get(i + 1).getHeight());
+ frameProcessors.get(i).updateProgramAndDraw(presentationTimeNs);
+ }
+ if (!frameProcessors.isEmpty()) {
+ GlUtil.focusEglSurface(
+ eglDisplay,
+ eglContext,
+ eglSurface,
+ getLast(sizes).getWidth(),
+ getLast(sizes).getHeight());
+ getLast(frameProcessors).updateProgramAndDraw(presentationTimeNs);
+ }
+
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs);
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java
index 9e7c4029ae..726e1b1084 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java
@@ -32,6 +32,9 @@ import java.io.IOException;
*
*/
/* package */ interface GlFrameProcessor {
+ // TODO(b/214975934): Investigate whether all configuration can be moved to initialize by
+ // using a placeholder surface until the encoder surface is known. If so, convert
+ // configureOutputSize to a simple getter.
/**
* Returns the output {@link Size} of frames processed through {@link
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java
index e8198a6e3e..e47781a2ad 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java
@@ -27,9 +27,7 @@ import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.util.GlUtil;
import java.io.IOException;
-import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
-import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/**
* Applies a simple rotation and/or scale in the vertex shader. All input frames' pixels will be
@@ -173,15 +171,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
*
* This method can only be called after {@link #configureOutputSize(int, int)}.
*/
- @RequiresNonNull("adjustedTransformationMatrix")
public boolean shouldProcess() {
+ checkStateNotNull(adjustedTransformationMatrix);
return inputWidth != outputWidth
|| inputHeight != outputHeight
|| !adjustedTransformationMatrix.isIdentity();
}
@Override
- @EnsuresNonNull("adjustedTransformationMatrix")
public Size configureOutputSize(int inputWidth, int inputHeight) {
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
@@ -191,7 +188,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
int displayHeight = inputHeight;
if (!transformationMatrix.isIdentity()) {
float inputAspectRatio = (float) inputWidth / inputHeight;
- // Scale frames by inputAspectRatio, to account for FrameEditor's normalized device
+ // Scale frames by inputAspectRatio, to account for FrameProcessorChain's normalized device
// coordinates (NDC) (a square from -1 to 1 for both x and y) and preserve rectangular
// display of input pixels during transformations (ex. rotations). With scaling,
// transformationMatrix operations operate on a rectangle for x from -inputAspectRatio to
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java
index bf4bfa7ebb..8361d1d7d0 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java
@@ -274,15 +274,15 @@ public final class TransformationException extends Exception {
}
/**
- * Creates an instance for a {@link FrameEditor} related exception.
+ * Creates an instance for a {@link FrameProcessorChain} related exception.
*
* @param cause The cause of the failure.
* @param errorCode See {@link #errorCode}.
* @return The created instance.
*/
- /* package */ static TransformationException createForFrameEditor(
+ /* package */ static TransformationException createForFrameProcessorChain(
Throwable cause, int errorCode) {
- return new TransformationException("FrameEditor error", cause, errorCode);
+ return new TransformationException("FrameProcessorChain error", cause, errorCode);
}
/**
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
index 9c52664402..6f044c18ec 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
@@ -28,6 +28,8 @@ import androidx.annotation.RequiresApi;
import androidx.media3.common.Format;
import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
import java.util.List;
import org.checkerframework.dataflow.qual.Pure;
@@ -40,7 +42,7 @@ import org.checkerframework.dataflow.qual.Pure;
private final DecoderInputBuffer decoderInputBuffer;
private final Codec decoder;
- @Nullable private final FrameEditor frameEditor;
+ @Nullable private final FrameProcessorChain frameProcessorChain;
private final Codec encoder;
private final DecoderInputBuffer encoderOutputBuffer;
@@ -74,14 +76,18 @@ import org.checkerframework.dataflow.qual.Pure;
.setRotationDegrees(transformationRequest.rotationDegrees)
.setResolution(transformationRequest.outputHeight)
.build();
- Size requestedEncoderDimensions =
- scaleToFitFrameProcessor.configureOutputSize(decodedWidth, decodedHeight);
+ // TODO(b/214975934): Allow a list of frame processors to be passed into the sample pipeline.
+ ImmutableList In this method the decoder could decode multiple frames in one invocation; as compared to
* {@link #processDataDefault()}, in which one frame is decoded in each invocation. Consequently,
- * if {@link FrameEditor} processes frames slower than the decoder, decoded frames are queued up
- * in the decoder's output surface.
+ * if {@link FrameProcessorChain} processes frames slower than the decoder, decoded frames are
+ * queued up in the decoder's output surface.
*
* Prior to API 29, decoders may drop frames to keep their output surface from growing out of
* bound; while after API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame
@@ -195,12 +205,12 @@ import org.checkerframework.dataflow.qual.Pure;
/**
* Processes at most one input frame and returns whether a frame was processed.
*
- * Only renders decoder output to the {@link FrameEditor}'s input surface if the {@link
- * FrameEditor} has finished processing the previous frame.
+ * Only renders decoder output to the {@link FrameProcessorChain}'s input surface if the {@link
+ * FrameProcessorChain} has finished processing the previous frame.
*/
private boolean processDataDefault() throws TransformationException {
// TODO(b/214975934): Check whether this can be converted to a while-loop like processDataV29.
- if (frameEditor != null && frameEditor.hasPendingFrames()) {
+ if (frameProcessorChain != null && frameProcessorChain.hasPendingFrames()) {
return false;
}
return maybeProcessDecoderOutput();
@@ -240,8 +250,8 @@ import org.checkerframework.dataflow.qual.Pure;
@Override
public void release() {
- if (frameEditor != null) {
- frameEditor.release();
+ if (frameProcessorChain != null) {
+ frameProcessorChain.release();
}
decoder.release();
encoder.release();
@@ -299,8 +309,8 @@ import org.checkerframework.dataflow.qual.Pure;
return false;
}
- if (frameEditor != null) {
- frameEditor.registerInputFrame();
+ if (frameProcessorChain != null) {
+ frameProcessorChain.registerInputFrame();
}
decoder.releaseOutputBuffer(/* render= */ true);
return true;
diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java
new file mode 100644
index 0000000000..b2b9a1a3d0
--- /dev/null
+++ b/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.transformer;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import android.util.Size;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/**
+ * Robolectric tests for {@link FrameProcessorChain}.
+ *
+ * See {@code FrameProcessorChainTest} and {@code FrameProcessorChainPixelTest} in the
+ * androidTest directory for instrumentation tests.
+ */
+@RunWith(AndroidJUnit4.class)
+public final class FrameProcessorChainTest {
+
+ @Test
+ public void configureOutputDimensions_withEmptyList_returnsInputSize() {
+ Size inputSize = new Size(200, 100);
+
+ List