diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/rotate45_then_scale2w.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/rotate45_then_scale2w.png new file mode 100644 index 0000000000..b68ddf2035 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/rotate45_then_scale2w.png differ diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java index 91cf57e840..7cec1d5a6a 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java @@ -63,6 +63,8 @@ public final class FrameProcessorChainPixelTest { "media/bitmap/sample_mp4_first_frame/translate_right.png"; public static final String ROTATE_THEN_TRANSLATE_PNG_ASSET_PATH = "media/bitmap/sample_mp4_first_frame/rotate_then_translate.png"; + public static final String ROTATE_THEN_SCALE_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/rotate45_then_scale2w.png"; public static final String TRANSLATE_THEN_ROTATE_PNG_ASSET_PATH = "media/bitmap/sample_mp4_first_frame/translate_then_rotate.png"; public static final String REQUEST_OUTPUT_HEIGHT_PNG_ASSET_PATH = @@ -88,7 +90,7 @@ public final class FrameProcessorChainPixelTest { new AtomicReference<>(); private @MonotonicNonNull FrameProcessorChain frameProcessorChain; - private @MonotonicNonNull ImageReader outputImageReader; + private volatile @MonotonicNonNull ImageReader outputImageReader; private @MonotonicNonNull MediaFormat mediaFormat; @After @@ -260,6 +262,30 @@ public final class FrameProcessorChainPixelTest { assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } + @Test + public void processData_withTwoWrappedScaleToFitTransformations_producesExpectedOutput() + throws Exception { + String testId = "processData_withTwoWrappedScaleToFitTransformations"; + setUpAndPrepareFirstFrame( + DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, + new GlEffectWrapper(new ScaleToFitTransformation.Builder().setRotationDegrees(45).build()), + new GlEffectWrapper( + new ScaleToFitTransformation.Builder() + .setScale(/* scaleX= */ 2, /* scaleY= */ 1) + .build())); + Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_THEN_SCALE_PNG_ASSET_PATH); + + Bitmap actualBitmap = processFirstFrameAndEnd(); + + BitmapTestUtil.maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap); + // TODO(b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + @Test public void processData_withManyComposedMatrixTransformations_producesSameOutputAsCombinedTransformation() @@ -325,27 +351,27 @@ public final class FrameProcessorChainPixelTest { int inputWidth = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH); int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); frameProcessorChain = - FrameProcessorChain.create( - context, - /* listener= */ this.frameProcessingException::set, - pixelWidthHeightRatio, - inputWidth, - inputHeight, - /* streamOffsetUs= */ 0L, - effects, - /* enableExperimentalHdrEditing= */ false); - Size outputSize = frameProcessorChain.getOutputSize(); - outputImageReader = - ImageReader.newInstance( - outputSize.getWidth(), - outputSize.getHeight(), - PixelFormat.RGBA_8888, - /* maxImages= */ 1); - frameProcessorChain.setOutputSurface( - outputImageReader.getSurface(), - outputSize.getWidth(), - outputSize.getHeight(), - /* debugSurfaceView= */ null); + checkNotNull( + FrameProcessorChain.create( + context, + /* listener= */ this.frameProcessingException::set, + pixelWidthHeightRatio, + inputWidth, + inputHeight, + /* streamOffsetUs= */ 0L, + effects, + /* outputSurfaceProvider= */ (requestedWidth, requestedHeight) -> { + outputImageReader = + ImageReader.newInstance( + requestedWidth, + requestedHeight, + PixelFormat.RGBA_8888, + /* maxImages= */ 1); + return new SurfaceInfo( + outputImageReader.getSurface(), requestedWidth, requestedHeight); + }, + Transformer.DebugViewProvider.NONE, + /* enableExperimentalHdrEditing= */ false)); frameProcessorChain.registerInputFrame(); // Queue the first video frame from the extractor. @@ -437,4 +463,27 @@ public final class FrameProcessorChainPixelTest { return checkStateNotNull(adjustedTransformationMatrix); } } + + /** + * Wraps a {@link GlEffect} to prevent the {@link FrameProcessorChain} from detecting its class + * and optimizing it. + * + *

This ensures that {@link FrameProcessorChain} uses a separate {@link GlTextureProcessor} for + * the wrapped {@link GlEffect} rather than merging it with preceding or subsequent {@link + * GlEffect} instances and applying them in one combined {@link GlTextureProcessor}. + */ + private static final class GlEffectWrapper implements GlEffect { + + private final GlEffect effect; + + public GlEffectWrapper(GlEffect effect) { + this.effect = effect; + } + + @Override + public SingleFrameGlTextureProcessor toGlTextureProcessor(Context context) + throws FrameProcessingException { + return effect.toGlTextureProcessor(context); + } + } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java deleted file mode 100644 index 5dfe72e5b1..0000000000 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright 2021 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.transformer; - -import static androidx.test.core.app.ApplicationProvider.getApplicationContext; -import static com.google.common.truth.Truth.assertThat; - -import android.content.Context; -import android.util.Size; -import androidx.test.ext.junit.runners.AndroidJUnit4; -import com.google.common.collect.ImmutableList; -import java.util.List; -import java.util.concurrent.atomic.AtomicReference; -import org.junit.Test; -import org.junit.runner.RunWith; - -/** - * Tests for creating and configuring a {@link FrameProcessorChain}. - * - *

See {@link FrameProcessorChainPixelTest} for data processing tests. - */ -@RunWith(AndroidJUnit4.class) -public final class FrameProcessorChainTest { - private final AtomicReference frameProcessingException = - new AtomicReference<>(); - - @Test - public void getOutputSize_noOperation_returnsInputSize() throws Exception { - Size inputSize = new Size(200, 100); - FrameProcessorChain frameProcessorChain = - createFrameProcessorChainWithFakeTextureProcessors( - /* pixelWidthHeightRatio= */ 1f, - inputSize, - /* textureProcessorOutputSizes= */ ImmutableList.of()); - - Size outputSize = frameProcessorChain.getOutputSize(); - - assertThat(outputSize).isEqualTo(inputSize); - assertThat(frameProcessingException.get()).isNull(); - } - - @Test - public void getOutputSize_withWidePixels_returnsWiderOutputSize() throws Exception { - Size inputSize = new Size(200, 100); - FrameProcessorChain frameProcessorChain = - createFrameProcessorChainWithFakeTextureProcessors( - /* pixelWidthHeightRatio= */ 2f, - inputSize, - /* textureProcessorOutputSizes= */ ImmutableList.of()); - - Size outputSize = frameProcessorChain.getOutputSize(); - - assertThat(outputSize).isEqualTo(new Size(400, 100)); - assertThat(frameProcessingException.get()).isNull(); - } - - @Test - public void getOutputSize_withTallPixels_returnsTallerOutputSize() throws Exception { - Size inputSize = new Size(200, 100); - FrameProcessorChain frameProcessorChain = - createFrameProcessorChainWithFakeTextureProcessors( - /* pixelWidthHeightRatio= */ .5f, - inputSize, - /* textureProcessorOutputSizes= */ ImmutableList.of()); - - Size outputSize = frameProcessorChain.getOutputSize(); - - assertThat(outputSize).isEqualTo(new Size(200, 200)); - assertThat(frameProcessingException.get()).isNull(); - } - - @Test - public void getOutputSize_withOneTextureProcessor_returnsItsOutputSize() throws Exception { - Size inputSize = new Size(200, 100); - Size textureProcessorOutputSize = new Size(300, 250); - FrameProcessorChain frameProcessorChain = - createFrameProcessorChainWithFakeTextureProcessors( - /* pixelWidthHeightRatio= */ 1f, - inputSize, - /* textureProcessorOutputSizes= */ ImmutableList.of(textureProcessorOutputSize)); - - Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize(); - - assertThat(frameProcessorChainOutputSize).isEqualTo(textureProcessorOutputSize); - assertThat(frameProcessingException.get()).isNull(); - } - - @Test - public void getOutputSize_withThreeTextureProcessors_returnsLastOutputSize() throws Exception { - Size inputSize = new Size(200, 100); - Size outputSize1 = new Size(300, 250); - Size outputSize2 = new Size(400, 244); - Size outputSize3 = new Size(150, 160); - FrameProcessorChain frameProcessorChain = - createFrameProcessorChainWithFakeTextureProcessors( - /* pixelWidthHeightRatio= */ 1f, - inputSize, - /* textureProcessorOutputSizes= */ ImmutableList.of( - outputSize1, outputSize2, outputSize3)); - - Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize(); - - assertThat(frameProcessorChainOutputSize).isEqualTo(outputSize3); - assertThat(frameProcessingException.get()).isNull(); - } - - private FrameProcessorChain createFrameProcessorChainWithFakeTextureProcessors( - float pixelWidthHeightRatio, Size inputSize, List textureProcessorOutputSizes) - throws FrameProcessingException { - ImmutableList.Builder effects = new ImmutableList.Builder<>(); - for (Size element : textureProcessorOutputSizes) { - effects.add((Context context) -> new FakeTextureProcessor(element)); - } - return FrameProcessorChain.create( - getApplicationContext(), - /* listener= */ this.frameProcessingException::set, - pixelWidthHeightRatio, - inputSize.getWidth(), - inputSize.getHeight(), - /* streamOffsetUs= */ 0L, - effects.build(), - /* enableExperimentalHdrEditing= */ false); - } - - private static class FakeTextureProcessor extends SingleFrameGlTextureProcessor { - - private final Size outputSize; - - private FakeTextureProcessor(Size outputSize) { - this.outputSize = outputSize; - } - - @Override - public Size configure(int inputWidth, int inputHeight) { - return outputSize; - } - - @Override - public void drawFrame(int inputTexId, long presentationTimeNs) {} - } -} diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 1b6af6de70..f7aeb0ca91 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -18,12 +18,15 @@ package androidx.media3.transformer; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; import android.content.Context; import android.net.Uri; +import androidx.media3.common.Format; import androidx.media3.common.MediaItem; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import java.util.List; import org.junit.Test; import org.junit.runner.RunWith; @@ -34,9 +37,10 @@ import org.junit.runner.RunWith; @RunWith(AndroidJUnit4.class) public class TransformerEndToEndTest { + private final Context context = ApplicationProvider.getApplicationContext(); + @Test public void videoEditing_completesWithConsistentFrameCount() throws Exception { - Context context = ApplicationProvider.getApplicationContext(); Transformer transformer = new Transformer.Builder(context) .setTransformationRequest( @@ -61,7 +65,6 @@ public class TransformerEndToEndTest { @Test public void videoOnly_completesWithConsistentDuration() throws Exception { - Context context = ApplicationProvider.getApplicationContext(); Transformer transformer = new Transformer.Builder(context) .setRemoveAudio(true) @@ -85,7 +88,6 @@ public class TransformerEndToEndTest { @Test public void clippedMedia_completesWithClippedDuration() throws Exception { - Context context = ApplicationProvider.getApplicationContext(); Transformer transformer = new Transformer.Builder(context).build(); long clippingStartMs = 10_000; long clippingEndMs = 11_000; @@ -106,4 +108,65 @@ public class TransformerEndToEndTest { assertThat(result.transformationResult.durationMs).isAtMost(clippingEndMs - clippingStartMs); } + + @Test + public void videoEncoderFormatUnsupported_completesWithError() { + Transformer transformer = + new Transformer.Builder(context) + .setEncoderFactory(new VideoUnsupportedEncoderFactory(context)) + .setRemoveAudio(true) + .build(); + + TransformationException exception = + assertThrows( + TransformationException.class, + () -> + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run( + /* testId= */ "videoEncoderFormatUnsupported_completesWithError", + MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)))); + + assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException.class); + assertThat(exception.errorCode) + .isEqualTo(TransformationException.ERROR_CODE_ENCODER_INIT_FAILED); + assertThat(exception).hasMessageThat().contains("video"); + } + + private static final class VideoUnsupportedEncoderFactory implements Codec.EncoderFactory { + + private final Codec.EncoderFactory encoderFactory; + + public VideoUnsupportedEncoderFactory(Context context) { + encoderFactory = new DefaultEncoderFactory(context); + } + + @Override + public Codec createForAudioEncoding(Format format, List allowedMimeTypes) + throws TransformationException { + return encoderFactory.createForAudioEncoding(format, allowedMimeTypes); + } + + @Override + public Codec createForVideoEncoding(Format format, List allowedMimeTypes) + throws TransformationException { + throw TransformationException.createForCodec( + new IllegalArgumentException(), + /* isVideo= */ true, + /* isDecoder= */ false, + format, + /* mediaCodecName= */ null, + TransformationException.ERROR_CODE_ENCODER_INIT_FAILED); + } + + @Override + public boolean audioNeedsEncoding() { + return false; + } + + @Override + public boolean videoNeedsEncoding() { + return true; + } + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderCompatibilityTransformation.java b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderCompatibilityTransformation.java deleted file mode 100644 index 05ff98fe0a..0000000000 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderCompatibilityTransformation.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2022 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.transformer; - -import static androidx.media3.common.util.Assertions.checkArgument; -import static androidx.media3.common.util.Assertions.checkState; -import static androidx.media3.common.util.Assertions.checkStateNotNull; - -import android.graphics.Matrix; -import android.util.Size; -import androidx.media3.common.C; -import androidx.media3.common.Format; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; - -/** - * Specifies a {@link Format#rotationDegrees} to apply to each frame for encoder compatibility, if - * needed. - * - *

Encoders commonly support higher maximum widths than maximum heights. This may rotate the - * decoded frame before encoding, so the encoded frame's width >= height, and set {@link - * Format#rotationDegrees} to ensure the frame is displayed in the correct orientation. - */ -/* package */ class EncoderCompatibilityTransformation implements MatrixTransformation { - // TODO(b/218488308): Allow reconfiguration of the output size, as encoders may not support the - // requested output resolution. - - private int outputRotationDegrees; - private @MonotonicNonNull Matrix transformationMatrix; - - /** Creates a new instance. */ - public EncoderCompatibilityTransformation() { - outputRotationDegrees = C.LENGTH_UNSET; - } - - @Override - public Size configure(int inputWidth, int inputHeight) { - checkArgument(inputWidth > 0, "inputWidth must be positive"); - checkArgument(inputHeight > 0, "inputHeight must be positive"); - - transformationMatrix = new Matrix(); - if (inputHeight > inputWidth) { - outputRotationDegrees = 90; - transformationMatrix.postRotate(outputRotationDegrees); - return new Size(inputHeight, inputWidth); - } else { - outputRotationDegrees = 0; - return new Size(inputWidth, inputHeight); - } - } - - @Override - public Matrix getMatrix(long presentationTimeUs) { - return checkStateNotNull(transformationMatrix, "configure must be called first"); - } - - /** - * Returns {@link Format#rotationDegrees} for the output frame. - * - *

Return values may be {@code 0} or {@code 90} degrees. - * - *

Should only be called after {@linkplain #configure(int, int) configuration}. - */ - public int getOutputRotationDegrees() { - checkState( - outputRotationDegrees != C.LENGTH_UNSET, - "configure must be called before getOutputRotationDegrees"); - return outputRotationDegrees; - } -} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java index 0b2bea5a34..63421d6d65 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java @@ -18,7 +18,6 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; -import static androidx.media3.common.util.Assertions.checkStateNotNull; import static com.google.common.collect.Iterables.getLast; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -40,6 +39,7 @@ import androidx.media3.common.C; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Log; import androidx.media3.common.util.Util; +import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import java.util.List; import java.util.concurrent.ConcurrentLinkedQueue; @@ -58,8 +58,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * and is processed on a background thread as it becomes available. All input frames should be * {@linkplain #registerInputFrame() registered} before they are rendered to the input surface. * {@link #getPendingFrameCount()} can be used to check whether there are frames that have not been - * fully processed yet. Output is written to its {@linkplain #setOutputSurface(Surface, int, int, - * SurfaceView) output surface}. + * fully processed yet. Output is written to the provided {@linkplain #create(Context, Listener, + * float, int, int, long, List, SurfaceInfo.Provider, Transformer.DebugViewProvider, boolean) output + * surface}. */ // TODO(b/227625423): Factor out FrameProcessor interface and rename this class to GlFrameProcessor. /* package */ final class FrameProcessorChain { @@ -84,11 +85,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param inputWidth The input frame width, in pixels. * @param inputHeight The input frame height, in pixels. * @param effects The {@link GlEffect GlEffects} to apply to each frame. + * @param outputSurfaceProvider A {@link SurfaceInfo.Provider} managing the output {@link + * Surface}. + * @param debugViewProvider A {@link Transformer.DebugViewProvider}. * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. - * @return A new instance. + * @return A new instance or {@code null}, if no output surface was provided. * @throws FrameProcessingException If reading shader files fails, or an OpenGL error occurs while * creating and configuring the OpenGL components. */ + // TODO(b/227625423): Remove @Nullable here and allow the output surface to be @Nullable until + // the output surface is requested when the output size becomes available asynchronously + // via the final GlTextureProcessor. + @Nullable public static FrameProcessorChain create( Context context, Listener listener, @@ -97,6 +105,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; int inputHeight, long streamOffsetUs, List effects, + SurfaceInfo.Provider outputSurfaceProvider, + Transformer.DebugViewProvider debugViewProvider, boolean enableExperimentalHdrEditing) throws FrameProcessingException { checkArgument(inputWidth > 0, "inputWidth must be positive"); @@ -104,21 +114,25 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); + Future> frameProcessorChainFuture = + singleThreadExecutorService.submit( + () -> + Optional.fromNullable( + createOpenGlObjectsAndFrameProcessorChain( + context, + listener, + pixelWidthHeightRatio, + inputWidth, + inputHeight, + streamOffsetUs, + effects, + outputSurfaceProvider, + debugViewProvider, + enableExperimentalHdrEditing, + singleThreadExecutorService))); + try { - return singleThreadExecutorService - .submit( - () -> - createOpenGlObjectsAndFrameProcessorChain( - context, - listener, - pixelWidthHeightRatio, - inputWidth, - inputHeight, - streamOffsetUs, - effects, - enableExperimentalHdrEditing, - singleThreadExecutorService)) - .get(); + return frameProcessorChainFuture.get().orNull(); } catch (ExecutionException e) { throw new FrameProcessingException(e); } catch (InterruptedException e) { @@ -135,6 +149,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; *

This method must be executed using the {@code singleThreadExecutorService}. */ @WorkerThread + @Nullable private static FrameProcessorChain createOpenGlObjectsAndFrameProcessorChain( Context context, Listener listener, @@ -143,6 +158,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; int inputHeight, long streamOffsetUs, List effects, + SurfaceInfo.Provider outputSurfaceProvider, + Transformer.DebugViewProvider debugViewProvider, boolean enableExperimentalHdrEditing, ExecutorService singleThreadExecutorService) throws GlUtil.GlException, FrameProcessingException { @@ -164,45 +181,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay); } - ExternalTextureProcessor externalTextureProcessor = - new ExternalTextureProcessor(context, enableExperimentalHdrEditing); - ImmutableList textureProcessors = - getTextureProcessors(context, externalTextureProcessor, pixelWidthHeightRatio, effects); - - // Initialize texture processors. - int inputExternalTexId = GlUtil.createExternalTexture(); - Size outputSize = externalTextureProcessor.configure(inputWidth, inputHeight); - ImmutableList.Builder intermediateTextures = new ImmutableList.Builder<>(); - for (int i = 1; i < textureProcessors.size(); i++) { - int texId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight()); - int fboId = GlUtil.createFboForTexture(texId); - intermediateTextures.add( - new TextureInfo(texId, fboId, outputSize.getWidth(), outputSize.getHeight())); - SingleFrameGlTextureProcessor textureProcessor = textureProcessors.get(i); - outputSize = textureProcessor.configure(outputSize.getWidth(), outputSize.getHeight()); - } - return new FrameProcessorChain( - eglDisplay, - eglContext, - singleThreadExecutorService, - inputExternalTexId, - streamOffsetUs, - intermediateTextures.build(), - textureProcessors, - outputSize, - listener, - enableExperimentalHdrEditing); - } - - private static ImmutableList getTextureProcessors( - Context context, - ExternalTextureProcessor externalTextureProcessor, - float pixelWidthHeightRatio, - List effects) - throws FrameProcessingException { - ImmutableList.Builder textureProcessors = - new ImmutableList.Builder().add(externalTextureProcessor); - ImmutableList.Builder matrixTransformationListBuilder = new ImmutableList.Builder<>(); // Scale to expand the frame to apply the pixelWidthHeightRatio. @@ -218,6 +196,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; .build()); } + ExternalTextureProcessor externalTextureProcessor = + new ExternalTextureProcessor(context, enableExperimentalHdrEditing); + int inputExternalTexId = GlUtil.createExternalTexture(); + Size outputSize = externalTextureProcessor.configure(inputWidth, inputHeight); + ImmutableList.Builder intermediateTextures = new ImmutableList.Builder<>(); + ImmutableList.Builder textureProcessors = + new ImmutableList.Builder().add(externalTextureProcessor); + // Combine consecutive GlMatrixTransformations into a single SingleFrameGlTextureProcessor and // convert all other GlEffects to SingleFrameGlTextureProcessors. for (int i = 0; i < effects.size(); i++) { @@ -226,21 +212,100 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; matrixTransformationListBuilder.add((GlMatrixTransformation) effect); continue; } + ImmutableList matrixTransformations = matrixTransformationListBuilder.build(); if (!matrixTransformations.isEmpty()) { - textureProcessors.add(new MatrixTransformationProcessor(context, matrixTransformations)); + MatrixTransformationProcessor matrixTransformationProcessor = + new MatrixTransformationProcessor(context, matrixTransformations); + intermediateTextures.add(createTexture(outputSize.getWidth(), outputSize.getHeight())); + outputSize = + matrixTransformationProcessor.configure(outputSize.getWidth(), outputSize.getHeight()); + textureProcessors.add(matrixTransformationProcessor); matrixTransformationListBuilder = new ImmutableList.Builder<>(); } - textureProcessors.add(effect.toGlTextureProcessor(context)); + intermediateTextures.add(createTexture(outputSize.getWidth(), outputSize.getHeight())); + SingleFrameGlTextureProcessor textureProcessor = effect.toGlTextureProcessor(context); + outputSize = textureProcessor.configure(outputSize.getWidth(), outputSize.getHeight()); + textureProcessors.add(textureProcessor); } + + // TODO(b/227625423): Request the output surface during processing when the output size becomes + // available asynchronously via the final GlTextureProcessor instead of requesting it here. + // This will also avoid needing to return null here when no surface is provided. + Size requestedOutputSize = + MatrixUtils.configureAndGetOutputSize( + outputSize.getWidth(), outputSize.getHeight(), matrixTransformationListBuilder.build()); + @Nullable + SurfaceInfo outputSurfaceInfo = + outputSurfaceProvider.getSurfaceInfo( + requestedOutputSize.getWidth(), requestedOutputSize.getHeight()); + if (outputSurfaceInfo == null) { + Log.d(TAG, "No output surface provided."); + return null; + } + + if (outputSurfaceInfo.orientationDegrees != 0) { + matrixTransformationListBuilder.add( + new ScaleToFitTransformation.Builder() + .setRotationDegrees(outputSurfaceInfo.orientationDegrees) + .build()); + } + if (outputSurfaceInfo.width != outputSize.getWidth() + || outputSurfaceInfo.height != outputSize.getHeight()) { + matrixTransformationListBuilder.add( + new Presentation.Builder() + .setAspectRatio( + outputSurfaceInfo.width / (float) outputSurfaceInfo.height, + Presentation.LAYOUT_SCALE_TO_FIT) + .setResolution(outputSurfaceInfo.height) + .build()); + } + + // Convert final list of matrix transformations (including additional transformations for the + // output surface) to a SingleFrameGlTextureProcessors. ImmutableList matrixTransformations = matrixTransformationListBuilder.build(); if (!matrixTransformations.isEmpty()) { - textureProcessors.add(new MatrixTransformationProcessor(context, matrixTransformations)); + intermediateTextures.add(createTexture(outputSize.getWidth(), outputSize.getHeight())); + MatrixTransformationProcessor matrixTransformationProcessor = + new MatrixTransformationProcessor(context, matrixTransformations); + outputSize = + matrixTransformationProcessor.configure(outputSize.getWidth(), outputSize.getHeight()); + checkState(outputSize.getWidth() == outputSurfaceInfo.width); + checkState(outputSize.getHeight() == outputSurfaceInfo.height); + textureProcessors.add(matrixTransformationProcessor); } - return textureProcessors.build(); + EGLSurface outputEglSurface; + if (enableExperimentalHdrEditing) { + // TODO(b/227624622): Don't assume BT.2020 PQ input/output. + outputEglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurfaceInfo.surface); + } else { + outputEglSurface = GlUtil.getEglSurface(eglDisplay, outputSurfaceInfo.surface); + } + return new FrameProcessorChain( + eglDisplay, + eglContext, + singleThreadExecutorService, + inputExternalTexId, + streamOffsetUs, + intermediateTextures.build(), + textureProcessors.build(), + outputSurfaceInfo.width, + outputSurfaceInfo.height, + outputEglSurface, + listener, + debugViewProvider.getDebugPreviewSurfaceView( + outputSurfaceInfo.width, outputSurfaceInfo.height), + enableExperimentalHdrEditing); + } + + private static TextureInfo createTexture(int outputWidth, int outputHeight) + throws GlUtil.GlException { + int texId = GlUtil.createTexture(outputWidth, outputHeight); + int fboId = GlUtil.createFboForTexture(texId); + return new TextureInfo(texId, fboId, outputWidth, outputHeight); } private static final String TAG = "FrameProcessorChain"; @@ -282,8 +347,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * SingleFrameGlTextureProcessor}. */ private final ImmutableList intermediateTextures; - /** The last texture processor's output {@link Size}. */ - private final Size recommendedOutputSize; private final Listener listener; @@ -293,15 +356,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; */ private final AtomicBoolean stopProcessing; - private int outputWidth; - private int outputHeight; - private @MonotonicNonNull Surface outputSurface; - + private final int outputWidth; + private final int outputHeight; /** * Wraps the output {@link Surface} that is populated with the output of the final {@link * SingleFrameGlTextureProcessor} for each frame. */ - private @MonotonicNonNull EGLSurface outputEglSurface; + private final EGLSurface outputEglSurface; /** * Wraps a debug {@link SurfaceView} that is populated with the output of the final {@link * SingleFrameGlTextureProcessor} for each frame. @@ -320,8 +381,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; long streamOffsetUs, ImmutableList intermediateTextures, ImmutableList textureProcessors, - Size recommendedOutputSize, + int outputWidth, + int outputHeight, + EGLSurface outputEglSurface, Listener listener, + @Nullable SurfaceView debugSurfaceView, boolean enableExperimentalHdrEditing) { checkState(!textureProcessors.isEmpty()); @@ -332,7 +396,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.streamOffsetUs = streamOffsetUs; this.intermediateTextures = intermediateTextures; this.textureProcessors = textureProcessors; - this.recommendedOutputSize = recommendedOutputSize; + this.outputWidth = outputWidth; + this.outputHeight = outputHeight; + this.outputEglSurface = outputEglSurface; this.listener = listener; this.stopProcessing = new AtomicBoolean(); this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; @@ -342,47 +408,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; inputSurfaceTexture = new SurfaceTexture(inputExternalTexId); inputSurface = new Surface(inputSurfaceTexture); textureTransformMatrix = new float[16]; - outputWidth = C.LENGTH_UNSET; - outputHeight = C.LENGTH_UNSET; - } - - /** - * Returns the recommended output size. - * - *

This is the recommended size to use for the {@linkplain #setOutputSurface(Surface, int, int, - * SurfaceView) output surface}. - */ - public Size getOutputSize() { - return recommendedOutputSize; - } - - /** - * Sets the output {@link Surface}. - * - *

The recommended output size is given by {@link #getOutputSize()}. Setting a different output - * size may cause poor quality or distortion. - * - * @param outputSurface The output {@link Surface}. - * @param outputWidth The output width, in pixels. - * @param outputHeight The output height, in pixels. - * @param debugSurfaceView Optional debug {@link SurfaceView} to show output. - */ - public void setOutputSurface( - Surface outputSurface, - int outputWidth, - int outputHeight, - @Nullable SurfaceView debugSurfaceView) { - // TODO(b/218488308): Don't override output size for encoder fallback. Instead allow the final - // SingleFrameGlTextureProcessor to be re-configured or append another - // SingleFrameGlTextureProcessor. - this.outputSurface = outputSurface; - this.outputWidth = outputWidth; - this.outputHeight = outputHeight; - if (debugSurfaceView != null) { debugSurfaceViewWrapper = new SurfaceViewWrapper(debugSurfaceView); } + } + /** Returns the input {@link Surface}. */ + public Surface getInputSurface() { + // TODO(b/227625423): Allow input surface to be recreated for input size change. inputSurfaceTexture.setOnFrameAvailableListener( surfaceTexture -> { if (stopProcessing.get()) { @@ -398,10 +431,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } }); - } - - /** Returns the input {@link Surface}. */ - public Surface getInputSurface() { return inputSurface; } @@ -479,16 +508,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; try { checkState(Thread.currentThread().getName().equals(THREAD_NAME)); - if (outputEglSurface == null) { - checkStateNotNull(outputSurface); - if (enableExperimentalHdrEditing) { - // TODO(b/227624622): Don't assume BT.2020 PQ input/output. - outputEglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurface); - } else { - outputEglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface); - } - } - inputSurfaceTexture.updateTexImage(); long inputFrameTimeNs = inputSurfaceTexture.getTimestamp(); // Correct for the stream offset so processors see original media presentation timestamps. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java index 542f79c45e..9f551f0a3b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java @@ -15,7 +15,6 @@ */ package androidx.media3.transformer; -import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkState; import android.content.Context; @@ -133,16 +132,7 @@ import java.util.Arrays; @Override public Size configure(int inputWidth, int inputHeight) { - checkArgument(inputWidth > 0, "inputWidth must be positive"); - checkArgument(inputHeight > 0, "inputHeight must be positive"); - - Size outputSize = new Size(inputWidth, inputHeight); - for (int i = 0; i < matrixTransformations.size(); i++) { - outputSize = - matrixTransformations.get(i).configure(outputSize.getWidth(), outputSize.getHeight()); - } - - return outputSize; + return MatrixUtils.configureAndGetOutputSize(inputWidth, inputHeight, matrixTransformations); } @Override diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixUtils.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixUtils.java index 206d7cf16d..5a48570e53 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixUtils.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MatrixUtils.java @@ -18,6 +18,7 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkArgument; import android.opengl.Matrix; +import android.util.Size; import com.google.common.collect.ImmutableList; import java.util.Arrays; @@ -217,6 +218,26 @@ import java.util.Arrays; return transformedPoints.build(); } + /** + * Returns the output frame {@link Size} after applying the given list of {@link + * GlMatrixTransformation GlMatrixTransformations} to an input frame with the given size. + */ + public static Size configureAndGetOutputSize( + int inputWidth, + int inputHeight, + ImmutableList matrixTransformations) { + checkArgument(inputWidth > 0, "inputWidth must be positive"); + checkArgument(inputHeight > 0, "inputHeight must be positive"); + + Size outputSize = new Size(inputWidth, inputHeight); + for (int i = 0; i < matrixTransformations.size(); i++) { + outputSize = + matrixTransformations.get(i).configure(outputSize.getWidth(), outputSize.getHeight()); + } + + return outputSize; + } + /** Class only contains static methods. */ private MatrixUtils() {} } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SurfaceInfo.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SurfaceInfo.java new file mode 100644 index 0000000000..09bc801058 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SurfaceInfo.java @@ -0,0 +1,70 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkArgument; + +import android.view.Surface; +import androidx.annotation.Nullable; + +/** Immutable value class for a {@link Surface} and supporting information. */ +/* package */ final class SurfaceInfo { + + /** The {@link Surface}. */ + public final Surface surface; + /** The width of frames rendered to the {@link #surface}, in pixels. */ + public final int width; + /** The height of frames rendered to the {@link #surface}, in pixels. */ + public final int height; + /** + * A counter-clockwise rotation to apply to frames before rendering them to the {@link #surface}. + * + *

Must be 0, 90, 180, or 270 degrees. Default is 0. + */ + public final int orientationDegrees; + + /** Creates a new instance. */ + public SurfaceInfo(Surface surface, int width, int height) { + this(surface, width, height, /* orientationDegrees= */ 0); + } + + /** Creates a new instance. */ + public SurfaceInfo(Surface surface, int width, int height, int orientationDegrees) { + checkArgument( + orientationDegrees == 0 + || orientationDegrees == 90 + || orientationDegrees == 180 + || orientationDegrees == 270, + "orientationDegrees must be 0, 90, 180, or 270"); + this.surface = surface; + this.width = width; + this.height = height; + this.orientationDegrees = orientationDegrees; + } + + /** A provider for a {@link SurfaceInfo} instance. */ + public interface Provider { + /** + * Provides a {@linkplain SurfaceInfo surface} for the requested dimensions. + * + *

The dimensions given in the provided {@link SurfaceInfo} may differ from the requested + * dimensions. It is up to the caller to transform frames from the requested dimensions to the + * provided dimensions before rendering them to the {@link SurfaceInfo#surface}. + */ + @Nullable + SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java index 6ea492fcb9..b97b14c57e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java @@ -106,10 +106,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; encoderFactory, muxerWrapper.getSupportedSampleMimeTypes(getTrackType()), fallbackListener, - /* frameProcessorChainListener= */ exception -> - asyncErrorListener.onTransformationException( - TransformationException.createForFrameProcessorChain( - exception, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED)), + asyncErrorListener, debugViewProvider); } if (transformationRequest.flattenForSlowMotion) { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 83ea088026..32d5c628aa 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -17,25 +17,29 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.content.Context; import android.media.MediaCodec; -import android.util.Size; +import android.view.Surface; import androidx.annotation.Nullable; +import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; import com.google.common.collect.ImmutableList; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.dataflow.qual.Pure; /** * Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them. */ /* package */ final class VideoTranscodingSamplePipeline implements SamplePipeline { - private final int outputRotationDegrees; private final int maxPendingFrameCount; private final DecoderInputBuffer decoderInputBuffer; @@ -44,7 +48,7 @@ import org.checkerframework.dataflow.qual.Pure; private final FrameProcessorChain frameProcessorChain; - private final Codec encoder; + private final EncoderWrapper encoderWrapper; private final DecoderInputBuffer encoderOutputBuffer; private boolean signaledEndOfStreamToEncoder; @@ -59,7 +63,7 @@ import org.checkerframework.dataflow.qual.Pure; Codec.EncoderFactory encoderFactory, List allowedOutputMimeTypes, FallbackListener fallbackListener, - FrameProcessorChain.Listener frameProcessorChainListener, + Transformer.AsyncErrorListener asyncErrorListener, Transformer.DebugViewProvider debugViewProvider) throws TransformationException { decoderInputBuffer = @@ -89,54 +93,45 @@ import org.checkerframework.dataflow.qual.Pure; effectsListBuilder.add( new Presentation.Builder().setResolution(transformationRequest.outputHeight).build()); } - EncoderCompatibilityTransformation encoderCompatibilityTransformation = - new EncoderCompatibilityTransformation(); - effectsListBuilder.add(encoderCompatibilityTransformation); + + AtomicReference encoderInitializationException = + new AtomicReference<>(); + encoderWrapper = + new EncoderWrapper( + encoderFactory, + inputFormat, + allowedOutputMimeTypes, + transformationRequest, + fallbackListener, + encoderInitializationException); + + @Nullable FrameProcessorChain frameProcessorChain; try { frameProcessorChain = FrameProcessorChain.create( context, - frameProcessorChainListener, + /* listener= */ exception -> + asyncErrorListener.onTransformationException( + TransformationException.createForFrameProcessorChain( + exception, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED)), inputFormat.pixelWidthHeightRatio, /* inputWidth= */ decodedWidth, /* inputHeight= */ decodedHeight, streamOffsetUs, effectsListBuilder.build(), + /* outputSurfaceProvider= */ encoderWrapper, + debugViewProvider, transformationRequest.enableHdrEditing); } catch (FrameProcessingException e) { throw TransformationException.createForFrameProcessorChain( e, TransformationException.ERROR_CODE_GL_INIT_FAILED); } - Size requestedEncoderSize = frameProcessorChain.getOutputSize(); - outputRotationDegrees = encoderCompatibilityTransformation.getOutputRotationDegrees(); - Format requestedEncoderFormat = - new Format.Builder() - .setWidth(requestedEncoderSize.getWidth()) - .setHeight(requestedEncoderSize.getHeight()) - .setRotationDegrees(0) - .setFrameRate(inputFormat.frameRate) - .setSampleMimeType( - transformationRequest.videoMimeType != null - ? transformationRequest.videoMimeType - : inputFormat.sampleMimeType) - .build(); - - encoder = encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes); - Format encoderSupportedFormat = encoder.getConfigurationFormat(); - fallbackListener.onTransformationRequestFinalized( - createFallbackTransformationRequest( - transformationRequest, - /* hasOutputFormatRotation= */ outputRotationDegrees == 0, - requestedEncoderFormat, - encoderSupportedFormat)); - - frameProcessorChain.setOutputSurface( - /* outputSurface= */ encoder.getInputSurface(), - /* outputWidth= */ encoderSupportedFormat.width, - /* outputHeight= */ encoderSupportedFormat.height, - debugViewProvider.getDebugPreviewSurfaceView( - encoderSupportedFormat.width, encoderSupportedFormat.height)); + if (frameProcessorChain == null) { + // Failed to create FrameProcessorChain because the encoder could not provide a surface. + throw checkStateNotNull(encoderInitializationException.get()); + } + this.frameProcessorChain = frameProcessorChain; decoder = decoderFactory.createForVideoDecoding( @@ -164,7 +159,7 @@ import org.checkerframework.dataflow.qual.Pure; public boolean processData() throws TransformationException { if (frameProcessorChain.isEnded()) { if (!signaledEndOfStreamToEncoder) { - encoder.signalEndOfInputStream(); + encoderWrapper.signalEndOfInputStream(); signaledEndOfStreamToEncoder = true; } return false; @@ -187,20 +182,17 @@ import org.checkerframework.dataflow.qual.Pure; @Override @Nullable public Format getOutputFormat() throws TransformationException { - @Nullable Format format = encoder.getOutputFormat(); - return format == null - ? null - : format.buildUpon().setRotationDegrees(outputRotationDegrees).build(); + return encoderWrapper.getOutputFormat(); } @Override @Nullable public DecoderInputBuffer getOutputBuffer() throws TransformationException { - encoderOutputBuffer.data = encoder.getOutputBuffer(); + encoderOutputBuffer.data = encoderWrapper.getOutputBuffer(); if (encoderOutputBuffer.data == null) { return null; } - MediaCodec.BufferInfo bufferInfo = checkNotNull(encoder.getOutputBufferInfo()); + MediaCodec.BufferInfo bufferInfo = checkNotNull(encoderWrapper.getOutputBufferInfo()); encoderOutputBuffer.timeUs = bufferInfo.presentationTimeUs; encoderOutputBuffer.setFlags(bufferInfo.flags); return encoderOutputBuffer; @@ -208,19 +200,19 @@ import org.checkerframework.dataflow.qual.Pure; @Override public void releaseOutputBuffer() throws TransformationException { - encoder.releaseOutputBuffer(/* render= */ false); + encoderWrapper.releaseOutputBuffer(/* render= */ false); } @Override public boolean isEnded() { - return encoder.isEnded(); + return encoderWrapper.isEnded(); } @Override public void release() { frameProcessorChain.release(); decoder.release(); - encoder.release(); + encoderWrapper.release(); } /** @@ -292,4 +284,151 @@ import org.checkerframework.dataflow.qual.Pure; } return false; } + + /** + * Wraps an {@linkplain Codec encoder} and provides its input {@link Surface}. + * + *

The encoder is created once the {@link Surface} is {@linkplain #getSurfaceInfo(int, int) + * requested}. If it is {@linkplain #getSurfaceInfo(int, int) requested} again with different + * dimensions, the same encoder is used and the provided dimensions stay fixed. + */ + @VisibleForTesting + /* package */ static final class EncoderWrapper implements SurfaceInfo.Provider { + + private final Codec.EncoderFactory encoderFactory; + private final Format inputFormat; + private final List allowedOutputMimeTypes; + private final TransformationRequest transformationRequest; + private final FallbackListener fallbackListener; + private final AtomicReference encoderInitializationException; + + private @MonotonicNonNull SurfaceInfo encoderSurfaceInfo; + + private volatile @MonotonicNonNull Codec encoder; + private volatile int outputRotationDegrees; + private volatile boolean releaseEncoder; + + public EncoderWrapper( + Codec.EncoderFactory encoderFactory, + Format inputFormat, + List allowedOutputMimeTypes, + TransformationRequest transformationRequest, + FallbackListener fallbackListener, + AtomicReference encoderInitializationException) { + + this.encoderFactory = encoderFactory; + this.inputFormat = inputFormat; + this.allowedOutputMimeTypes = allowedOutputMimeTypes; + this.transformationRequest = transformationRequest; + this.fallbackListener = fallbackListener; + this.encoderInitializationException = encoderInitializationException; + } + + @Override + @Nullable + public SurfaceInfo getSurfaceInfo(int requestedWidth, int requestedHeight) { + if (releaseEncoder) { + return null; + } + if (encoderSurfaceInfo != null) { + return encoderSurfaceInfo; + } + + // Encoders commonly support higher maximum widths than maximum heights. This may rotate the + // frame before encoding, so the encoded frame's width >= height, and sets + // rotationDegrees in the output Format to ensure the frame is displayed in the correct + // orientation. + boolean flipOrientation = requestedWidth < requestedHeight; + if (flipOrientation) { + int temp = requestedWidth; + requestedWidth = requestedHeight; + requestedHeight = temp; + outputRotationDegrees = 90; + } + + Format requestedEncoderFormat = + new Format.Builder() + .setWidth(requestedWidth) + .setHeight(requestedHeight) + .setRotationDegrees(0) + .setFrameRate(inputFormat.frameRate) + .setSampleMimeType( + transformationRequest.videoMimeType != null + ? transformationRequest.videoMimeType + : inputFormat.sampleMimeType) + .build(); + + try { + encoder = + encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes); + } catch (TransformationException e) { + encoderInitializationException.set(e); + return null; + } + Format encoderSupportedFormat = encoder.getConfigurationFormat(); + fallbackListener.onTransformationRequestFinalized( + createFallbackTransformationRequest( + transformationRequest, + /* hasOutputFormatRotation= */ flipOrientation, + requestedEncoderFormat, + encoderSupportedFormat)); + + encoderSurfaceInfo = + new SurfaceInfo( + encoder.getInputSurface(), + encoderSupportedFormat.width, + encoderSupportedFormat.height, + outputRotationDegrees); + + if (releaseEncoder) { + encoder.release(); + } + return encoderSurfaceInfo; + } + + public void signalEndOfInputStream() throws TransformationException { + if (encoder != null) { + encoder.signalEndOfInputStream(); + } + } + + @Nullable + public Format getOutputFormat() throws TransformationException { + if (encoder == null) { + return null; + } + @Nullable Format outputFormat = encoder.getOutputFormat(); + if (outputFormat != null && outputRotationDegrees != 0) { + outputFormat = outputFormat.buildUpon().setRotationDegrees(outputRotationDegrees).build(); + } + return outputFormat; + } + + @Nullable + public ByteBuffer getOutputBuffer() throws TransformationException { + return encoder != null ? encoder.getOutputBuffer() : null; + } + + @Nullable + public MediaCodec.BufferInfo getOutputBufferInfo() throws TransformationException { + return encoder != null ? encoder.getOutputBufferInfo() : null; + } + + public void releaseOutputBuffer(boolean render) throws TransformationException { + if (encoder != null) { + encoder.releaseOutputBuffer(render); + } + } + + public boolean isEnded() { + return encoder != null && encoder.isEnded(); + } + + public void release() { + if (encoder != null) { + encoder.release(); + } + releaseEncoder = true; + } + } } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/EncoderCompatibilityTransformationTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/EncoderCompatibilityTransformationTest.java deleted file mode 100644 index 3eb95008d1..0000000000 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/EncoderCompatibilityTransformationTest.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2022 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.transformer; - -import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertThrows; - -import android.util.Size; -import androidx.test.ext.junit.runners.AndroidJUnit4; -import org.junit.Test; -import org.junit.runner.RunWith; - -/** Unit tests for {@link EncoderCompatibilityTransformation}. */ -@RunWith(AndroidJUnit4.class) -public final class EncoderCompatibilityTransformationTest { - @Test - public void configure_noEditsLandscape_leavesOrientationUnchanged() { - int inputWidth = 200; - int inputHeight = 150; - EncoderCompatibilityTransformation encoderCompatibilityTransformation = - new EncoderCompatibilityTransformation(); - - Size outputSize = encoderCompatibilityTransformation.configure(inputWidth, inputHeight); - - assertThat(encoderCompatibilityTransformation.getOutputRotationDegrees()).isEqualTo(0); - assertThat(outputSize.getWidth()).isEqualTo(inputWidth); - assertThat(outputSize.getHeight()).isEqualTo(inputHeight); - } - - @Test - public void configure_noEditsSquare_leavesOrientationUnchanged() { - int inputWidth = 150; - int inputHeight = 150; - EncoderCompatibilityTransformation encoderCompatibilityTransformation = - new EncoderCompatibilityTransformation(); - - Size outputSize = encoderCompatibilityTransformation.configure(inputWidth, inputHeight); - - assertThat(encoderCompatibilityTransformation.getOutputRotationDegrees()).isEqualTo(0); - assertThat(outputSize.getWidth()).isEqualTo(inputWidth); - assertThat(outputSize.getHeight()).isEqualTo(inputHeight); - } - - @Test - public void configure_noEditsPortrait_flipsOrientation() { - int inputWidth = 150; - int inputHeight = 200; - EncoderCompatibilityTransformation encoderCompatibilityTransformation = - new EncoderCompatibilityTransformation(); - - Size outputSize = encoderCompatibilityTransformation.configure(inputWidth, inputHeight); - - assertThat(encoderCompatibilityTransformation.getOutputRotationDegrees()).isEqualTo(90); - assertThat(outputSize.getWidth()).isEqualTo(inputHeight); - assertThat(outputSize.getHeight()).isEqualTo(inputWidth); - } - - @Test - public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() { - EncoderCompatibilityTransformation encoderCompatibilityTransformation = - new EncoderCompatibilityTransformation(); - - // configure not called before getOutputRotationDegrees. - assertThrows( - IllegalStateException.class, encoderCompatibilityTransformation::getOutputRotationDegrees); - } -} diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java new file mode 100644 index 0000000000..e7e7c5e4f6 --- /dev/null +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java @@ -0,0 +1,146 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import android.net.Uri; +import android.os.Looper; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.MediaItem; +import androidx.media3.common.util.Clock; +import androidx.media3.common.util.ListenerSet; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link VideoTranscodingSamplePipeline.EncoderWrapper}. */ +@RunWith(AndroidJUnit4.class) +public final class VideoEncoderWrapperTest { + private final TransformationRequest emptyTransformationRequest = + new TransformationRequest.Builder().build(); + private final FakeVideoEncoderFactory fakeEncoderFactory = new FakeVideoEncoderFactory(); + private final FallbackListener fallbackListener = + new FallbackListener( + MediaItem.fromUri(Uri.EMPTY), + new ListenerSet<>(Looper.myLooper(), Clock.DEFAULT, (listener, flags) -> {}), + emptyTransformationRequest); + private final VideoTranscodingSamplePipeline.EncoderWrapper encoderWrapper = + new VideoTranscodingSamplePipeline.EncoderWrapper( + fakeEncoderFactory, + /* inputFormat= */ new Format.Builder().build(), + /* allowedOutputMimeTypes= */ ImmutableList.of(), + emptyTransformationRequest, + fallbackListener, + new AtomicReference<>()); + + @Before + public void registerTrack() { + fallbackListener.registerTrack(); + } + + @Test + public void getSurfaceInfo_landscape_leavesOrientationUnchanged() { + int inputWidth = 200; + int inputHeight = 150; + + SurfaceInfo surfaceInfo = encoderWrapper.getSurfaceInfo(inputWidth, inputHeight); + + assertThat(surfaceInfo.orientationDegrees).isEqualTo(0); + assertThat(surfaceInfo.width).isEqualTo(inputWidth); + assertThat(surfaceInfo.height).isEqualTo(inputHeight); + } + + @Test + public void getSurfaceInfo_square_leavesOrientationUnchanged() { + int inputWidth = 150; + int inputHeight = 150; + + SurfaceInfo surfaceInfo = encoderWrapper.getSurfaceInfo(inputWidth, inputHeight); + + assertThat(surfaceInfo.orientationDegrees).isEqualTo(0); + assertThat(surfaceInfo.width).isEqualTo(inputWidth); + assertThat(surfaceInfo.height).isEqualTo(inputHeight); + } + + @Test + public void getSurfaceInfo_portrait_flipsOrientation() { + int inputWidth = 150; + int inputHeight = 200; + + SurfaceInfo surfaceInfo = encoderWrapper.getSurfaceInfo(inputWidth, inputHeight); + + assertThat(surfaceInfo.orientationDegrees).isEqualTo(90); + assertThat(surfaceInfo.width).isEqualTo(inputHeight); + assertThat(surfaceInfo.height).isEqualTo(inputWidth); + } + + @Test + public void getSurfaceInfo_withEncoderFallback_usesFallbackResolution() { + int inputWidth = 200; + int inputHeight = 150; + int fallbackWidth = 100; + int fallbackHeight = 75; + fakeEncoderFactory.setFallbackResolution(fallbackWidth, fallbackHeight); + + SurfaceInfo surfaceInfo = encoderWrapper.getSurfaceInfo(inputWidth, inputHeight); + + assertThat(surfaceInfo.orientationDegrees).isEqualTo(0); + assertThat(surfaceInfo.width).isEqualTo(fallbackWidth); + assertThat(surfaceInfo.height).isEqualTo(fallbackHeight); + } + + private static class FakeVideoEncoderFactory implements Codec.EncoderFactory { + + private int fallbackWidth; + private int fallbackHeight; + + public FakeVideoEncoderFactory() { + fallbackWidth = C.LENGTH_UNSET; + fallbackHeight = C.LENGTH_UNSET; + } + + public void setFallbackResolution(int fallbackWidth, int fallbackHeight) { + this.fallbackWidth = fallbackWidth; + this.fallbackHeight = fallbackHeight; + } + + @Override + public Codec createForAudioEncoding(Format format, List allowedMimeTypes) { + throw new UnsupportedOperationException(); + } + + @Override + public Codec createForVideoEncoding(Format format, List allowedMimeTypes) { + Codec mockEncoder = mock(Codec.class); + if (fallbackWidth != C.LENGTH_UNSET) { + format = format.buildUpon().setWidth(fallbackWidth).build(); + } + if (fallbackHeight != C.LENGTH_UNSET) { + format = format.buildUpon().setHeight(fallbackHeight).build(); + } + when(mockEncoder.getConfigurationFormat()).thenReturn(format); + return mockEncoder; + } + } +}