diff --git a/libraries/test_data/src/test/assets/media/bitmap/transformer_multi_sequence_composition_test/export_withTwoSequencesWithVideoCompositorSettings_succeeds_0.png b/libraries/test_data/src/test/assets/media/bitmap/transformer_multi_sequence_composition_test/export_withTwoSequencesWithVideoCompositorSettings_succeeds_0.png new file mode 100644 index 0000000000..b8dc88a75e Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/transformer_multi_sequence_composition_test/export_withTwoSequencesWithVideoCompositorSettings_succeeds_0.png differ diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerMultiSequenceCompositionTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerMultiSequenceCompositionTest.java index 956ee7421d..321e3b50df 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerMultiSequenceCompositionTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerMultiSequenceCompositionTest.java @@ -32,11 +32,14 @@ import android.content.Context; import android.graphics.Bitmap; import androidx.media3.common.Effect; import androidx.media3.common.MediaItem; +import androidx.media3.common.util.Size; import androidx.media3.common.util.Util; import androidx.media3.effect.AlphaScale; import androidx.media3.effect.Contrast; +import androidx.media3.effect.OverlaySettings; import androidx.media3.effect.Presentation; import androidx.media3.effect.ScaleAndRotateTransformation; +import androidx.media3.effect.VideoCompositorSettings; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; @@ -49,6 +52,9 @@ import org.junit.runner.RunWith; @RunWith(AndroidJUnit4.class) public final class TransformerMultiSequenceCompositionTest { + // Bitmaps are generated on a Pixel 6 or 7 Pro instead of an emulator, due to an emulator bug. + // TODO: b/301242589 - Fix this test on the crow emulator, and re-generate bitmaps using the crow + // emulator, for consistency with other pixel tests. private static final String PNG_ASSET_BASE_PATH = "media/bitmap/transformer_multi_sequence_composition_test"; @@ -86,7 +92,8 @@ public final class TransformerMultiSequenceCompositionTest { .build()))), /* secondSequenceMediaItems= */ ImmutableList.of( editedMediaItemByClippingVideo( - MP4_ASSET_URI_STRING, /* effects= */ ImmutableList.of()))); + MP4_ASSET_URI_STRING, /* effects= */ ImmutableList.of())), + VideoCompositorSettings.DEFAULT); ExportTestResult result = new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) @@ -125,7 +132,70 @@ public final class TransformerMultiSequenceCompositionTest { .build()))), /* secondSequenceMediaItems= */ ImmutableList.of( editedMediaItemOfOneFrameImage( - JPG_ASSET_URI_STRING, /* effects= */ ImmutableList.of()))); + JPG_ASSET_URI_STRING, /* effects= */ ImmutableList.of())), + VideoCompositorSettings.DEFAULT); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build() + .run(testId, composition); + + assertThat(result.filePath).isNotNull(); + assertBitmapsMatchExpected( + extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId); + } + + @Test + public void export_withTwoSequencesWithVideoCompositorSettings_succeeds() throws Exception { + String testId = "export_withTwoSequencesWithVideoCompositorSettings_succeeds"; + if (AndroidTestUtil.skipAndLogIfFormatsUnsupported( + context, + testId, + /* inputFormat= */ MP4_ASSET_FORMAT, + /* outputFormat= */ MP4_ASSET_FORMAT)) { + return; + } + + VideoCompositorSettings pictureInPictureVideoCompositorSettings = + new VideoCompositorSettings() { + @Override + public Size getOutputSize(List inputSizes) { + return inputSizes.get(0); + } + + @Override + public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) { + if (inputId == 0) { + // This tests all OverlaySettings builder variables. + return new OverlaySettings.Builder() + .setScale(.25f, .25f) + .setOverlayFrameAnchor(1, -1) + .setBackgroundFrameAnchor(.9f, -.7f) + .build(); + } else { + return new OverlaySettings.Builder().build(); + } + } + }; + + Composition composition = + createComposition( + /* compositionEffects= */ ImmutableList.of( + new Contrast(0.1f), + Presentation.createForWidthAndHeight( + EXPORT_WIDTH, EXPORT_HEIGHT, Presentation.LAYOUT_SCALE_TO_FIT)), + /* firstSequenceMediaItems= */ ImmutableList.of( + editedMediaItemByClippingVideo( + MP4_ASSET_URI_STRING, + /* effects= */ ImmutableList.of( + new AlphaScale(0.5f), + new ScaleAndRotateTransformation.Builder() + .setRotationDegrees(180) + .build()))), + /* secondSequenceMediaItems= */ ImmutableList.of( + editedMediaItemByClippingVideo( + MP4_ASSET_URI_STRING, /* effects= */ ImmutableList.of())), + pictureInPictureVideoCompositorSettings); ExportTestResult result = new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) @@ -165,7 +235,8 @@ public final class TransformerMultiSequenceCompositionTest { private static Composition createComposition( List compositionEffects, List firstSequenceMediaItems, - List secondSequenceMediaItems) { + List secondSequenceMediaItems, + VideoCompositorSettings videoCompositorSettings) { return new Composition.Builder( ImmutableList.of( @@ -174,6 +245,7 @@ public final class TransformerMultiSequenceCompositionTest { .setEffects( new Effects( /* audioProcessors= */ ImmutableList.of(), /* videoEffects= */ compositionEffects)) + .setVideoCompositorSettings(videoCompositorSettings) .build(); } @@ -181,12 +253,11 @@ public final class TransformerMultiSequenceCompositionTest { throws IOException { for (int i = 0; i < actualBitmaps.size(); i++) { Bitmap actualBitmap = actualBitmaps.get(i); + maybeSaveTestBitmap( + testId, /* bitmapLabel= */ String.valueOf(i), actualBitmap, /* path= */ null); String subTestId = testId + "_" + i; Bitmap expectedBitmap = readBitmap(Util.formatInvariant("%s/%s.png", PNG_ASSET_BASE_PATH, subTestId)); - - maybeSaveTestBitmap( - testId, /* bitmapLabel= */ String.valueOf(i), actualBitmap, /* path= */ null); float averagePixelAbsoluteDifference = getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, subTestId); assertThat(averagePixelAbsoluteDifference) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java index 8f8c051b14..9e80b538d4 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java @@ -23,6 +23,7 @@ import androidx.annotation.IntDef; import androidx.media3.common.MediaItem; import androidx.media3.common.audio.AudioProcessor; import androidx.media3.common.util.UnstableApi; +import androidx.media3.effect.VideoCompositorSettings; import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.lang.annotation.Documented; @@ -43,6 +44,7 @@ public final class Composition { public static final class Builder { private ImmutableList sequences; + private VideoCompositorSettings videoCompositorSettings; private Effects effects; private boolean forceAudioTrack; private boolean transmuxAudio; @@ -69,12 +71,14 @@ public final class Composition { !sequences.isEmpty(), "The composition must contain at least one EditedMediaItemSequence."); this.sequences = ImmutableList.copyOf(sequences); + videoCompositorSettings = VideoCompositorSettings.DEFAULT; effects = Effects.EMPTY; } /** Creates a new instance to build upon the provided {@link Composition}. */ private Builder(Composition composition) { sequences = composition.sequences; + videoCompositorSettings = composition.videoCompositorSettings; effects = composition.effects; forceAudioTrack = composition.forceAudioTrack; transmuxAudio = composition.transmuxAudio; @@ -82,6 +86,20 @@ public final class Composition { hdrMode = composition.hdrMode; } + /** + * Sets the {@link VideoCompositorSettings} to apply to the {@link Composition}. + * + *

The default value is {@link VideoCompositorSettings#DEFAULT}. + * + * @param videoCompositorSettings The {@link VideoCompositorSettings}. + * @return This builder. + */ + @CanIgnoreReturnValue + public Builder setVideoCompositorSettings(VideoCompositorSettings videoCompositorSettings) { + this.videoCompositorSettings = videoCompositorSettings; + return this; + } + /** * Sets the {@link Effects} to apply to the {@link Composition}. * @@ -207,7 +225,13 @@ public final class Composition { /** Builds a {@link Composition} instance. */ public Composition build() { return new Composition( - sequences, effects, forceAudioTrack, transmuxAudio, transmuxVideo, hdrMode); + sequences, + videoCompositorSettings, + effects, + forceAudioTrack, + transmuxAudio, + transmuxVideo, + hdrMode); } /** @@ -309,6 +333,9 @@ public final class Composition { */ public final ImmutableList sequences; + /** The {@link VideoCompositorSettings} to apply to the composition. */ + public final VideoCompositorSettings videoCompositorSettings; + /** The {@link Effects} to apply to the composition. */ public final Effects effects; @@ -347,6 +374,7 @@ public final class Composition { private Composition( List sequences, + VideoCompositorSettings videoCompositorSettings, Effects effects, boolean forceAudioTrack, boolean transmuxAudio, @@ -356,6 +384,7 @@ public final class Composition { !transmuxAudio || !forceAudioTrack, "Audio transmuxing and audio track forcing are not allowed together."); this.sequences = ImmutableList.copyOf(sequences); + this.videoCompositorSettings = videoCompositorSettings; this.effects = effects; this.transmuxAudio = transmuxAudio; this.transmuxVideo = transmuxVideo; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java index cecfa85f66..6fedfce7a5 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java @@ -72,6 +72,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; DebugViewProvider debugViewProvider, Listener listener, Executor listenerExecutor, + VideoCompositorSettings videoCompositorSettings, List compositionEffects, long initialTimestampOffsetUs) { return new MultipleInputVideoGraph( @@ -81,6 +82,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; debugViewProvider, listener, listenerExecutor, + videoCompositorSettings, compositionEffects, initialTimestampOffsetUs); } @@ -99,6 +101,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final DebugViewProvider debugViewProvider; private final Listener listener; private final Executor listenerExecutor; + private final VideoCompositorSettings videoCompositorSettings; private final List compositionEffects; private final List preProcessingWrappers; @@ -128,6 +131,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; DebugViewProvider debugViewProvider, Listener listener, Executor listenerExecutor, + VideoCompositorSettings videoCompositorSettings, List compositionEffects, long initialTimestampOffsetUs) { this.context = context; @@ -136,6 +140,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.debugViewProvider = debugViewProvider; this.listener = listener; this.listenerExecutor = listenerExecutor; + this.videoCompositorSettings = videoCompositorSettings; this.compositionEffects = new ArrayList<>(compositionEffects); this.initialTimestampOffsetUs = initialTimestampOffsetUs; lastRenderedPresentationTimeUs = C.TIME_UNSET; @@ -219,7 +224,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; new DefaultVideoCompositor( context, glObjectsProvider, - VideoCompositorSettings.DEFAULT, + videoCompositorSettings, sharedExecutorService, new VideoCompositor.Listener() { // All of this listener's methods are called on the sharedExecutorService. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java index b8dce37abb..4934956ef6 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java @@ -17,6 +17,7 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.content.Context; @@ -30,6 +31,7 @@ import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; import androidx.media3.effect.Presentation; +import androidx.media3.effect.VideoCompositorSettings; import java.util.List; import java.util.concurrent.Executor; @@ -52,6 +54,11 @@ import java.util.concurrent.Executor; private boolean released; private volatile boolean hasProducedFrameWithTimestampZero; + /** + * Creates an instance. + * + *

{@code videoCompositorSettings} must be {@link VideoCompositorSettings#DEFAULT}. + */ public SingleInputVideoGraph( Context context, VideoFrameProcessor.Factory videoFrameProcessorFactory, @@ -60,9 +67,14 @@ import java.util.concurrent.Executor; Listener listener, DebugViewProvider debugViewProvider, Executor listenerExecutor, + VideoCompositorSettings videoCompositorSettings, boolean renderFramesAutomatically, @Nullable Presentation presentation, long initialTimestampOffsetUs) { + checkState( + VideoCompositorSettings.DEFAULT.equals(videoCompositorSettings), + "SingleInputVideoGraph does not use VideoCompositor, and therefore cannot apply" + + " VideoCompositorSettings"); this.context = context; this.videoFrameProcessorFactory = videoFrameProcessorFactory; this.inputColorInfo = inputColorInfo; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index 5bb443d11c..5a70e7d50b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -582,6 +582,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; context, firstAssetLoaderInputFormat, transformationRequest, + composition.videoCompositorSettings, composition.effects.videoEffects, videoFrameProcessorFactory, encoderFactory, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java index cc5c7619b0..e8c97c7401 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java @@ -23,6 +23,7 @@ import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.effect.Presentation; +import androidx.media3.effect.VideoCompositorSettings; import java.util.List; import java.util.concurrent.Executor; @@ -46,6 +47,7 @@ import java.util.concurrent.Executor; DebugViewProvider debugViewProvider, Listener listener, Executor listenerExecutor, + VideoCompositorSettings videoCompositorSettings, List compositionEffects, long initialTimestampOffsetUs) { @Nullable Presentation presentation = null; @@ -63,6 +65,7 @@ import java.util.concurrent.Executor; listener, debugViewProvider, listenerExecutor, + videoCompositorSettings, /* renderFramesAutomatically= */ true, presentation, initialTimestampOffsetUs); @@ -77,6 +80,7 @@ import java.util.concurrent.Executor; Listener listener, DebugViewProvider debugViewProvider, Executor listenerExecutor, + VideoCompositorSettings videoCompositorSettings, boolean renderFramesAutomatically, @Nullable Presentation presentation, long initialTimestampOffsetUs) { @@ -88,6 +92,7 @@ import java.util.concurrent.Executor; listener, debugViewProvider, listenerExecutor, + videoCompositorSettings, renderFramesAutomatically, presentation, initialTimestampOffsetUs); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java index f85598b474..97e2dcd164 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java @@ -23,6 +23,7 @@ import androidx.media3.common.Effect; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; +import androidx.media3.effect.VideoCompositorSettings; import java.util.List; import java.util.concurrent.Executor; @@ -40,7 +41,10 @@ import java.util.concurrent.Executor; * @param debugViewProvider A {@link DebugViewProvider}. * @param listener A {@link Listener}. * @param listenerExecutor The {@link Executor} on which the {@code listener} is invoked. + * @param videoCompositorSettings The {@link VideoCompositorSettings} to apply to the + * composition. * @param compositionEffects A list of {@linkplain Effect effects} to apply to the composition. + * @param initialTimestampOffsetUs The timestamp offset for the first frame, in microseconds. * @return A new instance. * @throws VideoFrameProcessingException If a problem occurs while creating the {@link * VideoFrameProcessor}. @@ -52,6 +56,7 @@ import java.util.concurrent.Executor; DebugViewProvider debugViewProvider, Listener listener, Executor listenerExecutor, + VideoCompositorSettings videoCompositorSettings, List compositionEffects, long initialTimestampOffsetUs) throws VideoFrameProcessingException; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index 75e8f34723..24f95794f1 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -48,6 +48,7 @@ import androidx.media3.common.util.Log; import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.effect.DebugTraceUtil; +import androidx.media3.effect.VideoCompositorSettings; import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; @@ -78,6 +79,7 @@ import org.checkerframework.dataflow.qual.Pure; Context context, Format firstInputFormat, TransformationRequest transformationRequest, + VideoCompositorSettings videoCompositorSettings, List compositionEffects, VideoFrameProcessor.Factory videoFrameProcessorFactory, Codec.EncoderFactory encoderFactory, @@ -151,6 +153,7 @@ import org.checkerframework.dataflow.qual.Pure; videoGraphOutputColor, errorConsumer, debugViewProvider, + videoCompositorSettings, compositionEffects); videoGraph.initialize(); } catch (VideoFrameProcessingException e) { @@ -477,6 +480,7 @@ import org.checkerframework.dataflow.qual.Pure; ColorInfo videoFrameProcessorOutputColor, Consumer errorConsumer, DebugViewProvider debugViewProvider, + VideoCompositorSettings videoCompositorSettings, List compositionEffects) throws VideoFrameProcessingException { this.errorConsumer = errorConsumer; @@ -493,6 +497,7 @@ import org.checkerframework.dataflow.qual.Pure; debugViewProvider, /* listener= */ thisRef, /* listenerExecutor= */ MoreExecutors.directExecutor(), + videoCompositorSettings, compositionEffects, initialTimestampOffsetUs); }