Compositor: Add VideoCompositorSettings to Composition.

This allows apps using Transformer to customize how a Composition is used.

PiperOrigin-RevId: 567633129

libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerMultiSequenceCompositionTest.java

@@ -32,11 +32,14 @@ import android.content.Context;
 import android.graphics.Bitmap;
 import androidx.media3.common.Effect;
 import androidx.media3.common.MediaItem;
+import androidx.media3.common.util.Size;
 import androidx.media3.common.util.Util;
 import androidx.media3.effect.AlphaScale;
 import androidx.media3.effect.Contrast;
+import androidx.media3.effect.OverlaySettings;
 import androidx.media3.effect.Presentation;
 import androidx.media3.effect.ScaleAndRotateTransformation;
+import androidx.media3.effect.VideoCompositorSettings;
 import androidx.test.core.app.ApplicationProvider;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import com.google.common.collect.ImmutableList;
@@ -49,6 +52,9 @@ import org.junit.runner.RunWith;
 @RunWith(AndroidJUnit4.class)
 public final class TransformerMultiSequenceCompositionTest {
 
+  // Bitmaps are generated on a Pixel 6 or 7 Pro instead of an emulator, due to an emulator bug.
+  // TODO: b/301242589 - Fix this test on the crow emulator, and re-generate bitmaps using the crow
+  //  emulator, for consistency with other pixel tests.
   private static final String PNG_ASSET_BASE_PATH =
       "media/bitmap/transformer_multi_sequence_composition_test";
 
@@ -86,7 +92,8 @@ public final class TransformerMultiSequenceCompositionTest {
                             .build()))),
             /* secondSequenceMediaItems= */ ImmutableList.of(
                 editedMediaItemByClippingVideo(
-                    MP4_ASSET_URI_STRING, /* effects= */ ImmutableList.of())));
+                    MP4_ASSET_URI_STRING, /* effects= */ ImmutableList.of())),
+            VideoCompositorSettings.DEFAULT);
 
     ExportTestResult result =
         new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
@@ -125,7 +132,70 @@ public final class TransformerMultiSequenceCompositionTest {
                             .build()))),
             /* secondSequenceMediaItems= */ ImmutableList.of(
                 editedMediaItemOfOneFrameImage(
-                    JPG_ASSET_URI_STRING, /* effects= */ ImmutableList.of())));
+                    JPG_ASSET_URI_STRING, /* effects= */ ImmutableList.of())),
+            VideoCompositorSettings.DEFAULT);
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+            .build()
+            .run(testId, composition);
+
+    assertThat(result.filePath).isNotNull();
+    assertBitmapsMatchExpected(
+        extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId);
+  }
+
+  @Test
+  public void export_withTwoSequencesWithVideoCompositorSettings_succeeds() throws Exception {
+    String testId = "export_withTwoSequencesWithVideoCompositorSettings_succeeds";
+    if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
+        context,
+        testId,
+        /* inputFormat= */ MP4_ASSET_FORMAT,
+        /* outputFormat= */ MP4_ASSET_FORMAT)) {
+      return;
+    }
+
+    VideoCompositorSettings pictureInPictureVideoCompositorSettings =
+        new VideoCompositorSettings() {
+          @Override
+          public Size getOutputSize(List<Size> inputSizes) {
+            return inputSizes.get(0);
+          }
+
+          @Override
+          public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) {
+            if (inputId == 0) {
+              // This tests all OverlaySettings builder variables.
+              return new OverlaySettings.Builder()
+                  .setScale(.25f, .25f)
+                  .setOverlayFrameAnchor(1, -1)
+                  .setBackgroundFrameAnchor(.9f, -.7f)
+                  .build();
+            } else {
+              return new OverlaySettings.Builder().build();
+            }
+          }
+        };
+
+    Composition composition =
+        createComposition(
+            /* compositionEffects= */ ImmutableList.of(
+                new Contrast(0.1f),
+                Presentation.createForWidthAndHeight(
+                    EXPORT_WIDTH, EXPORT_HEIGHT, Presentation.LAYOUT_SCALE_TO_FIT)),
+            /* firstSequenceMediaItems= */ ImmutableList.of(
+                editedMediaItemByClippingVideo(
+                    MP4_ASSET_URI_STRING,
+                    /* effects= */ ImmutableList.of(
+                        new AlphaScale(0.5f),
+                        new ScaleAndRotateTransformation.Builder()
+                            .setRotationDegrees(180)
+                            .build()))),
+            /* secondSequenceMediaItems= */ ImmutableList.of(
+                editedMediaItemByClippingVideo(
+                    MP4_ASSET_URI_STRING, /* effects= */ ImmutableList.of())),
+            pictureInPictureVideoCompositorSettings);
 
     ExportTestResult result =
         new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
@@ -165,7 +235,8 @@ public final class TransformerMultiSequenceCompositionTest {
   private static Composition createComposition(
       List<Effect> compositionEffects,
       List<EditedMediaItem> firstSequenceMediaItems,
-      List<EditedMediaItem> secondSequenceMediaItems) {
+      List<EditedMediaItem> secondSequenceMediaItems,
+      VideoCompositorSettings videoCompositorSettings) {
 
     return new Composition.Builder(
             ImmutableList.of(
@@ -174,6 +245,7 @@ public final class TransformerMultiSequenceCompositionTest {
         .setEffects(
             new Effects(
                 /* audioProcessors= */ ImmutableList.of(), /* videoEffects= */ compositionEffects))
+        .setVideoCompositorSettings(videoCompositorSettings)
         .build();
   }
 
@@ -181,12 +253,11 @@ public final class TransformerMultiSequenceCompositionTest {
       throws IOException {
     for (int i = 0; i < actualBitmaps.size(); i++) {
       Bitmap actualBitmap = actualBitmaps.get(i);
+      maybeSaveTestBitmap(
+          testId, /* bitmapLabel= */ String.valueOf(i), actualBitmap, /* path= */ null);
       String subTestId = testId + "_" + i;
       Bitmap expectedBitmap =
           readBitmap(Util.formatInvariant("%s/%s.png", PNG_ASSET_BASE_PATH, subTestId));
-
-      maybeSaveTestBitmap(
-          testId, /* bitmapLabel= */ String.valueOf(i), actualBitmap, /* path= */ null);
       float averagePixelAbsoluteDifference =
           getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, subTestId);
       assertThat(averagePixelAbsoluteDifference)

libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java

@@ -23,6 +23,7 @@ import androidx.annotation.IntDef;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.audio.AudioProcessor;
 import androidx.media3.common.util.UnstableApi;
+import androidx.media3.effect.VideoCompositorSettings;
 import com.google.common.collect.ImmutableList;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 import java.lang.annotation.Documented;
@@ -43,6 +44,7 @@ public final class Composition {
   public static final class Builder {
 
     private ImmutableList<EditedMediaItemSequence> sequences;
+    private VideoCompositorSettings videoCompositorSettings;
     private Effects effects;
     private boolean forceAudioTrack;
     private boolean transmuxAudio;
@@ -69,12 +71,14 @@ public final class Composition {
           !sequences.isEmpty(),
           "The composition must contain at least one EditedMediaItemSequence.");
       this.sequences = ImmutableList.copyOf(sequences);
+      videoCompositorSettings = VideoCompositorSettings.DEFAULT;
       effects = Effects.EMPTY;
     }
 
     /** Creates a new instance to build upon the provided {@link Composition}. */
     private Builder(Composition composition) {
       sequences = composition.sequences;
+      videoCompositorSettings = composition.videoCompositorSettings;
       effects = composition.effects;
       forceAudioTrack = composition.forceAudioTrack;
       transmuxAudio = composition.transmuxAudio;
@@ -82,6 +86,20 @@ public final class Composition {
       hdrMode = composition.hdrMode;
     }
 
+    /**
+     * Sets the {@link VideoCompositorSettings} to apply to the {@link Composition}.
+     *
+     * <p>The default value is {@link VideoCompositorSettings#DEFAULT}.
+     *
+     * @param videoCompositorSettings The {@link VideoCompositorSettings}.
+     * @return This builder.
+     */
+    @CanIgnoreReturnValue
+    public Builder setVideoCompositorSettings(VideoCompositorSettings videoCompositorSettings) {
+      this.videoCompositorSettings = videoCompositorSettings;
+      return this;
+    }
+
     /**
      * Sets the {@link Effects} to apply to the {@link Composition}.
      *
@@ -207,7 +225,13 @@ public final class Composition {
     /** Builds a {@link Composition} instance. */
     public Composition build() {
       return new Composition(
-          sequences, effects, forceAudioTrack, transmuxAudio, transmuxVideo, hdrMode);
+          sequences,
+          videoCompositorSettings,
+          effects,
+          forceAudioTrack,
+          transmuxAudio,
+          transmuxVideo,
+          hdrMode);
     }
 
     /**
@@ -309,6 +333,9 @@ public final class Composition {
    */
   public final ImmutableList<EditedMediaItemSequence> sequences;
 
+  /** The {@link VideoCompositorSettings} to apply to the composition. */
+  public final VideoCompositorSettings videoCompositorSettings;
+
   /** The {@link Effects} to apply to the composition. */
   public final Effects effects;
 
@@ -347,6 +374,7 @@ public final class Composition {
 
   private Composition(
       List<EditedMediaItemSequence> sequences,
+      VideoCompositorSettings videoCompositorSettings,
       Effects effects,
       boolean forceAudioTrack,
       boolean transmuxAudio,
@@ -356,6 +384,7 @@ public final class Composition {
         !transmuxAudio || !forceAudioTrack,
         "Audio transmuxing and audio track forcing are not allowed together.");
     this.sequences = ImmutableList.copyOf(sequences);
+    this.videoCompositorSettings = videoCompositorSettings;
     this.effects = effects;
     this.transmuxAudio = transmuxAudio;
     this.transmuxVideo = transmuxVideo;

libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java

@@ -72,6 +72,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
         DebugViewProvider debugViewProvider,
         Listener listener,
         Executor listenerExecutor,
+        VideoCompositorSettings videoCompositorSettings,
         List<Effect> compositionEffects,
         long initialTimestampOffsetUs) {
       return new MultipleInputVideoGraph(
@@ -81,6 +82,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
           debugViewProvider,
           listener,
           listenerExecutor,
+          videoCompositorSettings,
           compositionEffects,
           initialTimestampOffsetUs);
     }
@@ -99,6 +101,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private final DebugViewProvider debugViewProvider;
   private final Listener listener;
   private final Executor listenerExecutor;
+  private final VideoCompositorSettings videoCompositorSettings;
   private final List<Effect> compositionEffects;
   private final List<VideoFrameProcessingWrapper> preProcessingWrappers;
 
@@ -128,6 +131,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       DebugViewProvider debugViewProvider,
       Listener listener,
       Executor listenerExecutor,
+      VideoCompositorSettings videoCompositorSettings,
       List<Effect> compositionEffects,
       long initialTimestampOffsetUs) {
     this.context = context;
@@ -136,6 +140,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     this.debugViewProvider = debugViewProvider;
     this.listener = listener;
     this.listenerExecutor = listenerExecutor;
+    this.videoCompositorSettings = videoCompositorSettings;
     this.compositionEffects = new ArrayList<>(compositionEffects);
     this.initialTimestampOffsetUs = initialTimestampOffsetUs;
     lastRenderedPresentationTimeUs = C.TIME_UNSET;
@@ -219,7 +224,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
         new DefaultVideoCompositor(
             context,
             glObjectsProvider,
-            VideoCompositorSettings.DEFAULT,
+            videoCompositorSettings,
             sharedExecutorService,
             new VideoCompositor.Listener() {
               // All of this listener's methods are called on the sharedExecutorService.

libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java

@@ -17,6 +17,7 @@
 package androidx.media3.transformer;
 
 import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.common.util.Assertions.checkState;
 import static androidx.media3.common.util.Assertions.checkStateNotNull;
 
 import android.content.Context;
@@ -30,6 +31,7 @@ import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.VideoGraph;
 import androidx.media3.effect.Presentation;
+import androidx.media3.effect.VideoCompositorSettings;
 import java.util.List;
 import java.util.concurrent.Executor;
 
@@ -52,6 +54,11 @@ import java.util.concurrent.Executor;
   private boolean released;
   private volatile boolean hasProducedFrameWithTimestampZero;
 
+  /**
+   * Creates an instance.
+   *
+   * <p>{@code videoCompositorSettings} must be {@link VideoCompositorSettings#DEFAULT}.
+   */
   public SingleInputVideoGraph(
       Context context,
       VideoFrameProcessor.Factory videoFrameProcessorFactory,
@@ -60,9 +67,14 @@ import java.util.concurrent.Executor;
       Listener listener,
       DebugViewProvider debugViewProvider,
       Executor listenerExecutor,
+      VideoCompositorSettings videoCompositorSettings,
       boolean renderFramesAutomatically,
       @Nullable Presentation presentation,
       long initialTimestampOffsetUs) {
+    checkState(
+        VideoCompositorSettings.DEFAULT.equals(videoCompositorSettings),
+        "SingleInputVideoGraph does not use VideoCompositor, and therefore cannot apply"
+            + " VideoCompositorSettings");
     this.context = context;
     this.videoFrameProcessorFactory = videoFrameProcessorFactory;
     this.inputColorInfo = inputColorInfo;

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java

@@ -582,6 +582,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
                 context,
                 firstAssetLoaderInputFormat,
                 transformationRequest,
+                composition.videoCompositorSettings,
                 composition.effects.videoEffects,
                 videoFrameProcessorFactory,
                 encoderFactory,

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java

@@ -23,6 +23,7 @@ import androidx.media3.common.DebugViewProvider;
 import androidx.media3.common.Effect;
 import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.effect.Presentation;
+import androidx.media3.effect.VideoCompositorSettings;
 import java.util.List;
 import java.util.concurrent.Executor;
 
@@ -46,6 +47,7 @@ import java.util.concurrent.Executor;
         DebugViewProvider debugViewProvider,
         Listener listener,
         Executor listenerExecutor,
+        VideoCompositorSettings videoCompositorSettings,
         List<Effect> compositionEffects,
         long initialTimestampOffsetUs) {
       @Nullable Presentation presentation = null;
@@ -63,6 +65,7 @@ import java.util.concurrent.Executor;
           listener,
           debugViewProvider,
           listenerExecutor,
+          videoCompositorSettings,
           /* renderFramesAutomatically= */ true,
           presentation,
           initialTimestampOffsetUs);
@@ -77,6 +80,7 @@ import java.util.concurrent.Executor;
       Listener listener,
       DebugViewProvider debugViewProvider,
       Executor listenerExecutor,
+      VideoCompositorSettings videoCompositorSettings,
       boolean renderFramesAutomatically,
       @Nullable Presentation presentation,
       long initialTimestampOffsetUs) {
@@ -88,6 +92,7 @@ import java.util.concurrent.Executor;
         listener,
         debugViewProvider,
         listenerExecutor,
+        videoCompositorSettings,
         renderFramesAutomatically,
         presentation,
         initialTimestampOffsetUs);

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java

@@ -23,6 +23,7 @@ import androidx.media3.common.Effect;
 import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.VideoGraph;
+import androidx.media3.effect.VideoCompositorSettings;
 import java.util.List;
 import java.util.concurrent.Executor;
 
@@ -40,7 +41,10 @@ import java.util.concurrent.Executor;
      * @param debugViewProvider A {@link DebugViewProvider}.
      * @param listener A {@link Listener}.
      * @param listenerExecutor The {@link Executor} on which the {@code listener} is invoked.
+     * @param videoCompositorSettings The {@link VideoCompositorSettings} to apply to the
+     *     composition.
      * @param compositionEffects A list of {@linkplain Effect effects} to apply to the composition.
+     * @param initialTimestampOffsetUs The timestamp offset for the first frame, in microseconds.
      * @return A new instance.
      * @throws VideoFrameProcessingException If a problem occurs while creating the {@link
      *     VideoFrameProcessor}.
@@ -52,6 +56,7 @@ import java.util.concurrent.Executor;
         DebugViewProvider debugViewProvider,
         Listener listener,
         Executor listenerExecutor,
+        VideoCompositorSettings videoCompositorSettings,
         List<Effect> compositionEffects,
         long initialTimestampOffsetUs)
         throws VideoFrameProcessingException;

libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java

@@ -48,6 +48,7 @@ import androidx.media3.common.util.Log;
 import androidx.media3.common.util.Util;
 import androidx.media3.decoder.DecoderInputBuffer;
 import androidx.media3.effect.DebugTraceUtil;
+import androidx.media3.effect.VideoCompositorSettings;
 import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
 import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.MoreExecutors;
@@ -78,6 +79,7 @@ import org.checkerframework.dataflow.qual.Pure;
       Context context,
       Format firstInputFormat,
       TransformationRequest transformationRequest,
+      VideoCompositorSettings videoCompositorSettings,
       List<Effect> compositionEffects,
       VideoFrameProcessor.Factory videoFrameProcessorFactory,
       Codec.EncoderFactory encoderFactory,
@@ -151,6 +153,7 @@ import org.checkerframework.dataflow.qual.Pure;
               videoGraphOutputColor,
               errorConsumer,
               debugViewProvider,
+              videoCompositorSettings,
               compositionEffects);
       videoGraph.initialize();
     } catch (VideoFrameProcessingException e) {
@@ -477,6 +480,7 @@ import org.checkerframework.dataflow.qual.Pure;
         ColorInfo videoFrameProcessorOutputColor,
         Consumer<ExportException> errorConsumer,
         DebugViewProvider debugViewProvider,
+        VideoCompositorSettings videoCompositorSettings,
         List<Effect> compositionEffects)
         throws VideoFrameProcessingException {
       this.errorConsumer = errorConsumer;
@@ -493,6 +497,7 @@ import org.checkerframework.dataflow.qual.Pure;
               debugViewProvider,
               /* listener= */ thisRef,
               /* listenerExecutor= */ MoreExecutors.directExecutor(),
+              videoCompositorSettings,
               compositionEffects,
               initialTimestampOffsetUs);
     }