Compositor: Implement OverlaySettings and custom in/out size support.

Implement VideoCompositor support of:
* Different input and output sizes
* CompositorSettings, to customize output size based on input texture sizes
* OverlaySettings, to place an input frame in an arbitrary position on
  the output frame.

Also, refactor Overlay's matrix logic to make it more reusable between
Compositor and Overlays

PiperOrigin-RevId: 561931854

libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java

@@ -36,6 +36,7 @@ import androidx.media3.common.util.GlProgram;
 import androidx.media3.common.util.GlUtil;
 import androidx.media3.common.util.Log;
 import androidx.media3.common.util.LongArrayQueue;
+import androidx.media3.common.util.Size;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
 import com.google.common.collect.ImmutableList;
@@ -50,8 +51,8 @@ import java.util.concurrent.ExecutorService;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 /**
- * A basic {@link VideoCompositor} implementation that takes in frames from exactly 2 input sources'
+ * A basic {@link VideoCompositor} implementation that takes in frames from exactly 2 SDR input
- * streams and combines them into one output stream.
+ * sources' streams and combines them into one output stream.
  *
  * <p>The first {@linkplain #registerInputSource registered source} will be the primary stream,
  * which is used to determine the output frames' timestamps and dimensions.
@@ -62,17 +63,33 @@ public final class DefaultVideoCompositor implements VideoCompositor {
   //  * Use a lock to synchronize inputFrameInfos more narrowly, to reduce blocking.
   //  * If the primary stream ends, consider setting the secondary stream as the new primary stream,
   //    so that secondary stream frames aren't dropped.
+  //  * Add support for HDR input.
+
+  /** A default implementation of {@link VideoCompositor.Settings}. */
+  public static final class Settings implements VideoCompositor.Settings {
+    @Override
+    public Size getOutputSize(List<Size> inputSizes) {
+      return inputSizes.get(PRIMARY_INPUT_ID);
+    }
+
+    @Override
+    public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) {
+      return new OverlaySettings.Builder().build();
+    }
+  }
 
   private static final String THREAD_NAME = "Effect:DefaultVideoCompositor:GlThread";
   private static final String TAG = "DefaultVideoCompositor";
   private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl";
-  private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl";
+  private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_alpha_scale_es2.glsl";
   private static final int PRIMARY_INPUT_ID = 0;
 
   private final Context context;
   private final VideoCompositor.Listener listener;
   private final GlTextureProducer.Listener textureOutputListener;
   private final GlObjectsProvider glObjectsProvider;
+  private final VideoCompositor.Settings settings;
+  private final OverlayMatrixProvider overlayMatrixProvider;
   private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
 
   @GuardedBy("this")
@@ -100,6 +117,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
   public DefaultVideoCompositor(
       Context context,
       GlObjectsProvider glObjectsProvider,
+      VideoCompositor.Settings settings,
       @Nullable ExecutorService executorService,
       VideoCompositor.Listener listener,
       GlTextureProducer.Listener textureOutputListener,
@@ -108,6 +126,8 @@ public final class DefaultVideoCompositor implements VideoCompositor {
     this.listener = listener;
     this.textureOutputListener = textureOutputListener;
     this.glObjectsProvider = glObjectsProvider;
+    this.settings = settings;
+    this.overlayMatrixProvider = new OverlayMatrixProvider();
 
     inputSources = new ArrayList<>();
     outputTexturePool =
@@ -180,7 +200,11 @@ public final class DefaultVideoCompositor implements VideoCompositor {
     checkState(!inputSource.isInputEnded);
 
     InputFrameInfo inputFrameInfo =
-        new InputFrameInfo(textureProducer, inputTexture, presentationTimeUs);
+        new InputFrameInfo(
+            textureProducer,
+            inputTexture,
+            presentationTimeUs,
+            settings.getOverlaySettings(inputId, presentationTimeUs));
     inputSource.frameInfos.add(inputFrameInfo);
 
     if (inputId == PRIMARY_INPUT_ID) {
@@ -277,17 +301,17 @@ public final class DefaultVideoCompositor implements VideoCompositor {
 
     ensureGlProgramConfigured();
 
-    // TODO: b/262694346 - Allow different input frame dimensions.
     InputFrameInfo primaryInputFrame = framesToComposite.get(PRIMARY_INPUT_ID);
-    GlTextureInfo primaryInputTexture = primaryInputFrame.texture;
-    outputTexturePool.ensureConfigured(
-        glObjectsProvider, primaryInputTexture.width, primaryInputTexture.height);
 
-    for (int i = 1; i < framesToComposite.size(); i++) {
+    ImmutableList.Builder<Size> inputSizes = new ImmutableList.Builder<>();
-      GlTextureInfo textureToComposite = framesToComposite.get(i).texture;
+    for (int i = 0; i < framesToComposite.size(); i++) {
-      checkState(primaryInputTexture.width == textureToComposite.width);
+      GlTextureInfo texture = framesToComposite.get(i).texture;
-      checkState(primaryInputTexture.height == textureToComposite.height);
+      inputSizes.add(new Size(texture.width, texture.height));
     }
+    Size outputSize = settings.getOutputSize(inputSizes.build());
+    outputTexturePool.ensureConfigured(
+        glObjectsProvider, outputSize.getWidth(), outputSize.getHeight());
+
     GlTextureInfo outputTexture = outputTexturePool.useTexture();
     long outputPresentationTimestampUs = primaryInputFrame.presentationTimeUs;
     outputTextureTimestamps.add(outputPresentationTimestampUs);
@@ -394,16 +418,18 @@ public final class DefaultVideoCompositor implements VideoCompositor {
           GlUtil.getNormalizedCoordinateBounds(),
           GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
       glProgram.setFloatsUniform("uTexTransformationMatrix", GlUtil.create4x4IdentityMatrix());
-      glProgram.setFloatsUniform("uTransformationMatrix", GlUtil.create4x4IdentityMatrix());
     } catch (IOException e) {
       throw new VideoFrameProcessingException(e);
     }
   }
 
+  // Enhanced for-loops are discouraged in media3.effect due to short-lived allocations.
+  @SuppressWarnings("ListReverse")
   private void drawFrame(List<InputFrameInfo> framesToComposite, GlTextureInfo outputTexture)
       throws GlUtil.GlException {
     GlUtil.focusFramebufferUsingCurrentContext(
         outputTexture.fboId, outputTexture.width, outputTexture.height);
+    overlayMatrixProvider.configure(new Size(outputTexture.width, outputTexture.height));
     GlUtil.clearFocusedBuffers();
 
     GlProgram glProgram = checkNotNull(this.glProgram);
@@ -423,7 +449,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
 
     // Draw textures from back to front.
     for (int i = framesToComposite.size() - 1; i >= 0; i--) {
-      blendOntoFocusedTexture(framesToComposite.get(i).texture.texId);
+      blendOntoFocusedTexture(framesToComposite.get(i));
     }
 
     GLES20.glDisable(GLES20.GL_BLEND);
@@ -431,9 +457,16 @@ public final class DefaultVideoCompositor implements VideoCompositor {
     GlUtil.checkGlError();
   }
 
-  private void blendOntoFocusedTexture(int texId) throws GlUtil.GlException {
+  private void blendOntoFocusedTexture(InputFrameInfo inputFrameInfo) throws GlUtil.GlException {
     GlProgram glProgram = checkNotNull(this.glProgram);
-    glProgram.setSamplerTexIdUniform("uTexSampler", texId, /* texUnitIndex= */ 0);
+    GlTextureInfo inputTexture = inputFrameInfo.texture;
+    glProgram.setSamplerTexIdUniform("uTexSampler", inputTexture.texId, /* texUnitIndex= */ 0);
+    float[] transformationMatrix =
+        overlayMatrixProvider.getTransformationMatrix(
+            /* overlaySize= */ new Size(inputTexture.width, inputTexture.height),
+            inputFrameInfo.overlaySettings);
+    glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrix);
+    glProgram.setFloatUniform("uAlphaScale", inputFrameInfo.overlaySettings.alphaScale);
     glProgram.bindAttributesAndUniforms();
 
     // The four-vertex triangle strip forms a quad.
@@ -480,12 +513,17 @@ public final class DefaultVideoCompositor implements VideoCompositor {
     public final GlTextureProducer textureProducer;
     public final GlTextureInfo texture;
     public final long presentationTimeUs;
+    public final OverlaySettings overlaySettings;
 
     public InputFrameInfo(
-        GlTextureProducer textureProducer, GlTextureInfo texture, long presentationTimeUs) {
+        GlTextureProducer textureProducer,
+        GlTextureInfo texture,
+        long presentationTimeUs,
+        OverlaySettings overlaySettings) {
       this.textureProducer = textureProducer;
       this.texture = texture;
       this.presentationTimeUs = presentationTimeUs;
+      this.overlaySettings = overlaySettings;
     }
   }
 }

libraries/effect/src/main/java/androidx/media3/effect/OverlayMatrixProvider.java

@@ -15,7 +15,7 @@
  */
 package androidx.media3.effect;
 
-import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.common.util.Assertions.checkStateNotNull;
 
 import android.opengl.Matrix;
 import android.util.Pair;
@@ -23,15 +23,14 @@ import androidx.media3.common.util.GlUtil;
 import androidx.media3.common.util.Size;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
-/* package */ final class OverlayMatrixProvider {
+/** Provides a matrix for {@link OverlaySettings}, to be applied on a vertex. */
-  private static final int MATRIX_OFFSET = 0;
+/* package */ class OverlayMatrixProvider {
+  protected static final int MATRIX_OFFSET = 0;
   private final float[] videoFrameAnchorMatrix;
-  private final float[] videoFrameAnchorMatrixInv;
   private final float[] aspectRatioMatrix;
   private final float[] scaleMatrix;
   private final float[] scaleMatrixInv;
   private final float[] overlayAnchorMatrix;
-  private final float[] overlayAnchorMatrixInv;
   private final float[] rotateMatrix;
   private final float[] overlayAspectRatioMatrix;
   private final float[] overlayAspectRatioMatrixInv;
@@ -41,9 +40,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   public OverlayMatrixProvider() {
     aspectRatioMatrix = GlUtil.create4x4IdentityMatrix();
     videoFrameAnchorMatrix = GlUtil.create4x4IdentityMatrix();
-    videoFrameAnchorMatrixInv = GlUtil.create4x4IdentityMatrix();
     overlayAnchorMatrix = GlUtil.create4x4IdentityMatrix();
-    overlayAnchorMatrixInv = GlUtil.create4x4IdentityMatrix();
     rotateMatrix = GlUtil.create4x4IdentityMatrix();
     scaleMatrix = GlUtil.create4x4IdentityMatrix();
     scaleMatrixInv = GlUtil.create4x4IdentityMatrix();
@@ -56,6 +53,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     this.backgroundSize = backgroundSize;
   }
 
+  /**
+   * Returns the transformation matrix.
+   *
+   * <p>This instance must be {@linkplain #configure configured} before this method is called.
+   */
   public float[] getTransformationMatrix(Size overlaySize, OverlaySettings overlaySettings) {
     reset();
 
@@ -67,44 +69,38 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
         videoFrameAnchor.first,
         videoFrameAnchor.second,
         /* z= */ 0f);
-    Matrix.invertM(videoFrameAnchorMatrixInv, MATRIX_OFFSET, videoFrameAnchorMatrix, MATRIX_OFFSET);
 
+    checkStateNotNull(backgroundSize);
     Matrix.scaleM(
         aspectRatioMatrix,
         MATRIX_OFFSET,
-        checkNotNull(backgroundSize).getWidth() / (float) overlaySize.getWidth(),
+        (float) overlaySize.getWidth() / backgroundSize.getWidth(),
-        checkNotNull(backgroundSize).getHeight() / (float) overlaySize.getHeight(),
+        (float) overlaySize.getHeight() / backgroundSize.getHeight(),
         /* z= */ 1f);
 
     // Scale the image.
     Pair<Float, Float> scale = overlaySettings.scale;
-    Matrix.scaleM(
+    Matrix.scaleM(scaleMatrix, MATRIX_OFFSET, scale.first, scale.second, /* z= */ 1f);
-        scaleMatrix,
-        MATRIX_OFFSET,
-        scaleMatrix,
-        MATRIX_OFFSET,
-        scale.first,
-        scale.second,
-        /* z= */ 1f);
     Matrix.invertM(scaleMatrixInv, MATRIX_OFFSET, scaleMatrix, MATRIX_OFFSET);
 
-    // Translate the overlay within its frame.
+    // Translate the overlay within its frame. To position the overlay's anchor at the correct
+    // position, it must be translated the opposite direction by the same magnitude.
     Pair<Float, Float> overlayAnchor = overlaySettings.overlayAnchor;
     Matrix.translateM(
-        overlayAnchorMatrix, MATRIX_OFFSET, overlayAnchor.first, overlayAnchor.second, /* z= */ 0f);
+        overlayAnchorMatrix,
-    Matrix.invertM(overlayAnchorMatrixInv, MATRIX_OFFSET, overlayAnchorMatrix, MATRIX_OFFSET);
+        MATRIX_OFFSET,
+        -1 * overlayAnchor.first,
+        -1 * overlayAnchor.second,
+        /* z= */ 0f);
 
     // Rotate the image.
     Matrix.rotateM(
-        rotateMatrix,
-        MATRIX_OFFSET,
         rotateMatrix,
         MATRIX_OFFSET,
         overlaySettings.rotationDegrees,
         /* x= */ 0f,
         /* y= */ 0f,
         /* z= */ 1f);
-    Matrix.invertM(rotateMatrix, MATRIX_OFFSET, rotateMatrix, MATRIX_OFFSET);
 
     // Rotation matrix needs to account for overlay aspect ratio to prevent stretching.
     Matrix.scaleM(
@@ -116,67 +112,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     Matrix.invertM(
         overlayAspectRatioMatrixInv, MATRIX_OFFSET, overlayAspectRatioMatrix, MATRIX_OFFSET);
 
-    // Rotation needs to be agnostic of the scaling matrix and the aspect ratios.
+    // transformationMatrix = videoFrameAnchorMatrix * aspectRatioMatrix
-    // transformationMatrix = scaleMatrixInv * overlayAspectRatioMatrix * rotateMatrix *
+    //   * scaleMatrix * overlayAnchorMatrix * scaleMatrixInv
-    //   overlayAspectRatioInv * scaleMatrix * overlayAnchorMatrixInv * scaleMatrixInv *
+    //   * overlayAspectRatioMatrix * rotateMatrix * overlayAspectRatioMatrixInv
-    //   aspectRatioMatrix * videoFrameAnchorMatrixInv
+    //   * scaleMatrix.
-    Matrix.multiplyMM(
-        transformationMatrix,
-        MATRIX_OFFSET,
-        transformationMatrix,
-        MATRIX_OFFSET,
-        scaleMatrixInv,
-        MATRIX_OFFSET);
 
+    // Anchor position in output frame.
     Matrix.multiplyMM(
         transformationMatrix,
         MATRIX_OFFSET,
         transformationMatrix,
         MATRIX_OFFSET,
-        overlayAspectRatioMatrix,
+        videoFrameAnchorMatrix,
-        MATRIX_OFFSET);
-
-    // Rotation matrix.
-    Matrix.multiplyMM(
-        transformationMatrix,
-        MATRIX_OFFSET,
-        transformationMatrix,
-        MATRIX_OFFSET,
-        rotateMatrix,
-        MATRIX_OFFSET);
-
-    Matrix.multiplyMM(
-        transformationMatrix,
-        MATRIX_OFFSET,
-        transformationMatrix,
-        MATRIX_OFFSET,
-        overlayAspectRatioMatrixInv,
-        MATRIX_OFFSET);
-
-    Matrix.multiplyMM(
-        transformationMatrix,
-        MATRIX_OFFSET,
-        transformationMatrix,
-        MATRIX_OFFSET,
-        scaleMatrix,
-        MATRIX_OFFSET);
-
-    // Translate image.
-    Matrix.multiplyMM(
-        transformationMatrix,
-        MATRIX_OFFSET,
-        transformationMatrix,
-        MATRIX_OFFSET,
-        overlayAnchorMatrixInv,
-        MATRIX_OFFSET);
-
-    // Scale image.
-    Matrix.multiplyMM(
-        transformationMatrix,
-        MATRIX_OFFSET,
-        transformationMatrix,
-        MATRIX_OFFSET,
-        scaleMatrixInv,
         MATRIX_OFFSET);
 
     // Correct for aspect ratio of image in output frame.
@@ -188,23 +135,67 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
         aspectRatioMatrix,
         MATRIX_OFFSET);
 
-    // Anchor position in output frame.
     Matrix.multiplyMM(
         transformationMatrix,
         MATRIX_OFFSET,
         transformationMatrix,
         MATRIX_OFFSET,
-        videoFrameAnchorMatrixInv,
+        scaleMatrix,
         MATRIX_OFFSET);
+    Matrix.multiplyMM(
+        transformationMatrix,
+        MATRIX_OFFSET,
+        transformationMatrix,
+        MATRIX_OFFSET,
+        overlayAnchorMatrix,
+        MATRIX_OFFSET);
+    Matrix.multiplyMM(
+        transformationMatrix,
+        MATRIX_OFFSET,
+        transformationMatrix,
+        MATRIX_OFFSET,
+        scaleMatrixInv,
+        MATRIX_OFFSET);
+
+    // Rotation needs to be agnostic of the scaling matrix and the aspect ratios.
+    Matrix.multiplyMM(
+        transformationMatrix,
+        MATRIX_OFFSET,
+        transformationMatrix,
+        MATRIX_OFFSET,
+        overlayAspectRatioMatrix,
+        MATRIX_OFFSET);
+    Matrix.multiplyMM(
+        transformationMatrix,
+        MATRIX_OFFSET,
+        transformationMatrix,
+        MATRIX_OFFSET,
+        rotateMatrix,
+        MATRIX_OFFSET);
+    Matrix.multiplyMM(
+        transformationMatrix,
+        MATRIX_OFFSET,
+        transformationMatrix,
+        MATRIX_OFFSET,
+        overlayAspectRatioMatrixInv,
+        MATRIX_OFFSET);
+
+    // Scale image.
+    Matrix.multiplyMM(
+        transformationMatrix,
+        MATRIX_OFFSET,
+        transformationMatrix,
+        MATRIX_OFFSET,
+        scaleMatrix,
+        MATRIX_OFFSET);
+
     return transformationMatrix;
   }
 
   private void reset() {
     GlUtil.setToIdentity(aspectRatioMatrix);
     GlUtil.setToIdentity(videoFrameAnchorMatrix);
-    GlUtil.setToIdentity(videoFrameAnchorMatrixInv);
     GlUtil.setToIdentity(overlayAnchorMatrix);
-    GlUtil.setToIdentity(overlayAnchorMatrixInv);
     GlUtil.setToIdentity(scaleMatrix);
     GlUtil.setToIdentity(scaleMatrixInv);
     GlUtil.setToIdentity(rotateMatrix);

libraries/effect/src/main/java/androidx/media3/effect/OverlaySettings.java

@@ -22,7 +22,10 @@ import androidx.annotation.FloatRange;
 import androidx.media3.common.util.UnstableApi;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
 
-/** Contains information to control how a {@link TextureOverlay} is displayed on the screen. */
+/**
+ * Contains information to control how an input texture (for example, a {@link VideoCompositor} or
+ * {@link TextureOverlay}) is displayed on a background.
+ */
 @UnstableApi
 public final class OverlaySettings {
   public final boolean useHdr;
@@ -47,6 +50,11 @@ public final class OverlaySettings {
     this.rotationDegrees = rotationDegrees;
   }
 
+  /** Returns a new {@link Builder} initialized with the values of this instance. */
+  /* package */ Builder buildUpon() {
+    return new Builder(this);
+  }
+
   /** A builder for {@link OverlaySettings} instances. */
   public static final class Builder {
     private boolean useHdr;
@@ -65,6 +73,15 @@ public final class OverlaySettings {
       rotationDegrees = 0f;
     }
 
+    private Builder(OverlaySettings overlaySettings) {
+      this.useHdr = overlaySettings.useHdr;
+      this.alphaScale = overlaySettings.alphaScale;
+      this.videoFrameAnchor = overlaySettings.videoFrameAnchor;
+      this.overlayAnchor = overlaySettings.overlayAnchor;
+      this.scale = overlaySettings.scale;
+      this.rotationDegrees = overlaySettings.rotationDegrees;
+    }
+
     /**
      * Sets whether input overlay comes from an HDR source. If {@code true}, colors will be in
      * linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709.
@@ -92,6 +109,7 @@ public final class OverlaySettings {
       return this;
     }
 
+    // TODO: b/262694346 - Rename this method to setBackgroundAnchor in a follow-up CL.
     /**
      * Sets the coordinates for the anchor point of the overlay within the video frame.
      *

libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java

@@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList;
 /* package */ final class OverlayShaderProgram extends BaseGlShaderProgram {
 
   private final GlProgram glProgram;
-  private final OverlayMatrixProvider overlayMatrixProvider;
+  private final SamplerOverlayMatrixProvider samplerOverlayMatrixProvider;
   private final ImmutableList<TextureOverlay> overlays;
 
   /**
@@ -49,7 +49,7 @@ import com.google.common.collect.ImmutableList;
         overlays.size() <= 15,
         "OverlayShaderProgram does not support more than 15 overlays in the same instance.");
     this.overlays = overlays;
-    this.overlayMatrixProvider = new OverlayMatrixProvider();
+    this.samplerOverlayMatrixProvider = new SamplerOverlayMatrixProvider();
     try {
       glProgram =
           new GlProgram(createVertexShader(overlays.size()), createFragmentShader(overlays.size()));
@@ -66,7 +66,7 @@ import com.google.common.collect.ImmutableList;
   @Override
   public Size configure(int inputWidth, int inputHeight) {
     Size videoSize = new Size(inputWidth, inputHeight);
-    overlayMatrixProvider.configure(/* backgroundSize= */ videoSize);
+    samplerOverlayMatrixProvider.configure(/* backgroundSize= */ videoSize);
     for (TextureOverlay overlay : overlays) {
       overlay.configure(videoSize);
     }
@@ -91,7 +91,7 @@ import com.google.common.collect.ImmutableList;
 
           glProgram.setFloatsUniform(
               Util.formatInvariant("uTransformationMatrix%d", texUnitIndex),
-              overlayMatrixProvider.getTransformationMatrix(overlaySize, overlaySettings));
+              samplerOverlayMatrixProvider.getTransformationMatrix(overlaySize, overlaySettings));
 
           glProgram.setFloatUniform(
               Util.formatInvariant("uOverlayAlphaScale%d", texUnitIndex),

libraries/effect/src/main/java/androidx/media3/effect/SamplerOverlayMatrixProvider.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.effect;
+
+import android.opengl.Matrix;
+import androidx.media3.common.util.GlUtil;
+import androidx.media3.common.util.Size;
+
+/**
+ * Provides a matrix based on {@link OverlaySettings} to be applied on a texture sampling
+ * coordinate.
+ */
+/* package */ final class SamplerOverlayMatrixProvider extends OverlayMatrixProvider {
+  private final float[] transformationMatrixInv;
+
+  public SamplerOverlayMatrixProvider() {
+    super();
+    transformationMatrixInv = GlUtil.create4x4IdentityMatrix();
+  }
+
+  @Override
+  public float[] getTransformationMatrix(Size overlaySize, OverlaySettings overlaySettings) {
+    // When sampling from a (for example, texture) sampler, the overlay anchor's x and y coordinates
+    // are flipped.
+    OverlaySettings samplerOverlaySettings =
+        overlaySettings
+            .buildUpon()
+            .setOverlayAnchor(
+                /* x= */ -1 * overlaySettings.overlayAnchor.first,
+                /* y= */ -1 * overlaySettings.overlayAnchor.second)
+            .build();
+
+    // When sampling from a (for example, texture) sampler, the transformation matrix applied to a
+    // sampler's coordinate should be the inverse of the transformation matrix that would otherwise
+    // be applied to a vertex.
+    Matrix.invertM(
+        transformationMatrixInv,
+        MATRIX_OFFSET,
+        super.getTransformationMatrix(overlaySize, samplerOverlaySettings),
+        MATRIX_OFFSET);
+    return transformationMatrixInv;
+  }
+}

libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java

@@ -17,10 +17,12 @@ package androidx.media3.effect;
 
 import androidx.media3.common.GlTextureInfo;
 import androidx.media3.common.VideoFrameProcessingException;
+import androidx.media3.common.util.Size;
 import androidx.media3.common.util.UnstableApi;
+import java.util.List;
 
 /**
- * Interface for a video compositor that combines frames from mutliple input sources to produce
+ * Interface for a video compositor that combines frames from multiple input sources to produce
  * output frames.
  *
  * <p>Input and output are provided via OpenGL textures.
@@ -41,6 +43,23 @@ public interface VideoCompositor extends GlTextureProducer {
     void onEnded();
   }
 
+  /** Settings for the {@link VideoCompositor}. */
+  interface Settings {
+    // TODO: b/262694346 - Consider adding more features, like selecting a:
+    //  * custom order for drawing (instead of primary stream on top), and
+    //  * different primary source.
+
+    /**
+     * Returns an output texture {@link Size}, based on {@code inputSizes}.
+     *
+     * @param inputSizes The {@link Size} of each input frame, ordered by {@code inputId}.
+     */
+    Size getOutputSize(List<Size> inputSizes);
+
+    /** Returns {@link OverlaySettings} for {@code inputId} at time {@code presentationTimeUs}. */
+    OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs);
+  }
+
   /**
    * Registers a new input source, and returns a unique {@code inputId} corresponding to this
    * source, to be used in {@link #queueInputTexture}.

libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java

@@ -24,6 +24,7 @@ import static androidx.media3.test.utils.VideoFrameProcessorTestRunner.createTim
 import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertWithMessage;
+import static java.lang.Math.max;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 import android.graphics.Bitmap;
@@ -44,6 +45,7 @@ import androidx.media3.common.Effect;
 import androidx.media3.common.GlObjectsProvider;
 import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.util.GlUtil;
+import androidx.media3.common.util.Size;
 import androidx.media3.common.util.Util;
 import androidx.media3.effect.AlphaScale;
 import androidx.media3.effect.DefaultGlObjectsProvider;
@@ -51,6 +53,7 @@ import androidx.media3.effect.DefaultVideoCompositor;
 import androidx.media3.effect.DefaultVideoFrameProcessor;
 import androidx.media3.effect.OverlayEffect;
 import androidx.media3.effect.OverlaySettings;
+import androidx.media3.effect.Presentation;
 import androidx.media3.effect.RgbFilter;
 import androidx.media3.effect.ScaleAndRotateTransformation;
 import androidx.media3.effect.TextOverlay;
@@ -60,6 +63,7 @@ import androidx.media3.test.utils.TextureBitmapReader;
 import androidx.media3.test.utils.VideoFrameProcessorTestRunner;
 import com.google.common.base.Ascii;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
@@ -103,15 +107,15 @@ public final class DefaultVideoCompositorPixelTest {
   private static final String ORIGINAL_PNG_ASSET_PATH =
       "media/bitmap/input_images/media3test_srgb.png";
   private static final String TEST_DIRECTORY = "media/bitmap/CompositorTestTimestamps/";
-
-  private @MonotonicNonNull String testId;
-  private @MonotonicNonNull VideoCompositorTestRunner compositorTestRunner;
   private static final ImmutableList<ImmutableList<Effect>> TWO_INPUT_COMPOSITOR_EFFECT_LISTS =
       ImmutableList.of(
           ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(0.7f)),
           ImmutableList.of(
               new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build()));
 
+  private @MonotonicNonNull String testId;
+  private @MonotonicNonNull VideoCompositorTestRunner compositorTestRunner;
+
   @Before
   @EnsuresNonNull("testId")
   public void setUpTestId() {
@@ -125,6 +129,8 @@ public final class DefaultVideoCompositorPixelTest {
     }
   }
 
+  // Tests for alpha and frame alpha/occlusion.
+
   @Test
   @RequiresNonNull("testId")
   public void compositeTwoInputs_withOneFrameFromEach_differentTimestamp_matchesExpectedBitmap()
@@ -155,12 +161,13 @@ public final class DefaultVideoCompositorPixelTest {
   @RequiresNonNull("testId")
   public void compositeTwoInputs_withPrimaryTransparent_differentTimestamp_matchesExpectedBitmap()
       throws Exception {
-    ImmutableList<ImmutableList<Effect>> inputEffects =
+    ImmutableList<ImmutableList<Effect>> inputEffectLists =
         ImmutableList.of(
             ImmutableList.of(new AlphaScale(0f)),
             ImmutableList.of(
                 new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build()));
-    compositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffects);
+    compositorTestRunner =
+        new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffectLists);
 
     compositorTestRunner.queueBitmapToInput(
         /* inputId= */ 0, /* timestamps= */ ImmutableList.of(0L));
@@ -186,12 +193,13 @@ public final class DefaultVideoCompositorPixelTest {
   @RequiresNonNull("testId")
   public void compositeTwoInputs_withPrimaryOpaque_differentTimestamp_matchesExpectedBitmap()
       throws Exception {
-    ImmutableList<ImmutableList<Effect>> inputEffects =
+    ImmutableList<ImmutableList<Effect>> inputEffectLists =
         ImmutableList.of(
             ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(100f)),
             ImmutableList.of(
                 new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build()));
-    compositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffects);
+    compositorTestRunner =
+        new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffectLists);
 
     compositorTestRunner.queueBitmapToInput(
         /* inputId= */ 0, /* timestamps= */ ImmutableList.of(0L));
@@ -217,11 +225,12 @@ public final class DefaultVideoCompositorPixelTest {
   @RequiresNonNull("testId")
   public void compositeTwoInputs_withSecondaryTransparent_differentTimestamp_matchesExpectedBitmap()
       throws Exception {
-    ImmutableList<ImmutableList<Effect>> inputEffects =
+    ImmutableList<ImmutableList<Effect>> inputEffectLists =
         ImmutableList.of(
             ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(0.7f)),
             ImmutableList.of(new AlphaScale(0f)));
-    compositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffects);
+    compositorTestRunner =
+        new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffectLists);
 
     compositorTestRunner.queueBitmapToInput(
         /* inputId= */ 0, /* timestamps= */ ImmutableList.of(0L));
@@ -243,6 +252,8 @@ public final class DefaultVideoCompositorPixelTest {
         ImmutableList.of("0s_1s_transparent"));
   }
 
+  // Tests for mixing different frame rates and timestamps.
+
   @Test
   @RequiresNonNull("testId")
   public void compositeTwoInputs_withFiveFramesFromEach_matchesExpectedTimestamps()
@@ -428,6 +439,8 @@ public final class DefaultVideoCompositorPixelTest {
         ImmutableList.of("0s_1s", "1s_1s", "2s_1s", "3s_3s", "4s_4s"));
   }
 
+  // Tests for "many" inputs/frames.
+
   @Test
   @RequiresNonNull("testId")
   public void compositeTwoInputs_withTenFramesFromEach_matchesExpectedFrameCount()
@@ -468,6 +481,8 @@ public final class DefaultVideoCompositorPixelTest {
     assertThat(compositorTestRunner.getCompositedTimestamps()).hasSize(numberOfFramesToQueue);
   }
 
+  // Tests for different amounts of inputs.
+
   @Test
   @RequiresNonNull("testId")
   public void compositeOneInput_matchesExpectedBitmap() throws Exception {
@@ -526,6 +541,120 @@ public final class DefaultVideoCompositorPixelTest {
         ImmutableList.of("0s_1s_0s", "1s_1s_0s", "2s_1s_2s"));
   }
 
+  // Tests for different layouts.
+
+  @Test
+  @RequiresNonNull("testId")
+  public void compositeTwoInputs_pictureInPicture_matchesExpectedBitmap() throws Exception {
+    ImmutableList<ImmutableList<Effect>> inputEffectLists =
+        ImmutableList.of(ImmutableList.of(), ImmutableList.of(RgbFilter.createGrayscaleFilter()));
+    VideoCompositor.Settings pictureInPictureSettings =
+        new VideoCompositor.Settings() {
+          @Override
+          public Size getOutputSize(List<Size> inputSizes) {
+            return inputSizes.get(0);
+          }
+
+          @Override
+          public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) {
+            if (inputId == 0) {
+              // This tests all OverlaySettings builder variables.
+              return new OverlaySettings.Builder()
+                  .setScale(.25f, .5f)
+                  .setOverlayAnchor(1, -1)
+                  .setVideoFrameAnchor(.9f, -.7f)
+                  .setRotationDegrees(20)
+                  .setAlphaScale(.5f)
+                  .build();
+            } else {
+              return new OverlaySettings.Builder().build();
+            }
+          }
+        };
+    compositorTestRunner =
+        new VideoCompositorTestRunner(
+            testId, useSharedExecutor, inputEffectLists, pictureInPictureSettings);
+
+    compositorTestRunner.queueBitmapToAllInputs(1);
+    compositorTestRunner.endCompositing();
+
+    compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected(
+        ImmutableList.of("picture_in_picture"));
+  }
+
+  @Test
+  @RequiresNonNull("testId")
+  public void compositeTwoInputs_differentDimensions_matchesExpectedBitmap() throws Exception {
+    ImmutableList<ImmutableList<Effect>> inputEffectLists =
+        ImmutableList.of(
+            ImmutableList.of(
+                Presentation.createForWidthAndHeight(100, 100, Presentation.LAYOUT_STRETCH_TO_FIT)),
+            ImmutableList.of(RgbFilter.createGrayscaleFilter()));
+    VideoCompositor.Settings secondStreamAsOutputSizeSettings =
+        new VideoCompositor.Settings() {
+          @Override
+          public Size getOutputSize(List<Size> inputSizes) {
+            return Iterables.getLast(inputSizes);
+          }
+
+          @Override
+          public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) {
+            return new OverlaySettings.Builder().build();
+          }
+        };
+    compositorTestRunner =
+        new VideoCompositorTestRunner(
+            testId, useSharedExecutor, inputEffectLists, secondStreamAsOutputSizeSettings);
+
+    compositorTestRunner.queueBitmapToAllInputs(1);
+    compositorTestRunner.endCompositing();
+
+    compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected(
+        ImmutableList.of("different_dimensions"));
+  }
+
+  @Test
+  @RequiresNonNull("testId")
+  public void compositeTwoInputs_stacked_matchesExpectedBitmap() throws Exception {
+    ImmutableList<ImmutableList<Effect>> inputEffectLists =
+        ImmutableList.of(
+            ImmutableList.of(RgbFilter.createGrayscaleFilter()),
+            ImmutableList.of(),
+            ImmutableList.of(RgbFilter.createInvertedFilter()));
+    VideoCompositor.Settings stackedFrameSettings =
+        new VideoCompositor.Settings() {
+          private static final int NUMBER_OF_INPUT_STREAMS = 3;
+
+          @Override
+          public Size getOutputSize(List<Size> inputSizes) {
+            // Return the maximum width and sum of all heights.
+            int width = 0;
+            int height = 0;
+            for (int i = 0; i < inputSizes.size(); i++) {
+              width = max(width, inputSizes.get(i).getWidth());
+              height += inputSizes.get(i).getHeight();
+            }
+            return new Size(width, height);
+          }
+
+          @Override
+          public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) {
+            return new OverlaySettings.Builder()
+                .setOverlayAnchor(-1, -1)
+                .setVideoFrameAnchor(-1, -1 + 2f * inputId / NUMBER_OF_INPUT_STREAMS)
+                .build();
+          }
+        };
+    compositorTestRunner =
+        new VideoCompositorTestRunner(
+            testId, useSharedExecutor, inputEffectLists, stackedFrameSettings);
+
+    compositorTestRunner.queueBitmapToAllInputs(1);
+    compositorTestRunner.endCompositing();
+
+    compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected(ImmutableList.of("stacked"));
+  }
+
   /**
    * A test runner for {@link DefaultVideoCompositor} tests.
    *
@@ -544,7 +673,7 @@ public final class DefaultVideoCompositorPixelTest {
     private final String testId;
 
     /**
-     * Creates an instance.
+     * Creates an instance using {@link DefaultVideoCompositor.Settings}.
      *
      * @param testId The {@link String} identifier for the test, used to name output files.
      * @param useSharedExecutor Whether to use a shared executor for {@link
@@ -559,6 +688,27 @@ public final class DefaultVideoCompositorPixelTest {
         boolean useSharedExecutor,
         ImmutableList<ImmutableList<Effect>> inputEffectLists)
         throws GlUtil.GlException, VideoFrameProcessingException {
+      this(testId, useSharedExecutor, inputEffectLists, new DefaultVideoCompositor.Settings());
+    }
+
+    /**
+     * Creates an instance.
+     *
+     * @param testId The {@link String} identifier for the test, used to name output files.
+     * @param useSharedExecutor Whether to use a shared executor for {@link
+     *     VideoFrameProcessorTestRunner} and {@link VideoCompositor} instances.
+     * @param inputEffectLists {@link Effect}s to apply for {@link VideoCompositor} input sources.
+     *     The size of this outer {@link List} is the amount of inputs. One inner list of {@link
+     *     Effect}s is used for each input. For each input, the frame timestamp and {@code inputId}
+     *     are overlaid via {@link TextOverlay} prior to its effects being applied.
+     * @param settings The {@link VideoCompositor.Settings}.
+     */
+    public VideoCompositorTestRunner(
+        String testId,
+        boolean useSharedExecutor,
+        ImmutableList<ImmutableList<Effect>> inputEffectLists,
+        VideoCompositor.Settings settings)
+        throws GlUtil.GlException, VideoFrameProcessingException {
       this.testId = testId;
       timeoutMs = inputEffectLists.size() * VIDEO_FRAME_PROCESSING_WAIT_MS;
       sharedExecutorService =
@@ -575,6 +725,7 @@ public final class DefaultVideoCompositorPixelTest {
           new DefaultVideoCompositor(
               getApplicationContext(),
               glObjectsProvider,
+              settings,
               sharedExecutorService,
               new VideoCompositor.Listener() {
                 @Override