diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java index 0534a71d77..ccbe001669 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java @@ -36,6 +36,7 @@ import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Log; import androidx.media3.common.util.LongArrayQueue; +import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import com.google.common.collect.ImmutableList; @@ -50,8 +51,8 @@ import java.util.concurrent.ExecutorService; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** - * A basic {@link VideoCompositor} implementation that takes in frames from exactly 2 input sources' - * streams and combines them into one output stream. + * A basic {@link VideoCompositor} implementation that takes in frames from exactly 2 SDR input + * sources' streams and combines them into one output stream. * *

The first {@linkplain #registerInputSource registered source} will be the primary stream, * which is used to determine the output frames' timestamps and dimensions. @@ -62,17 +63,33 @@ public final class DefaultVideoCompositor implements VideoCompositor { // * Use a lock to synchronize inputFrameInfos more narrowly, to reduce blocking. // * If the primary stream ends, consider setting the secondary stream as the new primary stream, // so that secondary stream frames aren't dropped. + // * Add support for HDR input. + + /** A default implementation of {@link VideoCompositor.Settings}. */ + public static final class Settings implements VideoCompositor.Settings { + @Override + public Size getOutputSize(List inputSizes) { + return inputSizes.get(PRIMARY_INPUT_ID); + } + + @Override + public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) { + return new OverlaySettings.Builder().build(); + } + } private static final String THREAD_NAME = "Effect:DefaultVideoCompositor:GlThread"; private static final String TAG = "DefaultVideoCompositor"; private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl"; - private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl"; + private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_alpha_scale_es2.glsl"; private static final int PRIMARY_INPUT_ID = 0; private final Context context; private final VideoCompositor.Listener listener; private final GlTextureProducer.Listener textureOutputListener; private final GlObjectsProvider glObjectsProvider; + private final VideoCompositor.Settings settings; + private final OverlayMatrixProvider overlayMatrixProvider; private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; @GuardedBy("this") @@ -100,6 +117,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { public DefaultVideoCompositor( Context context, GlObjectsProvider glObjectsProvider, + VideoCompositor.Settings settings, @Nullable ExecutorService executorService, VideoCompositor.Listener listener, GlTextureProducer.Listener textureOutputListener, @@ -108,6 +126,8 @@ public final class DefaultVideoCompositor implements VideoCompositor { this.listener = listener; this.textureOutputListener = textureOutputListener; this.glObjectsProvider = glObjectsProvider; + this.settings = settings; + this.overlayMatrixProvider = new OverlayMatrixProvider(); inputSources = new ArrayList<>(); outputTexturePool = @@ -180,7 +200,11 @@ public final class DefaultVideoCompositor implements VideoCompositor { checkState(!inputSource.isInputEnded); InputFrameInfo inputFrameInfo = - new InputFrameInfo(textureProducer, inputTexture, presentationTimeUs); + new InputFrameInfo( + textureProducer, + inputTexture, + presentationTimeUs, + settings.getOverlaySettings(inputId, presentationTimeUs)); inputSource.frameInfos.add(inputFrameInfo); if (inputId == PRIMARY_INPUT_ID) { @@ -277,17 +301,17 @@ public final class DefaultVideoCompositor implements VideoCompositor { ensureGlProgramConfigured(); - // TODO: b/262694346 - Allow different input frame dimensions. InputFrameInfo primaryInputFrame = framesToComposite.get(PRIMARY_INPUT_ID); - GlTextureInfo primaryInputTexture = primaryInputFrame.texture; - outputTexturePool.ensureConfigured( - glObjectsProvider, primaryInputTexture.width, primaryInputTexture.height); - for (int i = 1; i < framesToComposite.size(); i++) { - GlTextureInfo textureToComposite = framesToComposite.get(i).texture; - checkState(primaryInputTexture.width == textureToComposite.width); - checkState(primaryInputTexture.height == textureToComposite.height); + ImmutableList.Builder inputSizes = new ImmutableList.Builder<>(); + for (int i = 0; i < framesToComposite.size(); i++) { + GlTextureInfo texture = framesToComposite.get(i).texture; + inputSizes.add(new Size(texture.width, texture.height)); } + Size outputSize = settings.getOutputSize(inputSizes.build()); + outputTexturePool.ensureConfigured( + glObjectsProvider, outputSize.getWidth(), outputSize.getHeight()); + GlTextureInfo outputTexture = outputTexturePool.useTexture(); long outputPresentationTimestampUs = primaryInputFrame.presentationTimeUs; outputTextureTimestamps.add(outputPresentationTimestampUs); @@ -394,16 +418,18 @@ public final class DefaultVideoCompositor implements VideoCompositor { GlUtil.getNormalizedCoordinateBounds(), GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); glProgram.setFloatsUniform("uTexTransformationMatrix", GlUtil.create4x4IdentityMatrix()); - glProgram.setFloatsUniform("uTransformationMatrix", GlUtil.create4x4IdentityMatrix()); } catch (IOException e) { throw new VideoFrameProcessingException(e); } } + // Enhanced for-loops are discouraged in media3.effect due to short-lived allocations. + @SuppressWarnings("ListReverse") private void drawFrame(List framesToComposite, GlTextureInfo outputTexture) throws GlUtil.GlException { GlUtil.focusFramebufferUsingCurrentContext( outputTexture.fboId, outputTexture.width, outputTexture.height); + overlayMatrixProvider.configure(new Size(outputTexture.width, outputTexture.height)); GlUtil.clearFocusedBuffers(); GlProgram glProgram = checkNotNull(this.glProgram); @@ -423,7 +449,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { // Draw textures from back to front. for (int i = framesToComposite.size() - 1; i >= 0; i--) { - blendOntoFocusedTexture(framesToComposite.get(i).texture.texId); + blendOntoFocusedTexture(framesToComposite.get(i)); } GLES20.glDisable(GLES20.GL_BLEND); @@ -431,9 +457,16 @@ public final class DefaultVideoCompositor implements VideoCompositor { GlUtil.checkGlError(); } - private void blendOntoFocusedTexture(int texId) throws GlUtil.GlException { + private void blendOntoFocusedTexture(InputFrameInfo inputFrameInfo) throws GlUtil.GlException { GlProgram glProgram = checkNotNull(this.glProgram); - glProgram.setSamplerTexIdUniform("uTexSampler", texId, /* texUnitIndex= */ 0); + GlTextureInfo inputTexture = inputFrameInfo.texture; + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexture.texId, /* texUnitIndex= */ 0); + float[] transformationMatrix = + overlayMatrixProvider.getTransformationMatrix( + /* overlaySize= */ new Size(inputTexture.width, inputTexture.height), + inputFrameInfo.overlaySettings); + glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrix); + glProgram.setFloatUniform("uAlphaScale", inputFrameInfo.overlaySettings.alphaScale); glProgram.bindAttributesAndUniforms(); // The four-vertex triangle strip forms a quad. @@ -480,12 +513,17 @@ public final class DefaultVideoCompositor implements VideoCompositor { public final GlTextureProducer textureProducer; public final GlTextureInfo texture; public final long presentationTimeUs; + public final OverlaySettings overlaySettings; public InputFrameInfo( - GlTextureProducer textureProducer, GlTextureInfo texture, long presentationTimeUs) { + GlTextureProducer textureProducer, + GlTextureInfo texture, + long presentationTimeUs, + OverlaySettings overlaySettings) { this.textureProducer = textureProducer; this.texture = texture; this.presentationTimeUs = presentationTimeUs; + this.overlaySettings = overlaySettings; } } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/OverlayMatrixProvider.java b/libraries/effect/src/main/java/androidx/media3/effect/OverlayMatrixProvider.java index 035dec0b97..91df8d8205 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/OverlayMatrixProvider.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/OverlayMatrixProvider.java @@ -15,7 +15,7 @@ */ package androidx.media3.effect; -import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.opengl.Matrix; import android.util.Pair; @@ -23,15 +23,14 @@ import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Size; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -/* package */ final class OverlayMatrixProvider { - private static final int MATRIX_OFFSET = 0; +/** Provides a matrix for {@link OverlaySettings}, to be applied on a vertex. */ +/* package */ class OverlayMatrixProvider { + protected static final int MATRIX_OFFSET = 0; private final float[] videoFrameAnchorMatrix; - private final float[] videoFrameAnchorMatrixInv; private final float[] aspectRatioMatrix; private final float[] scaleMatrix; private final float[] scaleMatrixInv; private final float[] overlayAnchorMatrix; - private final float[] overlayAnchorMatrixInv; private final float[] rotateMatrix; private final float[] overlayAspectRatioMatrix; private final float[] overlayAspectRatioMatrixInv; @@ -41,9 +40,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public OverlayMatrixProvider() { aspectRatioMatrix = GlUtil.create4x4IdentityMatrix(); videoFrameAnchorMatrix = GlUtil.create4x4IdentityMatrix(); - videoFrameAnchorMatrixInv = GlUtil.create4x4IdentityMatrix(); overlayAnchorMatrix = GlUtil.create4x4IdentityMatrix(); - overlayAnchorMatrixInv = GlUtil.create4x4IdentityMatrix(); rotateMatrix = GlUtil.create4x4IdentityMatrix(); scaleMatrix = GlUtil.create4x4IdentityMatrix(); scaleMatrixInv = GlUtil.create4x4IdentityMatrix(); @@ -56,6 +53,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.backgroundSize = backgroundSize; } + /** + * Returns the transformation matrix. + * + *

This instance must be {@linkplain #configure configured} before this method is called. + */ public float[] getTransformationMatrix(Size overlaySize, OverlaySettings overlaySettings) { reset(); @@ -67,44 +69,38 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; videoFrameAnchor.first, videoFrameAnchor.second, /* z= */ 0f); - Matrix.invertM(videoFrameAnchorMatrixInv, MATRIX_OFFSET, videoFrameAnchorMatrix, MATRIX_OFFSET); + checkStateNotNull(backgroundSize); Matrix.scaleM( aspectRatioMatrix, MATRIX_OFFSET, - checkNotNull(backgroundSize).getWidth() / (float) overlaySize.getWidth(), - checkNotNull(backgroundSize).getHeight() / (float) overlaySize.getHeight(), + (float) overlaySize.getWidth() / backgroundSize.getWidth(), + (float) overlaySize.getHeight() / backgroundSize.getHeight(), /* z= */ 1f); // Scale the image. Pair scale = overlaySettings.scale; - Matrix.scaleM( - scaleMatrix, - MATRIX_OFFSET, - scaleMatrix, - MATRIX_OFFSET, - scale.first, - scale.second, - /* z= */ 1f); + Matrix.scaleM(scaleMatrix, MATRIX_OFFSET, scale.first, scale.second, /* z= */ 1f); Matrix.invertM(scaleMatrixInv, MATRIX_OFFSET, scaleMatrix, MATRIX_OFFSET); - // Translate the overlay within its frame. + // Translate the overlay within its frame. To position the overlay's anchor at the correct + // position, it must be translated the opposite direction by the same magnitude. Pair overlayAnchor = overlaySettings.overlayAnchor; Matrix.translateM( - overlayAnchorMatrix, MATRIX_OFFSET, overlayAnchor.first, overlayAnchor.second, /* z= */ 0f); - Matrix.invertM(overlayAnchorMatrixInv, MATRIX_OFFSET, overlayAnchorMatrix, MATRIX_OFFSET); + overlayAnchorMatrix, + MATRIX_OFFSET, + -1 * overlayAnchor.first, + -1 * overlayAnchor.second, + /* z= */ 0f); // Rotate the image. Matrix.rotateM( - rotateMatrix, - MATRIX_OFFSET, rotateMatrix, MATRIX_OFFSET, overlaySettings.rotationDegrees, /* x= */ 0f, /* y= */ 0f, /* z= */ 1f); - Matrix.invertM(rotateMatrix, MATRIX_OFFSET, rotateMatrix, MATRIX_OFFSET); // Rotation matrix needs to account for overlay aspect ratio to prevent stretching. Matrix.scaleM( @@ -116,67 +112,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Matrix.invertM( overlayAspectRatioMatrixInv, MATRIX_OFFSET, overlayAspectRatioMatrix, MATRIX_OFFSET); - // Rotation needs to be agnostic of the scaling matrix and the aspect ratios. - // transformationMatrix = scaleMatrixInv * overlayAspectRatioMatrix * rotateMatrix * - // overlayAspectRatioInv * scaleMatrix * overlayAnchorMatrixInv * scaleMatrixInv * - // aspectRatioMatrix * videoFrameAnchorMatrixInv - Matrix.multiplyMM( - transformationMatrix, - MATRIX_OFFSET, - transformationMatrix, - MATRIX_OFFSET, - scaleMatrixInv, - MATRIX_OFFSET); + // transformationMatrix = videoFrameAnchorMatrix * aspectRatioMatrix + // * scaleMatrix * overlayAnchorMatrix * scaleMatrixInv + // * overlayAspectRatioMatrix * rotateMatrix * overlayAspectRatioMatrixInv + // * scaleMatrix. + // Anchor position in output frame. Matrix.multiplyMM( transformationMatrix, MATRIX_OFFSET, transformationMatrix, MATRIX_OFFSET, - overlayAspectRatioMatrix, - MATRIX_OFFSET); - - // Rotation matrix. - Matrix.multiplyMM( - transformationMatrix, - MATRIX_OFFSET, - transformationMatrix, - MATRIX_OFFSET, - rotateMatrix, - MATRIX_OFFSET); - - Matrix.multiplyMM( - transformationMatrix, - MATRIX_OFFSET, - transformationMatrix, - MATRIX_OFFSET, - overlayAspectRatioMatrixInv, - MATRIX_OFFSET); - - Matrix.multiplyMM( - transformationMatrix, - MATRIX_OFFSET, - transformationMatrix, - MATRIX_OFFSET, - scaleMatrix, - MATRIX_OFFSET); - - // Translate image. - Matrix.multiplyMM( - transformationMatrix, - MATRIX_OFFSET, - transformationMatrix, - MATRIX_OFFSET, - overlayAnchorMatrixInv, - MATRIX_OFFSET); - - // Scale image. - Matrix.multiplyMM( - transformationMatrix, - MATRIX_OFFSET, - transformationMatrix, - MATRIX_OFFSET, - scaleMatrixInv, + videoFrameAnchorMatrix, MATRIX_OFFSET); // Correct for aspect ratio of image in output frame. @@ -188,23 +135,67 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; aspectRatioMatrix, MATRIX_OFFSET); - // Anchor position in output frame. Matrix.multiplyMM( transformationMatrix, MATRIX_OFFSET, transformationMatrix, MATRIX_OFFSET, - videoFrameAnchorMatrixInv, + scaleMatrix, MATRIX_OFFSET); + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + overlayAnchorMatrix, + MATRIX_OFFSET); + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + scaleMatrixInv, + MATRIX_OFFSET); + + // Rotation needs to be agnostic of the scaling matrix and the aspect ratios. + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + overlayAspectRatioMatrix, + MATRIX_OFFSET); + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + rotateMatrix, + MATRIX_OFFSET); + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + overlayAspectRatioMatrixInv, + MATRIX_OFFSET); + + // Scale image. + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + scaleMatrix, + MATRIX_OFFSET); + return transformationMatrix; } private void reset() { GlUtil.setToIdentity(aspectRatioMatrix); GlUtil.setToIdentity(videoFrameAnchorMatrix); - GlUtil.setToIdentity(videoFrameAnchorMatrixInv); GlUtil.setToIdentity(overlayAnchorMatrix); - GlUtil.setToIdentity(overlayAnchorMatrixInv); GlUtil.setToIdentity(scaleMatrix); GlUtil.setToIdentity(scaleMatrixInv); GlUtil.setToIdentity(rotateMatrix); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/OverlaySettings.java b/libraries/effect/src/main/java/androidx/media3/effect/OverlaySettings.java index 18b3f5881c..97d9d17186 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/OverlaySettings.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/OverlaySettings.java @@ -22,7 +22,10 @@ import androidx.annotation.FloatRange; import androidx.media3.common.util.UnstableApi; import com.google.errorprone.annotations.CanIgnoreReturnValue; -/** Contains information to control how a {@link TextureOverlay} is displayed on the screen. */ +/** + * Contains information to control how an input texture (for example, a {@link VideoCompositor} or + * {@link TextureOverlay}) is displayed on a background. + */ @UnstableApi public final class OverlaySettings { public final boolean useHdr; @@ -47,6 +50,11 @@ public final class OverlaySettings { this.rotationDegrees = rotationDegrees; } + /** Returns a new {@link Builder} initialized with the values of this instance. */ + /* package */ Builder buildUpon() { + return new Builder(this); + } + /** A builder for {@link OverlaySettings} instances. */ public static final class Builder { private boolean useHdr; @@ -65,6 +73,15 @@ public final class OverlaySettings { rotationDegrees = 0f; } + private Builder(OverlaySettings overlaySettings) { + this.useHdr = overlaySettings.useHdr; + this.alphaScale = overlaySettings.alphaScale; + this.videoFrameAnchor = overlaySettings.videoFrameAnchor; + this.overlayAnchor = overlaySettings.overlayAnchor; + this.scale = overlaySettings.scale; + this.rotationDegrees = overlaySettings.rotationDegrees; + } + /** * Sets whether input overlay comes from an HDR source. If {@code true}, colors will be in * linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709. @@ -92,6 +109,7 @@ public final class OverlaySettings { return this; } + // TODO: b/262694346 - Rename this method to setBackgroundAnchor in a follow-up CL. /** * Sets the coordinates for the anchor point of the overlay within the video frame. * diff --git a/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java index 4c3d281aad..7a9dd98444 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java @@ -29,7 +29,7 @@ import com.google.common.collect.ImmutableList; /* package */ final class OverlayShaderProgram extends BaseGlShaderProgram { private final GlProgram glProgram; - private final OverlayMatrixProvider overlayMatrixProvider; + private final SamplerOverlayMatrixProvider samplerOverlayMatrixProvider; private final ImmutableList overlays; /** @@ -49,7 +49,7 @@ import com.google.common.collect.ImmutableList; overlays.size() <= 15, "OverlayShaderProgram does not support more than 15 overlays in the same instance."); this.overlays = overlays; - this.overlayMatrixProvider = new OverlayMatrixProvider(); + this.samplerOverlayMatrixProvider = new SamplerOverlayMatrixProvider(); try { glProgram = new GlProgram(createVertexShader(overlays.size()), createFragmentShader(overlays.size())); @@ -66,7 +66,7 @@ import com.google.common.collect.ImmutableList; @Override public Size configure(int inputWidth, int inputHeight) { Size videoSize = new Size(inputWidth, inputHeight); - overlayMatrixProvider.configure(/* backgroundSize= */ videoSize); + samplerOverlayMatrixProvider.configure(/* backgroundSize= */ videoSize); for (TextureOverlay overlay : overlays) { overlay.configure(videoSize); } @@ -91,7 +91,7 @@ import com.google.common.collect.ImmutableList; glProgram.setFloatsUniform( Util.formatInvariant("uTransformationMatrix%d", texUnitIndex), - overlayMatrixProvider.getTransformationMatrix(overlaySize, overlaySettings)); + samplerOverlayMatrixProvider.getTransformationMatrix(overlaySize, overlaySettings)); glProgram.setFloatUniform( Util.formatInvariant("uOverlayAlphaScale%d", texUnitIndex), diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SamplerOverlayMatrixProvider.java b/libraries/effect/src/main/java/androidx/media3/effect/SamplerOverlayMatrixProvider.java new file mode 100644 index 0000000000..5fe575c1dd --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/SamplerOverlayMatrixProvider.java @@ -0,0 +1,56 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import android.opengl.Matrix; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.Size; + +/** + * Provides a matrix based on {@link OverlaySettings} to be applied on a texture sampling + * coordinate. + */ +/* package */ final class SamplerOverlayMatrixProvider extends OverlayMatrixProvider { + private final float[] transformationMatrixInv; + + public SamplerOverlayMatrixProvider() { + super(); + transformationMatrixInv = GlUtil.create4x4IdentityMatrix(); + } + + @Override + public float[] getTransformationMatrix(Size overlaySize, OverlaySettings overlaySettings) { + // When sampling from a (for example, texture) sampler, the overlay anchor's x and y coordinates + // are flipped. + OverlaySettings samplerOverlaySettings = + overlaySettings + .buildUpon() + .setOverlayAnchor( + /* x= */ -1 * overlaySettings.overlayAnchor.first, + /* y= */ -1 * overlaySettings.overlayAnchor.second) + .build(); + + // When sampling from a (for example, texture) sampler, the transformation matrix applied to a + // sampler's coordinate should be the inverse of the transformation matrix that would otherwise + // be applied to a vertex. + Matrix.invertM( + transformationMatrixInv, + MATRIX_OFFSET, + super.getTransformationMatrix(overlaySize, samplerOverlaySettings), + MATRIX_OFFSET); + return transformationMatrixInv; + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java index c108b6fc62..63734893e7 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java @@ -17,10 +17,12 @@ package androidx.media3.effect; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; +import java.util.List; /** - * Interface for a video compositor that combines frames from mutliple input sources to produce + * Interface for a video compositor that combines frames from multiple input sources to produce * output frames. * *

Input and output are provided via OpenGL textures. @@ -41,6 +43,23 @@ public interface VideoCompositor extends GlTextureProducer { void onEnded(); } + /** Settings for the {@link VideoCompositor}. */ + interface Settings { + // TODO: b/262694346 - Consider adding more features, like selecting a: + // * custom order for drawing (instead of primary stream on top), and + // * different primary source. + + /** + * Returns an output texture {@link Size}, based on {@code inputSizes}. + * + * @param inputSizes The {@link Size} of each input frame, ordered by {@code inputId}. + */ + Size getOutputSize(List inputSizes); + + /** Returns {@link OverlaySettings} for {@code inputId} at time {@code presentationTimeUs}. */ + OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs); + } + /** * Registers a new input source, and returns a unique {@code inputId} corresponding to this * source, to be used in {@link #queueInputTexture}. diff --git a/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_different_dimensions.png b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_different_dimensions.png new file mode 100644 index 0000000000..a97da19841 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_different_dimensions.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_picture_in_picture.png b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_picture_in_picture.png new file mode 100644 index 0000000000..f6126f22d2 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_picture_in_picture.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_stacked.png b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_stacked.png new file mode 100644 index 0000000000..77827e8f13 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_stacked.png differ diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java index f9e6fa0342..ddc818a018 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java @@ -24,6 +24,7 @@ import static androidx.media3.test.utils.VideoFrameProcessorTestRunner.createTim import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; +import static java.lang.Math.max; import static java.util.concurrent.TimeUnit.MILLISECONDS; import android.graphics.Bitmap; @@ -44,6 +45,7 @@ import androidx.media3.common.Effect; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.Size; import androidx.media3.common.util.Util; import androidx.media3.effect.AlphaScale; import androidx.media3.effect.DefaultGlObjectsProvider; @@ -51,6 +53,7 @@ import androidx.media3.effect.DefaultVideoCompositor; import androidx.media3.effect.DefaultVideoFrameProcessor; import androidx.media3.effect.OverlayEffect; import androidx.media3.effect.OverlaySettings; +import androidx.media3.effect.Presentation; import androidx.media3.effect.RgbFilter; import androidx.media3.effect.ScaleAndRotateTransformation; import androidx.media3.effect.TextOverlay; @@ -60,6 +63,7 @@ import androidx.media3.test.utils.TextureBitmapReader; import androidx.media3.test.utils.VideoFrameProcessorTestRunner; import com.google.common.base.Ascii; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; import java.io.IOException; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -103,15 +107,15 @@ public final class DefaultVideoCompositorPixelTest { private static final String ORIGINAL_PNG_ASSET_PATH = "media/bitmap/input_images/media3test_srgb.png"; private static final String TEST_DIRECTORY = "media/bitmap/CompositorTestTimestamps/"; - - private @MonotonicNonNull String testId; - private @MonotonicNonNull VideoCompositorTestRunner compositorTestRunner; private static final ImmutableList> TWO_INPUT_COMPOSITOR_EFFECT_LISTS = ImmutableList.of( ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(0.7f)), ImmutableList.of( new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build())); + private @MonotonicNonNull String testId; + private @MonotonicNonNull VideoCompositorTestRunner compositorTestRunner; + @Before @EnsuresNonNull("testId") public void setUpTestId() { @@ -125,6 +129,8 @@ public final class DefaultVideoCompositorPixelTest { } } + // Tests for alpha and frame alpha/occlusion. + @Test @RequiresNonNull("testId") public void compositeTwoInputs_withOneFrameFromEach_differentTimestamp_matchesExpectedBitmap() @@ -155,12 +161,13 @@ public final class DefaultVideoCompositorPixelTest { @RequiresNonNull("testId") public void compositeTwoInputs_withPrimaryTransparent_differentTimestamp_matchesExpectedBitmap() throws Exception { - ImmutableList> inputEffects = + ImmutableList> inputEffectLists = ImmutableList.of( ImmutableList.of(new AlphaScale(0f)), ImmutableList.of( new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build())); - compositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffects); + compositorTestRunner = + new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffectLists); compositorTestRunner.queueBitmapToInput( /* inputId= */ 0, /* timestamps= */ ImmutableList.of(0L)); @@ -186,12 +193,13 @@ public final class DefaultVideoCompositorPixelTest { @RequiresNonNull("testId") public void compositeTwoInputs_withPrimaryOpaque_differentTimestamp_matchesExpectedBitmap() throws Exception { - ImmutableList> inputEffects = + ImmutableList> inputEffectLists = ImmutableList.of( ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(100f)), ImmutableList.of( new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build())); - compositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffects); + compositorTestRunner = + new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffectLists); compositorTestRunner.queueBitmapToInput( /* inputId= */ 0, /* timestamps= */ ImmutableList.of(0L)); @@ -217,11 +225,12 @@ public final class DefaultVideoCompositorPixelTest { @RequiresNonNull("testId") public void compositeTwoInputs_withSecondaryTransparent_differentTimestamp_matchesExpectedBitmap() throws Exception { - ImmutableList> inputEffects = + ImmutableList> inputEffectLists = ImmutableList.of( ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(0.7f)), ImmutableList.of(new AlphaScale(0f))); - compositorTestRunner = new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffects); + compositorTestRunner = + new VideoCompositorTestRunner(testId, useSharedExecutor, inputEffectLists); compositorTestRunner.queueBitmapToInput( /* inputId= */ 0, /* timestamps= */ ImmutableList.of(0L)); @@ -243,6 +252,8 @@ public final class DefaultVideoCompositorPixelTest { ImmutableList.of("0s_1s_transparent")); } + // Tests for mixing different frame rates and timestamps. + @Test @RequiresNonNull("testId") public void compositeTwoInputs_withFiveFramesFromEach_matchesExpectedTimestamps() @@ -428,6 +439,8 @@ public final class DefaultVideoCompositorPixelTest { ImmutableList.of("0s_1s", "1s_1s", "2s_1s", "3s_3s", "4s_4s")); } + // Tests for "many" inputs/frames. + @Test @RequiresNonNull("testId") public void compositeTwoInputs_withTenFramesFromEach_matchesExpectedFrameCount() @@ -468,6 +481,8 @@ public final class DefaultVideoCompositorPixelTest { assertThat(compositorTestRunner.getCompositedTimestamps()).hasSize(numberOfFramesToQueue); } + // Tests for different amounts of inputs. + @Test @RequiresNonNull("testId") public void compositeOneInput_matchesExpectedBitmap() throws Exception { @@ -526,6 +541,120 @@ public final class DefaultVideoCompositorPixelTest { ImmutableList.of("0s_1s_0s", "1s_1s_0s", "2s_1s_2s")); } + // Tests for different layouts. + + @Test + @RequiresNonNull("testId") + public void compositeTwoInputs_pictureInPicture_matchesExpectedBitmap() throws Exception { + ImmutableList> inputEffectLists = + ImmutableList.of(ImmutableList.of(), ImmutableList.of(RgbFilter.createGrayscaleFilter())); + VideoCompositor.Settings pictureInPictureSettings = + new VideoCompositor.Settings() { + @Override + public Size getOutputSize(List inputSizes) { + return inputSizes.get(0); + } + + @Override + public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) { + if (inputId == 0) { + // This tests all OverlaySettings builder variables. + return new OverlaySettings.Builder() + .setScale(.25f, .5f) + .setOverlayAnchor(1, -1) + .setVideoFrameAnchor(.9f, -.7f) + .setRotationDegrees(20) + .setAlphaScale(.5f) + .build(); + } else { + return new OverlaySettings.Builder().build(); + } + } + }; + compositorTestRunner = + new VideoCompositorTestRunner( + testId, useSharedExecutor, inputEffectLists, pictureInPictureSettings); + + compositorTestRunner.queueBitmapToAllInputs(1); + compositorTestRunner.endCompositing(); + + compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected( + ImmutableList.of("picture_in_picture")); + } + + @Test + @RequiresNonNull("testId") + public void compositeTwoInputs_differentDimensions_matchesExpectedBitmap() throws Exception { + ImmutableList> inputEffectLists = + ImmutableList.of( + ImmutableList.of( + Presentation.createForWidthAndHeight(100, 100, Presentation.LAYOUT_STRETCH_TO_FIT)), + ImmutableList.of(RgbFilter.createGrayscaleFilter())); + VideoCompositor.Settings secondStreamAsOutputSizeSettings = + new VideoCompositor.Settings() { + @Override + public Size getOutputSize(List inputSizes) { + return Iterables.getLast(inputSizes); + } + + @Override + public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) { + return new OverlaySettings.Builder().build(); + } + }; + compositorTestRunner = + new VideoCompositorTestRunner( + testId, useSharedExecutor, inputEffectLists, secondStreamAsOutputSizeSettings); + + compositorTestRunner.queueBitmapToAllInputs(1); + compositorTestRunner.endCompositing(); + + compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected( + ImmutableList.of("different_dimensions")); + } + + @Test + @RequiresNonNull("testId") + public void compositeTwoInputs_stacked_matchesExpectedBitmap() throws Exception { + ImmutableList> inputEffectLists = + ImmutableList.of( + ImmutableList.of(RgbFilter.createGrayscaleFilter()), + ImmutableList.of(), + ImmutableList.of(RgbFilter.createInvertedFilter())); + VideoCompositor.Settings stackedFrameSettings = + new VideoCompositor.Settings() { + private static final int NUMBER_OF_INPUT_STREAMS = 3; + + @Override + public Size getOutputSize(List inputSizes) { + // Return the maximum width and sum of all heights. + int width = 0; + int height = 0; + for (int i = 0; i < inputSizes.size(); i++) { + width = max(width, inputSizes.get(i).getWidth()); + height += inputSizes.get(i).getHeight(); + } + return new Size(width, height); + } + + @Override + public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) { + return new OverlaySettings.Builder() + .setOverlayAnchor(-1, -1) + .setVideoFrameAnchor(-1, -1 + 2f * inputId / NUMBER_OF_INPUT_STREAMS) + .build(); + } + }; + compositorTestRunner = + new VideoCompositorTestRunner( + testId, useSharedExecutor, inputEffectLists, stackedFrameSettings); + + compositorTestRunner.queueBitmapToAllInputs(1); + compositorTestRunner.endCompositing(); + + compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected(ImmutableList.of("stacked")); + } + /** * A test runner for {@link DefaultVideoCompositor} tests. * @@ -544,7 +673,7 @@ public final class DefaultVideoCompositorPixelTest { private final String testId; /** - * Creates an instance. + * Creates an instance using {@link DefaultVideoCompositor.Settings}. * * @param testId The {@link String} identifier for the test, used to name output files. * @param useSharedExecutor Whether to use a shared executor for {@link @@ -559,6 +688,27 @@ public final class DefaultVideoCompositorPixelTest { boolean useSharedExecutor, ImmutableList> inputEffectLists) throws GlUtil.GlException, VideoFrameProcessingException { + this(testId, useSharedExecutor, inputEffectLists, new DefaultVideoCompositor.Settings()); + } + + /** + * Creates an instance. + * + * @param testId The {@link String} identifier for the test, used to name output files. + * @param useSharedExecutor Whether to use a shared executor for {@link + * VideoFrameProcessorTestRunner} and {@link VideoCompositor} instances. + * @param inputEffectLists {@link Effect}s to apply for {@link VideoCompositor} input sources. + * The size of this outer {@link List} is the amount of inputs. One inner list of {@link + * Effect}s is used for each input. For each input, the frame timestamp and {@code inputId} + * are overlaid via {@link TextOverlay} prior to its effects being applied. + * @param settings The {@link VideoCompositor.Settings}. + */ + public VideoCompositorTestRunner( + String testId, + boolean useSharedExecutor, + ImmutableList> inputEffectLists, + VideoCompositor.Settings settings) + throws GlUtil.GlException, VideoFrameProcessingException { this.testId = testId; timeoutMs = inputEffectLists.size() * VIDEO_FRAME_PROCESSING_WAIT_MS; sharedExecutorService = @@ -575,6 +725,7 @@ public final class DefaultVideoCompositorPixelTest { new DefaultVideoCompositor( getApplicationContext(), glObjectsProvider, + settings, sharedExecutorService, new VideoCompositor.Listener() { @Override