diff --git a/libraries/effect/src/main/java/androidx/media3/effect/OverlayEffect.java b/libraries/effect/src/main/java/androidx/media3/effect/OverlayEffect.java index ba025b6347..68997dcdaa 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/OverlayEffect.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/OverlayEffect.java @@ -41,6 +41,6 @@ public final class OverlayEffect implements GlEffect { @Override public BaseGlShaderProgram toGlShaderProgram(Context context, boolean useHdr) throws VideoFrameProcessingException { - return new OverlayShaderProgram(context, useHdr, overlays); + return new OverlayShaderProgram(useHdr, overlays); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/OverlayMatrixProvider.java b/libraries/effect/src/main/java/androidx/media3/effect/OverlayMatrixProvider.java new file mode 100644 index 0000000000..035dec0b97 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/OverlayMatrixProvider.java @@ -0,0 +1,215 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; + +import android.opengl.Matrix; +import android.util.Pair; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.Size; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/* package */ final class OverlayMatrixProvider { + private static final int MATRIX_OFFSET = 0; + private final float[] videoFrameAnchorMatrix; + private final float[] videoFrameAnchorMatrixInv; + private final float[] aspectRatioMatrix; + private final float[] scaleMatrix; + private final float[] scaleMatrixInv; + private final float[] overlayAnchorMatrix; + private final float[] overlayAnchorMatrixInv; + private final float[] rotateMatrix; + private final float[] overlayAspectRatioMatrix; + private final float[] overlayAspectRatioMatrixInv; + private final float[] transformationMatrix; + private @MonotonicNonNull Size backgroundSize; + + public OverlayMatrixProvider() { + aspectRatioMatrix = GlUtil.create4x4IdentityMatrix(); + videoFrameAnchorMatrix = GlUtil.create4x4IdentityMatrix(); + videoFrameAnchorMatrixInv = GlUtil.create4x4IdentityMatrix(); + overlayAnchorMatrix = GlUtil.create4x4IdentityMatrix(); + overlayAnchorMatrixInv = GlUtil.create4x4IdentityMatrix(); + rotateMatrix = GlUtil.create4x4IdentityMatrix(); + scaleMatrix = GlUtil.create4x4IdentityMatrix(); + scaleMatrixInv = GlUtil.create4x4IdentityMatrix(); + overlayAspectRatioMatrix = GlUtil.create4x4IdentityMatrix(); + overlayAspectRatioMatrixInv = GlUtil.create4x4IdentityMatrix(); + transformationMatrix = GlUtil.create4x4IdentityMatrix(); + } + + public void configure(Size backgroundSize) { + this.backgroundSize = backgroundSize; + } + + public float[] getTransformationMatrix(Size overlaySize, OverlaySettings overlaySettings) { + reset(); + + // Anchor point of overlay within output frame. + Pair videoFrameAnchor = overlaySettings.videoFrameAnchor; + Matrix.translateM( + videoFrameAnchorMatrix, + MATRIX_OFFSET, + videoFrameAnchor.first, + videoFrameAnchor.second, + /* z= */ 0f); + Matrix.invertM(videoFrameAnchorMatrixInv, MATRIX_OFFSET, videoFrameAnchorMatrix, MATRIX_OFFSET); + + Matrix.scaleM( + aspectRatioMatrix, + MATRIX_OFFSET, + checkNotNull(backgroundSize).getWidth() / (float) overlaySize.getWidth(), + checkNotNull(backgroundSize).getHeight() / (float) overlaySize.getHeight(), + /* z= */ 1f); + + // Scale the image. + Pair scale = overlaySettings.scale; + Matrix.scaleM( + scaleMatrix, + MATRIX_OFFSET, + scaleMatrix, + MATRIX_OFFSET, + scale.first, + scale.second, + /* z= */ 1f); + Matrix.invertM(scaleMatrixInv, MATRIX_OFFSET, scaleMatrix, MATRIX_OFFSET); + + // Translate the overlay within its frame. + Pair overlayAnchor = overlaySettings.overlayAnchor; + Matrix.translateM( + overlayAnchorMatrix, MATRIX_OFFSET, overlayAnchor.first, overlayAnchor.second, /* z= */ 0f); + Matrix.invertM(overlayAnchorMatrixInv, MATRIX_OFFSET, overlayAnchorMatrix, MATRIX_OFFSET); + + // Rotate the image. + Matrix.rotateM( + rotateMatrix, + MATRIX_OFFSET, + rotateMatrix, + MATRIX_OFFSET, + overlaySettings.rotationDegrees, + /* x= */ 0f, + /* y= */ 0f, + /* z= */ 1f); + Matrix.invertM(rotateMatrix, MATRIX_OFFSET, rotateMatrix, MATRIX_OFFSET); + + // Rotation matrix needs to account for overlay aspect ratio to prevent stretching. + Matrix.scaleM( + overlayAspectRatioMatrix, + MATRIX_OFFSET, + (float) overlaySize.getHeight() / (float) overlaySize.getWidth(), + /* y= */ 1f, + /* z= */ 1f); + Matrix.invertM( + overlayAspectRatioMatrixInv, MATRIX_OFFSET, overlayAspectRatioMatrix, MATRIX_OFFSET); + + // Rotation needs to be agnostic of the scaling matrix and the aspect ratios. + // transformationMatrix = scaleMatrixInv * overlayAspectRatioMatrix * rotateMatrix * + // overlayAspectRatioInv * scaleMatrix * overlayAnchorMatrixInv * scaleMatrixInv * + // aspectRatioMatrix * videoFrameAnchorMatrixInv + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + scaleMatrixInv, + MATRIX_OFFSET); + + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + overlayAspectRatioMatrix, + MATRIX_OFFSET); + + // Rotation matrix. + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + rotateMatrix, + MATRIX_OFFSET); + + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + overlayAspectRatioMatrixInv, + MATRIX_OFFSET); + + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + scaleMatrix, + MATRIX_OFFSET); + + // Translate image. + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + overlayAnchorMatrixInv, + MATRIX_OFFSET); + + // Scale image. + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + scaleMatrixInv, + MATRIX_OFFSET); + + // Correct for aspect ratio of image in output frame. + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + aspectRatioMatrix, + MATRIX_OFFSET); + + // Anchor position in output frame. + Matrix.multiplyMM( + transformationMatrix, + MATRIX_OFFSET, + transformationMatrix, + MATRIX_OFFSET, + videoFrameAnchorMatrixInv, + MATRIX_OFFSET); + return transformationMatrix; + } + + private void reset() { + GlUtil.setToIdentity(aspectRatioMatrix); + GlUtil.setToIdentity(videoFrameAnchorMatrix); + GlUtil.setToIdentity(videoFrameAnchorMatrixInv); + GlUtil.setToIdentity(overlayAnchorMatrix); + GlUtil.setToIdentity(overlayAnchorMatrixInv); + GlUtil.setToIdentity(scaleMatrix); + GlUtil.setToIdentity(scaleMatrixInv); + GlUtil.setToIdentity(rotateMatrix); + GlUtil.setToIdentity(overlayAspectRatioMatrix); + GlUtil.setToIdentity(overlayAspectRatioMatrixInv); + GlUtil.setToIdentity(transformationMatrix); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java index 3346cdbd8e..4c3d281aad 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java @@ -17,10 +17,7 @@ package androidx.media3.effect; import static androidx.media3.common.util.Assertions.checkArgument; -import android.content.Context; import android.opengl.GLES20; -import android.opengl.Matrix; -import android.util.Pair; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; @@ -31,35 +28,18 @@ import com.google.common.collect.ImmutableList; /** Applies zero or more {@link TextureOverlay}s onto each frame. */ /* package */ final class OverlayShaderProgram extends BaseGlShaderProgram { - private static final int MATRIX_OFFSET = 0; - private final GlProgram glProgram; + private final OverlayMatrixProvider overlayMatrixProvider; private final ImmutableList overlays; - private final float[] videoFrameAnchorMatrix; - private final float[] videoFrameAnchorMatrixInv; - private final float[] aspectRatioMatrix; - private final float[] scaleMatrix; - private final float[] scaleMatrixInv; - private final float[] overlayAnchorMatrix; - private final float[] overlayAnchorMatrixInv; - private final float[] rotateMatrix; - private final float[] overlayAspectRatioMatrix; - private final float[] overlayAspectRatioMatrixInv; - private final float[] transformationMatrix; - - private int videoWidth; - private int videoHeight; /** * Creates a new instance. * - * @param context The {@link Context}. * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be * in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709. * @throws VideoFrameProcessingException If a problem occurs while reading shader files. */ - public OverlayShaderProgram( - Context context, boolean useHdr, ImmutableList overlays) + public OverlayShaderProgram(boolean useHdr, ImmutableList overlays) throws VideoFrameProcessingException { super(/* useHighPrecisionColorComponents= */ useHdr, /* texturePoolCapacity= */ 1); checkArgument(!useHdr, "OverlayShaderProgram does not support HDR colors yet."); @@ -69,17 +49,7 @@ import com.google.common.collect.ImmutableList; overlays.size() <= 15, "OverlayShaderProgram does not support more than 15 overlays in the same instance."); this.overlays = overlays; - aspectRatioMatrix = GlUtil.create4x4IdentityMatrix(); - videoFrameAnchorMatrix = GlUtil.create4x4IdentityMatrix(); - videoFrameAnchorMatrixInv = GlUtil.create4x4IdentityMatrix(); - overlayAnchorMatrix = GlUtil.create4x4IdentityMatrix(); - overlayAnchorMatrixInv = GlUtil.create4x4IdentityMatrix(); - rotateMatrix = GlUtil.create4x4IdentityMatrix(); - scaleMatrix = GlUtil.create4x4IdentityMatrix(); - scaleMatrixInv = GlUtil.create4x4IdentityMatrix(); - overlayAspectRatioMatrix = GlUtil.create4x4IdentityMatrix(); - overlayAspectRatioMatrixInv = GlUtil.create4x4IdentityMatrix(); - transformationMatrix = GlUtil.create4x4IdentityMatrix(); + this.overlayMatrixProvider = new OverlayMatrixProvider(); try { glProgram = new GlProgram(createVertexShader(overlays.size()), createFragmentShader(overlays.size())); @@ -95,9 +65,8 @@ import com.google.common.collect.ImmutableList; @Override public Size configure(int inputWidth, int inputHeight) { - videoWidth = inputWidth; - videoHeight = inputHeight; Size videoSize = new Size(inputWidth, inputHeight); + overlayMatrixProvider.configure(/* backgroundSize= */ videoSize); for (TextureOverlay overlay : overlays) { overlay.configure(videoSize); } @@ -120,160 +89,9 @@ import com.google.common.collect.ImmutableList; OverlaySettings overlaySettings = overlay.getOverlaySettings(presentationTimeUs); Size overlaySize = overlay.getTextureSize(presentationTimeUs); - GlUtil.setToIdentity(aspectRatioMatrix); - GlUtil.setToIdentity(videoFrameAnchorMatrix); - GlUtil.setToIdentity(videoFrameAnchorMatrixInv); - GlUtil.setToIdentity(overlayAnchorMatrix); - GlUtil.setToIdentity(overlayAnchorMatrixInv); - GlUtil.setToIdentity(scaleMatrix); - GlUtil.setToIdentity(scaleMatrixInv); - GlUtil.setToIdentity(rotateMatrix); - GlUtil.setToIdentity(overlayAspectRatioMatrix); - GlUtil.setToIdentity(overlayAspectRatioMatrixInv); - GlUtil.setToIdentity(transformationMatrix); - - // Anchor point of overlay within output frame. - Pair videoFrameAnchor = overlaySettings.videoFrameAnchor; - Matrix.translateM( - videoFrameAnchorMatrix, - MATRIX_OFFSET, - videoFrameAnchor.first, - videoFrameAnchor.second, - /* z= */ 0f); - Matrix.invertM( - videoFrameAnchorMatrixInv, MATRIX_OFFSET, videoFrameAnchorMatrix, MATRIX_OFFSET); - - Matrix.scaleM( - aspectRatioMatrix, - MATRIX_OFFSET, - videoWidth / (float) overlaySize.getWidth(), - videoHeight / (float) overlaySize.getHeight(), - /* z= */ 1f); - - // Scale the image. - Pair scale = overlaySettings.scale; - Matrix.scaleM( - scaleMatrix, - MATRIX_OFFSET, - scaleMatrix, - MATRIX_OFFSET, - scale.first, - scale.second, - /* z= */ 1f); - Matrix.invertM(scaleMatrixInv, MATRIX_OFFSET, scaleMatrix, MATRIX_OFFSET); - - // Translate the overlay within its frame. - Pair overlayAnchor = overlaySettings.overlayAnchor; - Matrix.translateM( - overlayAnchorMatrix, - MATRIX_OFFSET, - overlayAnchor.first, - overlayAnchor.second, - /* z= */ 0f); - Matrix.invertM(overlayAnchorMatrixInv, MATRIX_OFFSET, overlayAnchorMatrix, MATRIX_OFFSET); - - // Rotate the image. - Matrix.rotateM( - rotateMatrix, - MATRIX_OFFSET, - rotateMatrix, - MATRIX_OFFSET, - overlaySettings.rotationDegrees, - /* x= */ 0f, - /* y= */ 0f, - /* z= */ 1f); - Matrix.invertM(rotateMatrix, MATRIX_OFFSET, rotateMatrix, MATRIX_OFFSET); - - // Rotation matrix needs to account for overlay aspect ratio to prevent stretching. - Matrix.scaleM( - overlayAspectRatioMatrix, - MATRIX_OFFSET, - (float) overlaySize.getHeight() / (float) overlaySize.getWidth(), - /* y= */ 1f, - /* z= */ 1f); - Matrix.invertM( - overlayAspectRatioMatrixInv, MATRIX_OFFSET, overlayAspectRatioMatrix, MATRIX_OFFSET); - - // Rotation needs to be agnostic of the scaling matrix and the aspect ratios. - Matrix.multiplyMM( - transformationMatrix, - MATRIX_OFFSET, - transformationMatrix, - MATRIX_OFFSET, - scaleMatrixInv, - MATRIX_OFFSET); - - Matrix.multiplyMM( - transformationMatrix, - MATRIX_OFFSET, - transformationMatrix, - MATRIX_OFFSET, - overlayAspectRatioMatrix, - MATRIX_OFFSET); - - // Rotation matrix. - Matrix.multiplyMM( - transformationMatrix, - MATRIX_OFFSET, - transformationMatrix, - MATRIX_OFFSET, - rotateMatrix, - MATRIX_OFFSET); - - Matrix.multiplyMM( - transformationMatrix, - MATRIX_OFFSET, - transformationMatrix, - MATRIX_OFFSET, - overlayAspectRatioMatrixInv, - MATRIX_OFFSET); - - Matrix.multiplyMM( - transformationMatrix, - MATRIX_OFFSET, - transformationMatrix, - MATRIX_OFFSET, - scaleMatrix, - MATRIX_OFFSET); - - // Translate image. - Matrix.multiplyMM( - transformationMatrix, - MATRIX_OFFSET, - transformationMatrix, - MATRIX_OFFSET, - overlayAnchorMatrixInv, - MATRIX_OFFSET); - - // Scale image. - Matrix.multiplyMM( - transformationMatrix, - MATRIX_OFFSET, - transformationMatrix, - MATRIX_OFFSET, - scaleMatrixInv, - MATRIX_OFFSET); - - // Correct for aspect ratio of image in output frame. - Matrix.multiplyMM( - transformationMatrix, - MATRIX_OFFSET, - transformationMatrix, - MATRIX_OFFSET, - aspectRatioMatrix, - MATRIX_OFFSET); - - // Anchor position in output frame. - Matrix.multiplyMM( - transformationMatrix, - MATRIX_OFFSET, - transformationMatrix, - MATRIX_OFFSET, - videoFrameAnchorMatrixInv, - MATRIX_OFFSET); - glProgram.setFloatsUniform( - Util.formatInvariant("uTransformationMatrix%d", texUnitIndex), transformationMatrix); + Util.formatInvariant("uTransformationMatrix%d", texUnitIndex), + overlayMatrixProvider.getTransformationMatrix(overlaySize, overlaySettings)); glProgram.setFloatUniform( Util.formatInvariant("uOverlayAlphaScale%d", texUnitIndex),