From 5ef5d46708a359942d82187ced585ddf3917c619 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Thu, 14 Sep 2023 10:31:14 -0700 Subject: [PATCH] Compositor: Move Settings to DefaultVideoCompositor. This previously was in the VideoCompositor class, but wasn't referenced at all from that interface. PiperOrigin-RevId: 565409646 --- .../media3/effect/DefaultVideoCompositor.java | 17 +----- .../media3/effect/VideoCompositor.java | 19 ------ .../effect/VideoCompositorSettings.java | 61 +++++++++++++++++++ .../DefaultVideoCompositorPixelTest.java | 32 +++++----- .../transformer/MultipleInputVideoGraph.java | 3 +- 5 files changed, 83 insertions(+), 49 deletions(-) create mode 100644 libraries/effect/src/main/java/androidx/media3/effect/VideoCompositorSettings.java diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java index 72864d4926..082dbcbdf5 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java @@ -76,19 +76,6 @@ public final class DefaultVideoCompositor implements VideoCompositor { // * Add support for mixing SDR streams with different ColorInfo. // * Add support for HDR input. - /** A default implementation of {@link VideoCompositor.Settings}. */ - public static final class Settings implements VideoCompositor.Settings { - @Override - public Size getOutputSize(List inputSizes) { - return inputSizes.get(PRIMARY_INPUT_ID); - } - - @Override - public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) { - return new OverlaySettings.Builder().build(); - } - } - private static final String THREAD_NAME = "Effect:DefaultVideoCompositor:GlThread"; private static final String TAG = "DefaultVideoCompositor"; private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl"; @@ -99,7 +86,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { private final VideoCompositor.Listener listener; private final GlTextureProducer.Listener textureOutputListener; private final GlObjectsProvider glObjectsProvider; - private final VideoCompositor.Settings settings; + private final VideoCompositorSettings settings; private final OverlayMatrixProvider overlayMatrixProvider; private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; @@ -130,7 +117,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { public DefaultVideoCompositor( Context context, GlObjectsProvider glObjectsProvider, - VideoCompositor.Settings settings, + VideoCompositorSettings settings, @Nullable ExecutorService executorService, VideoCompositor.Listener listener, GlTextureProducer.Listener textureOutputListener, diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java index f9c78e3ad4..5ec3e68047 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java @@ -18,9 +18,7 @@ package androidx.media3.effect; import androidx.media3.common.ColorInfo; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; -import java.util.List; /** * Interface for a video compositor that combines frames from multiple input sources to produce @@ -47,23 +45,6 @@ public interface VideoCompositor extends GlTextureProducer { void onEnded(); } - /** Settings for the {@link VideoCompositor}. */ - interface Settings { - // TODO: b/262694346 - Consider adding more features, like selecting a: - // * custom order for drawing (instead of primary stream on top), and - // * different primary source. - - /** - * Returns an output texture {@link Size}, based on {@code inputSizes}. - * - * @param inputSizes The {@link Size} of each input frame, ordered by {@code inputId}. - */ - Size getOutputSize(List inputSizes); - - /** Returns {@link OverlaySettings} for {@code inputId} at time {@code presentationTimeUs}. */ - OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs); - } - /** * Registers a new input source, and returns a unique {@code inputId} corresponding to this * source, to be used in {@link #queueInputTexture}. diff --git a/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositorSettings.java b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositorSettings.java new file mode 100644 index 0000000000..4326105120 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/VideoCompositorSettings.java @@ -0,0 +1,61 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import androidx.media3.common.util.Size; +import androidx.media3.common.util.UnstableApi; +import java.util.List; + +/** Settings for the {@link VideoCompositor}. */ +@UnstableApi +public interface VideoCompositorSettings { + // TODO: b/262694346 - Consider adding more features, like selecting a: + // * custom order for drawing (instead of primary stream on top), and + // * different primary source. + + VideoCompositorSettings DEFAULT = + new VideoCompositorSettings() { + /** + * {@inheritDoc} + * + *

Returns the primary stream's {@link Size}. + */ + @Override + public Size getOutputSize(List inputSizes) { + return inputSizes.get(0); + } + + /** + * {@inheritDoc} + * + *

Returns a default {@link OverlaySettings} instance. + */ + @Override + public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) { + return new OverlaySettings.Builder().build(); + } + }; + + /** + * Returns an output texture {@link Size}, based on {@code inputSizes}. + * + * @param inputSizes The {@link Size} of each input frame, ordered by {@code inputId}. + */ + Size getOutputSize(List inputSizes); + + /** Returns {@link OverlaySettings} for {@code inputId} at time {@code presentationTimeUs}. */ + OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs); +} diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java index 2dd47b7527..0401756cde 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java @@ -58,6 +58,7 @@ import androidx.media3.effect.RgbFilter; import androidx.media3.effect.ScaleAndRotateTransformation; import androidx.media3.effect.TextOverlay; import androidx.media3.effect.VideoCompositor; +import androidx.media3.effect.VideoCompositorSettings; import androidx.media3.test.utils.BitmapPixelTestUtil; import androidx.media3.test.utils.TextureBitmapReader; import androidx.media3.test.utils.VideoFrameProcessorTestRunner; @@ -548,8 +549,8 @@ public final class DefaultVideoCompositorPixelTest { public void compositeTwoInputs_pictureInPicture_matchesExpectedBitmap() throws Exception { ImmutableList> inputEffectLists = ImmutableList.of(ImmutableList.of(), ImmutableList.of(RgbFilter.createGrayscaleFilter())); - VideoCompositor.Settings pictureInPictureSettings = - new VideoCompositor.Settings() { + VideoCompositorSettings pictureInPictureVideoCompositorSettings = + new VideoCompositorSettings() { @Override public Size getOutputSize(List inputSizes) { return inputSizes.get(0); @@ -573,7 +574,7 @@ public final class DefaultVideoCompositorPixelTest { }; compositorTestRunner = new VideoCompositorTestRunner( - testId, useSharedExecutor, inputEffectLists, pictureInPictureSettings); + testId, useSharedExecutor, inputEffectLists, pictureInPictureVideoCompositorSettings); compositorTestRunner.queueBitmapToAllInputs(1); compositorTestRunner.endCompositing(); @@ -590,8 +591,8 @@ public final class DefaultVideoCompositorPixelTest { ImmutableList.of( Presentation.createForWidthAndHeight(100, 100, Presentation.LAYOUT_STRETCH_TO_FIT)), ImmutableList.of(RgbFilter.createGrayscaleFilter())); - VideoCompositor.Settings secondStreamAsOutputSizeSettings = - new VideoCompositor.Settings() { + VideoCompositorSettings secondStreamAsOutputSizeVideoCompositorSettings = + new VideoCompositorSettings() { @Override public Size getOutputSize(List inputSizes) { return Iterables.getLast(inputSizes); @@ -604,7 +605,10 @@ public final class DefaultVideoCompositorPixelTest { }; compositorTestRunner = new VideoCompositorTestRunner( - testId, useSharedExecutor, inputEffectLists, secondStreamAsOutputSizeSettings); + testId, + useSharedExecutor, + inputEffectLists, + secondStreamAsOutputSizeVideoCompositorSettings); compositorTestRunner.queueBitmapToAllInputs(1); compositorTestRunner.endCompositing(); @@ -621,8 +625,8 @@ public final class DefaultVideoCompositorPixelTest { ImmutableList.of(RgbFilter.createGrayscaleFilter()), ImmutableList.of(), ImmutableList.of(RgbFilter.createInvertedFilter())); - VideoCompositor.Settings stackedFrameSettings = - new VideoCompositor.Settings() { + VideoCompositorSettings stackedFrameVideoCompositorSettings = + new VideoCompositorSettings() { private static final int NUMBER_OF_INPUT_STREAMS = 3; @Override @@ -647,7 +651,7 @@ public final class DefaultVideoCompositorPixelTest { }; compositorTestRunner = new VideoCompositorTestRunner( - testId, useSharedExecutor, inputEffectLists, stackedFrameSettings); + testId, useSharedExecutor, inputEffectLists, stackedFrameVideoCompositorSettings); compositorTestRunner.queueBitmapToAllInputs(1); compositorTestRunner.endCompositing(); @@ -673,7 +677,7 @@ public final class DefaultVideoCompositorPixelTest { private final String testId; /** - * Creates an instance using {@link DefaultVideoCompositor.Settings}. + * Creates an instance using {@link VideoCompositorSettings}. * * @param testId The {@link String} identifier for the test, used to name output files. * @param useSharedExecutor Whether to use a shared executor for {@link @@ -688,7 +692,7 @@ public final class DefaultVideoCompositorPixelTest { boolean useSharedExecutor, ImmutableList> inputEffectLists) throws GlUtil.GlException, VideoFrameProcessingException { - this(testId, useSharedExecutor, inputEffectLists, new DefaultVideoCompositor.Settings()); + this(testId, useSharedExecutor, inputEffectLists, VideoCompositorSettings.DEFAULT); } /** @@ -701,13 +705,13 @@ public final class DefaultVideoCompositorPixelTest { * The size of this outer {@link List} is the amount of inputs. One inner list of {@link * Effect}s is used for each input. For each input, the frame timestamp and {@code inputId} * are overlaid via {@link TextOverlay} prior to its effects being applied. - * @param settings The {@link VideoCompositor.Settings}. + * @param videoCompositorSettings The {@link VideoCompositorSettings}. */ public VideoCompositorTestRunner( String testId, boolean useSharedExecutor, ImmutableList> inputEffectLists, - VideoCompositor.Settings settings) + VideoCompositorSettings videoCompositorSettings) throws GlUtil.GlException, VideoFrameProcessingException { this.testId = testId; timeoutMs = inputEffectLists.size() * VIDEO_FRAME_PROCESSING_WAIT_MS; @@ -725,7 +729,7 @@ public final class DefaultVideoCompositorPixelTest { new DefaultVideoCompositor( getApplicationContext(), glObjectsProvider, - settings, + videoCompositorSettings, sharedExecutorService, new VideoCompositor.Listener() { @Override diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java index 1c5de414e0..7d76833bb0 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java @@ -49,6 +49,7 @@ import androidx.media3.effect.DefaultVideoCompositor; import androidx.media3.effect.DefaultVideoFrameProcessor; import androidx.media3.effect.GlTextureProducer; import androidx.media3.effect.VideoCompositor; +import androidx.media3.effect.VideoCompositorSettings; import com.google.common.util.concurrent.MoreExecutors; import java.util.ArrayDeque; import java.util.ArrayList; @@ -225,7 +226,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; new DefaultVideoCompositor( context, glObjectsProvider, - new DefaultVideoCompositor.Settings(), + VideoCompositorSettings.DEFAULT, sharedExecutorService, new VideoCompositor.Listener() { // All of this listener's methods are called on the sharedExecutorService.