diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java index 3755c08468..b041abf15a 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java @@ -61,8 +61,6 @@ public final class DefaultVideoCompositor implements VideoCompositor { // * Use a lock to synchronize inputFrameInfos more narrowly, to reduce blocking. // * If the primary stream ends, consider setting the secondary stream as the new primary stream, // so that secondary stream frames aren't dropped. - // * Consider adding info about the timestamps for each input frame used to composite an output - // frame, to aid debugging and testing. private static final String THREAD_NAME = "Effect:DefaultVideoCompositor:GlThread"; private static final String TAG = "DefaultVideoCompositor"; @@ -133,6 +131,9 @@ public final class DefaultVideoCompositor implements VideoCompositor { *

The input source must be able to have at least two {@linkplain #queueInputTexture queued * textures} before one texture is {@linkplain * DefaultVideoFrameProcessor.ReleaseOutputTextureCallback released}. + * + *

When composited, textures are drawn in the reverse order of their registration order, so + * that the first registered source is on the very top. */ @Override public synchronized int registerInputSource() { @@ -269,9 +270,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { ensureGlProgramConfigured(); - // TODO: b/262694346 - - // * Support an arbitrary number of inputs. - // * Allow different input frame dimensions. + // TODO: b/262694346 - Allow different input frame dimensions. InputFrameInfo primaryInputFrame = framesToComposite.get(PRIMARY_INPUT_ID); GlTextureInfo primaryInputTexture = primaryInputFrame.texture; outputTexturePool.ensureConfigured( diff --git a/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_0s_1s_0s.png b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_0s_1s_0s.png new file mode 100644 index 0000000000..2a3525e510 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_0s_1s_0s.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_1s_1s_0s.png b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_1s_1s_0s.png new file mode 100644 index 0000000000..eb6a1284dc Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_1s_1s_0s.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_2s_1s_2s.png b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_2s_1s_2s.png new file mode 100644 index 0000000000..7c0990f0c0 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_2s_1s_2s.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_grayscale_opaque_1s.png b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_grayscale_opaque_1s.png new file mode 100644 index 0000000000..7e8196a62b Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_grayscale_opaque_1s.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_grayscale_opaque_2s.png b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_grayscale_opaque_2s.png new file mode 100644 index 0000000000..10039b5628 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_grayscale_opaque_2s.png differ diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java index 45177e5996..05bd10ab6a 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java @@ -82,6 +82,8 @@ import org.junit.runners.Parameterized; /** Pixel test for {@link DefaultVideoCompositor} compositing 2 input frames into 1 output frame. */ @RunWith(Parameterized.class) public final class DefaultVideoCompositorPixelTest { + // TODO: b/262694346 - Have CompositorTestRunner queueBitmapToInput queue bitmaps at specified + // timestamps instead of frame rates. @Parameterized.Parameters(name = "useSharedExecutor={0}") public static ImmutableList useSharedExecutor() { return ImmutableList.of(true, false); @@ -89,7 +91,7 @@ public final class DefaultVideoCompositorPixelTest { // Golden images were generated on an API 33 emulator. API 26 emulators have a different text // rendering implementation that leads to a larger pixel difference. - public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_WITH_OVERLAY = + public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_WITH_TEXT_OVERLAY = (Ascii.toLowerCase(Util.DEVICE).contains("emulator") || Ascii.toLowerCase(Util.DEVICE).contains("generic")) && SDK_INT <= 26 @@ -214,7 +216,7 @@ public final class DefaultVideoCompositorPixelTest { @Test @RequiresNonNull("testId") - public void compositeTwoInputs_withSecondaryAlphaZero_differentTimestamp_matchesExpectedBitmap() + public void compositeTwoInputs_withSecondaryTransparent_differentTimestamp_matchesExpectedBitmap() throws Exception { ImmutableList> inputEffects = ImmutableList.of( @@ -498,17 +500,101 @@ public final class DefaultVideoCompositorPixelTest { assertThat(compositorTestRunner.getCompositedTimestamps()).hasSize(numberOfFramesToQueue); } + @Test + @RequiresNonNull("testId") + public void compositeFiveInputs_withFiveFramesFromEach_matchesExpectedFrameCount() + throws Exception { + compositorTestRunner = + new VideoCompositorTestRunner( + testId, + useSharedExecutor, + /* inputEffectLists= */ ImmutableList.of( + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of())); + int numberOfFramesToQueue = 5; + + compositorTestRunner.queueBitmapToAllInputs(/* durationSec= */ numberOfFramesToQueue); + compositorTestRunner.endCompositing(); + + assertThat(compositorTestRunner.getCompositedTimestamps()).hasSize(numberOfFramesToQueue); + } + + @Test + @RequiresNonNull("testId") + public void compositeOneInput_matchesExpectedBitmap() throws Exception { + compositorTestRunner = + new VideoCompositorTestRunner( + testId, + useSharedExecutor, + ImmutableList.of( + ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(100f)))); + + compositorTestRunner.queueBitmapToAllInputs(/* durationSec= */ 3); + compositorTestRunner.endCompositing(); + + ImmutableList primaryTimestamps = + ImmutableList.of(0 * C.MICROS_PER_SECOND, 1 * C.MICROS_PER_SECOND, 2 * C.MICROS_PER_SECOND); + assertThat(compositorTestRunner.inputBitmapReaders.get(0).getOutputTimestamps()) + .containsExactlyElementsIn(primaryTimestamps) + .inOrder(); + compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected( + ImmutableList.of("grayscale_opaque_0s", "grayscale_opaque_1s", "grayscale_opaque_2s")); + } + + @Test + @RequiresNonNull("testId") + public void compositeThreeInputs_matchesExpectedBitmap() throws Exception { + compositorTestRunner = + new VideoCompositorTestRunner( + testId, + useSharedExecutor, + ImmutableList.of( + ImmutableList.of(RgbFilter.createInvertedFilter(), new AlphaScale(0.4f)), + ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(0.7f)), + ImmutableList.of( + new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build()))); + + compositorTestRunner.queueBitmapToInput( + /* inputId= */ 0, /* durationSec= */ 3, /* offsetToAddSec= */ 0, /* frameRate= */ 1); + compositorTestRunner.queueBitmapToInput( + /* inputId= */ 1, /* durationSec= */ 1, /* offsetToAddSec= */ 1, /* frameRate= */ 1); + compositorTestRunner.queueBitmapToInput( + /* inputId= */ 2, /* durationSec= */ 3, /* offsetToAddSec= */ 0, /* frameRate= */ 0.5f); + compositorTestRunner.endCompositing(); + + ImmutableList primaryTimestamps = + ImmutableList.of(0 * C.MICROS_PER_SECOND, 1 * C.MICROS_PER_SECOND, 2 * C.MICROS_PER_SECOND); + ImmutableList secondary1Timestamps = ImmutableList.of(1 * C.MICROS_PER_SECOND); + ImmutableList secondary2Timestamps = + ImmutableList.of(0 * C.MICROS_PER_SECOND, 2 * C.MICROS_PER_SECOND); + assertThat(compositorTestRunner.inputBitmapReaders.get(0).getOutputTimestamps()) + .containsExactlyElementsIn(primaryTimestamps) + .inOrder(); + assertThat(compositorTestRunner.inputBitmapReaders.get(1).getOutputTimestamps()) + .containsExactlyElementsIn(secondary1Timestamps) + .inOrder(); + assertThat(compositorTestRunner.inputBitmapReaders.get(2).getOutputTimestamps()) + .containsExactlyElementsIn(secondary2Timestamps) + .inOrder(); + assertThat(compositorTestRunner.getCompositedTimestamps()) + .containsExactlyElementsIn(primaryTimestamps) + .inOrder(); + compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected( + ImmutableList.of("0s_1s_0s", "1s_1s_0s", "2s_1s_2s")); + } + /** * A test runner for {@link DefaultVideoCompositor} tests. * *

Composites input bitmaps from two input sources. */ private static final class VideoCompositorTestRunner { - // Compositor tests rely on 2 VideoFrameProcessor instances, plus the compositor. - private static final int COMPOSITOR_TIMEOUT_MS = 2 * VIDEO_FRAME_PROCESSING_WAIT_MS; - private static final int COMPOSITOR_INPUT_SIZE = 2; public final List inputBitmapReaders; + private final int timeoutMs; private final LinkedHashMap outputTimestampsToBitmaps; private final List inputVideoFrameProcessorTestRunners; private final VideoCompositor videoCompositor; @@ -534,6 +620,7 @@ public final class DefaultVideoCompositorPixelTest { ImmutableList> inputEffectLists) throws GlUtil.GlException, VideoFrameProcessingException { this.testId = testId; + timeoutMs = inputEffectLists.size() * VIDEO_FRAME_PROCESSING_WAIT_MS; sharedExecutorService = useSharedExecutor ? Util.newSingleThreadExecutor("Effect:Shared:GlThread") : null; EGLContext sharedEglContext = AndroidTestUtil.createOpenGlObjects(); @@ -578,7 +665,6 @@ public final class DefaultVideoCompositorPixelTest { /* textureOutputCapacity= */ 1); inputBitmapReaders = new ArrayList<>(); inputVideoFrameProcessorTestRunners = new ArrayList<>(); - assertThat(inputEffectLists).hasSize(COMPOSITOR_INPUT_SIZE); for (int i = 0; i < inputEffectLists.size(); i++) { TextureBitmapReader textureBitmapReader = new TextureBitmapReader(); inputBitmapReaders.add(textureBitmapReader); @@ -629,11 +715,11 @@ public final class DefaultVideoCompositorPixelTest { inputVideoFrameProcessorTestRunners.get(i).signalEndOfInput(); } for (int i = 0; i < inputVideoFrameProcessorTestRunners.size(); i++) { - inputVideoFrameProcessorTestRunners.get(i).awaitFrameProcessingEnd(COMPOSITOR_TIMEOUT_MS); + inputVideoFrameProcessorTestRunners.get(i).awaitFrameProcessingEnd(timeoutMs); } @Nullable Exception endCompositingException = null; try { - if (!compositorEnded.await(COMPOSITOR_TIMEOUT_MS, MILLISECONDS)) { + if (!compositorEnded.await(timeoutMs, MILLISECONDS)) { endCompositingException = new IllegalStateException("Compositing timed out."); } } catch (InterruptedException e) { @@ -685,7 +771,7 @@ public final class DefaultVideoCompositorPixelTest { if (sharedExecutorService != null) { try { sharedExecutorService.shutdown(); - if (!sharedExecutorService.awaitTermination(COMPOSITOR_TIMEOUT_MS, MILLISECONDS)) { + if (!sharedExecutorService.awaitTermination(timeoutMs, MILLISECONDS)) { throw new IllegalStateException("Missed shutdown timeout."); } } catch (InterruptedException unexpected) { @@ -806,6 +892,6 @@ public final class DefaultVideoCompositorPixelTest { readBitmapUnpremultipliedAlpha(expectedBitmapAssetPath), actualBitmap, testId); assertWithMessage("Pixel difference for bitmapLabel = " + actualBitmapLabel) .that(averagePixelAbsoluteDifference) - .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_WITH_OVERLAY); + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_WITH_TEXT_OVERLAY); } }