diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java index b0a35a8e07..e351f5e0c8 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java @@ -221,7 +221,9 @@ public final class VideoFrameProcessorTestRunner { return new VideoFrameProcessorTestRunner( testId, videoFrameProcessorFactory, - bitmapReader == null ? new SurfaceBitmapReader() : bitmapReader, + bitmapReader == null + ? new SurfaceBitmapReader(/* releaseOutputSurface= */ false) + : bitmapReader, videoAssetPath, outputFileLabel == null ? "" : outputFileLabel, effects == null ? ImmutableList.of() : effects, @@ -289,6 +291,11 @@ public final class VideoFrameProcessorTestRunner { @Override public void onOutputSizeChanged(int width, int height) { boolean useHighPrecisionColorComponents = ColorInfo.isTransferHdr(outputColorInfo); + if (bitmapReader instanceof SurfaceBitmapReader) { + if (((SurfaceBitmapReader) bitmapReader).releaseOutputSurface) { + checkNotNull(videoFrameProcessor).setOutputSurfaceInfo(null); + } + } @Nullable Surface outputSurface = bitmapReader.getSurface(width, height, useHighPrecisionColorComponents); @@ -515,6 +522,18 @@ public final class VideoFrameProcessorTestRunner { public static final class SurfaceBitmapReader implements VideoFrameProcessorTestRunner.BitmapReader { + public final boolean releaseOutputSurface; + + /** + * Creates an instance. + * + * @param releaseOutputSurface Whether the {@link VideoFrameProcessor} output Surface must be + * released at calls to {@link #getSurface(int, int, boolean)}. + */ + public SurfaceBitmapReader(boolean releaseOutputSurface) { + this.releaseOutputSurface = releaseOutputSurface; + } + // ImageReader only supports SDR input. private @MonotonicNonNull ImageReader imageReader; @@ -522,6 +541,9 @@ public final class VideoFrameProcessorTestRunner { @SuppressLint("WrongConstant") @Nullable public Surface getSurface(int width, int height, boolean useHighPrecisionColorComponents) { + if (imageReader != null && releaseOutputSurface) { + imageReader.close(); + } imageReader = ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); return imageReader.getSurface(); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorParametrizedSurfaceOutputTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorParametrizedSurfaceOutputTest.java new file mode 100644 index 0000000000..522dbb6e4c --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorParametrizedSurfaceOutputTest.java @@ -0,0 +1,226 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer.mh; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; +import static androidx.media3.test.utils.VideoFrameProcessorTestRunner.createTimestampIterator; +import static com.google.common.truth.Truth.assertThat; + +import android.graphics.Bitmap; +import android.util.Pair; +import androidx.media3.common.C; +import androidx.media3.effect.DefaultVideoFrameProcessor; +import androidx.media3.test.utils.VideoFrameProcessorTestRunner; +import androidx.media3.test.utils.VideoFrameProcessorTestRunner.SurfaceBitmapReader; +import com.google.common.collect.ImmutableList; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; + +/** + * Tests for frame queuing and output in {@link DefaultVideoFrameProcessor} given image input. Uses + * a {@link SurfaceBitmapReader} to verify that releasing the output surface during processing + * succeeds. + */ +// TODO(b/263395272): Move this test to effects/mh tests +@RunWith(Parameterized.class) +public class DefaultVideoFrameProcessorParametrizedSurfaceOutputTest { + @Rule public final TestName testName = new TestName(); + + private static final String ORIGINAL_PNG_ASSET_PATH = + "test-generated-goldens/sample_mp4_first_frame/electrical_colors/original.png"; + private static final String SCALE_WIDE_PNG_ASSET_PATH = + "test-generated-goldens/sample_mp4_first_frame/electrical_colors/scale_wide.png"; + private static final String BITMAP_OVERLAY_PNG_ASSET_PATH = + "test-generated-goldens/sample_mp4_first_frame/electrical_colors/overlay_bitmap_FrameProcessor.png"; + + @Parameters(name = "{0}") + public static ImmutableList releaseOutputSurface() { + return ImmutableList.of(false, true); + } + + private String testId; + private @MonotonicNonNull VideoFrameProcessorTestRunner videoFrameProcessorTestRunner; + private @MonotonicNonNull AtomicInteger framesProduced; + + @Parameter public boolean releaseOutputSurface; + + @Before + @EnsuresNonNull({"framesProduced"}) + public void setUp() { + framesProduced = new AtomicInteger(); + testId = testName.getMethodName(); + } + + @After + public void release() { + checkNotNull(videoFrameProcessorTestRunner).release(); + } + + @Test + @RequiresNonNull({"framesProduced"}) + public void imageInput_queueThreeBitmaps_outputsCorrectNumberOfFrames() throws Exception { + videoFrameProcessorTestRunner = + getDefaultFrameProcessorTestRunnerBuilder(testId) + .setBitmapReader(new SurfaceBitmapReader(releaseOutputSurface)) + .build(); + + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(ORIGINAL_PNG_ASSET_PATH), + C.MICROS_PER_SECOND, + /* offsetToAddUs= */ 0L, + /* frameRate= */ 2); + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(SCALE_WIDE_PNG_ASSET_PATH), + 2 * C.MICROS_PER_SECOND, + /* offsetToAddUs= */ 0L, + /* frameRate= */ 3); + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH), + 3 * C.MICROS_PER_SECOND, + /* offsetToAddUs= */ 0L, + /* frameRate= */ 4); + videoFrameProcessorTestRunner.endFrameProcessing(); + + int actualFrameCount = framesProduced.get(); + assertThat(actualFrameCount).isEqualTo(/* expected= */ 20); + } + + @Test + @RequiresNonNull({"framesProduced"}) + public void imageInput_queueTwentyBitmaps_outputsCorrectNumberOfFrames() throws Exception { + videoFrameProcessorTestRunner = + getDefaultFrameProcessorTestRunnerBuilder(testId) + .setBitmapReader(new SurfaceBitmapReader(releaseOutputSurface)) + .build(); + + for (int i = 0; i < 20; i++) { + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(ORIGINAL_PNG_ASSET_PATH), + /* durationUs= */ C.MICROS_PER_SECOND, + /* offsetToAddUs= */ 0L, + /* frameRate= */ 1); + } + videoFrameProcessorTestRunner.endFrameProcessing(); + + int actualFrameCount = framesProduced.get(); + assertThat(actualFrameCount).isEqualTo(/* expected= */ 20); + } + + @Test + @RequiresNonNull({"framesProduced"}) + public void imageInput_queueOneWithStartOffset_outputsFramesAtTheCorrectPresentationTimesUs() + throws Exception { + Queue actualPresentationTimesUs = new ConcurrentLinkedQueue<>(); + videoFrameProcessorTestRunner = + getDefaultFrameProcessorTestRunnerBuilder(testId) + .setOnOutputFrameAvailableForRenderingListener(actualPresentationTimesUs::add) + .setBitmapReader(new SurfaceBitmapReader(releaseOutputSurface)) + .build(); + + long offsetUs = 1_000_000L; + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(ORIGINAL_PNG_ASSET_PATH), + /* durationUs= */ C.MICROS_PER_SECOND, + /* offsetToAddUs= */ offsetUs, + /* frameRate= */ 2); + videoFrameProcessorTestRunner.endFrameProcessing(); + assertThat(actualPresentationTimesUs) + .containsExactly(offsetUs, offsetUs + C.MICROS_PER_SECOND / 2) + .inOrder(); + } + + @Test + @RequiresNonNull({"framesProduced"}) + public void imageInput_queueWithStartOffsets_outputsFramesAtTheCorrectPresentationTimesUs() + throws Exception { + Queue actualPresentationTimesUs = new ConcurrentLinkedQueue<>(); + videoFrameProcessorTestRunner = + getDefaultFrameProcessorTestRunnerBuilder(testId) + .setOnOutputFrameAvailableForRenderingListener(actualPresentationTimesUs::add) + .setBitmapReader(new SurfaceBitmapReader(releaseOutputSurface)) + .build(); + + long offsetUs1 = 1_000_000L; + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(ORIGINAL_PNG_ASSET_PATH), + /* durationUs= */ C.MICROS_PER_SECOND, + /* offsetToAddUs= */ offsetUs1, + /* frameRate= */ 2); + long offsetUs2 = 2_000_000L; + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(SCALE_WIDE_PNG_ASSET_PATH), + /* durationUs= */ C.MICROS_PER_SECOND, + /* offsetToAddUs= */ offsetUs2, + /* frameRate= */ 2); + videoFrameProcessorTestRunner.endFrameProcessing(); + assertThat(actualPresentationTimesUs) + .containsExactly( + offsetUs1, + offsetUs1 + C.MICROS_PER_SECOND / 2, + offsetUs2, + offsetUs2 + C.MICROS_PER_SECOND / 2) + .inOrder(); + } + + @Test + @RequiresNonNull({"framesProduced"}) + public void queueBitmapsWithTimestamps_outputsFramesAtTheCorrectPresentationTimesUs() + throws Exception { + Queue actualPresentationTimesUs = new ConcurrentLinkedQueue<>(); + videoFrameProcessorTestRunner = + getDefaultFrameProcessorTestRunnerBuilder(testId) + .setOnOutputFrameAvailableForRenderingListener(actualPresentationTimesUs::add) + .setBitmapReader(new SurfaceBitmapReader(releaseOutputSurface)) + .build(); + Bitmap bitmap1 = readBitmap(ORIGINAL_PNG_ASSET_PATH); + Bitmap bitmap2 = readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH); + Long offset1 = 0L; + Long offset2 = C.MICROS_PER_SECOND; + Long offset3 = 2 * C.MICROS_PER_SECOND; + + videoFrameProcessorTestRunner.queueInputBitmaps( + bitmap1.getWidth(), + bitmap1.getHeight(), + Pair.create(bitmap1, createTimestampIterator(ImmutableList.of(offset1))), + Pair.create(bitmap2, createTimestampIterator(ImmutableList.of(offset2, offset3)))); + videoFrameProcessorTestRunner.endFrameProcessing(); + + assertThat(actualPresentationTimesUs).containsExactly(offset1, offset2, offset3).inOrder(); + } + + private VideoFrameProcessorTestRunner.Builder getDefaultFrameProcessorTestRunnerBuilder( + String testId) { + return new VideoFrameProcessorTestRunner.Builder() + .setTestId(testId) + .setVideoFrameProcessorFactory(new DefaultVideoFrameProcessor.Factory.Builder().build()) + .setOnOutputFrameAvailableForRenderingListener( + unused -> checkNotNull(framesProduced).incrementAndGet()); + } +}