diff --git a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java new file mode 100644 index 0000000000..eef3bf1d71 --- /dev/null +++ b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java @@ -0,0 +1,122 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.effect; + +import static com.google.android.exoplayer2.testutil.BitmapPixelTestUtil.readBitmap; +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; +import static com.google.common.truth.Truth.assertThat; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.testutil.VideoFrameProcessorTestRunner; +import java.util.concurrent.atomic.AtomicInteger; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Tests for frame queuing and output in {@link DefaultVideoFrameProcessor} given image input. */ +@RunWith(AndroidJUnit4.class) +public class DefaultVideoFrameProcessorImageFrameOutputTest { + public static final String ORIGINAL_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png"; + public static final String WRAPPED_CROP_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/electrical_colors/image_input_with_wrapped_crop.png"; + public static final String BITMAP_OVERLAY_PNG_ASSET_PATH = + "media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_FrameProcessor.png"; + + private @MonotonicNonNull VideoFrameProcessorTestRunner videoFrameProcessorTestRunner; + private @MonotonicNonNull AtomicInteger framesProduced; + + @EnsuresNonNull("framesProduced") + @Before + public void setUp() { + framesProduced = new AtomicInteger(); + } + + @After + public void release() { + checkNotNull(videoFrameProcessorTestRunner).release(); + } + + @RequiresNonNull("framesProduced") + @Test + public void imageInput_queueThreeBitmaps_outputsAllFrames() throws Exception { + String testId = "imageInput_withThreeBitmaps_outputsAllFrames"; + videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId).build(); + + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(ORIGINAL_PNG_ASSET_PATH), C.MICROS_PER_SECOND, /* frameRate= */ 2); + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(WRAPPED_CROP_PNG_ASSET_PATH), 2 * C.MICROS_PER_SECOND, /* frameRate= */ 3); + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH), 3 * C.MICROS_PER_SECOND, /* frameRate= */ 4); + videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); + + int actualFrameCount = framesProduced.get(); + assertThat(actualFrameCount).isEqualTo(/* expected= */ 20); + } + + @RequiresNonNull("framesProduced") + @Test + public void imageInput_queueTwentyBitmaps_outputsAllFrames() throws Exception { + String testId = "imageInput_queueTwentyBitmaps_outputsAllFrames"; + videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId).build(); + + for (int i = 0; i < 20; i++) { + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(ORIGINAL_PNG_ASSET_PATH), + /* durationUs= */ C.MICROS_PER_SECOND, + /* frameRate= */ 1); + } + videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); + + int actualFrameCount = framesProduced.get(); + assertThat(actualFrameCount).isEqualTo(/* expected= */ 20); + } + + @RequiresNonNull("framesProduced") + @Test + public void imageInput_queueEndAndQueueAgain_outputsFirstSetOfFramesOnly() throws Exception { + String testId = "imageInput_queueEndAndQueueAgain_outputsFirstSetOfFramesOnly"; + videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId).build(); + + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(ORIGINAL_PNG_ASSET_PATH), + /* durationUs= */ C.MICROS_PER_SECOND, + /* frameRate= */ 2); + videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); + videoFrameProcessorTestRunner.queueInputBitmap( + readBitmap(ORIGINAL_PNG_ASSET_PATH), + /* durationUs= */ 2 * C.MICROS_PER_SECOND, + /* frameRate= */ 3); + + int actualFrameCount = framesProduced.get(); + assertThat(actualFrameCount).isEqualTo(/* expected= */ 2); + } + + private VideoFrameProcessorTestRunner.Builder getDefaultFrameProcessorTestRunnerBuilder( + String testId) { + return new VideoFrameProcessorTestRunner.Builder() + .setTestId(testId) + .setVideoFrameProcessorFactory(new DefaultVideoFrameProcessor.Factory()) + .setIsInputTextureExternal(false) + .setOnFrameAvailableListener((unused) -> checkNotNull(framesProduced).incrementAndGet()); + } +} diff --git a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorPixelTest.java b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorPixelTest.java index 81fce8653d..34f502ce92 100644 --- a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorPixelTest.java +++ b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorPixelTest.java @@ -27,6 +27,7 @@ import android.content.Context; import android.graphics.Bitmap; import android.graphics.Matrix; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.testutil.BitmapPixelTestUtil; import com.google.android.exoplayer2.testutil.VideoFrameProcessorTestRunner; import com.google.android.exoplayer2.util.Effect; @@ -109,7 +110,9 @@ public final class DefaultVideoFrameProcessorPixelTest { getDefaultFrameProcessorTestRunnerBuilder(testId).setIsInputTextureExternal(false).build(); Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH); - Bitmap actualBitmap = videoFrameProcessorTestRunner.processImageFrameAndEnd(expectedBitmap); + videoFrameProcessorTestRunner.queueInputBitmap( + expectedBitmap, C.MICROS_PER_SECOND, /* frameRate= */ 1); + Bitmap actualBitmap = videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); // TODO(b/207848601): Switch to using proper tooling for testing against golden data. float averagePixelAbsoluteDifference = @@ -134,16 +137,15 @@ public final class DefaultVideoFrameProcessorPixelTest { Bitmap originalBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH); Bitmap expectedBitmap = readBitmap(WRAPPED_CROP_PNG_ASSET_PATH); - Bitmap actualBitmap = videoFrameProcessorTestRunner.processImageFrameAndEnd(originalBitmap); + videoFrameProcessorTestRunner.queueInputBitmap( + originalBitmap, C.MICROS_PER_SECOND, /* frameRate= */ 1); + Bitmap actualBitmap = videoFrameProcessorTestRunner.endFrameProcessingAndGetImage(); // TODO(b/207848601): Switch to using proper tooling for testing against golden data. float averagePixelAbsoluteDifference = getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } - // TODO(b/262693274): Once texture deletion is added to InternalTextureManager.java, add a test - // queuing multiple input bitmaps to ensure the operation successfully completes and that the - // correct number of frames has been queued. @Test public void noEffects_withFrameCache_matchesGoldenFile() throws Exception { diff --git a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java index e2f3cc770d..d1efb6c0cb 100644 --- a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java +++ b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java @@ -273,14 +273,14 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { assertThat(actualReleaseTimesNs).isEqualTo(releaseTimesNs); } - private interface OnFrameAvailableListener { + private interface OnOutputFrameAvailableListener { void onFrameAvailable(long presentationTimeUs); } @EnsuresNonNull("defaultVideoFrameProcessor") private void processFramesToEndOfStream( long[] inputPresentationTimesUs, - OnFrameAvailableListener onFrameAvailableListener, + OnOutputFrameAvailableListener onFrameAvailableListener, boolean releaseFramesAutomatically) throws Exception { AtomicReference<@NullableType VideoFrameProcessingException> diff --git a/testutils/src/main/java/com/google/android/exoplayer2/testutil/VideoFrameProcessorTestRunner.java b/testutils/src/main/java/com/google/android/exoplayer2/testutil/VideoFrameProcessorTestRunner.java index 4d236a1b1d..c7a339efc1 100644 --- a/testutils/src/main/java/com/google/android/exoplayer2/testutil/VideoFrameProcessorTestRunner.java +++ b/testutils/src/main/java/com/google/android/exoplayer2/testutil/VideoFrameProcessorTestRunner.java @@ -27,8 +27,8 @@ import android.graphics.PixelFormat; import android.media.Image; import android.media.ImageReader; import android.media.MediaFormat; +import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; -import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.util.DebugViewProvider; import com.google.android.exoplayer2.util.Effect; import com.google.android.exoplayer2.util.FrameInfo; @@ -61,11 +61,13 @@ public final class VideoFrameProcessorTestRunner { private @MonotonicNonNull ColorInfo inputColorInfo; private @MonotonicNonNull ColorInfo outputColorInfo; private boolean isInputTextureExternal; + private OnOutputFrameAvailableListener onOutputFrameAvailableListener; /** Creates a new instance with default values. */ public Builder() { pixelWidthHeightRatio = DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO; isInputTextureExternal = true; + onOutputFrameAvailableListener = (unused) -> {}; } /** @@ -94,7 +96,7 @@ public final class VideoFrameProcessorTestRunner { /** * Sets the input video asset path. * - *

This is a required value. + *

No default value is set. Must be set when the input is a video file. */ @CanIgnoreReturnValue public Builder setVideoAssetPath(String videoAssetPath) { @@ -170,7 +172,8 @@ public final class VideoFrameProcessorTestRunner { return this; } /** - * Sets the input track type. See {@link VideoFrameProcessor.Factory#create}. + * Sets whether input comes from an external texture. See {@link + * VideoFrameProcessor.Factory#create}. * *

The default value is {@code true}. */ @@ -180,10 +183,21 @@ public final class VideoFrameProcessorTestRunner { return this; } + /** + * Sets the method to be called in {@link VideoFrameProcessor.Listener#onOutputFrameAvailable}. + * + *

The default value is a no-op. + */ + @CanIgnoreReturnValue + public Builder setOnFrameAvailableListener( + OnOutputFrameAvailableListener onOutputFrameAvailableListener) { + this.onOutputFrameAvailableListener = onOutputFrameAvailableListener; + return this; + } + public VideoFrameProcessorTestRunner build() throws VideoFrameProcessingException { checkStateNotNull(testId, "testId must be set."); checkStateNotNull(videoFrameProcessorFactory, "videoFrameProcessorFactory must be set."); - checkStateNotNull(videoAssetPath, "videoAssetPath must be set."); return new VideoFrameProcessorTestRunner( testId, @@ -194,7 +208,8 @@ public final class VideoFrameProcessorTestRunner { pixelWidthHeightRatio, inputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : inputColorInfo, outputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : outputColorInfo, - isInputTextureExternal); + isInputTextureExternal, + onOutputFrameAvailableListener); } } @@ -205,7 +220,7 @@ public final class VideoFrameProcessorTestRunner { private static final int VIDEO_FRAME_PROCESSING_WAIT_MS = 5000; private final String testId; - private final String videoAssetPath; + private final @MonotonicNonNull String videoAssetPath; private final String outputFileLabel; private final float pixelWidthHeightRatio; private final AtomicReference videoFrameProcessingException; @@ -218,13 +233,14 @@ public final class VideoFrameProcessorTestRunner { private VideoFrameProcessorTestRunner( String testId, VideoFrameProcessor.Factory videoFrameProcessorFactory, - String videoAssetPath, + @Nullable String videoAssetPath, String outputFileLabel, ImmutableList effects, float pixelWidthHeightRatio, ColorInfo inputColorInfo, ColorInfo outputColorInfo, - boolean isInputTextureExternal) + boolean isInputTextureExternal, + OnOutputFrameAvailableListener onOutputFrameAvailableListener) throws VideoFrameProcessingException { this.testId = testId; this.videoAssetPath = videoAssetPath; @@ -256,6 +272,7 @@ public final class VideoFrameProcessorTestRunner { @Override public void onOutputFrameAvailable(long presentationTimeUs) { // Do nothing as frames are released automatically. + onOutputFrameAvailableListener.onFrameAvailable(presentationTimeUs); } @Override @@ -272,7 +289,7 @@ public final class VideoFrameProcessorTestRunner { public Bitmap processFirstFrameAndEnd() throws Exception { DecodeOneFrameUtil.decodeOneAssetFileFrame( - videoAssetPath, + checkNotNull(videoAssetPath), new DecodeOneFrameUtil.Listener() { @Override public void onContainerExtracted(MediaFormat mediaFormat) { @@ -294,16 +311,15 @@ public final class VideoFrameProcessorTestRunner { return endFrameProcessingAndGetImage(); } - public Bitmap processImageFrameAndEnd(Bitmap inputBitmap) throws Exception { + public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) { videoFrameProcessor.setInputFrameInfo( new FrameInfo.Builder(inputBitmap.getWidth(), inputBitmap.getHeight()) .setPixelWidthHeightRatio(pixelWidthHeightRatio) .build()); - videoFrameProcessor.queueInputBitmap(inputBitmap, C.MICROS_PER_SECOND, /* frameRate= */ 1); - return endFrameProcessingAndGetImage(); + videoFrameProcessor.queueInputBitmap(inputBitmap, durationUs, frameRate); } - private Bitmap endFrameProcessingAndGetImage() throws Exception { + public Bitmap endFrameProcessingAndGetImage() throws Exception { videoFrameProcessor.signalEndOfInput(); Thread.sleep(VIDEO_FRAME_PROCESSING_WAIT_MS); @@ -322,4 +338,8 @@ public final class VideoFrameProcessorTestRunner { videoFrameProcessor.release(); } } + + public interface OnOutputFrameAvailableListener { + void onFrameAvailable(long presentationTimeUs); + } }