diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorFlushTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorFlushTest.java new file mode 100644 index 0000000000..119335191a --- /dev/null +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorFlushTest.java @@ -0,0 +1,131 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmapUnpremultipliedAlpha; +import static com.google.common.truth.Truth.assertThat; + +import android.graphics.Bitmap; +import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.test.utils.VideoFrameProcessorTestRunner; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +/** Test for {@link DefaultVideoFrameProcessor} flushing. */ +@RunWith(AndroidJUnit4.class) +public class DefaultVideoFrameProcessorFlushTest { + private static final String ORIGINAL_PNG_ASSET_PATH = + "media/bitmap/input_images/media3test_srgb.png"; + + @Rule public final TestName testName = new TestName(); + + private int outputFrameCount; + private @MonotonicNonNull String testId; + private @MonotonicNonNull VideoFrameProcessorTestRunner videoFrameProcessorTestRunner; + + @Before + @EnsuresNonNull({"testId"}) + public void setUp() { + testId = testName.getMethodName(); + } + + @After + public void release() { + checkNotNull(videoFrameProcessorTestRunner).release(); + } + + @Test + @RequiresNonNull({"testId"}) + public void imageInput_flushBeforeInput_outputsAllFrames() throws Exception { + videoFrameProcessorTestRunner = createDefaultVideoFrameProcessorTestRunner(testId); + Bitmap bitmap = readBitmapUnpremultipliedAlpha(ORIGINAL_PNG_ASSET_PATH); + int inputFrameCount = 3; + + videoFrameProcessorTestRunner.flush(); + videoFrameProcessorTestRunner.queueInputBitmap( + bitmap, + /* durationUs= */ inputFrameCount * C.MICROS_PER_SECOND, + /* offsetToAddUs= */ 0L, + /* frameRate= */ 1); + videoFrameProcessorTestRunner.endFrameProcessing(); + + assertThat(outputFrameCount).isEqualTo(inputFrameCount); + } + + // This tests a condition that is difficult to synchronize, and is subject to a race condition. It + // may flake/fail if any queued frames are processed in the VideoFrameProcessor thread, before + // flush begins and cancels these pending frames. However, this is better than not testing this + // behavior at all, and in practice has succeeded every time on a 1000-time run. + // TODO: b/302695659 - Make this test more deterministic. + @Test + @RequiresNonNull({"testId"}) + public void imageInput_flushRightAfterInput_outputsPartialFrames() throws Exception { + videoFrameProcessorTestRunner = createDefaultVideoFrameProcessorTestRunner(testId); + Bitmap bitmap = readBitmapUnpremultipliedAlpha(ORIGINAL_PNG_ASSET_PATH); + int inputFrameCount = 3; + + videoFrameProcessorTestRunner.queueInputBitmap( + bitmap, + /* durationUs= */ inputFrameCount * C.MICROS_PER_SECOND, + /* offsetToAddUs= */ 0L, + /* frameRate= */ 1); + videoFrameProcessorTestRunner.flush(); + videoFrameProcessorTestRunner.endFrameProcessing(); + + // This assertion is subject to flaking, per test comments. If it flakes, consider increasing + // inputFrameCount. + assertThat(outputFrameCount).isLessThan(inputFrameCount); + } + + @Test + @RequiresNonNull({"testId"}) + public void imageInput_flushAfterAllFramesOutput_outputsAllFrames() throws Exception { + videoFrameProcessorTestRunner = createDefaultVideoFrameProcessorTestRunner(testId); + Bitmap bitmap = readBitmapUnpremultipliedAlpha(ORIGINAL_PNG_ASSET_PATH); + int inputFrameCount = 3; + + videoFrameProcessorTestRunner.queueInputBitmap( + bitmap, + /* durationUs= */ inputFrameCount * C.MICROS_PER_SECOND, + /* offsetToAddUs= */ 0L, + /* frameRate= */ 1); + videoFrameProcessorTestRunner.endFrameProcessing(); + videoFrameProcessorTestRunner.flush(); + + assertThat(outputFrameCount).isEqualTo(inputFrameCount); + } + + private VideoFrameProcessorTestRunner createDefaultVideoFrameProcessorTestRunner(String testId) + throws VideoFrameProcessingException { + return new VideoFrameProcessorTestRunner.Builder() + .setTestId(testId) + .setVideoFrameProcessorFactory(new DefaultVideoFrameProcessor.Factory.Builder().build()) + .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) + .setOnOutputFrameAvailableForRenderingListener(unused -> outputFrameCount++) + .build(); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java index a9b8141504..83f0d857b6 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java @@ -61,6 +61,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private boolean currentInputStreamEnded; private boolean isNextFrameInTexture; + @Nullable private volatile VideoFrameProcessingTaskExecutor.Task onFlushCompleteTask; + /** * Creates a new instance. * @@ -89,6 +91,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; }); } + @Override + public void onFlush() { + videoFrameProcessingTaskExecutor.submit(this::flush); + } + @Override public void queueInputBitmap( Bitmap inputBitmap, @@ -124,7 +131,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public void setOnFlushCompleteListener(@Nullable VideoFrameProcessingTaskExecutor.Task task) { - // Do nothing. + onFlushCompleteTask = task; } @Override @@ -192,6 +199,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } + private void flush() { + pendingBitmaps.clear(); + if (onFlushCompleteTask != null) { + videoFrameProcessingTaskExecutor.submitWithHighPriority(onFlushCompleteTask); + } + } + /** Information needed to generate all the frames associated with a specific {@link Bitmap}. */ private static final class BitmapFrameSequenceInfo { public final Bitmap bitmap; diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 89f6a69e5e..fc202d14a8 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -382,9 +382,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { if (pendingInputStreamInfo != null) { InputStreamInfo pendingInputStreamInfo = this.pendingInputStreamInfo; videoFrameProcessingTaskExecutor.submit( - () -> { - configureEffects(pendingInputStreamInfo, /* forceReconfigure= */ false); - }); + () -> configureEffects(pendingInputStreamInfo, /* forceReconfigure= */ false)); this.pendingInputStreamInfo = null; } } @@ -551,6 +549,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { @Override public void flush() { + if (!inputSwitcher.hasActiveInput()) { + return; + } try { videoFrameProcessingTaskExecutor.flush(); CountDownLatch latch = new CountDownLatch(1); diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java index 70e04c61cb..1f5a03f466 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java @@ -205,6 +205,9 @@ public final class VideoFrameProcessorTestRunner { * Sets the method to be called in {@link * VideoFrameProcessor.Listener#onOutputFrameAvailableForRendering}. * + *

The method will be called on the thread the {@link VideoFrameProcessorTestRunner} is + * created on. + * *

The default value is a no-op. */ @CanIgnoreReturnValue @@ -290,7 +293,7 @@ public final class VideoFrameProcessorTestRunner { inputColorInfo, outputColorInfo, /* renderFramesAutomatically= */ true, - MoreExecutors.directExecutor(), + /* listenerExecutor= */ MoreExecutors.directExecutor(), new VideoFrameProcessor.Listener() { @Override public void onInputStreamRegistered( @@ -444,6 +447,11 @@ public final class VideoFrameProcessorTestRunner { videoFrameProcessor.signalEndOfInput(); } + /** Calls {@link VideoFrameProcessor#flush}. */ + public void flush() { + videoFrameProcessor.flush(); + } + /** After {@link #signalEndOfInput}, is called, wait for this instance to end. */ public void awaitFrameProcessingEnd(long videoFrameProcessingWaitTimeMs) { @Nullable Exception endFrameProcessingException = null;