diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java index 6a9a9c3802..65d1982baa 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorTest.java @@ -16,17 +16,23 @@ package androidx.media3.effect; import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmapUnpremultipliedAlpha; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import android.graphics.Bitmap; import androidx.annotation.Nullable; +import androidx.media3.common.C; import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; import androidx.media3.common.FrameInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.util.ConditionVariable; +import androidx.media3.common.util.ConstantRateTimestampIterator; +import androidx.media3.common.util.SystemClock; import androidx.media3.common.util.Util; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; @@ -35,8 +41,10 @@ import java.util.List; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -46,20 +54,30 @@ import org.junit.runner.RunWith; public class DefaultVideoFrameProcessorTest { private static final long INPUT_REGISTRATION_TIMEOUT_MS = 1_000L; + private static final String ORIGINAL_PNG_ASSET_PATH = "media/png/media3test_srgb.png"; + private static final long TEST_TIMEOUT_MS = 10_000L; private DefaultVideoFrameProcessor.@MonotonicNonNull Factory factory; + private @MonotonicNonNull DefaultVideoFrameProcessor defaultVideoFrameProcessor; @Before public void setUp() { factory = new DefaultVideoFrameProcessor.Factory.Builder().build(); } + @After + public void tearDown() { + if (defaultVideoFrameProcessor != null) { + defaultVideoFrameProcessor.release(); + } + } + @Test public void registerInputStream_withBlockingVideoFrameProcessorConfiguration_succeeds() throws Exception { AtomicReference videoFrameProcessingException = new AtomicReference<>(); CountDownLatch inputStreamRegisteredCountDownLatch = new CountDownLatch(1); - DefaultVideoFrameProcessor defaultVideoFrameProcessor = + defaultVideoFrameProcessor = createDefaultVideoFrameProcessor( new VideoFrameProcessor.Listener() { @Override @@ -119,7 +137,7 @@ public class DefaultVideoFrameProcessorTest { AtomicReference videoFrameProcessingException = new AtomicReference<>(); CountDownLatch countDownLatch = new CountDownLatch(3); Queue registeredInputStreamInfoWidths = new ConcurrentLinkedQueue<>(); - DefaultVideoFrameProcessor defaultVideoFrameProcessor = + defaultVideoFrameProcessor = createDefaultVideoFrameProcessor( new VideoFrameProcessor.Listener() { @Override @@ -177,6 +195,99 @@ public class DefaultVideoFrameProcessorTest { .inOrder(); } + @Test + public void + registerInputStream_withManualFrameRendering_configuresTheSecondStreamAfterRenderingAllFramesFromTheFirst() + throws Exception { + AtomicReference videoFrameProcessingException = new AtomicReference<>(); + AtomicLong firstStreamLastFrameAvailableTimeMs = new AtomicLong(); + AtomicLong secondStreamConfigurationTimeMs = new AtomicLong(); + ConditionVariable inputStreamRegisteredCondition = new ConditionVariable(); + CountDownLatch frameProcessorEnded = new CountDownLatch(1); + defaultVideoFrameProcessor = + factory.create( + getApplicationContext(), + DebugViewProvider.NONE, + /* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, + /* renderFramesAutomatically= */ false, + Util.newSingleThreadExecutor("DVFPTest"), + new VideoFrameProcessor.Listener() { + + int outputFrameCount = 0; + + @Override + public void onInputStreamRegistered( + @VideoFrameProcessor.InputType int inputType, + List effects, + FrameInfo frameInfo) { + inputStreamRegisteredCondition.open(); + } + + @Override + public void onOutputSizeChanged(int width, int height) {} + + @Override + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + outputFrameCount++; + if (outputFrameCount == 30) { + firstStreamLastFrameAvailableTimeMs.set(SystemClock.DEFAULT.elapsedRealtime()); + } + defaultVideoFrameProcessor.renderOutputFrame( + VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY); + } + + @Override + public void onError(VideoFrameProcessingException exception) { + videoFrameProcessingException.set(exception); + } + + @Override + public void onEnded() { + frameProcessorEnded.countDown(); + } + }); + + Bitmap bitmap1 = readBitmapUnpremultipliedAlpha(ORIGINAL_PNG_ASSET_PATH); + // Needs a different bitmap as the bitmap is recycled after single use. + Bitmap bitmap2 = readBitmapUnpremultipliedAlpha(ORIGINAL_PNG_ASSET_PATH); + + // First image + inputStreamRegisteredCondition.close(); + defaultVideoFrameProcessor.registerInputStream( + VideoFrameProcessor.INPUT_TYPE_BITMAP, + ImmutableList.of(), + new FrameInfo.Builder(ColorInfo.SRGB_BT709_FULL, bitmap1.getWidth(), bitmap1.getHeight()) + .build()); + inputStreamRegisteredCondition.block(); + defaultVideoFrameProcessor.queueInputBitmap( + bitmap1, new ConstantRateTimestampIterator(C.MICROS_PER_SECOND, 30.f)); + + // Second image + inputStreamRegisteredCondition.close(); + defaultVideoFrameProcessor.registerInputStream( + VideoFrameProcessor.INPUT_TYPE_BITMAP, + ImmutableList.of( + (GlEffect) + (context, useHdr) -> { + secondStreamConfigurationTimeMs.set(SystemClock.DEFAULT.elapsedRealtime()); + return new PassthroughShaderProgram(); + }), + new FrameInfo.Builder(ColorInfo.SRGB_BT709_FULL, bitmap2.getWidth(), bitmap2.getHeight()) + .build()); + inputStreamRegisteredCondition.block(); + defaultVideoFrameProcessor.queueInputBitmap( + bitmap2, new ConstantRateTimestampIterator(C.MICROS_PER_SECOND, 30.f)); + + defaultVideoFrameProcessor.signalEndOfInput(); + + if (!frameProcessorEnded.await(TEST_TIMEOUT_MS, MILLISECONDS)) { + throw new IllegalStateException("Test timeout", videoFrameProcessingException.get()); + } + + assertThat(secondStreamConfigurationTimeMs.get()) + .isAtLeast(firstStreamLastFrameAvailableTimeMs.get()); + } + private DefaultVideoFrameProcessor createDefaultVideoFrameProcessor( VideoFrameProcessor.Listener listener) throws Exception { return checkNotNull(factory) diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java index fea005d1d2..a90768e23c 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java @@ -56,6 +56,7 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest { private static final int WIDTH = 200; private static final int HEIGHT = 100; + private static final long TEST_TIMEOUT_MS = 10_000L; /** * Time to wait between rendering frames to avoid frame drops between GL and the {@link @@ -238,37 +239,6 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest { assertThat(actualRenderTimesNs).containsExactlyElementsIn(renderTimesNs).inOrder(); } - @Test - public void controlledFrameRendering_withThreeFramesAtOnce_usesGivenTimestamps() - throws Exception { - ImmutableList originalPresentationTimesUs = ImmutableList.of(1234L, 3456L, 4567L); - long offsetNs = System.nanoTime(); - ImmutableList renderTimesNs = - ImmutableList.of(offsetNs + 123456, offsetNs + 234567, offsetNs + 345678); - ArrayList actualPresentationTimesUs = new ArrayList<>(); - processFramesToEndOfStream( - /* inputPresentationTimesUs= */ originalPresentationTimesUs, - /* onFrameAvailableListener= */ actualPresentationTimesUs::add, - /* renderFramesAutomatically= */ false); - - // TODO(b/264252759): Investigate output frames being dropped and remove sleep. - // Frames can be dropped silently between EGL and the ImageReader. Sleep after each call - // to swap buffers, to avoid this behavior. - defaultVideoFrameProcessor.renderOutputFrame(renderTimesNs.get(0)); - Thread.sleep(PER_FRAME_RENDERING_WAIT_TIME_MS); - defaultVideoFrameProcessor.renderOutputFrame(renderTimesNs.get(1)); - Thread.sleep(PER_FRAME_RENDERING_WAIT_TIME_MS); - defaultVideoFrameProcessor.renderOutputFrame(renderTimesNs.get(2)); - Thread.sleep(PER_FRAME_RENDERING_WAIT_TIME_MS); - - assertThat(actualPresentationTimesUs) - .containsExactlyElementsIn(originalPresentationTimesUs) - .inOrder(); - ImmutableList actualRenderTimesNs = - waitForFrameRenderingAndGetRenderTimesNs(/* expectedFrameCount= */ 3); - assertThat(actualRenderTimesNs).containsExactlyElementsIn(renderTimesNs).inOrder(); - } - private interface OnOutputFrameAvailableForRenderingListener { void onFrameAvailableForRendering(long presentationTimeUs); } @@ -347,14 +317,19 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest { INPUT_TYPE_SURFACE, /* effects= */ ImmutableList.of((GlEffect) (context, useHdr) -> blankFrameProducer), new FrameInfo.Builder(ColorInfo.SDR_BT709_LIMITED, WIDTH, HEIGHT).build()); - videoFrameProcessorReadyCountDownLatch.await(); + boolean testTimedOut = false; + if (!videoFrameProcessorReadyCountDownLatch.await(TEST_TIMEOUT_MS, MILLISECONDS)) { + testTimedOut = true; + } blankFrameProducer.produceBlankFrames(inputPresentationTimesUs); defaultVideoFrameProcessor.signalEndOfInput(); - videoFrameProcessingEndedCountDownLatch.await(); + if (!videoFrameProcessingEndedCountDownLatch.await(TEST_TIMEOUT_MS, MILLISECONDS)) { + testTimedOut = true; + } @Nullable Exception videoFrameProcessingException = videoFrameProcessingExceptionReference.get(); - if (videoFrameProcessingException != null) { - throw videoFrameProcessingException; + if (videoFrameProcessingException != null || testTimedOut) { + throw new IllegalStateException(videoFrameProcessingException); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java index e72be53199..7bfed03025 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java @@ -102,6 +102,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private int outputHeight; @Nullable private DefaultShaderProgram defaultShaderProgram; @Nullable private SurfaceViewWrapper debugSurfaceViewWrapper; + // Whether the input stream has ended, but not all input has been released. This is relevant only + // when renderFramesAutomatically is false. Ensures all frames are rendered before reporting + // onInputStreamProcessed. + // TODO: b/320481157 - Apply isInputStreamEnded to texture output as well. + private boolean isInputStreamEndedWithPendingAvailableFrames; private InputListener inputListener; private @MonotonicNonNull Size outputSizeBeforeSurfaceTransformation; @Nullable private SurfaceView debugSurfaceView; @@ -183,7 +188,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public void signalEndOfCurrentInputStream() { - checkNotNull(onInputStreamProcessedListener).onInputStreamProcessed(); + if (availableFrames.isEmpty()) { + checkNotNull(onInputStreamProcessedListener).onInputStreamProcessed(); + isInputStreamEndedWithPendingAvailableFrames = false; + } else { + checkState(!renderFramesAutomatically); + isInputStreamEndedWithPendingAvailableFrames = true; + } } // Methods that must be called on the GL thread. @@ -266,6 +277,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // Drops all frames that aren't rendered yet. availableFrames.clear(); + isInputStreamEndedWithPendingAvailableFrames = false; if (defaultShaderProgram != null) { defaultShaderProgram.flush(); } @@ -308,6 +320,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* inputTexture= */ oldestAvailableFrame.first, /* presentationTimeUs= */ oldestAvailableFrame.second, renderTimeNs); + if (availableFrames.isEmpty() && isInputStreamEndedWithPendingAvailableFrames) { + checkNotNull(onInputStreamProcessedListener).onInputStreamProcessed(); + isInputStreamEndedWithPendingAvailableFrames = false; + } } /** See {@link DefaultVideoFrameProcessor#setOutputSurfaceInfo} */