From c22195888911d1780851279e832ba5dd907d6e50 Mon Sep 17 00:00:00 2001 From: claincly Date: Fri, 28 Jul 2023 10:52:28 +0100 Subject: [PATCH] Remove setInputFrameInfo After this change, every queued bitmap is treated as an individual input stream (like a new MediaItems). This change merges the FrameDropTest and FrameDropPixelTest into one (while maintaining all the test cases) - This is accomplished by generating bitmaps with timestamps on it in FrameDropTest and compare them with goldens (one may call this a pixel test, please lmk if you want this to be renamed) - The most part of the change comes from DefaultVideoFrameProcessorVideoFrameRenderingTest. The overall working is - We bypass the input manager - The TestFrameGenerator generates frames based on timestamps. In this case, we generate frames with timestamps on it - The generated frame is sent to texture output and in turn saved to bitmaps - We then compare the generated bitmap with the goldens PiperOrigin-RevId: 551795770 --- .../media3/common/VideoFrameProcessor.java | 30 +-- .../media3/effect/BlankFrameProducer.java | 97 +++++++ ...deoFrameProcessorImageFrameOutputTest.java | 2 - .../DefaultVideoFrameProcessorPixelTest.java | 4 - ...FrameProcessorVideoFrameRenderingTest.java | 4 +- .../androidx/media3/effect/FrameDropTest.java | 248 ++++++++++++------ .../DefaultFrameDroppingShaderProgram.java | 4 +- .../effect/DefaultVideoFrameProcessor.java | 23 +- .../androidx/media3/effect/InputSwitcher.java | 29 +- .../SimpleFrameDroppingShaderProgram.java | 2 +- .../media3/effect/TextureManager.java | 6 +- .../video/MediaCodecVideoRenderer.java | 6 +- .../media/bitmap/FrameDropTest/pts_0.png | Bin 0 -> 591 bytes .../media/bitmap/FrameDropTest/pts_32000.png | Bin 0 -> 1419 bytes .../media/bitmap/FrameDropTest/pts_333333.png | Bin 0 -> 1319 bytes .../media/bitmap/FrameDropTest/pts_666667.png | Bin 0 -> 1434 bytes .../media/bitmap/FrameDropTest/pts_71000.png | Bin 0 -> 1140 bytes .../media/bitmap/FrameDropTest/pts_750000.png | Bin 0 -> 1441 bytes .../test/utils}/TextureBitmapReader.java | 29 +- .../utils/VideoFrameProcessorTestRunner.java | 34 +-- .../transformer/VideoCompositorPixelTest.java | 3 +- ...ocessorMultipleTextureOutputPixelTest.java | 12 +- ...oFrameProcessorTextureOutputPixelTest.java | 18 +- .../transformer/mh/FrameDropPixelTest.java | 199 -------------- .../transformer/SingleInputVideoGraph.java | 3 +- 25 files changed, 371 insertions(+), 382 deletions(-) create mode 100644 libraries/effect/src/androidTest/java/androidx/media3/effect/BlankFrameProducer.java create mode 100644 libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_0.png create mode 100644 libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_32000.png create mode 100644 libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_333333.png create mode 100644 libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_666667.png create mode 100644 libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_71000.png create mode 100644 libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_750000.png rename libraries/{transformer/src/androidTest/java/androidx/media3/transformer => test_utils/src/main/java/androidx/media3/test/utils}/TextureBitmapReader.java (84%) delete mode 100644 libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/FrameDropPixelTest.java diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index 8cdcda9972..599ecfe4ac 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -168,8 +168,8 @@ public interface VideoFrameProcessor { /** * Provides an input texture ID to the {@code VideoFrameProcessor}. * - *

It must be called after the {@link #setOnInputFrameProcessedListener - * onInputFrameProcessedListener} and the {@link #setInputFrameInfo frameInfo} have been set. + *

It must be only called after {@link #setOnInputFrameProcessedListener} and {@link + * #registerInputStream} have been called. * *

Can be called on any thread. * @@ -191,6 +191,10 @@ public interface VideoFrameProcessor { * Returns the input {@link Surface}, where {@link VideoFrameProcessor} consumes input frames * from. * + *

The frames arriving on the {@link Surface} will not be consumed by the {@code + * VideoFrameProcessor} until {@link #registerInputStream} is called with {@link + * #INPUT_TYPE_SURFACE}. + * *

Can be called on any thread. * * @throws UnsupportedOperationException If the {@code VideoFrameProcessor} does not accept @@ -202,27 +206,11 @@ public interface VideoFrameProcessor { * Informs the {@code VideoFrameProcessor} that a new input stream will be queued with the list of * {@link Effect Effects} to apply to the new input stream. * - *

Call {@link #setInputFrameInfo} before this method if the {@link FrameInfo} of the new input - * stream differs from that of the current input stream. - * * @param inputType The {@link InputType} of the new input stream. * @param effects The list of {@link Effect effects} to apply to the new input stream. + * @param frameInfo The {@link FrameInfo} of the new input stream. */ - // TODO(b/286032822): Merge this and setInputFrameInfo. - void registerInputStream(@InputType int inputType, List effects); - - /** - * Sets information about the input frames. - * - *

The new input information is applied from the next frame {@linkplain #registerInputFrame() - * registered} or {@linkplain #queueInputTexture} queued} onwards. - * - *

Pixels are expanded using the {@link FrameInfo#pixelWidthHeightRatio} so that the output - * frames' pixels have a ratio of 1. - * - *

Can be called on any thread. - */ - void setInputFrameInfo(FrameInfo inputFrameInfo); + void registerInputStream(@InputType int inputType, List effects, FrameInfo frameInfo); /** * Informs the {@code VideoFrameProcessor} that a frame will be queued to its {@linkplain @@ -235,7 +223,7 @@ public interface VideoFrameProcessor { * @throws UnsupportedOperationException If the {@code VideoFrameProcessor} does not accept * {@linkplain #INPUT_TYPE_SURFACE surface input}. * @throws IllegalStateException If called after {@link #signalEndOfInput()} or before {@link - * #setInputFrameInfo(FrameInfo)}. + * #registerInputStream}. */ void registerInputFrame(); diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/BlankFrameProducer.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/BlankFrameProducer.java new file mode 100644 index 0000000000..e721d196d8 --- /dev/null +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/BlankFrameProducer.java @@ -0,0 +1,97 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; + +import androidx.media3.common.C; +import androidx.media3.common.GlObjectsProvider; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.GlUtil; +import java.util.List; +import java.util.concurrent.Executor; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** Produces blank frames with the given timestamps. */ +/* package */ final class BlankFrameProducer implements GlShaderProgram { + private final int width; + private final int height; + + private @MonotonicNonNull GlTextureInfo blankTexture; + private @MonotonicNonNull OutputListener outputListener; + + public BlankFrameProducer(int width, int height) { + this.width = width; + this.height = height; + } + + public void configureGlObjects() throws VideoFrameProcessingException { + try { + int texId = GlUtil.createTexture(width, height, /* useHighPrecisionColorComponents= */ false); + int fboId = GlUtil.createFboForTexture(texId); + blankTexture = new GlTextureInfo(texId, fboId, /* rboId= */ C.INDEX_UNSET, width, height); + GlUtil.focusFramebufferUsingCurrentContext(fboId, width, height); + GlUtil.clearFocusedBuffers(); + } catch (GlUtil.GlException e) { + throw new VideoFrameProcessingException(e); + } + } + + public void produceBlankFrames(List presentationTimesUs) { + checkNotNull(outputListener); + for (long presentationTimeUs : presentationTimesUs) { + outputListener.onOutputFrameAvailable(checkNotNull(blankTexture), presentationTimeUs); + } + } + + @Override + public void setInputListener(InputListener inputListener) {} + + @Override + public void setOutputListener(OutputListener outputListener) { + this.outputListener = outputListener; + } + + @Override + public void setErrorListener(Executor executor, ErrorListener errorListener) {} + + @Override + public void queueInputFrame( + GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { + // No input is queued in these tests. The BlankFrameProducer is used to produce frames. + throw new UnsupportedOperationException(); + } + + @Override + public void releaseOutputFrame(GlTextureInfo outputTexture) {} + + @Override + public void signalEndOfCurrentInputStream() { + checkNotNull(outputListener).onCurrentOutputStreamEnded(); + } + + @Override + public void flush() { + throw new UnsupportedOperationException(); + } + + @Override + public void release() { + // Do nothing as destroying the OpenGL context destroys the texture. + } +} diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java index be659202aa..18eea571ae 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java @@ -15,7 +15,6 @@ */ package androidx.media3.effect; -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; import static com.google.common.truth.Truth.assertThat; @@ -195,7 +194,6 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest { return new VideoFrameProcessorTestRunner.Builder() .setTestId(testId) .setVideoFrameProcessorFactory(new DefaultVideoFrameProcessor.Factory.Builder().build()) - .setInputType(INPUT_TYPE_BITMAP) .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) .setOnOutputFrameAvailableForRenderingListener( unused -> checkNotNull(framesProduced).incrementAndGet()); diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java index 32f4189be1..152efa7c0d 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java @@ -15,7 +15,6 @@ */ package androidx.media3.effect; -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; @@ -158,7 +157,6 @@ public final class DefaultVideoFrameProcessorPixelTest { String testId = "noEffects_withImageInput_matchesGoldenFile"; videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId) - .setInputType(INPUT_TYPE_BITMAP) .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) .build(); Bitmap originalBitmap = readBitmap(IMAGE_PNG_ASSET_PATH); @@ -180,7 +178,6 @@ public final class DefaultVideoFrameProcessorPixelTest { String testId = "wrappedCrop_withImageInput_matchesGoldenFile"; videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId) - .setInputType(INPUT_TYPE_BITMAP) .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) .setEffects( new GlEffectWrapper( @@ -214,7 +211,6 @@ public final class DefaultVideoFrameProcessorPixelTest { new DefaultVideoFrameProcessor.Factory.Builder() .setEnableColorTransfers(false) .build()) - .setInputType(INPUT_TYPE_BITMAP) .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) .setEffects(NO_OP_EFFECT) .build(); diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java index 29296a8c6a..8b7405cd70 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java @@ -350,8 +350,8 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest { checkNotNull(defaultVideoFrameProcessor) .registerInputStream( INPUT_TYPE_SURFACE, - /* effects= */ ImmutableList.of((GlEffect) (context, useHdr) -> blankFrameProducer)); - defaultVideoFrameProcessor.setInputFrameInfo(new FrameInfo.Builder(WIDTH, HEIGHT).build()); + /* effects= */ ImmutableList.of((GlEffect) (context, useHdr) -> blankFrameProducer), + new FrameInfo.Builder(WIDTH, HEIGHT).build()); blankFrameProducer.produceBlankFramesAndQueueEndOfStream(inputPresentationTimesUs); defaultVideoFrameProcessor.signalEndOfInput(); videoFrameProcessingEndedCountDownLatch.await(); diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/FrameDropTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/FrameDropTest.java index 0bfa1b5c15..1064c81064 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/FrameDropTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/FrameDropTest.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,18 +15,38 @@ */ package androidx.media3.effect; -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE; import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; +import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; +import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; -import androidx.media3.common.C; +import android.graphics.Bitmap; +import android.graphics.Color; +import android.text.Spannable; +import android.text.SpannableString; +import android.text.style.AbsoluteSizeSpan; +import android.text.style.ForegroundColorSpan; +import android.text.style.TypefaceSpan; +import androidx.annotation.Nullable; import androidx.media3.common.ColorInfo; -import androidx.media3.test.utils.VideoFrameProcessorTestRunner; +import androidx.media3.common.DebugViewProvider; +import androidx.media3.common.FrameInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.util.NullableType; +import androidx.media3.common.util.Util; +import androidx.media3.test.utils.TextureBitmapReader; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; -import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedQueue; +import com.google.common.util.concurrent.MoreExecutors; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -38,108 +58,184 @@ import org.junit.runner.RunWith; /** Tests for {@link FrameDropEffect}. */ @RunWith(AndroidJUnit4.class) public class FrameDropTest { - private static final String ORIGINAL_PNG_ASSET_PATH = - "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png"; + private static final String ASSET_PATH = "media/bitmap/FrameDropTest"; + private static final int BLANK_FRAME_WIDTH = 100; + private static final int BLANK_FRAME_HEIGHT = 50; - private static final String SCALE_WIDE_PNG_ASSET_PATH = - "media/bitmap/sample_mp4_first_frame/electrical_colors/scale_wide.png"; + private @MonotonicNonNull TextureBitmapReader textureBitmapReader; + private @MonotonicNonNull DefaultVideoFrameProcessor defaultVideoFrameProcessor; - private @MonotonicNonNull VideoFrameProcessorTestRunner videoFrameProcessorTestRunner; - - private @MonotonicNonNull Queue actualPresentationTimesUs; - - @EnsuresNonNull("actualPresentationTimesUs") + @EnsuresNonNull("textureBitmapReader") @Before public void setUp() { - actualPresentationTimesUs = new ConcurrentLinkedQueue<>(); + textureBitmapReader = new TextureBitmapReader(); } @After - public void release() { - checkNotNull(videoFrameProcessorTestRunner).release(); + public void tearDown() { + checkNotNull(defaultVideoFrameProcessor).release(); } - @RequiresNonNull("actualPresentationTimesUs") + @RequiresNonNull("textureBitmapReader") @Test public void frameDrop_withDefaultStrategy_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception { String testId = "frameDrop_withDefaultStrategy_outputsFramesAtTheCorrectPresentationTimesUs"; - videoFrameProcessorTestRunner = - getDefaultFrameProcessorTestRunnerBuilder( - testId, FrameDropEffect.createDefaultFrameDropEffect(/* targetFrameRate= */ 30)) - .setOnOutputFrameAvailableForRenderingListener(actualPresentationTimesUs::add) - .build(); + ImmutableList frameTimesUs = + ImmutableList.of(0L, 16_000L, 32_000L, 48_000L, 58_000L, 71_000L, 86_000L); - ImmutableList timestampsMs = ImmutableList.of(0, 16, 32, 48, 58, 71, 86); - for (int timestampMs : timestampsMs) { - videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(ORIGINAL_PNG_ASSET_PATH), - /* durationUs= */ C.MICROS_PER_SECOND, - /* offsetToAddUs= */ timestampMs * 1000L, - /* frameRate= */ 1); - } - videoFrameProcessorTestRunner.endFrameProcessing(); + ImmutableList actualPresentationTimesUs = + processFramesToEndOfStream( + frameTimesUs, FrameDropEffect.createDefaultFrameDropEffect(/* targetFrameRate= */ 30)); assertThat(actualPresentationTimesUs).containsExactly(0L, 32_000L, 71_000L).inOrder(); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId); } - @RequiresNonNull("actualPresentationTimesUs") + @RequiresNonNull("textureBitmapReader") @Test public void frameDrop_withSimpleStrategy_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception { String testId = "frameDrop_withSimpleStrategy_outputsFramesAtTheCorrectPresentationTimesUs"; - videoFrameProcessorTestRunner = - getDefaultFrameProcessorTestRunnerBuilder( - testId, - FrameDropEffect.createSimpleFrameDropEffect( - /* expectedFrameRate= */ 6, /* targetFrameRate= */ 2)) - .build(); + ImmutableList frameTimesUs = + ImmutableList.of(0L, 250_000L, 500_000L, 750_000L, 1_000_000L, 1_500_000L); - videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(ORIGINAL_PNG_ASSET_PATH), - /* durationUs= */ C.MICROS_PER_SECOND, - /* offsetToAddUs= */ 0L, - /* frameRate= */ 4); - videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(SCALE_WIDE_PNG_ASSET_PATH), - /* durationUs= */ C.MICROS_PER_SECOND, - /* offsetToAddUs= */ C.MICROS_PER_SECOND, - /* frameRate= */ 2); - videoFrameProcessorTestRunner.endFrameProcessing(); + ImmutableList actualPresentationTimesUs = + processFramesToEndOfStream( + frameTimesUs, + FrameDropEffect.createSimpleFrameDropEffect( + /* expectedFrameRate= */ 6, /* targetFrameRate= */ 2)); - assertThat(actualPresentationTimesUs).containsExactly(500_000L, 1_500_000L).inOrder(); + assertThat(actualPresentationTimesUs).containsExactly(0L, 750_000L).inOrder(); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId); } - @RequiresNonNull("actualPresentationTimesUs") + @RequiresNonNull("textureBitmapReader") @Test public void frameDrop_withSimpleStrategy_outputsAllFrames() throws Exception { - String testId = "frameDrop_withSimpleStrategy_outputsCorrectNumberOfFrames"; - videoFrameProcessorTestRunner = - getDefaultFrameProcessorTestRunnerBuilder( - testId, - FrameDropEffect.createSimpleFrameDropEffect( - /* expectedFrameRate= */ 3, /* targetFrameRate= */ 3)) - .build(); + String testId = "frameDrop_withSimpleStrategy_outputsAllFrames"; + ImmutableList frameTimesUs = ImmutableList.of(0L, 333_333L, 666_667L); - videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(ORIGINAL_PNG_ASSET_PATH), - /* durationUs= */ C.MICROS_PER_SECOND, - /* offsetToAddUs= */ 0L, - /* frameRate= */ 3); - videoFrameProcessorTestRunner.endFrameProcessing(); + ImmutableList actualPresentationTimesUs = + processFramesToEndOfStream( + frameTimesUs, + FrameDropEffect.createSimpleFrameDropEffect( + /* expectedFrameRate= */ 3, /* targetFrameRate= */ 3)); assertThat(actualPresentationTimesUs).containsExactly(0L, 333_333L, 666_667L).inOrder(); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId); } - @RequiresNonNull("actualPresentationTimesUs") - private VideoFrameProcessorTestRunner.Builder getDefaultFrameProcessorTestRunnerBuilder( - String testId, FrameDropEffect frameDropEffect) { - return new VideoFrameProcessorTestRunner.Builder() - .setTestId(testId) - .setVideoFrameProcessorFactory(new DefaultVideoFrameProcessor.Factory.Builder().build()) - .setInputType(INPUT_TYPE_BITMAP) - .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) - .setEffects(frameDropEffect) - .setOnOutputFrameAvailableForRenderingListener(actualPresentationTimesUs::add); + private static void getAndAssertOutputBitmaps( + TextureBitmapReader textureBitmapReader, List presentationTimesUs, String testId) + throws IOException { + for (int i = 0; i < presentationTimesUs.size(); i++) { + long presentationTimeUs = presentationTimesUs.get(i); + Bitmap actualBitmap = textureBitmapReader.getBitmap(presentationTimeUs); + Bitmap expectedBitmap = + readBitmap(Util.formatInvariant("%s/pts_%d.png", ASSET_PATH, presentationTimeUs)); + maybeSaveTestBitmap( + testId, String.valueOf(presentationTimeUs), actualBitmap, /* path= */ null); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference) + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + } + + @EnsuresNonNull("defaultVideoFrameProcessor") + private ImmutableList processFramesToEndOfStream( + List inputPresentationTimesUs, FrameDropEffect frameDropEffect) throws Exception { + AtomicReference<@NullableType VideoFrameProcessingException> + videoFrameProcessingExceptionReference = new AtomicReference<>(); + BlankFrameProducer blankFrameProducer = + new BlankFrameProducer(BLANK_FRAME_WIDTH, BLANK_FRAME_HEIGHT); + CountDownLatch videoFrameProcessingEndedCountDownLatch = new CountDownLatch(1); + ImmutableList.Builder actualPresentationTimesUs = new ImmutableList.Builder<>(); + + defaultVideoFrameProcessor = + checkNotNull( + new DefaultVideoFrameProcessor.Factory.Builder() + .setTextureOutput( + (outputTexture, presentationTimeUs, releaseOutputTextureCallback, token) -> { + checkNotNull(textureBitmapReader) + .readBitmap(outputTexture, presentationTimeUs); + releaseOutputTextureCallback.release(presentationTimeUs); + }, + /* textureOutputCapacity= */ 1) + .build() + .create( + getApplicationContext(), + DebugViewProvider.NONE, + /* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, + /* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, + /* renderFramesAutomatically= */ true, + MoreExecutors.directExecutor(), + new VideoFrameProcessor.Listener() { + @Override + public void onOutputSizeChanged(int width, int height) {} + + @Override + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + actualPresentationTimesUs.add(presentationTimeUs); + } + + @Override + public void onError(VideoFrameProcessingException exception) { + videoFrameProcessingExceptionReference.set(exception); + videoFrameProcessingEndedCountDownLatch.countDown(); + } + + @Override + public void onEnded() { + videoFrameProcessingEndedCountDownLatch.countDown(); + } + })); + + defaultVideoFrameProcessor.getTaskExecutor().submit(blankFrameProducer::configureGlObjects); + // A frame needs to be registered despite not queuing any external input to ensure + // that the video frame processor knows about the stream offset. + checkNotNull(defaultVideoFrameProcessor) + .registerInputStream( + INPUT_TYPE_SURFACE, + /* effects= */ ImmutableList.of( + (GlEffect) (context, useHdr) -> blankFrameProducer, + // Use an overlay effect to generate bitmaps with timestamps on it. + new OverlayEffect( + ImmutableList.of( + new TextOverlay() { + @Override + public SpannableString getText(long presentationTimeUs) { + SpannableString text = + new SpannableString(String.valueOf(presentationTimeUs)); + text.setSpan( + new ForegroundColorSpan(Color.BLACK), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new AbsoluteSizeSpan(/* size= */ 24), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new TypefaceSpan(/* family= */ "sans-serif"), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + return text; + } + })), + frameDropEffect), + new FrameInfo.Builder(BLANK_FRAME_WIDTH, BLANK_FRAME_HEIGHT).build()); + blankFrameProducer.produceBlankFrames(inputPresentationTimesUs); + defaultVideoFrameProcessor.signalEndOfInput(); + videoFrameProcessingEndedCountDownLatch.await(); + @Nullable + Exception videoFrameProcessingException = videoFrameProcessingExceptionReference.get(); + if (videoFrameProcessingException != null) { + throw videoFrameProcessingException; + } + return actualPresentationTimesUs.build(); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultFrameDroppingShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultFrameDroppingShaderProgram.java index 1ded34d7d5..77bc99ce41 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultFrameDroppingShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultFrameDroppingShaderProgram.java @@ -88,7 +88,9 @@ import androidx.media3.common.util.Size; copyTextureToPreviousFrame(glObjectsProvider, inputTexture, presentationTimeUs); getInputListener().onInputFrameProcessed(inputTexture); - getInputListener().onReadyToAcceptInputFrame(); + if (outputTexturePool.freeTextureCount() > 0) { + getInputListener().onReadyToAcceptInputFrame(); + } } @Override diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 9859e791ef..d51cea1945 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -418,7 +418,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) { checkState( hasRefreshedNextInputFrameInfo, - "setInputFrameInfo must be called before queueing another bitmap"); + "registerInputStream must be called before queueing another bitmap"); inputSwitcher .activeTextureManager() .queueInputBitmap( @@ -442,15 +442,19 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { @Override public Surface getInputSurface() { - return inputSwitcher.activeTextureManager().getInputSurface(); + return inputSwitcher.getInputSurface(); } @Override - public void registerInputStream(@InputType int inputType, List effects) { + public void registerInputStream( + @InputType int inputType, List effects, FrameInfo frameInfo) { + nextInputFrameInfo = adjustForPixelWidthHeightRatio(frameInfo); + hasRefreshedNextInputFrameInfo = true; synchronized (lock) { if (!processingInput) { videoFrameProcessingTaskExecutor.submitAndBlock(() -> configureEffects(effects)); - inputSwitcher.switchToInput(inputType); + inputSwitcher.switchToInput(inputType, nextInputFrameInfo); + inputSwitcher.activeTextureManager().setInputFrameInfo(nextInputFrameInfo); processingInput = true; return; } @@ -477,21 +481,14 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { // a new frame from the new input stream prematurely. videoFrameProcessingTaskExecutor.submitAndBlock(() -> configureEffects(effects)); } - inputSwitcher.switchToInput(inputType); - } - - @Override - public void setInputFrameInfo(FrameInfo inputFrameInfo) { - nextInputFrameInfo = adjustForPixelWidthHeightRatio(inputFrameInfo); - inputSwitcher.activeTextureManager().setInputFrameInfo(nextInputFrameInfo); - hasRefreshedNextInputFrameInfo = true; + inputSwitcher.switchToInput(inputType, nextInputFrameInfo); } @Override public void registerInputFrame() { checkState(!inputStreamEnded); checkStateNotNull( - nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames"); + nextInputFrameInfo, "registerInputStream must be called before registering input frames"); inputSwitcher.activeTextureManager().registerInputFrame(nextInputFrameInfo); hasRefreshedNextInputFrameInfo = false; diff --git a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java index 8edab8770b..c74a5a207b 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java @@ -17,6 +17,9 @@ package androidx.media3.effect; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; @@ -24,7 +27,9 @@ import static androidx.media3.common.util.Util.containsKey; import android.content.Context; import android.util.SparseArray; +import android.view.Surface; import androidx.media3.common.ColorInfo; +import androidx.media3.common.FrameInfo; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; @@ -84,7 +89,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; TextureManager textureManager; // TODO(b/274109008): Refactor DefaultShaderProgram to create a class just for sampling. switch (inputType) { - case VideoFrameProcessor.INPUT_TYPE_SURFACE: + case INPUT_TYPE_SURFACE: samplingShaderProgram = DefaultShaderProgram.createWithExternalSampler( context, @@ -98,7 +103,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; glObjectsProvider, samplingShaderProgram, videoFrameProcessingTaskExecutor); inputs.put(inputType, new Input(textureManager, samplingShaderProgram)); break; - case VideoFrameProcessor.INPUT_TYPE_BITMAP: + case INPUT_TYPE_BITMAP: samplingShaderProgram = DefaultShaderProgram.createWithInternalSampler( context, @@ -113,7 +118,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; glObjectsProvider, samplingShaderProgram, videoFrameProcessingTaskExecutor); inputs.put(inputType, new Input(textureManager, samplingShaderProgram)); break; - case VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID: + case INPUT_TYPE_TEXTURE_ID: samplingShaderProgram = DefaultShaderProgram.createWithInternalSampler( context, @@ -145,8 +150,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * registered}. * * @param newInputType The new {@link VideoFrameProcessor.InputType} to switch to. + * @param inputFrameInfo The {@link FrameInfo} associated with the new input. */ - public void switchToInput(@VideoFrameProcessor.InputType int newInputType) { + public void switchToInput( + @VideoFrameProcessor.InputType int newInputType, FrameInfo inputFrameInfo) { checkStateNotNull(downstreamShaderProgram); checkState(containsKey(inputs, newInputType), "Input type not registered: " + newInputType); @@ -167,6 +174,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; input.setActive(false); } } + checkNotNull(activeTextureManager).setInputFrameInfo(inputFrameInfo); } /** @@ -186,6 +194,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; checkNotNull(activeTextureManager).signalEndOfCurrentInputStream(); } + /** + * Returns the input {@link Surface}. + * + * @return The input {@link Surface}, regardless if the current input is {@linkplain + * #switchToInput set} to {@link VideoFrameProcessor#INPUT_TYPE_SURFACE}. + * @throws IllegalStateException If {@link VideoFrameProcessor#INPUT_TYPE_SURFACE} is not + * {@linkplain #registerInput registered}. + */ + public Surface getInputSurface() { + checkState(containsKey(inputs, INPUT_TYPE_SURFACE)); + return inputs.get(INPUT_TYPE_SURFACE).textureManager.getInputSurface(); + } + /** Releases the resources. */ public void release() throws VideoFrameProcessingException { for (int i = 0; i < inputs.size(); i++) { diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SimpleFrameDroppingShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/SimpleFrameDroppingShaderProgram.java index 32ac809e42..2ec6386d34 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SimpleFrameDroppingShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SimpleFrameDroppingShaderProgram.java @@ -57,13 +57,13 @@ import androidx.media3.common.VideoFrameProcessingException; @Override public void queueInputFrame( GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { - framesReceived++; if (framesReceived % n == 0) { super.queueInputFrame(glObjectsProvider, inputTexture, presentationTimeUs); } else { getInputListener().onInputFrameProcessed(inputTexture); getInputListener().onReadyToAcceptInputFrame(); } + framesReceived++; } @Override diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java index 14b98dc35e..9c5f6bfeb9 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java @@ -73,7 +73,11 @@ import androidx.media3.common.VideoFrameProcessor; /** * Sets information about the input frames. * - * @see VideoFrameProcessor#setInputFrameInfo + *

The new input information is applied from the next frame {@linkplain #registerInputFrame + * registered} or {@linkplain #queueInputTexture queued} onwards. + * + *

Pixels are expanded using the {@link FrameInfo#pixelWidthHeightRatio} so that the output + * frames' pixels have a ratio of 1. */ default void setInputFrameInfo(FrameInfo inputFrameInfo) { // Do nothing. diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 9bf9a88cb9..fd7d255ae6 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -2142,8 +2142,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } }); - videoFrameProcessor.registerInputStream( - VideoFrameProcessor.INPUT_TYPE_SURFACE, videoEffects); this.initialStreamOffsetUs = initialStreamOffsetUs; } catch (Exception e) { throw renderer.createRendererException( @@ -2226,7 +2224,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { */ public void setInputFormat(Format inputFormat) { checkNotNull(videoFrameProcessor) - .setInputFrameInfo( + .registerInputStream( + VideoFrameProcessor.INPUT_TYPE_SURFACE, + checkNotNull(videoEffects), new FrameInfo.Builder(inputFormat.width, inputFormat.height) .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio) .build()); diff --git a/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_0.png b/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_0.png new file mode 100644 index 0000000000000000000000000000000000000000..4ffb8030fd0cc9248247d2c2c83f484fb229831d GIT binary patch literal 591 zcmeAS@N?(olHy`uVBq!ia0vp^DL`z*!3HE(nbz$CQjEnx?oJHr&dIz4a#)I;JVQ8u zpoSx*11R^?)5S5QV$R!}w%H+$B5V)TUxyk>KNc72?CKJ15ozjJ(R(A}rZz{}wSR2& zP4$P41?6nrD8Rz0)U+bB!)XzlebS3;VG6qKheysqBKbz&QsZWv1Az>#t~QpK&ZI`K8vW()Kq8 zRPP+9?3wadiJMGoSdQ8@ z_}&}z{r%e;7T?F3nJf{%@j`OlN}1f3AI<}nX>O2QT@{}le)?kM?5Q%`mp*-ezx>U; zNK3Jl1y3U;n7&l_fBCxom91vlK#|B@_r$8^3%16~c1*2ny`^C(_N2RH_p2@5*QE81 zr%Ud;RIGENx@JP$i}pYF`lS!Yt=wsG_kPA#pgECyCfs_LUa%xPZ}t_vnMb_$ysO?d ztLV))J@5BDoGW)*Ywh(uR>&B>(LKxR-tp~eNnw-k+U~t_%r)k4;mbV{6Y5rp6^d{F zxC$6LPoo4b9kkW_`@b;a*4xh&UG7!icmAui6koDPx#1am@3R0s$N2z&@+hyVZuHAzH4RCt{2*x8R%RS?JVPY;75FrtEhAgF+-!Kk31 zXpC#(o@ineBXNyNeDYtwKS3XSQR9QVM&oWYaY@9W!MLE}h9o!$D&saVGAx1*bvk{y zG&4QZcZ|{VO-?#}?)}}ZPSvR@GMP*!lgVTtm#q|e1$9wp`$$bXXtVB;mnh{gAhs?oESS}3Gg~?civ&HJ#gx-Q? zZb5Sz4i=uZ80TOre!`C$D{>;Pz}+}ccwsjdru!V0?z5$EpF?qF6U}KO&f=7^_+m~aidsWo6~Pyb4$`Zlr(=8G}oXn9p9YpwFq|#Z+seWV>^z+ z(lq*n5q~Xc?!b|Frlh$(Y3k4RTN|2x4GZwR2s5{d{h&@4ZpS+D0Z&Tvi|~>#!p$Yk z!NPOa;!T{J=DV>1KjS{kENBkGb@&*siZEE$+>o_(p`VNojtX2y=Z{jUNh{o#|R@#VVfLqUNKd znb)SKZK@1e>+!I7*xn|`$EN#j$F4NDPK34|`@V3UJ!$L^2BKk{O-WPEbhf1X&%jY) z(M=K=(ho^jg{Bw3B+ZN_&pslmG)P|>&6q60$4uekJ!$^4G@8+Y**FH@6*S}1=(n#& z)fq@*g4m-v3z|QZt`*IMAvD`dno6kBAVs=yv`AqO6s{hFt!%`~0E)KtP~8Dt8s#reV@M+?_~03X=DrYJKBSL32|{u`n$ zA!?^q_r6+b3`f&3Oid+>mO%zYh*>M#zAycrE;jeo4s4WeT#98PPCp?w_BtbZ`OzqV zM}^Tc$QF^ezbInt9FezQCft0JMhGf04i|{rzb5LEl^RI^jg5Y=RbMNeyGLxwgBhk> zO(l$$L3U$n8oe5AjCrE1GT;6NIY_i8mWvAb0Z~hAEuFVh7^oiS%XDgVAa;onUA|fXgadV@%lye7iIbkVQ6Q50Hh#0Qw)WK2uv{4VyRxuR2BFXU#3 zgkYH{M4!Q1qRQVTyz;oDnJ8+EDm3-x=8pD1VYTUCO{|!uxJ1O=Ug7>nh>G`hJSm!q zdqsPx#1am@3R0s$N2z&@+hyVZt(Md!>RCt{2+3Rlk zkX>Iq&ty+e&&CEI1p*}0)PL{Olpe$ASK{*U?p&()0iKD&A=B-{9zKa0$AvoZZ-aa z6g5kme{0{8MFwFromSS#Jb3{g+ zs$(wFI-OpfwmK$Mrwur@j_HgrQ<8Z|4;TX;1TF!V0jq&~wIN^Xey-_O^AK>UXU5db z_sr%LbB|*dIOYW4;|wzjY(qx1Z${Nz=9%9dQ*z8lTIXTU-0hf!9QTJxY)1&+NbvEkpWYuXUa)cZfv$N&{U@I_~vb`B;@Sob4D)7EzLf6!0{rtnL^Z68* zl4m|dZoq@cNz79xk0~MN;9t!mq@lyW@|>m>If=u-xshfFIFEe2uodYPL%{hFW{a8& zBFtxwiMk3E86j?tMSg_C^?Ms|9PkA2RfcIrW+jQ413VdNjt8CsdI^t)+1j>g`n_H2 zJRM=$fVpb6MVe={PG|&8Syln>A+wZkkp{oO_!(Bnyc;m{bDABVy_; zGqlb~N}V(_H_}X5nt-E_p?WehHT(uyt*(eL{dG)ZhFL|pCx%R4q*)7WSF^f+xsYNr z*-1+oxC7}Sk3^aZ@F@9)c^SCfF?kL;F$aLh$ai%p1Gga~Fvi@f=5ZqL(?xFTry;%k zJLEx{n+h^3sRCzGoAxe7rh-`pU}>bOItDdO!1o!ZQotm2%E%9zn4QR_$~Rl}^9&;I zBSnVOb--Q7L%&d(^z*NPNo!!EW2%n1Gs7fx_BnsMR_nMYyk6U4OwzV%91}W+p*p!z zWQ3G~HORW*YsUAG7T_xV-s73e9kVB=S)===nG&^?v|A@J!=AZP_xU7XMj}nSV}?9) zh3@ky;i+&zTViq!tJAw7cP`(e?t!6K>=vzkZ$!KCakY__Xf%}17 zj#;k8uhZew37NAq%mDBJ@UwF~Zkz$0HXuWG9WrEh zAx}43kj2_|;CY~n+B=c5n)S#fXm%da6w$ax$mVSS%Kc#bU8oEEbE!VzF2( d7K`PO%|C_w@g`BaI9>n%002ovPDHLkV1hQoaw7l$ literal 0 HcmV?d00001 diff --git a/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_666667.png b/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_666667.png new file mode 100644 index 0000000000000000000000000000000000000000..cd7cfdcbd1f2f2945b73b05757da9cb19aa7168e GIT binary patch literal 1434 zcmV;L1!ek)P)Px#1am@3R0s$N2z&@+hyVZuL`g(JRCt{2nMrI_RT#&A9c&9$5uYSggFqWKv7iP7 zhlR6k72`m`DelA}YMjuG3M||i$0!RGjF`9)!Nk~_pfQ%}f-ZC;tyN5oR60;1B4Qaz zUHrexJKlHiefJGEhMZs0G`;u!?m71i-}%ldcsw4D$K&yMJRXn7L^ z22wa{P(yA*BW?f~A-)5XoVX5b23DKXZBFn#u5$=@0fn{>qOJ#*eI(3vpj@)S-IccOLYMM@j=sc8ic0$&01QaUZbUqv_v z%xNsp=|by38g2mobevK``6(!*jQ|&y_^#BfN8zH~{;mT~C3!TwgGTLgRLTF!dE1;7 z1vqb^*>ML7MQNNk+R$L#XzwYAVtTI*=ttq^8kAuzK&jwCU^|N2`$NtTC=6VS)}OG` z4g3*u`p}5G4#oI1PA_WAz?l!!n$u%-3p+c28Nl^uF%CQ3z`m4i^8uQrx1br9l+oo- zOouc9JJEvGiPnOnC}y4y>;g6eZ=x^|I6nfvMmXmIgE3COy{{8kf<|%JSx2_>8?BtM za|AfooGn(4>-5{b&w#}-&N}|xkx^U$bfCeq!ugie6f-5yvfmlhuv<``eHe|bhkz%6 zpHQX(a3<>fD5p8(Y)0WQ?0js_p^&rF%9)I)TiAIJc*2}b5zdkb=KAbC56ULoIC&DP{3Ke4Ywh|{^dWnIyJDPU5zd4;ccYv>%IR|MEk!dg z&Uwu4yUtCHb59;;dTgs`Pc{sE3(QJ*uIwyF4ZfA=BZLNEFB)NU>{>f&yscy#JnW1C z<7h(`I1}bXEshPS5y#19VC|^!K2LB)%n7ooWsWl~r4yzCc*)-LDCu|cIV5dv;t0-l zMo^p$GmJt)4GKH1GfMPX(l81MwPZt8TBjz$Nvd1mG}yJ^xukN!-xH<@4Tb^WQwm(j zU5e?DQTsh7;hFFbdep9+Lwq6;{s_06d`?haAx=QB<4>wv(s>yu49`ZV6@w|BFDtXq zN9;u(5hjB|$=}ZXGk_s;qB=5z8gtya?>cj1oDnN$p5yE`Cu;MVK_fnJD$MyUk27RW zBe|V+HcGXJQCeQoMzEL}GKMmt(}DX@RuNEzrfvrKn(P_X7|N_>0xQuz!F86QhUrEh z@0OF#`6lFS0Zv01&y0|ByW4+(36}hRX>+Fsw zXBD}Jnw997JnRgZ^K^s*T04%Qg}ju*$m5g}@2Y@HfU8jUQ3d=?_UPn50ZtFf^W&UF zz}1ek&nYKrH(ZJn)U7&&(}GThyX>dr*-?&@$V#-1RG|h<^Dok3oqE(z|2N0u@pwEQ okH_Qjcsw4D$K&yMJRXOC0PWql%6UBgg#Z8m07*qoM6N<$g2vyQ-~a#s literal 0 HcmV?d00001 diff --git a/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_71000.png b/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_71000.png new file mode 100644 index 0000000000000000000000000000000000000000..4fc6424832cfc70ff7870f8adf2eaa4be1ea51c1 GIT binary patch literal 1140 zcmV-)1dIELP)Px#1am@3R0s$N2z&@+hyVZt9!W$&RCt{2*~@QLRTRhZPs@c$c_<_nDZU7(7$9nV zfrzh(8V4pOMn^^m`ad{y=*YQK6EzrT4n@JJydp+|_yCQF0#ZOL&^CpFgSDF;j<=`x z_MS_8>~FFLJ$wJw>DglzGMP*!lgVT4>@l zavt9#n~wEjiToCyivffC@F))A6R}i1UEyb;2a0})V3Gncom!R5q6a{?=;bz zFKKqqQqwWLbvS8(K5WA6*ol3Gww-t#ui*_b3;(F_wG697K=>YZk5@5e4Q1EuF}nb^x#f$&p05Cx??r?y_l?O z*7ub9UKIOORbV>R)Iv7{vP5LZGLaqM)#P_v+yN%n`7#YtHMI-L#*?+9Ip3zH7P=Xb z_2Q^|SY*eUn*5%_S+NW*Y|=h?qBMRu$bcZ~ft z`aUVjAuGiFrd(kjNOmOIrGn;qQJyJlmf_lx=2Q!s0a3&}()niv-3rJWksU)KJ5E)1 zZ;K(Z{Pl_bX=R~pz9_}4#i3+n&4voiLHv-OpJ-8YNEAYAp&K{#evuvP@PQ~Xv~%&B zi4klU0pBc&dt1d4bE_!&^4~R9sAvV{2ViGi~KNU4Mi$R^O=BuPR)8iCX>lzGMP*!lgVTPx#1am@3R0s$N2z&@+hyVZuOG!jQRCt{2nOkgCMHGg=a!Nfdf=X#D6-=ZiUZ~Na z7%$XIP^(lQjJ^~#K8OKh)KC*mG=}Jl7ala4_(0JIW2{l1yjE*K)F?3;T6nM(6=_if z8^ud)m0o;UGw$hh=5+7FNomOXlV*DMp6{$Zd)CZavks_KDwRs5QmIrbl}e>jsZ=VJ zN~KaA+bS!9smRcG1Q>HZmjkn=VE&Cg7t1Fl89D5hHAAM^iW;6dOz;7_0*s07XgdVo#98%$hNCUd`oWY8;-6;_BD2{4C& z4+$qMP3C?F$&TVOA?LaVNlxC6JV#V%UQ8A>BHsKCviG{GHtn~_63!ukF!sRpru~ZW zK5Vyw^GRM*J>sotnAs_ro&dANVM0lTwX=+j6OuKC6jB~oi+Ik9On%S>$RNgmn~`j* z68H<)iDV`Nf&RY`-h+R0`c)w@sm5Fz2=uqixWf#a|0!VSkjD(Auw#=J%lsYbH%XgZn2du{A26JjcniBDt4Zcl0K z*vU;3FaKAmCZrt6II55!I27nJgiO>Bvd8Z+Tmx_;a5v&vPa>;ms$l4Pv)y{9gDET0 zPAZjZLK={KdmO2U>)$1`CN>OsXWXmJEaKfWx(ygd}MXR z4t@K8w~_OKdj|Ofcp2HYP9-_X+W;FIynB7bYh&XIFeaHg`<^xsahE!YxDjA?Kf>j1WdvOJ?5`)r|A z5@M!B_aGTj8-+8E9Pj}$@Y|T2Ok9n`o_$Ch0r9e4)dmh<^a=< z#DR4l(_@(HJ?13fl?d~-!{j2&U1Ths97&yJ0q_iR+HOT^NHL8gHLS~!Z7YYAS?Ykx zfqRj&$+N&)rvDH!aaSU7YQ)$&16XbTx`C%i4v8*P?=daFEyy7>#9R?zteqHR?TmU% z3sP9!h}65LMXM1HeUo@_b5#Z0hx{h-J#v8TL`o^Wz;nnJ=&A;(4DUj=iB9B1--ncE znwh-s*C2(>ogUL~7=INMV)~Gu4{j(CGo_9spKby!M$*`ADVZw6T!d6iXCYZmH*$XY zDZo@C-n*RSN1#67Q{wffQZfC=OU}wzJDKChbRDF8<$x1NCM0yS_mxLV+oQjsZ=VJ>X_AkMy;V0Rii4q00000NkvXXu0mjfxUZta literal 0 HcmV?d00001 diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TextureBitmapReader.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/TextureBitmapReader.java similarity index 84% rename from libraries/transformer/src/androidTest/java/androidx/media3/transformer/TextureBitmapReader.java rename to libraries/test_utils/src/main/java/androidx/media3/test/utils/TextureBitmapReader.java index 59b7c55972..5896478d08 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TextureBitmapReader.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/TextureBitmapReader.java @@ -13,32 +13,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package androidx.media3.transformer; + +package androidx.media3.test.utils; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.graphics.Bitmap; import android.view.Surface; +import androidx.annotation.Nullable; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; -import androidx.media3.effect.DefaultVideoFrameProcessor; -import androidx.media3.test.utils.BitmapPixelTestUtil; -import androidx.media3.test.utils.VideoFrameProcessorTestRunner; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; /** * {@inheritDoc} * *

Reads from an OpenGL texture. Only for use on physical devices. */ +@UnstableApi public final class TextureBitmapReader implements VideoFrameProcessorTestRunner.BitmapReader { + // TODO(b/239172735): This outputs an incorrect black output image on emulators. private final Map outputTimestampsToBitmaps; private boolean useHighPrecisionColorComponents; @@ -60,6 +61,10 @@ public final class TextureBitmapReader implements VideoFrameProcessorTestRunner. return checkStateNotNull(outputBitmap); } + /** + * @return The output {@link Bitmap} at a given {@code presentationTimeUs}. + * @throws IllegalStateException If no such bitmap is produced. + */ public Bitmap getBitmap(long presentationTimeUs) { return checkStateNotNull(outputTimestampsToBitmaps.get(presentationTimeUs)); } @@ -69,6 +74,11 @@ public final class TextureBitmapReader implements VideoFrameProcessorTestRunner. return outputTimestampsToBitmaps.keySet(); } + /** + * Reads the given {@code outputTexture}. + * + *

The read result can be fetched by calling one of me {@link #getBitmap} methods. + */ public void readBitmap(GlTextureInfo outputTexture, long presentationTimeUs) throws VideoFrameProcessingException { try { @@ -83,15 +93,6 @@ public final class TextureBitmapReader implements VideoFrameProcessorTestRunner. } } - public void readBitmapAndReleaseTexture( - GlTextureInfo outputTexture, - long presentationTimeUs, - DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseOutputTextureCallback) - throws VideoFrameProcessingException { - readBitmap(outputTexture, presentationTimeUs); - releaseOutputTextureCallback.release(presentationTimeUs); - } - private static Bitmap createBitmapFromCurrentGlFrameBuffer( int width, int height, boolean useHighPrecisionColorComponents) throws GlUtil.GlException { if (!useHighPrecisionColorComponents) { diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java index 79b232a9f1..bb25420e35 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java @@ -15,7 +15,9 @@ */ package androidx.media3.test.utils; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.test.utils.BitmapPixelTestUtil.createArgb8888BitmapFromRgba8888Image; @@ -41,7 +43,6 @@ import androidx.media3.common.GlTextureInfo; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; -import androidx.media3.common.VideoFrameProcessor.InputType; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.UnstableApi; import com.google.common.collect.ImmutableList; @@ -71,13 +72,11 @@ public final class VideoFrameProcessorTestRunner { private float pixelWidthHeightRatio; private @MonotonicNonNull ColorInfo inputColorInfo; private @MonotonicNonNull ColorInfo outputColorInfo; - private @InputType int inputType; private OnOutputFrameAvailableForRenderingListener onOutputFrameAvailableListener; /** Creates a new instance with default values. */ public Builder() { pixelWidthHeightRatio = DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO; - inputType = INPUT_TYPE_SURFACE; onOutputFrameAvailableListener = unused -> {}; } @@ -194,18 +193,6 @@ public final class VideoFrameProcessorTestRunner { return this; } - /** - * Sets whether input comes from an external texture. See {@link - * VideoFrameProcessor.Factory#create}. - * - *

The default value is {@link VideoFrameProcessor#INPUT_TYPE_SURFACE}. - */ - @CanIgnoreReturnValue - public Builder setInputType(@InputType int inputType) { - this.inputType = inputType; - return this; - } - /** * Sets the method to be called in {@link * VideoFrameProcessor.Listener#onOutputFrameAvailableForRendering}. @@ -233,7 +220,6 @@ public final class VideoFrameProcessorTestRunner { pixelWidthHeightRatio, inputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : inputColorInfo, outputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : outputColorInfo, - inputType, onOutputFrameAvailableListener); } } @@ -251,6 +237,7 @@ public final class VideoFrameProcessorTestRunner { private final @MonotonicNonNull CountDownLatch videoFrameProcessingEndedLatch; private final AtomicReference videoFrameProcessingException; private final VideoFrameProcessor videoFrameProcessor; + private final ImmutableList effects; private @MonotonicNonNull BitmapReader bitmapReader; @@ -264,7 +251,6 @@ public final class VideoFrameProcessorTestRunner { float pixelWidthHeightRatio, ColorInfo inputColorInfo, ColorInfo outputColorInfo, - @InputType int inputType, OnOutputFrameAvailableForRenderingListener onOutputFrameAvailableForRenderingListener) throws VideoFrameProcessingException { this.testId = testId; @@ -314,7 +300,7 @@ public final class VideoFrameProcessorTestRunner { checkNotNull(videoFrameProcessingEndedLatch).countDown(); } }); - videoFrameProcessor.registerInputStream(inputType, effects); + this.effects = effects; } public void processFirstFrameAndEnd() throws Exception { @@ -323,7 +309,9 @@ public final class VideoFrameProcessorTestRunner { new DecodeOneFrameUtil.Listener() { @Override public void onContainerExtracted(MediaFormat mediaFormat) { - videoFrameProcessor.setInputFrameInfo( + videoFrameProcessor.registerInputStream( + INPUT_TYPE_SURFACE, + effects, new FrameInfo.Builder( mediaFormat.getInteger(MediaFormat.KEY_WIDTH), mediaFormat.getInteger(MediaFormat.KEY_HEIGHT)) @@ -343,7 +331,9 @@ public final class VideoFrameProcessorTestRunner { public void queueInputBitmap( Bitmap inputBitmap, long durationUs, long offsetToAddUs, float frameRate) { - videoFrameProcessor.setInputFrameInfo( + videoFrameProcessor.registerInputStream( + INPUT_TYPE_BITMAP, + effects, new FrameInfo.Builder(inputBitmap.getWidth(), inputBitmap.getHeight()) .setPixelWidthHeightRatio(pixelWidthHeightRatio) .setOffsetToAddUs(offsetToAddUs) @@ -352,7 +342,9 @@ public final class VideoFrameProcessorTestRunner { } public void queueInputTexture(GlTextureInfo inputTexture, long pts) { - videoFrameProcessor.setInputFrameInfo( + videoFrameProcessor.registerInputStream( + INPUT_TYPE_TEXTURE_ID, + effects, new FrameInfo.Builder(inputTexture.width, inputTexture.height) .setPixelWidthHeightRatio(pixelWidthHeightRatio) .build()); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java index 75808369db..68f1d4441b 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java @@ -15,7 +15,6 @@ */ package androidx.media3.transformer; -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE; import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; @@ -39,6 +38,7 @@ import androidx.media3.effect.RgbFilter; import androidx.media3.effect.ScaleAndRotateTransformation; import androidx.media3.effect.VideoCompositor; import androidx.media3.test.utils.BitmapPixelTestUtil; +import androidx.media3.test.utils.TextureBitmapReader; import androidx.media3.test.utils.VideoFrameProcessorTestRunner; import com.google.common.collect.ImmutableList; import java.io.IOException; @@ -334,7 +334,6 @@ public final class VideoCompositorPixelTest { return new VideoFrameProcessorTestRunner.Builder() .setTestId(testId) .setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactoryBuilder.build()) - .setInputType(INPUT_TYPE_BITMAP) .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) .setBitmapReader(textureBitmapReader); } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorMultipleTextureOutputPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorMultipleTextureOutputPixelTest.java index 6adc4460c8..cd1c57fbca 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorMultipleTextureOutputPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorMultipleTextureOutputPixelTest.java @@ -15,7 +15,6 @@ */ package androidx.media3.transformer.mh; -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE; import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; @@ -28,8 +27,8 @@ import androidx.media3.common.ColorInfo; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.effect.DefaultVideoFrameProcessor; import androidx.media3.test.utils.BitmapPixelTestUtil; +import androidx.media3.test.utils.TextureBitmapReader; import androidx.media3.test.utils.VideoFrameProcessorTestRunner; -import androidx.media3.transformer.TextureBitmapReader; import androidx.test.ext.junit.runners.AndroidJUnit4; import java.util.Set; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -142,16 +141,15 @@ public class DefaultVideoFrameProcessorMultipleTextureOutputPixelTest { (outputTexture, presentationTimeUs, releaseOutputTextureCallback, - unusedSyncObject) -> - checkNotNull(textureBitmapReader) - .readBitmapAndReleaseTexture( - outputTexture, presentationTimeUs, releaseOutputTextureCallback), + unusedSyncObject) -> { + checkNotNull(textureBitmapReader).readBitmap(outputTexture, presentationTimeUs); + releaseOutputTextureCallback.release(presentationTimeUs); + }, /* textureOutputCapacity= */ 1) .build(); return new VideoFrameProcessorTestRunner.Builder() .setTestId(testId) .setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactory) - .setInputType(INPUT_TYPE_BITMAP) .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) .setBitmapReader(textureBitmapReader); } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java index 61f08c1287..03f1849324 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java @@ -38,17 +38,16 @@ import androidx.media3.common.Format; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.GlUtil; import androidx.media3.effect.BitmapOverlay; import androidx.media3.effect.DefaultGlObjectsProvider; import androidx.media3.effect.DefaultVideoFrameProcessor; import androidx.media3.effect.OverlayEffect; import androidx.media3.test.utils.BitmapPixelTestUtil; +import androidx.media3.test.utils.TextureBitmapReader; import androidx.media3.test.utils.VideoFrameProcessorTestRunner; import androidx.media3.transformer.AndroidTestUtil; import androidx.media3.transformer.EncoderUtil; -import androidx.media3.transformer.TextureBitmapReader; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; import java.util.List; @@ -537,9 +536,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { (outputTexture, presentationTimeUs1, releaseOutputTextureCallback1, - unusedSyncObject) -> - bitmapReader.readBitmapAndReleaseTexture( - outputTexture, presentationTimeUs1, releaseOutputTextureCallback1), + unusedSyncObject) -> { + bitmapReader.readBitmap(outputTexture, presentationTimeUs1); + releaseOutputTextureCallback1.release(presentationTimeUs1); + }, /* textureOutputCapacity= */ 1) .setGlObjectsProvider(contextSharingGlObjectsProvider) .build(); @@ -550,7 +550,6 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { .setInputColorInfo(colorInfo) .setOutputColorInfo(colorInfo) .setBitmapReader(bitmapReader) - .setInputType(VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID) .setEffects(effects) .build(); GlUtil.awaitSyncObject(syncObject); @@ -573,9 +572,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { (outputTexture, presentationTimeUs, releaseOutputTextureCallback, - unusedSyncObject) -> - textureBitmapReader.readBitmapAndReleaseTexture( - outputTexture, presentationTimeUs, releaseOutputTextureCallback), + unusedSyncObject) -> { + textureBitmapReader.readBitmap(outputTexture, presentationTimeUs); + releaseOutputTextureCallback.release(presentationTimeUs); + }, /* textureOutputCapacity= */ 1) .build(); return new VideoFrameProcessorTestRunner.Builder() diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/FrameDropPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/FrameDropPixelTest.java deleted file mode 100644 index 50c24e5439..0000000000 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/FrameDropPixelTest.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.transformer.mh; - -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; -import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE; -import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; -import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; -import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; -import static com.google.common.truth.Truth.assertThat; - -import android.graphics.Bitmap; -import androidx.media3.common.C; -import androidx.media3.common.ColorInfo; -import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.VideoFrameProcessor; -import androidx.media3.effect.DefaultVideoFrameProcessor; -import androidx.media3.effect.FrameDropEffect; -import androidx.media3.test.utils.VideoFrameProcessorTestRunner; -import androidx.media3.transformer.TextureBitmapReader; -import androidx.test.ext.junit.runners.AndroidJUnit4; -import org.checkerframework.checker.nullness.qual.EnsuresNonNull; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; - -/** - * Tests to ensure {@link FrameDropEffect} outputs the correct frame associated with a chosen - * timestamp. - */ -@RunWith(AndroidJUnit4.class) -public class FrameDropPixelTest { - private static final String ORIGINAL_PNG_ASSET_PATH = - "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png"; - private static final String MEDIA3_TEST_PNG_ASSET_PATH = - "media/bitmap/input_images/media3test.png"; - private static final String ROTATE_90_PNG_ASSET_PATH = - "media/bitmap/sample_mp4_first_frame/electrical_colors/rotate90.png"; - private static final String SRGB_TO_ELECTRICAL_ORIGINAL_PNG_ASSET_PATH = - "media/bitmap/sample_mp4_first_frame/electrical_colors/srgb_to_electrical_original.png"; - private static final String SRGB_TO_ELECTRICAL_MEDIA3_TEST_PNG_ASSET_PATH = - "media/bitmap/sample_mp4_first_frame/electrical_colors/srgb_to_electrical_media3test.png"; - - private @MonotonicNonNull TextureBitmapReader textureBitmapReader; - private @MonotonicNonNull VideoFrameProcessorTestRunner videoFrameProcessorTestRunner; - - @EnsuresNonNull("textureBitmapReader") - @Before - public void setUp() { - textureBitmapReader = new TextureBitmapReader(); - } - - @After - public void tearDown() { - checkNotNull(videoFrameProcessorTestRunner).release(); - } - - @RequiresNonNull("textureBitmapReader") - @Test - public void frameDrop_withDefaultStrategy_outputsCorrectFramesAtTheCorrectPresentationTimesUs() - throws Exception { - String testId = - "frameDrop_withDefaultStrategy_outputsCorrectFramesAtTheCorrectPresentationTimesUs"; - videoFrameProcessorTestRunner = - createDefaultFrameProcessorTestRunnerBuilder( - testId, FrameDropEffect.createDefaultFrameDropEffect(/* targetFrameRate= */ 30)); - - long expectedPresentationTimeUs1 = 0; - long expectedPresentationTimeUs2 = 32_000; - long expectedPresentationTimeUs3 = 71_000; - Bitmap chosenBitmap1 = readBitmap(ORIGINAL_PNG_ASSET_PATH); - Bitmap chosenBitmap2 = readBitmap(MEDIA3_TEST_PNG_ASSET_PATH); - Bitmap droppedFrameBitmap = readBitmap(ROTATE_90_PNG_ASSET_PATH); - queueOneFrameAt(chosenBitmap1, expectedPresentationTimeUs1); - queueOneFrameAt(droppedFrameBitmap, /* presentationTimeUs= */ 16_000L); - queueOneFrameAt(chosenBitmap2, expectedPresentationTimeUs2); - queueOneFrameAt(droppedFrameBitmap, /* presentationTimeUs= */ 48_000L); - queueOneFrameAt(droppedFrameBitmap, /* presentationTimeUs= */ 58_000L); - queueOneFrameAt(chosenBitmap1, expectedPresentationTimeUs3); - queueOneFrameAt(droppedFrameBitmap, /* presentationTimeUs= */ 86_000L); - videoFrameProcessorTestRunner.endFrameProcessing(); - - assertThat(textureBitmapReader.getOutputTimestamps()) - .containsExactly( - expectedPresentationTimeUs1, expectedPresentationTimeUs2, expectedPresentationTimeUs3) - .inOrder(); - assertThat( - getBitmapAveragePixelAbsoluteDifferenceArgb8888( - readBitmap(SRGB_TO_ELECTRICAL_ORIGINAL_PNG_ASSET_PATH), - textureBitmapReader.getBitmap(expectedPresentationTimeUs1), - testId)) - .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE); - assertThat( - getBitmapAveragePixelAbsoluteDifferenceArgb8888( - readBitmap(SRGB_TO_ELECTRICAL_MEDIA3_TEST_PNG_ASSET_PATH), - textureBitmapReader.getBitmap(expectedPresentationTimeUs2), - testId)) - .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE); - assertThat( - getBitmapAveragePixelAbsoluteDifferenceArgb8888( - readBitmap(SRGB_TO_ELECTRICAL_ORIGINAL_PNG_ASSET_PATH), - textureBitmapReader.getBitmap(expectedPresentationTimeUs3), - testId)) - .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE); - } - - @RequiresNonNull("textureBitmapReader") - @Test - public void frameDrop_withSimpleStrategy_outputsCorrectFramesAtTheCorrectPresentationTimesUs() - throws Exception { - String testId = - "frameDrop_withSimpleStrategy_outputsCorrectFramesAtTheCorrectPresentationTimesUs"; - videoFrameProcessorTestRunner = - createDefaultFrameProcessorTestRunnerBuilder( - testId, - FrameDropEffect.createSimpleFrameDropEffect( - /* expectedFrameRate= */ 6, /* targetFrameRate= */ 2)); - long expectedPresentationTimeUs1 = 500_000; - long expectedPresentationTimeUs2 = 1_500_000; - videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(ORIGINAL_PNG_ASSET_PATH), - /* durationUs= */ C.MICROS_PER_SECOND, - /* offsetToAddUs= */ 0L, - /* frameRate= */ 4); - videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(MEDIA3_TEST_PNG_ASSET_PATH), - /* durationUs= */ C.MICROS_PER_SECOND, - /* offsetToAddUs= */ C.MICROS_PER_SECOND, - /* frameRate= */ 2); - videoFrameProcessorTestRunner.endFrameProcessing(); - - assertThat(textureBitmapReader.getOutputTimestamps()) - .containsExactly(expectedPresentationTimeUs1, expectedPresentationTimeUs2) - .inOrder(); - Bitmap actualBitmap1 = textureBitmapReader.getBitmap(expectedPresentationTimeUs1); - maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual1", actualBitmap1, /* path= */ null); - Bitmap actualBitmap2 = textureBitmapReader.getBitmap(expectedPresentationTimeUs2); - maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual2", actualBitmap2, /* path= */ null); - assertThat( - getBitmapAveragePixelAbsoluteDifferenceArgb8888( - readBitmap(SRGB_TO_ELECTRICAL_ORIGINAL_PNG_ASSET_PATH), actualBitmap1, testId)) - .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE); - assertThat( - getBitmapAveragePixelAbsoluteDifferenceArgb8888( - readBitmap(SRGB_TO_ELECTRICAL_MEDIA3_TEST_PNG_ASSET_PATH), actualBitmap2, testId)) - .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE); - } - - @RequiresNonNull("textureBitmapReader") - private VideoFrameProcessorTestRunner createDefaultFrameProcessorTestRunnerBuilder( - String testId, FrameDropEffect frameDropEffect) throws VideoFrameProcessingException { - VideoFrameProcessor.Factory defaultVideoFrameProcessorFactory = - new DefaultVideoFrameProcessor.Factory.Builder() - .setTextureOutput( - (outputTexture, presentationTimeUs, releaseOutputTextureCallback, token) -> - checkNotNull(textureBitmapReader) - .readBitmapAndReleaseTexture( - outputTexture, presentationTimeUs, releaseOutputTextureCallback), - /* textureOutputCapacity= */ 1) - .build(); - return new VideoFrameProcessorTestRunner.Builder() - .setTestId(testId) - .setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactory) - .setInputType(INPUT_TYPE_BITMAP) - .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) - .setEffects(frameDropEffect) - .build(); - } - - /** - * Queues a {@link Bitmap} into the {@link VideoFrameProcessor} so that exactly one frame is - * produced at the given {@code presentationTimeUs}. - */ - private void queueOneFrameAt(Bitmap bitmap, long presentationTimeUs) { - checkNotNull(videoFrameProcessorTestRunner) - .queueInputBitmap( - bitmap, - /* durationUs= */ C.MICROS_PER_SECOND, - /* offsetToAddUs= */ presentationTimeUs, - /* frameRate= */ 1); - } -} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java index 958a55b2b6..ac15d05570 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java @@ -158,8 +158,7 @@ import java.util.concurrent.atomic.AtomicLong; Size decodedSize = getDecodedSize(trackFormat); videoFrameProcessor.registerInputStream( getInputType(checkNotNull(trackFormat.sampleMimeType)), - createEffectListWithPresentation(editedMediaItem.effects.videoEffects, presentation)); - videoFrameProcessor.setInputFrameInfo( + createEffectListWithPresentation(editedMediaItem.effects.videoEffects, presentation), new FrameInfo.Builder(decodedSize.getWidth(), decodedSize.getHeight()) .setPixelWidthHeightRatio(trackFormat.pixelWidthHeightRatio) .setOffsetToAddUs(mediaItemOffsetUs.get())