diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index 8cdcda9972..599ecfe4ac 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -168,8 +168,8 @@ public interface VideoFrameProcessor { /** * Provides an input texture ID to the {@code VideoFrameProcessor}. * - *

It must be called after the {@link #setOnInputFrameProcessedListener - * onInputFrameProcessedListener} and the {@link #setInputFrameInfo frameInfo} have been set. + *

It must be only called after {@link #setOnInputFrameProcessedListener} and {@link + * #registerInputStream} have been called. * *

Can be called on any thread. * @@ -191,6 +191,10 @@ public interface VideoFrameProcessor { * Returns the input {@link Surface}, where {@link VideoFrameProcessor} consumes input frames * from. * + *

The frames arriving on the {@link Surface} will not be consumed by the {@code + * VideoFrameProcessor} until {@link #registerInputStream} is called with {@link + * #INPUT_TYPE_SURFACE}. + * *

Can be called on any thread. * * @throws UnsupportedOperationException If the {@code VideoFrameProcessor} does not accept @@ -202,27 +206,11 @@ public interface VideoFrameProcessor { * Informs the {@code VideoFrameProcessor} that a new input stream will be queued with the list of * {@link Effect Effects} to apply to the new input stream. * - *

Call {@link #setInputFrameInfo} before this method if the {@link FrameInfo} of the new input - * stream differs from that of the current input stream. - * * @param inputType The {@link InputType} of the new input stream. * @param effects The list of {@link Effect effects} to apply to the new input stream. + * @param frameInfo The {@link FrameInfo} of the new input stream. */ - // TODO(b/286032822): Merge this and setInputFrameInfo. - void registerInputStream(@InputType int inputType, List effects); - - /** - * Sets information about the input frames. - * - *

The new input information is applied from the next frame {@linkplain #registerInputFrame() - * registered} or {@linkplain #queueInputTexture} queued} onwards. - * - *

Pixels are expanded using the {@link FrameInfo#pixelWidthHeightRatio} so that the output - * frames' pixels have a ratio of 1. - * - *

Can be called on any thread. - */ - void setInputFrameInfo(FrameInfo inputFrameInfo); + void registerInputStream(@InputType int inputType, List effects, FrameInfo frameInfo); /** * Informs the {@code VideoFrameProcessor} that a frame will be queued to its {@linkplain @@ -235,7 +223,7 @@ public interface VideoFrameProcessor { * @throws UnsupportedOperationException If the {@code VideoFrameProcessor} does not accept * {@linkplain #INPUT_TYPE_SURFACE surface input}. * @throws IllegalStateException If called after {@link #signalEndOfInput()} or before {@link - * #setInputFrameInfo(FrameInfo)}. + * #registerInputStream}. */ void registerInputFrame(); diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/BlankFrameProducer.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/BlankFrameProducer.java new file mode 100644 index 0000000000..e721d196d8 --- /dev/null +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/BlankFrameProducer.java @@ -0,0 +1,97 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; + +import androidx.media3.common.C; +import androidx.media3.common.GlObjectsProvider; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.GlUtil; +import java.util.List; +import java.util.concurrent.Executor; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** Produces blank frames with the given timestamps. */ +/* package */ final class BlankFrameProducer implements GlShaderProgram { + private final int width; + private final int height; + + private @MonotonicNonNull GlTextureInfo blankTexture; + private @MonotonicNonNull OutputListener outputListener; + + public BlankFrameProducer(int width, int height) { + this.width = width; + this.height = height; + } + + public void configureGlObjects() throws VideoFrameProcessingException { + try { + int texId = GlUtil.createTexture(width, height, /* useHighPrecisionColorComponents= */ false); + int fboId = GlUtil.createFboForTexture(texId); + blankTexture = new GlTextureInfo(texId, fboId, /* rboId= */ C.INDEX_UNSET, width, height); + GlUtil.focusFramebufferUsingCurrentContext(fboId, width, height); + GlUtil.clearFocusedBuffers(); + } catch (GlUtil.GlException e) { + throw new VideoFrameProcessingException(e); + } + } + + public void produceBlankFrames(List presentationTimesUs) { + checkNotNull(outputListener); + for (long presentationTimeUs : presentationTimesUs) { + outputListener.onOutputFrameAvailable(checkNotNull(blankTexture), presentationTimeUs); + } + } + + @Override + public void setInputListener(InputListener inputListener) {} + + @Override + public void setOutputListener(OutputListener outputListener) { + this.outputListener = outputListener; + } + + @Override + public void setErrorListener(Executor executor, ErrorListener errorListener) {} + + @Override + public void queueInputFrame( + GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { + // No input is queued in these tests. The BlankFrameProducer is used to produce frames. + throw new UnsupportedOperationException(); + } + + @Override + public void releaseOutputFrame(GlTextureInfo outputTexture) {} + + @Override + public void signalEndOfCurrentInputStream() { + checkNotNull(outputListener).onCurrentOutputStreamEnded(); + } + + @Override + public void flush() { + throw new UnsupportedOperationException(); + } + + @Override + public void release() { + // Do nothing as destroying the OpenGL context destroys the texture. + } +} diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java index be659202aa..18eea571ae 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java @@ -15,7 +15,6 @@ */ package androidx.media3.effect; -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; import static com.google.common.truth.Truth.assertThat; @@ -195,7 +194,6 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest { return new VideoFrameProcessorTestRunner.Builder() .setTestId(testId) .setVideoFrameProcessorFactory(new DefaultVideoFrameProcessor.Factory.Builder().build()) - .setInputType(INPUT_TYPE_BITMAP) .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) .setOnOutputFrameAvailableForRenderingListener( unused -> checkNotNull(framesProduced).incrementAndGet()); diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java index 32f4189be1..152efa7c0d 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java @@ -15,7 +15,6 @@ */ package androidx.media3.effect; -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; @@ -158,7 +157,6 @@ public final class DefaultVideoFrameProcessorPixelTest { String testId = "noEffects_withImageInput_matchesGoldenFile"; videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId) - .setInputType(INPUT_TYPE_BITMAP) .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) .build(); Bitmap originalBitmap = readBitmap(IMAGE_PNG_ASSET_PATH); @@ -180,7 +178,6 @@ public final class DefaultVideoFrameProcessorPixelTest { String testId = "wrappedCrop_withImageInput_matchesGoldenFile"; videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId) - .setInputType(INPUT_TYPE_BITMAP) .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) .setEffects( new GlEffectWrapper( @@ -214,7 +211,6 @@ public final class DefaultVideoFrameProcessorPixelTest { new DefaultVideoFrameProcessor.Factory.Builder() .setEnableColorTransfers(false) .build()) - .setInputType(INPUT_TYPE_BITMAP) .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) .setEffects(NO_OP_EFFECT) .build(); diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java index 29296a8c6a..8b7405cd70 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java @@ -350,8 +350,8 @@ public final class DefaultVideoFrameProcessorVideoFrameRenderingTest { checkNotNull(defaultVideoFrameProcessor) .registerInputStream( INPUT_TYPE_SURFACE, - /* effects= */ ImmutableList.of((GlEffect) (context, useHdr) -> blankFrameProducer)); - defaultVideoFrameProcessor.setInputFrameInfo(new FrameInfo.Builder(WIDTH, HEIGHT).build()); + /* effects= */ ImmutableList.of((GlEffect) (context, useHdr) -> blankFrameProducer), + new FrameInfo.Builder(WIDTH, HEIGHT).build()); blankFrameProducer.produceBlankFramesAndQueueEndOfStream(inputPresentationTimesUs); defaultVideoFrameProcessor.signalEndOfInput(); videoFrameProcessingEndedCountDownLatch.await(); diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/FrameDropTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/FrameDropTest.java index 0bfa1b5c15..1064c81064 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/FrameDropTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/FrameDropTest.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * https://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,18 +15,38 @@ */ package androidx.media3.effect; -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE; import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; +import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; +import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; -import androidx.media3.common.C; +import android.graphics.Bitmap; +import android.graphics.Color; +import android.text.Spannable; +import android.text.SpannableString; +import android.text.style.AbsoluteSizeSpan; +import android.text.style.ForegroundColorSpan; +import android.text.style.TypefaceSpan; +import androidx.annotation.Nullable; import androidx.media3.common.ColorInfo; -import androidx.media3.test.utils.VideoFrameProcessorTestRunner; +import androidx.media3.common.DebugViewProvider; +import androidx.media3.common.FrameInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.util.NullableType; +import androidx.media3.common.util.Util; +import androidx.media3.test.utils.TextureBitmapReader; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; -import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedQueue; +import com.google.common.util.concurrent.MoreExecutors; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -38,108 +58,184 @@ import org.junit.runner.RunWith; /** Tests for {@link FrameDropEffect}. */ @RunWith(AndroidJUnit4.class) public class FrameDropTest { - private static final String ORIGINAL_PNG_ASSET_PATH = - "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png"; + private static final String ASSET_PATH = "media/bitmap/FrameDropTest"; + private static final int BLANK_FRAME_WIDTH = 100; + private static final int BLANK_FRAME_HEIGHT = 50; - private static final String SCALE_WIDE_PNG_ASSET_PATH = - "media/bitmap/sample_mp4_first_frame/electrical_colors/scale_wide.png"; + private @MonotonicNonNull TextureBitmapReader textureBitmapReader; + private @MonotonicNonNull DefaultVideoFrameProcessor defaultVideoFrameProcessor; - private @MonotonicNonNull VideoFrameProcessorTestRunner videoFrameProcessorTestRunner; - - private @MonotonicNonNull Queue actualPresentationTimesUs; - - @EnsuresNonNull("actualPresentationTimesUs") + @EnsuresNonNull("textureBitmapReader") @Before public void setUp() { - actualPresentationTimesUs = new ConcurrentLinkedQueue<>(); + textureBitmapReader = new TextureBitmapReader(); } @After - public void release() { - checkNotNull(videoFrameProcessorTestRunner).release(); + public void tearDown() { + checkNotNull(defaultVideoFrameProcessor).release(); } - @RequiresNonNull("actualPresentationTimesUs") + @RequiresNonNull("textureBitmapReader") @Test public void frameDrop_withDefaultStrategy_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception { String testId = "frameDrop_withDefaultStrategy_outputsFramesAtTheCorrectPresentationTimesUs"; - videoFrameProcessorTestRunner = - getDefaultFrameProcessorTestRunnerBuilder( - testId, FrameDropEffect.createDefaultFrameDropEffect(/* targetFrameRate= */ 30)) - .setOnOutputFrameAvailableForRenderingListener(actualPresentationTimesUs::add) - .build(); + ImmutableList frameTimesUs = + ImmutableList.of(0L, 16_000L, 32_000L, 48_000L, 58_000L, 71_000L, 86_000L); - ImmutableList timestampsMs = ImmutableList.of(0, 16, 32, 48, 58, 71, 86); - for (int timestampMs : timestampsMs) { - videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(ORIGINAL_PNG_ASSET_PATH), - /* durationUs= */ C.MICROS_PER_SECOND, - /* offsetToAddUs= */ timestampMs * 1000L, - /* frameRate= */ 1); - } - videoFrameProcessorTestRunner.endFrameProcessing(); + ImmutableList actualPresentationTimesUs = + processFramesToEndOfStream( + frameTimesUs, FrameDropEffect.createDefaultFrameDropEffect(/* targetFrameRate= */ 30)); assertThat(actualPresentationTimesUs).containsExactly(0L, 32_000L, 71_000L).inOrder(); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId); } - @RequiresNonNull("actualPresentationTimesUs") + @RequiresNonNull("textureBitmapReader") @Test public void frameDrop_withSimpleStrategy_outputsFramesAtTheCorrectPresentationTimesUs() throws Exception { String testId = "frameDrop_withSimpleStrategy_outputsFramesAtTheCorrectPresentationTimesUs"; - videoFrameProcessorTestRunner = - getDefaultFrameProcessorTestRunnerBuilder( - testId, - FrameDropEffect.createSimpleFrameDropEffect( - /* expectedFrameRate= */ 6, /* targetFrameRate= */ 2)) - .build(); + ImmutableList frameTimesUs = + ImmutableList.of(0L, 250_000L, 500_000L, 750_000L, 1_000_000L, 1_500_000L); - videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(ORIGINAL_PNG_ASSET_PATH), - /* durationUs= */ C.MICROS_PER_SECOND, - /* offsetToAddUs= */ 0L, - /* frameRate= */ 4); - videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(SCALE_WIDE_PNG_ASSET_PATH), - /* durationUs= */ C.MICROS_PER_SECOND, - /* offsetToAddUs= */ C.MICROS_PER_SECOND, - /* frameRate= */ 2); - videoFrameProcessorTestRunner.endFrameProcessing(); + ImmutableList actualPresentationTimesUs = + processFramesToEndOfStream( + frameTimesUs, + FrameDropEffect.createSimpleFrameDropEffect( + /* expectedFrameRate= */ 6, /* targetFrameRate= */ 2)); - assertThat(actualPresentationTimesUs).containsExactly(500_000L, 1_500_000L).inOrder(); + assertThat(actualPresentationTimesUs).containsExactly(0L, 750_000L).inOrder(); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId); } - @RequiresNonNull("actualPresentationTimesUs") + @RequiresNonNull("textureBitmapReader") @Test public void frameDrop_withSimpleStrategy_outputsAllFrames() throws Exception { - String testId = "frameDrop_withSimpleStrategy_outputsCorrectNumberOfFrames"; - videoFrameProcessorTestRunner = - getDefaultFrameProcessorTestRunnerBuilder( - testId, - FrameDropEffect.createSimpleFrameDropEffect( - /* expectedFrameRate= */ 3, /* targetFrameRate= */ 3)) - .build(); + String testId = "frameDrop_withSimpleStrategy_outputsAllFrames"; + ImmutableList frameTimesUs = ImmutableList.of(0L, 333_333L, 666_667L); - videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(ORIGINAL_PNG_ASSET_PATH), - /* durationUs= */ C.MICROS_PER_SECOND, - /* offsetToAddUs= */ 0L, - /* frameRate= */ 3); - videoFrameProcessorTestRunner.endFrameProcessing(); + ImmutableList actualPresentationTimesUs = + processFramesToEndOfStream( + frameTimesUs, + FrameDropEffect.createSimpleFrameDropEffect( + /* expectedFrameRate= */ 3, /* targetFrameRate= */ 3)); assertThat(actualPresentationTimesUs).containsExactly(0L, 333_333L, 666_667L).inOrder(); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId); } - @RequiresNonNull("actualPresentationTimesUs") - private VideoFrameProcessorTestRunner.Builder getDefaultFrameProcessorTestRunnerBuilder( - String testId, FrameDropEffect frameDropEffect) { - return new VideoFrameProcessorTestRunner.Builder() - .setTestId(testId) - .setVideoFrameProcessorFactory(new DefaultVideoFrameProcessor.Factory.Builder().build()) - .setInputType(INPUT_TYPE_BITMAP) - .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) - .setEffects(frameDropEffect) - .setOnOutputFrameAvailableForRenderingListener(actualPresentationTimesUs::add); + private static void getAndAssertOutputBitmaps( + TextureBitmapReader textureBitmapReader, List presentationTimesUs, String testId) + throws IOException { + for (int i = 0; i < presentationTimesUs.size(); i++) { + long presentationTimeUs = presentationTimesUs.get(i); + Bitmap actualBitmap = textureBitmapReader.getBitmap(presentationTimeUs); + Bitmap expectedBitmap = + readBitmap(Util.formatInvariant("%s/pts_%d.png", ASSET_PATH, presentationTimeUs)); + maybeSaveTestBitmap( + testId, String.valueOf(presentationTimeUs), actualBitmap, /* path= */ null); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference) + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + } + + @EnsuresNonNull("defaultVideoFrameProcessor") + private ImmutableList processFramesToEndOfStream( + List inputPresentationTimesUs, FrameDropEffect frameDropEffect) throws Exception { + AtomicReference<@NullableType VideoFrameProcessingException> + videoFrameProcessingExceptionReference = new AtomicReference<>(); + BlankFrameProducer blankFrameProducer = + new BlankFrameProducer(BLANK_FRAME_WIDTH, BLANK_FRAME_HEIGHT); + CountDownLatch videoFrameProcessingEndedCountDownLatch = new CountDownLatch(1); + ImmutableList.Builder actualPresentationTimesUs = new ImmutableList.Builder<>(); + + defaultVideoFrameProcessor = + checkNotNull( + new DefaultVideoFrameProcessor.Factory.Builder() + .setTextureOutput( + (outputTexture, presentationTimeUs, releaseOutputTextureCallback, token) -> { + checkNotNull(textureBitmapReader) + .readBitmap(outputTexture, presentationTimeUs); + releaseOutputTextureCallback.release(presentationTimeUs); + }, + /* textureOutputCapacity= */ 1) + .build() + .create( + getApplicationContext(), + DebugViewProvider.NONE, + /* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, + /* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, + /* renderFramesAutomatically= */ true, + MoreExecutors.directExecutor(), + new VideoFrameProcessor.Listener() { + @Override + public void onOutputSizeChanged(int width, int height) {} + + @Override + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + actualPresentationTimesUs.add(presentationTimeUs); + } + + @Override + public void onError(VideoFrameProcessingException exception) { + videoFrameProcessingExceptionReference.set(exception); + videoFrameProcessingEndedCountDownLatch.countDown(); + } + + @Override + public void onEnded() { + videoFrameProcessingEndedCountDownLatch.countDown(); + } + })); + + defaultVideoFrameProcessor.getTaskExecutor().submit(blankFrameProducer::configureGlObjects); + // A frame needs to be registered despite not queuing any external input to ensure + // that the video frame processor knows about the stream offset. + checkNotNull(defaultVideoFrameProcessor) + .registerInputStream( + INPUT_TYPE_SURFACE, + /* effects= */ ImmutableList.of( + (GlEffect) (context, useHdr) -> blankFrameProducer, + // Use an overlay effect to generate bitmaps with timestamps on it. + new OverlayEffect( + ImmutableList.of( + new TextOverlay() { + @Override + public SpannableString getText(long presentationTimeUs) { + SpannableString text = + new SpannableString(String.valueOf(presentationTimeUs)); + text.setSpan( + new ForegroundColorSpan(Color.BLACK), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new AbsoluteSizeSpan(/* size= */ 24), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new TypefaceSpan(/* family= */ "sans-serif"), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + return text; + } + })), + frameDropEffect), + new FrameInfo.Builder(BLANK_FRAME_WIDTH, BLANK_FRAME_HEIGHT).build()); + blankFrameProducer.produceBlankFrames(inputPresentationTimesUs); + defaultVideoFrameProcessor.signalEndOfInput(); + videoFrameProcessingEndedCountDownLatch.await(); + @Nullable + Exception videoFrameProcessingException = videoFrameProcessingExceptionReference.get(); + if (videoFrameProcessingException != null) { + throw videoFrameProcessingException; + } + return actualPresentationTimesUs.build(); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultFrameDroppingShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultFrameDroppingShaderProgram.java index 1ded34d7d5..77bc99ce41 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultFrameDroppingShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultFrameDroppingShaderProgram.java @@ -88,7 +88,9 @@ import androidx.media3.common.util.Size; copyTextureToPreviousFrame(glObjectsProvider, inputTexture, presentationTimeUs); getInputListener().onInputFrameProcessed(inputTexture); - getInputListener().onReadyToAcceptInputFrame(); + if (outputTexturePool.freeTextureCount() > 0) { + getInputListener().onReadyToAcceptInputFrame(); + } } @Override diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 9859e791ef..d51cea1945 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -418,7 +418,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) { checkState( hasRefreshedNextInputFrameInfo, - "setInputFrameInfo must be called before queueing another bitmap"); + "registerInputStream must be called before queueing another bitmap"); inputSwitcher .activeTextureManager() .queueInputBitmap( @@ -442,15 +442,19 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { @Override public Surface getInputSurface() { - return inputSwitcher.activeTextureManager().getInputSurface(); + return inputSwitcher.getInputSurface(); } @Override - public void registerInputStream(@InputType int inputType, List effects) { + public void registerInputStream( + @InputType int inputType, List effects, FrameInfo frameInfo) { + nextInputFrameInfo = adjustForPixelWidthHeightRatio(frameInfo); + hasRefreshedNextInputFrameInfo = true; synchronized (lock) { if (!processingInput) { videoFrameProcessingTaskExecutor.submitAndBlock(() -> configureEffects(effects)); - inputSwitcher.switchToInput(inputType); + inputSwitcher.switchToInput(inputType, nextInputFrameInfo); + inputSwitcher.activeTextureManager().setInputFrameInfo(nextInputFrameInfo); processingInput = true; return; } @@ -477,21 +481,14 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { // a new frame from the new input stream prematurely. videoFrameProcessingTaskExecutor.submitAndBlock(() -> configureEffects(effects)); } - inputSwitcher.switchToInput(inputType); - } - - @Override - public void setInputFrameInfo(FrameInfo inputFrameInfo) { - nextInputFrameInfo = adjustForPixelWidthHeightRatio(inputFrameInfo); - inputSwitcher.activeTextureManager().setInputFrameInfo(nextInputFrameInfo); - hasRefreshedNextInputFrameInfo = true; + inputSwitcher.switchToInput(inputType, nextInputFrameInfo); } @Override public void registerInputFrame() { checkState(!inputStreamEnded); checkStateNotNull( - nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames"); + nextInputFrameInfo, "registerInputStream must be called before registering input frames"); inputSwitcher.activeTextureManager().registerInputFrame(nextInputFrameInfo); hasRefreshedNextInputFrameInfo = false; diff --git a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java index 8edab8770b..c74a5a207b 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java @@ -17,6 +17,9 @@ package androidx.media3.effect; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; @@ -24,7 +27,9 @@ import static androidx.media3.common.util.Util.containsKey; import android.content.Context; import android.util.SparseArray; +import android.view.Surface; import androidx.media3.common.ColorInfo; +import androidx.media3.common.FrameInfo; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; @@ -84,7 +89,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; TextureManager textureManager; // TODO(b/274109008): Refactor DefaultShaderProgram to create a class just for sampling. switch (inputType) { - case VideoFrameProcessor.INPUT_TYPE_SURFACE: + case INPUT_TYPE_SURFACE: samplingShaderProgram = DefaultShaderProgram.createWithExternalSampler( context, @@ -98,7 +103,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; glObjectsProvider, samplingShaderProgram, videoFrameProcessingTaskExecutor); inputs.put(inputType, new Input(textureManager, samplingShaderProgram)); break; - case VideoFrameProcessor.INPUT_TYPE_BITMAP: + case INPUT_TYPE_BITMAP: samplingShaderProgram = DefaultShaderProgram.createWithInternalSampler( context, @@ -113,7 +118,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; glObjectsProvider, samplingShaderProgram, videoFrameProcessingTaskExecutor); inputs.put(inputType, new Input(textureManager, samplingShaderProgram)); break; - case VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID: + case INPUT_TYPE_TEXTURE_ID: samplingShaderProgram = DefaultShaderProgram.createWithInternalSampler( context, @@ -145,8 +150,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * registered}. * * @param newInputType The new {@link VideoFrameProcessor.InputType} to switch to. + * @param inputFrameInfo The {@link FrameInfo} associated with the new input. */ - public void switchToInput(@VideoFrameProcessor.InputType int newInputType) { + public void switchToInput( + @VideoFrameProcessor.InputType int newInputType, FrameInfo inputFrameInfo) { checkStateNotNull(downstreamShaderProgram); checkState(containsKey(inputs, newInputType), "Input type not registered: " + newInputType); @@ -167,6 +174,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; input.setActive(false); } } + checkNotNull(activeTextureManager).setInputFrameInfo(inputFrameInfo); } /** @@ -186,6 +194,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; checkNotNull(activeTextureManager).signalEndOfCurrentInputStream(); } + /** + * Returns the input {@link Surface}. + * + * @return The input {@link Surface}, regardless if the current input is {@linkplain + * #switchToInput set} to {@link VideoFrameProcessor#INPUT_TYPE_SURFACE}. + * @throws IllegalStateException If {@link VideoFrameProcessor#INPUT_TYPE_SURFACE} is not + * {@linkplain #registerInput registered}. + */ + public Surface getInputSurface() { + checkState(containsKey(inputs, INPUT_TYPE_SURFACE)); + return inputs.get(INPUT_TYPE_SURFACE).textureManager.getInputSurface(); + } + /** Releases the resources. */ public void release() throws VideoFrameProcessingException { for (int i = 0; i < inputs.size(); i++) { diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SimpleFrameDroppingShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/SimpleFrameDroppingShaderProgram.java index 32ac809e42..2ec6386d34 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SimpleFrameDroppingShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SimpleFrameDroppingShaderProgram.java @@ -57,13 +57,13 @@ import androidx.media3.common.VideoFrameProcessingException; @Override public void queueInputFrame( GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { - framesReceived++; if (framesReceived % n == 0) { super.queueInputFrame(glObjectsProvider, inputTexture, presentationTimeUs); } else { getInputListener().onInputFrameProcessed(inputTexture); getInputListener().onReadyToAcceptInputFrame(); } + framesReceived++; } @Override diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java index 14b98dc35e..9c5f6bfeb9 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java @@ -73,7 +73,11 @@ import androidx.media3.common.VideoFrameProcessor; /** * Sets information about the input frames. * - * @see VideoFrameProcessor#setInputFrameInfo + *

The new input information is applied from the next frame {@linkplain #registerInputFrame + * registered} or {@linkplain #queueInputTexture queued} onwards. + * + *

Pixels are expanded using the {@link FrameInfo#pixelWidthHeightRatio} so that the output + * frames' pixels have a ratio of 1. */ default void setInputFrameInfo(FrameInfo inputFrameInfo) { // Do nothing. diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 9bf9a88cb9..fd7d255ae6 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -2142,8 +2142,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } }); - videoFrameProcessor.registerInputStream( - VideoFrameProcessor.INPUT_TYPE_SURFACE, videoEffects); this.initialStreamOffsetUs = initialStreamOffsetUs; } catch (Exception e) { throw renderer.createRendererException( @@ -2226,7 +2224,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { */ public void setInputFormat(Format inputFormat) { checkNotNull(videoFrameProcessor) - .setInputFrameInfo( + .registerInputStream( + VideoFrameProcessor.INPUT_TYPE_SURFACE, + checkNotNull(videoEffects), new FrameInfo.Builder(inputFormat.width, inputFormat.height) .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio) .build()); diff --git a/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_0.png b/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_0.png new file mode 100644 index 0000000000..4ffb8030fd Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_0.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_32000.png b/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_32000.png new file mode 100644 index 0000000000..a8dff723fb Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_32000.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_333333.png b/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_333333.png new file mode 100644 index 0000000000..c4faffefe2 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_333333.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_666667.png b/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_666667.png new file mode 100644 index 0000000000..cd7cfdcbd1 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_666667.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_71000.png b/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_71000.png new file mode 100644 index 0000000000..4fc6424832 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_71000.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_750000.png b/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_750000.png new file mode 100644 index 0000000000..a170bf112c Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/FrameDropTest/pts_750000.png differ diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TextureBitmapReader.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/TextureBitmapReader.java similarity index 84% rename from libraries/transformer/src/androidTest/java/androidx/media3/transformer/TextureBitmapReader.java rename to libraries/test_utils/src/main/java/androidx/media3/test/utils/TextureBitmapReader.java index 59b7c55972..5896478d08 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TextureBitmapReader.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/TextureBitmapReader.java @@ -13,32 +13,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package androidx.media3.transformer; + +package androidx.media3.test.utils; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.graphics.Bitmap; import android.view.Surface; +import androidx.annotation.Nullable; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; -import androidx.media3.effect.DefaultVideoFrameProcessor; -import androidx.media3.test.utils.BitmapPixelTestUtil; -import androidx.media3.test.utils.VideoFrameProcessorTestRunner; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.Nullable; /** * {@inheritDoc} * *

Reads from an OpenGL texture. Only for use on physical devices. */ +@UnstableApi public final class TextureBitmapReader implements VideoFrameProcessorTestRunner.BitmapReader { + // TODO(b/239172735): This outputs an incorrect black output image on emulators. private final Map outputTimestampsToBitmaps; private boolean useHighPrecisionColorComponents; @@ -60,6 +61,10 @@ public final class TextureBitmapReader implements VideoFrameProcessorTestRunner. return checkStateNotNull(outputBitmap); } + /** + * @return The output {@link Bitmap} at a given {@code presentationTimeUs}. + * @throws IllegalStateException If no such bitmap is produced. + */ public Bitmap getBitmap(long presentationTimeUs) { return checkStateNotNull(outputTimestampsToBitmaps.get(presentationTimeUs)); } @@ -69,6 +74,11 @@ public final class TextureBitmapReader implements VideoFrameProcessorTestRunner. return outputTimestampsToBitmaps.keySet(); } + /** + * Reads the given {@code outputTexture}. + * + *

The read result can be fetched by calling one of me {@link #getBitmap} methods. + */ public void readBitmap(GlTextureInfo outputTexture, long presentationTimeUs) throws VideoFrameProcessingException { try { @@ -83,15 +93,6 @@ public final class TextureBitmapReader implements VideoFrameProcessorTestRunner. } } - public void readBitmapAndReleaseTexture( - GlTextureInfo outputTexture, - long presentationTimeUs, - DefaultVideoFrameProcessor.ReleaseOutputTextureCallback releaseOutputTextureCallback) - throws VideoFrameProcessingException { - readBitmap(outputTexture, presentationTimeUs); - releaseOutputTextureCallback.release(presentationTimeUs); - } - private static Bitmap createBitmapFromCurrentGlFrameBuffer( int width, int height, boolean useHighPrecisionColorComponents) throws GlUtil.GlException { if (!useHighPrecisionColorComponents) { diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java index 79b232a9f1..bb25420e35 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java @@ -15,7 +15,9 @@ */ package androidx.media3.test.utils; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_SURFACE; +import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.test.utils.BitmapPixelTestUtil.createArgb8888BitmapFromRgba8888Image; @@ -41,7 +43,6 @@ import androidx.media3.common.GlTextureInfo; import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; -import androidx.media3.common.VideoFrameProcessor.InputType; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.UnstableApi; import com.google.common.collect.ImmutableList; @@ -71,13 +72,11 @@ public final class VideoFrameProcessorTestRunner { private float pixelWidthHeightRatio; private @MonotonicNonNull ColorInfo inputColorInfo; private @MonotonicNonNull ColorInfo outputColorInfo; - private @InputType int inputType; private OnOutputFrameAvailableForRenderingListener onOutputFrameAvailableListener; /** Creates a new instance with default values. */ public Builder() { pixelWidthHeightRatio = DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO; - inputType = INPUT_TYPE_SURFACE; onOutputFrameAvailableListener = unused -> {}; } @@ -194,18 +193,6 @@ public final class VideoFrameProcessorTestRunner { return this; } - /** - * Sets whether input comes from an external texture. See {@link - * VideoFrameProcessor.Factory#create}. - * - *

The default value is {@link VideoFrameProcessor#INPUT_TYPE_SURFACE}. - */ - @CanIgnoreReturnValue - public Builder setInputType(@InputType int inputType) { - this.inputType = inputType; - return this; - } - /** * Sets the method to be called in {@link * VideoFrameProcessor.Listener#onOutputFrameAvailableForRendering}. @@ -233,7 +220,6 @@ public final class VideoFrameProcessorTestRunner { pixelWidthHeightRatio, inputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : inputColorInfo, outputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : outputColorInfo, - inputType, onOutputFrameAvailableListener); } } @@ -251,6 +237,7 @@ public final class VideoFrameProcessorTestRunner { private final @MonotonicNonNull CountDownLatch videoFrameProcessingEndedLatch; private final AtomicReference videoFrameProcessingException; private final VideoFrameProcessor videoFrameProcessor; + private final ImmutableList effects; private @MonotonicNonNull BitmapReader bitmapReader; @@ -264,7 +251,6 @@ public final class VideoFrameProcessorTestRunner { float pixelWidthHeightRatio, ColorInfo inputColorInfo, ColorInfo outputColorInfo, - @InputType int inputType, OnOutputFrameAvailableForRenderingListener onOutputFrameAvailableForRenderingListener) throws VideoFrameProcessingException { this.testId = testId; @@ -314,7 +300,7 @@ public final class VideoFrameProcessorTestRunner { checkNotNull(videoFrameProcessingEndedLatch).countDown(); } }); - videoFrameProcessor.registerInputStream(inputType, effects); + this.effects = effects; } public void processFirstFrameAndEnd() throws Exception { @@ -323,7 +309,9 @@ public final class VideoFrameProcessorTestRunner { new DecodeOneFrameUtil.Listener() { @Override public void onContainerExtracted(MediaFormat mediaFormat) { - videoFrameProcessor.setInputFrameInfo( + videoFrameProcessor.registerInputStream( + INPUT_TYPE_SURFACE, + effects, new FrameInfo.Builder( mediaFormat.getInteger(MediaFormat.KEY_WIDTH), mediaFormat.getInteger(MediaFormat.KEY_HEIGHT)) @@ -343,7 +331,9 @@ public final class VideoFrameProcessorTestRunner { public void queueInputBitmap( Bitmap inputBitmap, long durationUs, long offsetToAddUs, float frameRate) { - videoFrameProcessor.setInputFrameInfo( + videoFrameProcessor.registerInputStream( + INPUT_TYPE_BITMAP, + effects, new FrameInfo.Builder(inputBitmap.getWidth(), inputBitmap.getHeight()) .setPixelWidthHeightRatio(pixelWidthHeightRatio) .setOffsetToAddUs(offsetToAddUs) @@ -352,7 +342,9 @@ public final class VideoFrameProcessorTestRunner { } public void queueInputTexture(GlTextureInfo inputTexture, long pts) { - videoFrameProcessor.setInputFrameInfo( + videoFrameProcessor.registerInputStream( + INPUT_TYPE_TEXTURE_ID, + effects, new FrameInfo.Builder(inputTexture.width, inputTexture.height) .setPixelWidthHeightRatio(pixelWidthHeightRatio) .build()); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java index 75808369db..68f1d4441b 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java @@ -15,7 +15,6 @@ */ package androidx.media3.transformer; -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE; import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; @@ -39,6 +38,7 @@ import androidx.media3.effect.RgbFilter; import androidx.media3.effect.ScaleAndRotateTransformation; import androidx.media3.effect.VideoCompositor; import androidx.media3.test.utils.BitmapPixelTestUtil; +import androidx.media3.test.utils.TextureBitmapReader; import androidx.media3.test.utils.VideoFrameProcessorTestRunner; import com.google.common.collect.ImmutableList; import java.io.IOException; @@ -334,7 +334,6 @@ public final class VideoCompositorPixelTest { return new VideoFrameProcessorTestRunner.Builder() .setTestId(testId) .setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactoryBuilder.build()) - .setInputType(INPUT_TYPE_BITMAP) .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) .setBitmapReader(textureBitmapReader); } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorMultipleTextureOutputPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorMultipleTextureOutputPixelTest.java index 6adc4460c8..cd1c57fbca 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorMultipleTextureOutputPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorMultipleTextureOutputPixelTest.java @@ -15,7 +15,6 @@ */ package androidx.media3.transformer.mh; -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE; import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; @@ -28,8 +27,8 @@ import androidx.media3.common.ColorInfo; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.effect.DefaultVideoFrameProcessor; import androidx.media3.test.utils.BitmapPixelTestUtil; +import androidx.media3.test.utils.TextureBitmapReader; import androidx.media3.test.utils.VideoFrameProcessorTestRunner; -import androidx.media3.transformer.TextureBitmapReader; import androidx.test.ext.junit.runners.AndroidJUnit4; import java.util.Set; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -142,16 +141,15 @@ public class DefaultVideoFrameProcessorMultipleTextureOutputPixelTest { (outputTexture, presentationTimeUs, releaseOutputTextureCallback, - unusedSyncObject) -> - checkNotNull(textureBitmapReader) - .readBitmapAndReleaseTexture( - outputTexture, presentationTimeUs, releaseOutputTextureCallback), + unusedSyncObject) -> { + checkNotNull(textureBitmapReader).readBitmap(outputTexture, presentationTimeUs); + releaseOutputTextureCallback.release(presentationTimeUs); + }, /* textureOutputCapacity= */ 1) .build(); return new VideoFrameProcessorTestRunner.Builder() .setTestId(testId) .setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactory) - .setInputType(INPUT_TYPE_BITMAP) .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) .setBitmapReader(textureBitmapReader); } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java index 61f08c1287..03f1849324 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java @@ -38,17 +38,16 @@ import androidx.media3.common.Format; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.GlUtil; import androidx.media3.effect.BitmapOverlay; import androidx.media3.effect.DefaultGlObjectsProvider; import androidx.media3.effect.DefaultVideoFrameProcessor; import androidx.media3.effect.OverlayEffect; import androidx.media3.test.utils.BitmapPixelTestUtil; +import androidx.media3.test.utils.TextureBitmapReader; import androidx.media3.test.utils.VideoFrameProcessorTestRunner; import androidx.media3.transformer.AndroidTestUtil; import androidx.media3.transformer.EncoderUtil; -import androidx.media3.transformer.TextureBitmapReader; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; import java.util.List; @@ -537,9 +536,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { (outputTexture, presentationTimeUs1, releaseOutputTextureCallback1, - unusedSyncObject) -> - bitmapReader.readBitmapAndReleaseTexture( - outputTexture, presentationTimeUs1, releaseOutputTextureCallback1), + unusedSyncObject) -> { + bitmapReader.readBitmap(outputTexture, presentationTimeUs1); + releaseOutputTextureCallback1.release(presentationTimeUs1); + }, /* textureOutputCapacity= */ 1) .setGlObjectsProvider(contextSharingGlObjectsProvider) .build(); @@ -550,7 +550,6 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { .setInputColorInfo(colorInfo) .setOutputColorInfo(colorInfo) .setBitmapReader(bitmapReader) - .setInputType(VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID) .setEffects(effects) .build(); GlUtil.awaitSyncObject(syncObject); @@ -573,9 +572,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { (outputTexture, presentationTimeUs, releaseOutputTextureCallback, - unusedSyncObject) -> - textureBitmapReader.readBitmapAndReleaseTexture( - outputTexture, presentationTimeUs, releaseOutputTextureCallback), + unusedSyncObject) -> { + textureBitmapReader.readBitmap(outputTexture, presentationTimeUs); + releaseOutputTextureCallback.release(presentationTimeUs); + }, /* textureOutputCapacity= */ 1) .build(); return new VideoFrameProcessorTestRunner.Builder() diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/FrameDropPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/FrameDropPixelTest.java deleted file mode 100644 index 50c24e5439..0000000000 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/FrameDropPixelTest.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.transformer.mh; - -import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP; -import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE; -import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; -import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; -import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; -import static com.google.common.truth.Truth.assertThat; - -import android.graphics.Bitmap; -import androidx.media3.common.C; -import androidx.media3.common.ColorInfo; -import androidx.media3.common.VideoFrameProcessingException; -import androidx.media3.common.VideoFrameProcessor; -import androidx.media3.effect.DefaultVideoFrameProcessor; -import androidx.media3.effect.FrameDropEffect; -import androidx.media3.test.utils.VideoFrameProcessorTestRunner; -import androidx.media3.transformer.TextureBitmapReader; -import androidx.test.ext.junit.runners.AndroidJUnit4; -import org.checkerframework.checker.nullness.qual.EnsuresNonNull; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; - -/** - * Tests to ensure {@link FrameDropEffect} outputs the correct frame associated with a chosen - * timestamp. - */ -@RunWith(AndroidJUnit4.class) -public class FrameDropPixelTest { - private static final String ORIGINAL_PNG_ASSET_PATH = - "media/bitmap/sample_mp4_first_frame/electrical_colors/original.png"; - private static final String MEDIA3_TEST_PNG_ASSET_PATH = - "media/bitmap/input_images/media3test.png"; - private static final String ROTATE_90_PNG_ASSET_PATH = - "media/bitmap/sample_mp4_first_frame/electrical_colors/rotate90.png"; - private static final String SRGB_TO_ELECTRICAL_ORIGINAL_PNG_ASSET_PATH = - "media/bitmap/sample_mp4_first_frame/electrical_colors/srgb_to_electrical_original.png"; - private static final String SRGB_TO_ELECTRICAL_MEDIA3_TEST_PNG_ASSET_PATH = - "media/bitmap/sample_mp4_first_frame/electrical_colors/srgb_to_electrical_media3test.png"; - - private @MonotonicNonNull TextureBitmapReader textureBitmapReader; - private @MonotonicNonNull VideoFrameProcessorTestRunner videoFrameProcessorTestRunner; - - @EnsuresNonNull("textureBitmapReader") - @Before - public void setUp() { - textureBitmapReader = new TextureBitmapReader(); - } - - @After - public void tearDown() { - checkNotNull(videoFrameProcessorTestRunner).release(); - } - - @RequiresNonNull("textureBitmapReader") - @Test - public void frameDrop_withDefaultStrategy_outputsCorrectFramesAtTheCorrectPresentationTimesUs() - throws Exception { - String testId = - "frameDrop_withDefaultStrategy_outputsCorrectFramesAtTheCorrectPresentationTimesUs"; - videoFrameProcessorTestRunner = - createDefaultFrameProcessorTestRunnerBuilder( - testId, FrameDropEffect.createDefaultFrameDropEffect(/* targetFrameRate= */ 30)); - - long expectedPresentationTimeUs1 = 0; - long expectedPresentationTimeUs2 = 32_000; - long expectedPresentationTimeUs3 = 71_000; - Bitmap chosenBitmap1 = readBitmap(ORIGINAL_PNG_ASSET_PATH); - Bitmap chosenBitmap2 = readBitmap(MEDIA3_TEST_PNG_ASSET_PATH); - Bitmap droppedFrameBitmap = readBitmap(ROTATE_90_PNG_ASSET_PATH); - queueOneFrameAt(chosenBitmap1, expectedPresentationTimeUs1); - queueOneFrameAt(droppedFrameBitmap, /* presentationTimeUs= */ 16_000L); - queueOneFrameAt(chosenBitmap2, expectedPresentationTimeUs2); - queueOneFrameAt(droppedFrameBitmap, /* presentationTimeUs= */ 48_000L); - queueOneFrameAt(droppedFrameBitmap, /* presentationTimeUs= */ 58_000L); - queueOneFrameAt(chosenBitmap1, expectedPresentationTimeUs3); - queueOneFrameAt(droppedFrameBitmap, /* presentationTimeUs= */ 86_000L); - videoFrameProcessorTestRunner.endFrameProcessing(); - - assertThat(textureBitmapReader.getOutputTimestamps()) - .containsExactly( - expectedPresentationTimeUs1, expectedPresentationTimeUs2, expectedPresentationTimeUs3) - .inOrder(); - assertThat( - getBitmapAveragePixelAbsoluteDifferenceArgb8888( - readBitmap(SRGB_TO_ELECTRICAL_ORIGINAL_PNG_ASSET_PATH), - textureBitmapReader.getBitmap(expectedPresentationTimeUs1), - testId)) - .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE); - assertThat( - getBitmapAveragePixelAbsoluteDifferenceArgb8888( - readBitmap(SRGB_TO_ELECTRICAL_MEDIA3_TEST_PNG_ASSET_PATH), - textureBitmapReader.getBitmap(expectedPresentationTimeUs2), - testId)) - .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE); - assertThat( - getBitmapAveragePixelAbsoluteDifferenceArgb8888( - readBitmap(SRGB_TO_ELECTRICAL_ORIGINAL_PNG_ASSET_PATH), - textureBitmapReader.getBitmap(expectedPresentationTimeUs3), - testId)) - .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE); - } - - @RequiresNonNull("textureBitmapReader") - @Test - public void frameDrop_withSimpleStrategy_outputsCorrectFramesAtTheCorrectPresentationTimesUs() - throws Exception { - String testId = - "frameDrop_withSimpleStrategy_outputsCorrectFramesAtTheCorrectPresentationTimesUs"; - videoFrameProcessorTestRunner = - createDefaultFrameProcessorTestRunnerBuilder( - testId, - FrameDropEffect.createSimpleFrameDropEffect( - /* expectedFrameRate= */ 6, /* targetFrameRate= */ 2)); - long expectedPresentationTimeUs1 = 500_000; - long expectedPresentationTimeUs2 = 1_500_000; - videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(ORIGINAL_PNG_ASSET_PATH), - /* durationUs= */ C.MICROS_PER_SECOND, - /* offsetToAddUs= */ 0L, - /* frameRate= */ 4); - videoFrameProcessorTestRunner.queueInputBitmap( - readBitmap(MEDIA3_TEST_PNG_ASSET_PATH), - /* durationUs= */ C.MICROS_PER_SECOND, - /* offsetToAddUs= */ C.MICROS_PER_SECOND, - /* frameRate= */ 2); - videoFrameProcessorTestRunner.endFrameProcessing(); - - assertThat(textureBitmapReader.getOutputTimestamps()) - .containsExactly(expectedPresentationTimeUs1, expectedPresentationTimeUs2) - .inOrder(); - Bitmap actualBitmap1 = textureBitmapReader.getBitmap(expectedPresentationTimeUs1); - maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual1", actualBitmap1, /* path= */ null); - Bitmap actualBitmap2 = textureBitmapReader.getBitmap(expectedPresentationTimeUs2); - maybeSaveTestBitmap(testId, /* bitmapLabel= */ "actual2", actualBitmap2, /* path= */ null); - assertThat( - getBitmapAveragePixelAbsoluteDifferenceArgb8888( - readBitmap(SRGB_TO_ELECTRICAL_ORIGINAL_PNG_ASSET_PATH), actualBitmap1, testId)) - .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE); - assertThat( - getBitmapAveragePixelAbsoluteDifferenceArgb8888( - readBitmap(SRGB_TO_ELECTRICAL_MEDIA3_TEST_PNG_ASSET_PATH), actualBitmap2, testId)) - .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE); - } - - @RequiresNonNull("textureBitmapReader") - private VideoFrameProcessorTestRunner createDefaultFrameProcessorTestRunnerBuilder( - String testId, FrameDropEffect frameDropEffect) throws VideoFrameProcessingException { - VideoFrameProcessor.Factory defaultVideoFrameProcessorFactory = - new DefaultVideoFrameProcessor.Factory.Builder() - .setTextureOutput( - (outputTexture, presentationTimeUs, releaseOutputTextureCallback, token) -> - checkNotNull(textureBitmapReader) - .readBitmapAndReleaseTexture( - outputTexture, presentationTimeUs, releaseOutputTextureCallback), - /* textureOutputCapacity= */ 1) - .build(); - return new VideoFrameProcessorTestRunner.Builder() - .setTestId(testId) - .setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactory) - .setInputType(INPUT_TYPE_BITMAP) - .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) - .setEffects(frameDropEffect) - .build(); - } - - /** - * Queues a {@link Bitmap} into the {@link VideoFrameProcessor} so that exactly one frame is - * produced at the given {@code presentationTimeUs}. - */ - private void queueOneFrameAt(Bitmap bitmap, long presentationTimeUs) { - checkNotNull(videoFrameProcessorTestRunner) - .queueInputBitmap( - bitmap, - /* durationUs= */ C.MICROS_PER_SECOND, - /* offsetToAddUs= */ presentationTimeUs, - /* frameRate= */ 1); - } -} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java index 958a55b2b6..ac15d05570 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java @@ -158,8 +158,7 @@ import java.util.concurrent.atomic.AtomicLong; Size decodedSize = getDecodedSize(trackFormat); videoFrameProcessor.registerInputStream( getInputType(checkNotNull(trackFormat.sampleMimeType)), - createEffectListWithPresentation(editedMediaItem.effects.videoEffects, presentation)); - videoFrameProcessor.setInputFrameInfo( + createEffectListWithPresentation(editedMediaItem.effects.videoEffects, presentation), new FrameInfo.Builder(decodedSize.getWidth(), decodedSize.getHeight()) .setPixelWidthHeightRatio(trackFormat.pixelWidthHeightRatio) .setOffsetToAddUs(mediaItemOffsetUs.get())