diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java index 496f0f576d..8ce51566d0 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java @@ -73,7 +73,8 @@ import java.util.concurrent.atomic.AtomicReference; maybeSaveTestBitmap( testId, String.valueOf(presentationTimeUs), actualBitmap, /* path= */ null); float averagePixelAbsoluteDifference = - getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + getBitmapAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId + "_" + i); assertThat(averagePixelAbsoluteDifference) .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/TimestampAdjustmentTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/TimestampAdjustmentTest.java new file mode 100644 index 0000000000..b403eb0310 --- /dev/null +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/TimestampAdjustmentTest.java @@ -0,0 +1,113 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.effect.EffectsTestUtil.generateAndProcessFrames; +import static androidx.media3.effect.EffectsTestUtil.getAndAssertOutputBitmaps; +import static com.google.common.truth.Truth.assertThat; + +import android.graphics.Color; +import android.text.Spannable; +import android.text.SpannableString; +import android.text.style.AbsoluteSizeSpan; +import android.text.style.ForegroundColorSpan; +import android.text.style.TypefaceSpan; +import androidx.media3.common.util.Consumer; +import androidx.media3.test.utils.TextureBitmapReader; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +/** Tests for {@link TimestampAdjustment}. */ +@RunWith(AndroidJUnit4.class) +public class TimestampAdjustmentTest { + @Rule public final TestName testName = new TestName(); + + private static final String ASSET_PATH = "test-generated-goldens/TimestampAdjustmentTest"; + + private @MonotonicNonNull TextureBitmapReader textureBitmapReader; + private @MonotonicNonNull String testId; + + @EnsuresNonNull({"textureBitmapReader", "testId"}) + @Before + public void setUp() { + textureBitmapReader = new TextureBitmapReader(); + testId = testName.getMethodName(); + } + + @Test + @RequiresNonNull({"textureBitmapReader", "testId"}) + public void timestampAdjustmentTest_outputsFramesAtTheCorrectPresentationTimesUs() + throws Exception { + ImmutableList frameTimesUs = ImmutableList.of(0L, 32_000L, 71_000L); + TimestampAdjustment timestampAdjustment = + new TimestampAdjustment( + (inputTimeUs, callback) -> { + try { + Thread.sleep(/* millis= */ 50); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + callback.accept(inputTimeUs / 2); + }); + + ImmutableList actualPresentationTimesUs = + generateAndProcessBlackTimeStampedFrames(frameTimesUs, timestampAdjustment); + + assertThat(actualPresentationTimesUs).containsExactly(0L, 16_000L, 35_500L).inOrder(); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); + } + + private ImmutableList generateAndProcessBlackTimeStampedFrames( + ImmutableList frameTimesUs, GlEffect effect) throws Exception { + int blankFrameWidth = 100; + int blankFrameHeight = 50; + Consumer textSpanConsumer = + (text) -> { + text.setSpan( + new ForegroundColorSpan(Color.BLACK), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new AbsoluteSizeSpan(/* size= */ 24), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new TypefaceSpan(/* family= */ "sans-serif"), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + }; + return generateAndProcessFrames( + blankFrameWidth, + blankFrameHeight, + frameTimesUs, + effect, + checkNotNull(textureBitmapReader), + textSpanConsumer); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustment.java b/libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustment.java new file mode 100644 index 0000000000..2539d375b7 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustment.java @@ -0,0 +1,60 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import android.content.Context; +import androidx.media3.common.util.UnstableApi; +import java.util.function.LongConsumer; + +/** + * Changes the frame timestamps using the {@link TimestampMap}. + * + *

This effect doesn't drop any frames. + * + *

This effect is not supported for effects previewing. + */ +@UnstableApi +public final class TimestampAdjustment implements GlEffect { + + /** + * Maps input timestamps to output timestamps asynchronously. + * + *

Implementation can choose to calculate the timestamp and invoke the consumer on another + * thread asynchronously. + */ + public interface TimestampMap { + + /** + * Calculates the output timestamp that corresponds to the input timestamp. + * + *

The implementation should invoke the {@code outputTimeConsumer} with the output timestamp, + * on any thread. + */ + void calculateOutputTimeUs(long inputTimeUs, LongConsumer outputTimeConsumer); + } + + private final TimestampMap timestampMap; + + /** Creates an instance. */ + public TimestampAdjustment(TimestampMap timestampMap) { + this.timestampMap = timestampMap; + } + + @Override + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) { + return new TimestampAdjustmentShaderProgram(timestampMap); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustmentShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustmentShaderProgram.java new file mode 100644 index 0000000000..547ca3f3c7 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustmentShaderProgram.java @@ -0,0 +1,114 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; + +import androidx.annotation.Nullable; +import androidx.media3.common.GlObjectsProvider; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.effect.TimestampAdjustment.TimestampMap; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** Changes the frame timestamps using the {@link TimestampMap}. */ +@UnstableApi +public class TimestampAdjustmentShaderProgram implements GlShaderProgram { + + private final TimestampMap timestampMap; + private final AtomicInteger pendingCallbacksCount; + private final AtomicBoolean pendingEndOfStream; + + @Nullable private GlTextureInfo inputTexture; + private InputListener inputListener; + private OutputListener outputListener; + + public TimestampAdjustmentShaderProgram(TimestampMap timestampMap) { + inputListener = new InputListener() {}; + outputListener = new OutputListener() {}; + + this.timestampMap = timestampMap; + pendingCallbacksCount = new AtomicInteger(); + pendingEndOfStream = new AtomicBoolean(); + } + + @Override + public void setInputListener(InputListener inputListener) { + this.inputListener = inputListener; + if (inputTexture == null) { + inputListener.onReadyToAcceptInputFrame(); + } + } + + @Override + public void setOutputListener(OutputListener outputListener) { + this.outputListener = outputListener; + } + + @Override + public void setErrorListener(Executor executor, ErrorListener errorListener) { + // No checked exceptions thrown. + } + + @Override + public void queueInputFrame( + GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { + this.inputTexture = inputTexture; + timestampMap.calculateOutputTimeUs( + presentationTimeUs, /* outputTimeConsumer= */ this::onOutputTimeAvailable); + pendingCallbacksCount.incrementAndGet(); + } + + @Override + public void signalEndOfCurrentInputStream() { + if (pendingCallbacksCount.get() == 0) { + outputListener.onCurrentOutputStreamEnded(); + } else { + pendingEndOfStream.set(true); + } + } + + @Override + public void releaseOutputFrame(GlTextureInfo outputTexture) { + checkState(outputTexture.texId == checkNotNull(inputTexture).texId); + inputListener.onInputFrameProcessed(outputTexture); + inputListener.onReadyToAcceptInputFrame(); + } + + @Override + public void flush() { + // TODO - b/320242819: Investigate support for previewing. + throw new UnsupportedOperationException("This effect is not supported for previewing."); + } + + @Override + public void release() throws VideoFrameProcessingException { + inputTexture = null; + } + + private void onOutputTimeAvailable(long outputTimeUs) { + outputListener.onOutputFrameAvailable(checkNotNull(inputTexture), outputTimeUs); + if (pendingEndOfStream.get()) { + outputListener.onCurrentOutputStreamEnded(); + pendingEndOfStream.set(false); + } + pendingCallbacksCount.decrementAndGet(); + } +} diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/TimestampAdjustmentTest/pts_0.png b/libraries/test_data/src/test/assets/test-generated-goldens/TimestampAdjustmentTest/pts_0.png new file mode 100644 index 0000000000..4ffb8030fd Binary files /dev/null and b/libraries/test_data/src/test/assets/test-generated-goldens/TimestampAdjustmentTest/pts_0.png differ diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/TimestampAdjustmentTest/pts_16000.png b/libraries/test_data/src/test/assets/test-generated-goldens/TimestampAdjustmentTest/pts_16000.png new file mode 100644 index 0000000000..a8dff723fb Binary files /dev/null and b/libraries/test_data/src/test/assets/test-generated-goldens/TimestampAdjustmentTest/pts_16000.png differ diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/TimestampAdjustmentTest/pts_35500.png b/libraries/test_data/src/test/assets/test-generated-goldens/TimestampAdjustmentTest/pts_35500.png new file mode 100644 index 0000000000..4fc6424832 Binary files /dev/null and b/libraries/test_data/src/test/assets/test-generated-goldens/TimestampAdjustmentTest/pts_35500.png differ