From c0ffc94d1589a853dcae98d7abccd508fcdb493c Mon Sep 17 00:00:00 2001 From: tofunmi Date: Fri, 15 Mar 2024 06:53:04 -0700 Subject: [PATCH] Add GlEffect that asynchronously adjusts input presentation times Single asset export only effect. PiperOrigin-RevId: 616114153 --- .../media3/effect/EffectsTestUtil.java | 3 +- .../effect/TimestampAdjustmentTest.java | 113 +++++++++++++++++ .../media3/effect/TimestampAdjustment.java | 60 +++++++++ .../TimestampAdjustmentShaderProgram.java | 114 ++++++++++++++++++ .../TimestampAdjustmentTest/pts_0.png | Bin 0 -> 591 bytes .../TimestampAdjustmentTest/pts_16000.png | Bin 0 -> 1419 bytes .../TimestampAdjustmentTest/pts_35500.png | Bin 0 -> 1140 bytes 7 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 libraries/effect/src/androidTest/java/androidx/media3/effect/TimestampAdjustmentTest.java create mode 100644 libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustment.java create mode 100644 libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustmentShaderProgram.java create mode 100644 libraries/test_data/src/test/assets/test-generated-goldens/TimestampAdjustmentTest/pts_0.png create mode 100644 libraries/test_data/src/test/assets/test-generated-goldens/TimestampAdjustmentTest/pts_16000.png create mode 100644 libraries/test_data/src/test/assets/test-generated-goldens/TimestampAdjustmentTest/pts_35500.png diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java index 496f0f576d..8ce51566d0 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/EffectsTestUtil.java @@ -73,7 +73,8 @@ import java.util.concurrent.atomic.AtomicReference; maybeSaveTestBitmap( testId, String.valueOf(presentationTimeUs), actualBitmap, /* path= */ null); float averagePixelAbsoluteDifference = - getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + getBitmapAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId + "_" + i); assertThat(averagePixelAbsoluteDifference) .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/TimestampAdjustmentTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/TimestampAdjustmentTest.java new file mode 100644 index 0000000000..b403eb0310 --- /dev/null +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/TimestampAdjustmentTest.java @@ -0,0 +1,113 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.effect.EffectsTestUtil.generateAndProcessFrames; +import static androidx.media3.effect.EffectsTestUtil.getAndAssertOutputBitmaps; +import static com.google.common.truth.Truth.assertThat; + +import android.graphics.Color; +import android.text.Spannable; +import android.text.SpannableString; +import android.text.style.AbsoluteSizeSpan; +import android.text.style.ForegroundColorSpan; +import android.text.style.TypefaceSpan; +import androidx.media3.common.util.Consumer; +import androidx.media3.test.utils.TextureBitmapReader; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +/** Tests for {@link TimestampAdjustment}. */ +@RunWith(AndroidJUnit4.class) +public class TimestampAdjustmentTest { + @Rule public final TestName testName = new TestName(); + + private static final String ASSET_PATH = "test-generated-goldens/TimestampAdjustmentTest"; + + private @MonotonicNonNull TextureBitmapReader textureBitmapReader; + private @MonotonicNonNull String testId; + + @EnsuresNonNull({"textureBitmapReader", "testId"}) + @Before + public void setUp() { + textureBitmapReader = new TextureBitmapReader(); + testId = testName.getMethodName(); + } + + @Test + @RequiresNonNull({"textureBitmapReader", "testId"}) + public void timestampAdjustmentTest_outputsFramesAtTheCorrectPresentationTimesUs() + throws Exception { + ImmutableList frameTimesUs = ImmutableList.of(0L, 32_000L, 71_000L); + TimestampAdjustment timestampAdjustment = + new TimestampAdjustment( + (inputTimeUs, callback) -> { + try { + Thread.sleep(/* millis= */ 50); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + callback.accept(inputTimeUs / 2); + }); + + ImmutableList actualPresentationTimesUs = + generateAndProcessBlackTimeStampedFrames(frameTimesUs, timestampAdjustment); + + assertThat(actualPresentationTimesUs).containsExactly(0L, 16_000L, 35_500L).inOrder(); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); + } + + private ImmutableList generateAndProcessBlackTimeStampedFrames( + ImmutableList frameTimesUs, GlEffect effect) throws Exception { + int blankFrameWidth = 100; + int blankFrameHeight = 50; + Consumer textSpanConsumer = + (text) -> { + text.setSpan( + new ForegroundColorSpan(Color.BLACK), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new AbsoluteSizeSpan(/* size= */ 24), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new TypefaceSpan(/* family= */ "sans-serif"), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + }; + return generateAndProcessFrames( + blankFrameWidth, + blankFrameHeight, + frameTimesUs, + effect, + checkNotNull(textureBitmapReader), + textSpanConsumer); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustment.java b/libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustment.java new file mode 100644 index 0000000000..2539d375b7 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustment.java @@ -0,0 +1,60 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import android.content.Context; +import androidx.media3.common.util.UnstableApi; +import java.util.function.LongConsumer; + +/** + * Changes the frame timestamps using the {@link TimestampMap}. + * + *

This effect doesn't drop any frames. + * + *

This effect is not supported for effects previewing. + */ +@UnstableApi +public final class TimestampAdjustment implements GlEffect { + + /** + * Maps input timestamps to output timestamps asynchronously. + * + *

Implementation can choose to calculate the timestamp and invoke the consumer on another + * thread asynchronously. + */ + public interface TimestampMap { + + /** + * Calculates the output timestamp that corresponds to the input timestamp. + * + *

The implementation should invoke the {@code outputTimeConsumer} with the output timestamp, + * on any thread. + */ + void calculateOutputTimeUs(long inputTimeUs, LongConsumer outputTimeConsumer); + } + + private final TimestampMap timestampMap; + + /** Creates an instance. */ + public TimestampAdjustment(TimestampMap timestampMap) { + this.timestampMap = timestampMap; + } + + @Override + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) { + return new TimestampAdjustmentShaderProgram(timestampMap); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustmentShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustmentShaderProgram.java new file mode 100644 index 0000000000..547ca3f3c7 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/TimestampAdjustmentShaderProgram.java @@ -0,0 +1,114 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; + +import androidx.annotation.Nullable; +import androidx.media3.common.GlObjectsProvider; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.effect.TimestampAdjustment.TimestampMap; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/** Changes the frame timestamps using the {@link TimestampMap}. */ +@UnstableApi +public class TimestampAdjustmentShaderProgram implements GlShaderProgram { + + private final TimestampMap timestampMap; + private final AtomicInteger pendingCallbacksCount; + private final AtomicBoolean pendingEndOfStream; + + @Nullable private GlTextureInfo inputTexture; + private InputListener inputListener; + private OutputListener outputListener; + + public TimestampAdjustmentShaderProgram(TimestampMap timestampMap) { + inputListener = new InputListener() {}; + outputListener = new OutputListener() {}; + + this.timestampMap = timestampMap; + pendingCallbacksCount = new AtomicInteger(); + pendingEndOfStream = new AtomicBoolean(); + } + + @Override + public void setInputListener(InputListener inputListener) { + this.inputListener = inputListener; + if (inputTexture == null) { + inputListener.onReadyToAcceptInputFrame(); + } + } + + @Override + public void setOutputListener(OutputListener outputListener) { + this.outputListener = outputListener; + } + + @Override + public void setErrorListener(Executor executor, ErrorListener errorListener) { + // No checked exceptions thrown. + } + + @Override + public void queueInputFrame( + GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { + this.inputTexture = inputTexture; + timestampMap.calculateOutputTimeUs( + presentationTimeUs, /* outputTimeConsumer= */ this::onOutputTimeAvailable); + pendingCallbacksCount.incrementAndGet(); + } + + @Override + public void signalEndOfCurrentInputStream() { + if (pendingCallbacksCount.get() == 0) { + outputListener.onCurrentOutputStreamEnded(); + } else { + pendingEndOfStream.set(true); + } + } + + @Override + public void releaseOutputFrame(GlTextureInfo outputTexture) { + checkState(outputTexture.texId == checkNotNull(inputTexture).texId); + inputListener.onInputFrameProcessed(outputTexture); + inputListener.onReadyToAcceptInputFrame(); + } + + @Override + public void flush() { + // TODO - b/320242819: Investigate support for previewing. + throw new UnsupportedOperationException("This effect is not supported for previewing."); + } + + @Override + public void release() throws VideoFrameProcessingException { + inputTexture = null; + } + + private void onOutputTimeAvailable(long outputTimeUs) { + outputListener.onOutputFrameAvailable(checkNotNull(inputTexture), outputTimeUs); + if (pendingEndOfStream.get()) { + outputListener.onCurrentOutputStreamEnded(); + pendingEndOfStream.set(false); + } + pendingCallbacksCount.decrementAndGet(); + } +} diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/TimestampAdjustmentTest/pts_0.png b/libraries/test_data/src/test/assets/test-generated-goldens/TimestampAdjustmentTest/pts_0.png new file mode 100644 index 0000000000000000000000000000000000000000..4ffb8030fd0cc9248247d2c2c83f484fb229831d GIT binary patch literal 591 zcmeAS@N?(olHy`uVBq!ia0vp^DL`z*!3HE(nbz$CQjEnx?oJHr&dIz4a#)I;JVQ8u zpoSx*11R^?)5S5QV$R!}w%H+$B5V)TUxyk>KNc72?CKJ15ozjJ(R(A}rZz{}wSR2& zP4$P41?6nrD8Rz0)U+bB!)XzlebS3;VG6qKheysqBKbz&QsZWv1Az>#t~QpK&ZI`K8vW()Kq8 zRPP+9?3wadiJMGoSdQ8@ z_}&}z{r%e;7T?F3nJf{%@j`OlN}1f3AI<}nX>O2QT@{}le)?kM?5Q%`mp*-ezx>U; zNK3Jl1y3U;n7&l_fBCxom91vlK#|B@_r$8^3%16~c1*2ny`^C(_N2RH_p2@5*QE81 zr%Ud;RIGENx@JP$i}pYF`lS!Yt=wsG_kPA#pgECyCfs_LUa%xPZ}t_vnMb_$ysO?d ztLV))J@5BDoGW)*Ywh(uR>&B>(LKxR-tp~eNnw-k+U~t_%r)k4;mbV{6Y5rp6^d{F zxC$6LPoo4b9kkW_`@b;a*4xh&UG7!icmAui6koDPx#1am@3R0s$N2z&@+hyVZuHAzH4RCt{2*x8R%RS?JVPY;75FrtEhAgF+-!Kk31 zXpC#(o@ineBXNyNeDYtwKS3XSQR9QVM&oWYaY@9W!MLE}h9o!$D&saVGAx1*bvk{y zG&4QZcZ|{VO-?#}?)}}ZPSvR@GMP*!lgVTtm#q|e1$9wp`$$bXXtVB;mnh{gAhs?oESS}3Gg~?civ&HJ#gx-Q? zZb5Sz4i=uZ80TOre!`C$D{>;Pz}+}ccwsjdru!V0?z5$EpF?qF6U}KO&f=7^_+m~aidsWo6~Pyb4$`Zlr(=8G}oXn9p9YpwFq|#Z+seWV>^z+ z(lq*n5q~Xc?!b|Frlh$(Y3k4RTN|2x4GZwR2s5{d{h&@4ZpS+D0Z&Tvi|~>#!p$Yk z!NPOa;!T{J=DV>1KjS{kENBkGb@&*siZEE$+>o_(p`VNojtX2y=Z{jUNh{o#|R@#VVfLqUNKd znb)SKZK@1e>+!I7*xn|`$EN#j$F4NDPK34|`@V3UJ!$L^2BKk{O-WPEbhf1X&%jY) z(M=K=(ho^jg{Bw3B+ZN_&pslmG)P|>&6q60$4uekJ!$^4G@8+Y**FH@6*S}1=(n#& z)fq@*g4m-v3z|QZt`*IMAvD`dno6kBAVs=yv`AqO6s{hFt!%`~0E)KtP~8Dt8s#reV@M+?_~03X=DrYJKBSL32|{u`n$ zA!?^q_r6+b3`f&3Oid+>mO%zYh*>M#zAycrE;jeo4s4WeT#98PPCp?w_BtbZ`OzqV zM}^Tc$QF^ezbInt9FezQCft0JMhGf04i|{rzb5LEl^RI^jg5Y=RbMNeyGLxwgBhk> zO(l$$L3U$n8oe5AjCrE1GT;6NIY_i8mWvAb0Z~hAEuFVh7^oiS%XDgVAa;onUA|fXgadV@%lye7iIbkVQ6Q50Hh#0Qw)WK2uv{4VyRxuR2BFXU#3 zgkYH{M4!Q1qRQVTyz;oDnJ8+EDm3-x=8pD1VYTUCO{|!uxJ1O=Ug7>nh>G`hJSm!q zdqsPx#1am@3R0s$N2z&@+hyVZt9!W$&RCt{2*~@QLRTRhZPs@c$c_<_nDZU7(7$9nV zfrzh(8V4pOMn^^m`ad{y=*YQK6EzrT4n@JJydp+|_yCQF0#ZOL&^CpFgSDF;j<=`x z_MS_8>~FFLJ$wJw>DglzGMP*!lgVT4>@l zavt9#n~wEjiToCyivffC@F))A6R}i1UEyb;2a0})V3Gncom!R5q6a{?=;bz zFKKqqQqwWLbvS8(K5WA6*ol3Gww-t#ui*_b3;(F_wG697K=>YZk5@5e4Q1EuF}nb^x#f$&p05Cx??r?y_l?O z*7ub9UKIOORbV>R)Iv7{vP5LZGLaqM)#P_v+yN%n`7#YtHMI-L#*?+9Ip3zH7P=Xb z_2Q^|SY*eUn*5%_S+NW*Y|=h?qBMRu$bcZ~ft z`aUVjAuGiFrd(kjNOmOIrGn;qQJyJlmf_lx=2Q!s0a3&}()niv-3rJWksU)KJ5E)1 zZ;K(Z{Pl_bX=R~pz9_}4#i3+n&4voiLHv-OpJ-8YNEAYAp&K{#evuvP@PQ~Xv~%&B zi4klU0pBc&dt1d4bE_!&^4~R9sAvV{2ViGi~KNU4Mi$R^O=BuPR)8iCX>lzGMP*!lgVT