diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ReplayableFrameCacheGlShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/ReplayableFrameCacheGlShaderProgram.java index b06cd110a1..66ce0cb979 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/ReplayableFrameCacheGlShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/ReplayableFrameCacheGlShaderProgram.java @@ -66,6 +66,17 @@ import androidx.media3.common.VideoFrameProcessingException; super.flush(); } + @Override + public void signalEndOfCurrentInputStream() { + // TODO: b/391109625 - Support mixed size buffers in the output texture pool to allow + // replaying the last frame in a sequence. + for (int i = 0; i < cacheSize; i++) { + super.releaseOutputFrame(cachedFrames[i].glTextureInfo); + } + cacheSize = 0; + super.signalEndOfCurrentInputStream(); + } + /** Returns whether there is no cached frame. */ public boolean isEmpty() { return cacheSize == 0; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java index 74936e71ef..9b7e735644 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java @@ -731,8 +731,15 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video @Override public void redraw() { checkState(isInitialized()); + // Resignal EOS only for the last item. + boolean needsResignalEndOfCurrentInputStream = signaledEndOfStream; + long replayedPresentationTimeUs = lastOutputBufferPresentationTimeUs; PlaybackVideoGraphWrapper.this.flush(/* resetPosition= */ false); checkNotNull(videoGraph).redraw(); + lastOutputBufferPresentationTimeUs = replayedPresentationTimeUs; + if (needsResignalEndOfCurrentInputStream) { + signalEndOfCurrentInputStream(); + } } @Override diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java index f145011298..4c8b9c1907 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java @@ -209,6 +209,13 @@ import androidx.media3.exoplayer.ExoPlaybackException; * this method, the end of input signal is ignored. */ public void signalEndOfInput() { + if (latestInputPresentationTimeUs == C.TIME_UNSET) { + // If EOS is signalled right after a flush without receiving a frame (could happen with frame + // replaying as available frame is not reported to the render control), set the latest input + // and output timestamp to end of source to ensure isEnded() returns true. + latestInputPresentationTimeUs = C.TIME_END_OF_SOURCE; + latestOutputPresentationTimeUs = C.TIME_END_OF_SOURCE; + } lastPresentationTimeUs = latestInputPresentationTimeUs; } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index 7ae068aa31..c10a14b8c2 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -69,6 +69,9 @@ import androidx.media3.effect.ScaleAndRotateTransformation; import androidx.media3.effect.SingleInputVideoGraph; import androidx.media3.exoplayer.mediacodec.MediaCodecSelector; import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; +import androidx.media3.exoplayer.video.MediaCodecVideoRenderer; +import androidx.media3.exoplayer.video.PlaybackVideoGraphWrapper; +import androidx.media3.exoplayer.video.VideoFrameReleaseControl; import androidx.media3.extractor.ExtractorOutput; import androidx.media3.muxer.MuxerException; import androidx.media3.test.utils.BitmapPixelTestUtil; @@ -98,6 +101,24 @@ import org.junit.AssumptionViolatedException; /** Utilities for instrumentation tests. */ public final class AndroidTestUtil { + + /** A {@link MediaCodecVideoRenderer} subclass that supports replaying a frame. */ + public static class ReplayVideoRenderer extends MediaCodecVideoRenderer { + + public ReplayVideoRenderer(Context context) { + super(new Builder(context).setMediaCodecSelector(MediaCodecSelector.DEFAULT)); + } + + @Override + protected PlaybackVideoGraphWrapper createPlaybackVideoGraphWrapper( + Context context, VideoFrameReleaseControl videoFrameReleaseControl) { + return new PlaybackVideoGraphWrapper.Builder(context, videoFrameReleaseControl) + .setClock(getClock()) + .setEnableReplayableCache(true) + .build(); + } + } + private static final String TAG = "AndroidTestUtil"; /** An {@link Effects} instance that forces video transcoding. */ diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ReplayCacheTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ReplayCacheTest.java new file mode 100644 index 0000000000..6fd569f1b8 --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ReplayCacheTest.java @@ -0,0 +1,162 @@ +/* + * Copyright 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer.mh; + +import static androidx.media3.common.util.Util.isRunningOnEmulator; +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET; +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS; +import static androidx.test.platform.app.InstrumentationRegistry.getInstrumentation; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assume.assumeTrue; + +import android.app.Instrumentation; +import android.content.Context; +import android.media.MediaFormat; +import android.os.Handler; +import android.view.SurfaceView; +import androidx.annotation.Nullable; +import androidx.media3.common.Format; +import androidx.media3.common.MediaItem; +import androidx.media3.common.PlaybackException; +import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.util.Util; +import androidx.media3.effect.Contrast; +import androidx.media3.exoplayer.DefaultRenderersFactory; +import androidx.media3.exoplayer.ExoPlayer; +import androidx.media3.exoplayer.Renderer; +import androidx.media3.exoplayer.mediacodec.MediaCodecSelector; +import androidx.media3.exoplayer.util.EventLogger; +import androidx.media3.exoplayer.video.VideoFrameMetadataListener; +import androidx.media3.exoplayer.video.VideoRendererEventListener; +import androidx.media3.transformer.AndroidTestUtil.ReplayVideoRenderer; +import androidx.media3.transformer.PlayerTestListener; +import androidx.media3.transformer.SurfaceTestActivity; +import androidx.test.ext.junit.rules.ActivityScenarioRule; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeoutException; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Instrumentation tests for frame replaying (dynamic effect update). */ +@RunWith(AndroidJUnit4.class) +public class ReplayCacheTest { + private static final long TEST_TIMEOUT_MS = isRunningOnEmulator() ? 20_000 : 10_000; + + private static final MediaItem VIDEO_MEDIA_ITEM_1 = MediaItem.fromUri(MP4_ASSET.uri); + private static final MediaItem VIDEO_MEDIA_ITEM_2 = + MediaItem.fromUri(MP4_ASSET_WITH_INCREASING_TIMESTAMPS.uri); + + @Rule + public ActivityScenarioRule rule = + new ActivityScenarioRule<>(SurfaceTestActivity.class); + + private final Context context = getInstrumentation().getContext().getApplicationContext(); + private final Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); + + private ExoPlayer exoPlayer; + private SurfaceView surfaceView; + + @Before + public void setUp() { + rule.getScenario().onActivity(activity -> surfaceView = activity.getSurfaceView()); + } + + @After + public void tearDown() { + rule.getScenario().close(); + getInstrumentation() + .runOnMainSync( + () -> { + if (exoPlayer != null) { + exoPlayer.release(); + } + }); + } + + @Test + public void replayOnEveryFrame_withExoPlayer_succeeds() + throws PlaybackException, TimeoutException { + assumeTrue( + "The MediaCodec decoder's output surface is sometimes dropping frames on emulator despite" + + " using MediaFormat.KEY_ALLOW_FRAME_DROP.", + !Util.isRunningOnEmulator()); + PlayerTestListener playerTestListener = new PlayerTestListener(TEST_TIMEOUT_MS); + List playedFrameTimestampsUs = new ArrayList<>(); + + instrumentation.runOnMainSync( + () -> { + Renderer videoRenderer = new ReplayVideoRenderer(context); + exoPlayer = + new ExoPlayer.Builder(context) + .setRenderersFactory( + new DefaultRenderersFactory(context) { + @Override + protected void buildVideoRenderers( + Context context, + @ExtensionRendererMode int extensionRendererMode, + MediaCodecSelector mediaCodecSelector, + boolean enableDecoderFallback, + Handler eventHandler, + VideoRendererEventListener eventListener, + long allowedVideoJoiningTimeMs, + ArrayList builtVideoRenderers) { + builtVideoRenderers.add(videoRenderer); + } + }) + .build(); + exoPlayer.setVideoSurfaceView(surfaceView); + // Adding an EventLogger to use its log output in case the test fails. + exoPlayer.addAnalyticsListener(new EventLogger()); + exoPlayer.addListener(playerTestListener); + exoPlayer.setVideoEffects(ImmutableList.of(new Contrast(0.5f))); + exoPlayer.setVideoFrameMetadataListener( + new VideoFrameMetadataListener() { + private final List replayedFrames = new ArrayList<>(); + + @Override + public void onVideoFrameAboutToBeRendered( + long presentationTimeUs, + long releaseTimeNs, + Format format, + @Nullable MediaFormat mediaFormat) { + playedFrameTimestampsUs.add(presentationTimeUs); + if (replayedFrames.contains(presentationTimeUs)) { + return; + } + replayedFrames.add(presentationTimeUs); + instrumentation.runOnMainSync( + () -> exoPlayer.setVideoEffects(VideoFrameProcessor.REDRAW)); + } + }); + exoPlayer.setMediaItems(ImmutableList.of(VIDEO_MEDIA_ITEM_1, VIDEO_MEDIA_ITEM_2)); + exoPlayer.prepare(); + exoPlayer.play(); + }); + + playerTestListener.waitUntilPlayerEnded(); + // VIDEO_1 has size 30, VIDEO_2 has size 30, every frame is replayed once, minus one frame that + // we don't currently replay (the frame at media item transition). + assertThat(playedFrameTimestampsUs).hasSize(119); + } +} diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/EffectPlaybackPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/EffectPlaybackPixelTest.java index 1e79d633e0..a6c0ce88bd 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/EffectPlaybackPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/EffectPlaybackPixelTest.java @@ -56,10 +56,9 @@ import androidx.media3.exoplayer.Renderer; import androidx.media3.exoplayer.mediacodec.MediaCodecSelector; import androidx.media3.exoplayer.util.EventLogger; import androidx.media3.exoplayer.video.MediaCodecVideoRenderer; -import androidx.media3.exoplayer.video.PlaybackVideoGraphWrapper; -import androidx.media3.exoplayer.video.VideoFrameReleaseControl; import androidx.media3.exoplayer.video.VideoRendererEventListener; import androidx.media3.test.utils.BitmapPixelTestUtil; +import androidx.media3.transformer.AndroidTestUtil.ReplayVideoRenderer; import androidx.media3.transformer.SurfaceTestActivity; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.rules.ActivityScenarioRule; @@ -294,7 +293,7 @@ public class EffectPlaybackPixelTest { instrumentation.runOnMainSync( () -> { Context context = ApplicationProvider.getApplicationContext(); - Renderer videoRenderer = new ReplayVideoRenderer(context, MediaCodecSelector.DEFAULT); + Renderer videoRenderer = new ReplayVideoRenderer(context); player = new ExoPlayer.Builder(context) .setRenderersFactory( @@ -573,22 +572,6 @@ public class EffectPlaybackPixelTest { } } - private static class ReplayVideoRenderer extends MediaCodecVideoRenderer { - - public ReplayVideoRenderer(Context context, MediaCodecSelector mediaCodecSelector) { - super(new Builder(context).setMediaCodecSelector(mediaCodecSelector)); - } - - @Override - protected PlaybackVideoGraphWrapper createPlaybackVideoGraphWrapper( - Context context, VideoFrameReleaseControl videoFrameReleaseControl) { - return new PlaybackVideoGraphWrapper.Builder(context, videoFrameReleaseControl) - .setClock(getClock()) - .setEnableReplayableCache(true) - .build(); - } - } - private static class NoFrameDroppedVideoRenderer extends MediaCodecVideoRenderer { public NoFrameDroppedVideoRenderer(Context context, MediaCodecSelector mediaCodecSelector) {