loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-29 22:36:54 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix playback not transition into next mediaItem

Browse Source 
This is because the replay cache needs to clear the cache after one mediaItem is fully played.

That is, if the last two frames are cached, we need to wait until they are both rendered before
receiving inputs from the next input stream because the texture size might change. And when the
texture size changes, we teardown all previous used textures, and this causes a state confusion
in the shader program in that, the cache thinks one texture id is in-use (because it's not released)
but the baseGlShaderProgram texturePool thinks it's already free (as a result of size change)

Also fixes an issue that, if replaying a frame after EOS is signalled, the EOS
signal is lost because we flush the pipeline.

PiperOrigin-RevId: 745191032
This commit is contained in:
claincly 2025-04-08 10:06:29 -07:00 committed by Copybara-Service
parent 2f1fc4773c
commit e11a8a1b19
6 changed files with 210 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
libraries/effect/src/main/java/androidx/media3/effect/ReplayableFrameCacheGlShaderProgram.java
View File

@ -66,6 +66,17 @@ import androidx.media3.common.VideoFrameProcessingException;
super.flush();
}
@Override
public void signalEndOfCurrentInputStream() {
// TODO: b/391109625 - Support mixed size buffers in the output texture pool to allow
// replaying the last frame in a sequence.
for (int i = 0; i < cacheSize; i++) {
super.releaseOutputFrame(cachedFrames[i].glTextureInfo);
}
cacheSize = 0;
super.signalEndOfCurrentInputStream();
}
/** Returns whether there is no cached frame. */
public boolean isEmpty() {
return cacheSize == 0;

7
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java
View File

@ -731,8 +731,15 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video
@Override
public void redraw() {
checkState(isInitialized());
// Resignal EOS only for the last item.
boolean needsResignalEndOfCurrentInputStream = signaledEndOfStream;
long replayedPresentationTimeUs = lastOutputBufferPresentationTimeUs;
PlaybackVideoGraphWrapper.this.flush(/* resetPosition= */ false);
checkNotNull(videoGraph).redraw();
lastOutputBufferPresentationTimeUs = replayedPresentationTimeUs;
if (needsResignalEndOfCurrentInputStream) {
signalEndOfCurrentInputStream();
}
}
@Override

7
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoFrameRenderControl.java
View File

@ -209,6 +209,13 @@ import androidx.media3.exoplayer.ExoPlaybackException;
* this method, the end of input signal is ignored.
*/
public void signalEndOfInput() {
if (latestInputPresentationTimeUs == C.TIME_UNSET) {
// If EOS is signalled right after a flush without receiving a frame (could happen with frame
// replaying as available frame is not reported to the render control), set the latest input
// and output timestamp to end of source to ensure isEnded() returns true.
latestInputPresentationTimeUs = C.TIME_END_OF_SOURCE;
latestOutputPresentationTimeUs = C.TIME_END_OF_SOURCE;
}
lastPresentationTimeUs = latestInputPresentationTimeUs;
}

21
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View File

@ -69,6 +69,9 @@ import androidx.media3.effect.ScaleAndRotateTransformation;
import androidx.media3.effect.SingleInputVideoGraph;
import androidx.media3.exoplayer.mediacodec.MediaCodecSelector;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
import androidx.media3.exoplayer.video.MediaCodecVideoRenderer;
import androidx.media3.exoplayer.video.PlaybackVideoGraphWrapper;
import androidx.media3.exoplayer.video.VideoFrameReleaseControl;
import androidx.media3.extractor.ExtractorOutput;
import androidx.media3.muxer.MuxerException;
import androidx.media3.test.utils.BitmapPixelTestUtil;
@ -98,6 +101,24 @@ import org.junit.AssumptionViolatedException;
/** Utilities for instrumentation tests. */
public final class AndroidTestUtil {
/** A {@link MediaCodecVideoRenderer} subclass that supports replaying a frame. */
public static class ReplayVideoRenderer extends MediaCodecVideoRenderer {
public ReplayVideoRenderer(Context context) {
super(new Builder(context).setMediaCodecSelector(MediaCodecSelector.DEFAULT));
}
@Override
protected PlaybackVideoGraphWrapper createPlaybackVideoGraphWrapper(
Context context, VideoFrameReleaseControl videoFrameReleaseControl) {
return new PlaybackVideoGraphWrapper.Builder(context, videoFrameReleaseControl)
.setClock(getClock())
.setEnableReplayableCache(true)
.build();
}
}
private static final String TAG = "AndroidTestUtil";
/** An {@link Effects} instance that forces video transcoding. */

162
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ReplayCacheTest.java Normal file
View File

@ -0,0 +1,162 @@
/*
* Copyright 2025 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer.mh;
import static androidx.media3.common.util.Util.isRunningOnEmulator;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS;
import static androidx.test.platform.app.InstrumentationRegistry.getInstrumentation;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assume.assumeTrue;
import android.app.Instrumentation;
import android.content.Context;
import android.media.MediaFormat;
import android.os.Handler;
import android.view.SurfaceView;
import androidx.annotation.Nullable;
import androidx.media3.common.Format;
import androidx.media3.common.MediaItem;
import androidx.media3.common.PlaybackException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.util.Util;
import androidx.media3.effect.Contrast;
import androidx.media3.exoplayer.DefaultRenderersFactory;
import androidx.media3.exoplayer.ExoPlayer;
import androidx.media3.exoplayer.Renderer;
import androidx.media3.exoplayer.mediacodec.MediaCodecSelector;
import androidx.media3.exoplayer.util.EventLogger;
import androidx.media3.exoplayer.video.VideoFrameMetadataListener;
import androidx.media3.exoplayer.video.VideoRendererEventListener;
import androidx.media3.transformer.AndroidTestUtil.ReplayVideoRenderer;
import androidx.media3.transformer.PlayerTestListener;
import androidx.media3.transformer.SurfaceTestActivity;
import androidx.test.ext.junit.rules.ActivityScenarioRule;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.platform.app.InstrumentationRegistry;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeoutException;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
/** Instrumentation tests for frame replaying (dynamic effect update). */
@RunWith(AndroidJUnit4.class)
public class ReplayCacheTest {
private static final long TEST_TIMEOUT_MS = isRunningOnEmulator() ? 20_000 : 10_000;
private static final MediaItem VIDEO_MEDIA_ITEM_1 = MediaItem.fromUri(MP4_ASSET.uri);
private static final MediaItem VIDEO_MEDIA_ITEM_2 =
MediaItem.fromUri(MP4_ASSET_WITH_INCREASING_TIMESTAMPS.uri);
@Rule
public ActivityScenarioRule<SurfaceTestActivity> rule =
new ActivityScenarioRule<>(SurfaceTestActivity.class);
private final Context context = getInstrumentation().getContext().getApplicationContext();
private final Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation();
private ExoPlayer exoPlayer;
private SurfaceView surfaceView;
@Before
public void setUp() {
rule.getScenario().onActivity(activity -> surfaceView = activity.getSurfaceView());
}
@After
public void tearDown() {
rule.getScenario().close();
getInstrumentation()
.runOnMainSync(
() -> {
if (exoPlayer != null) {
exoPlayer.release();
}
});
}
@Test
public void replayOnEveryFrame_withExoPlayer_succeeds()
throws PlaybackException, TimeoutException {
assumeTrue(
"The MediaCodec decoder's output surface is sometimes dropping frames on emulator despite"
+ " using MediaFormat.KEY_ALLOW_FRAME_DROP.",
!Util.isRunningOnEmulator());
PlayerTestListener playerTestListener = new PlayerTestListener(TEST_TIMEOUT_MS);
List<Long> playedFrameTimestampsUs = new ArrayList<>();
instrumentation.runOnMainSync(
() -> {
Renderer videoRenderer = new ReplayVideoRenderer(context);
exoPlayer =
new ExoPlayer.Builder(context)
.setRenderersFactory(
new DefaultRenderersFactory(context) {
@Override
protected void buildVideoRenderers(
Context context,
@ExtensionRendererMode int extensionRendererMode,
MediaCodecSelector mediaCodecSelector,
boolean enableDecoderFallback,
Handler eventHandler,
VideoRendererEventListener eventListener,
long allowedVideoJoiningTimeMs,
ArrayList<Renderer> builtVideoRenderers) {
builtVideoRenderers.add(videoRenderer);
}
})
.build();
exoPlayer.setVideoSurfaceView(surfaceView);
// Adding an EventLogger to use its log output in case the test fails.
exoPlayer.addAnalyticsListener(new EventLogger());
exoPlayer.addListener(playerTestListener);
exoPlayer.setVideoEffects(ImmutableList.of(new Contrast(0.5f)));
exoPlayer.setVideoFrameMetadataListener(
new VideoFrameMetadataListener() {
private final List<Long> replayedFrames = new ArrayList<>();
@Override
public void onVideoFrameAboutToBeRendered(
long presentationTimeUs,
long releaseTimeNs,
Format format,
@Nullable MediaFormat mediaFormat) {
playedFrameTimestampsUs.add(presentationTimeUs);
if (replayedFrames.contains(presentationTimeUs)) {
return;
}
replayedFrames.add(presentationTimeUs);
instrumentation.runOnMainSync(
() -> exoPlayer.setVideoEffects(VideoFrameProcessor.REDRAW));
}
});
exoPlayer.setMediaItems(ImmutableList.of(VIDEO_MEDIA_ITEM_1, VIDEO_MEDIA_ITEM_2));
exoPlayer.prepare();
exoPlayer.play();
});
playerTestListener.waitUntilPlayerEnded();
// VIDEO_1 has size 30, VIDEO_2 has size 30, every frame is replayed once, minus one frame that
// we don't currently replay (the frame at media item transition).
assertThat(playedFrameTimestampsUs).hasSize(119);
}
}

21
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/EffectPlaybackPixelTest.java
View File

@ -56,10 +56,9 @@ import androidx.media3.exoplayer.Renderer;
import androidx.media3.exoplayer.mediacodec.MediaCodecSelector;
import androidx.media3.exoplayer.util.EventLogger;
import androidx.media3.exoplayer.video.MediaCodecVideoRenderer;
import androidx.media3.exoplayer.video.PlaybackVideoGraphWrapper;
import androidx.media3.exoplayer.video.VideoFrameReleaseControl;
import androidx.media3.exoplayer.video.VideoRendererEventListener;
import androidx.media3.test.utils.BitmapPixelTestUtil;
import androidx.media3.transformer.AndroidTestUtil.ReplayVideoRenderer;
import androidx.media3.transformer.SurfaceTestActivity;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.rules.ActivityScenarioRule;
@ -294,7 +293,7 @@ public class EffectPlaybackPixelTest {
instrumentation.runOnMainSync(
() -> {
Context context = ApplicationProvider.getApplicationContext();
Renderer videoRenderer = new ReplayVideoRenderer(context, MediaCodecSelector.DEFAULT);
Renderer videoRenderer = new ReplayVideoRenderer(context);
player =
new ExoPlayer.Builder(context)
.setRenderersFactory(
@ -573,22 +572,6 @@ public class EffectPlaybackPixelTest {
}
}
private static class ReplayVideoRenderer extends MediaCodecVideoRenderer {
public ReplayVideoRenderer(Context context, MediaCodecSelector mediaCodecSelector) {
super(new Builder(context).setMediaCodecSelector(mediaCodecSelector));
}
@Override
protected PlaybackVideoGraphWrapper createPlaybackVideoGraphWrapper(
Context context, VideoFrameReleaseControl videoFrameReleaseControl) {
return new PlaybackVideoGraphWrapper.Builder(context, videoFrameReleaseControl)
.setClock(getClock())
.setEnableReplayableCache(true)
.build();
}
}
private static class NoFrameDroppedVideoRenderer extends MediaCodecVideoRenderer {
public NoFrameDroppedVideoRenderer(Context context, MediaCodecSelector mediaCodecSelector) {