loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix issue where subtitles starting before a seek position are skipped

Browse Source 
These subtitles were skipped because they are marked as shouldBeSkipped
based on their timestamps. The fix removes this flag entirely in
SimpleSubtitleDecoder because TextRenderer handles potential skipping
if needed.

PiperOrigin-RevId: 629717970
This commit is contained in:
tonihei 2024-05-01 07:31:43 -07:00 committed by Copybara-Service
parent fb982c2d54
commit 1af86d4c4d
4 changed files with 5962 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
RELEASENOTES.md
View File

@ -30,6 +30,8 @@
* Audio:
* Video:
* Text:
* Fix issue where subtitles starting before a seek position are skipped.
This issue was only introduced in Media3 1.4.0-alpha01.
* Metadata:
* Fix mapping of MP4 to ID3 sort tags. Previously the 'album sort'
(`soal`), 'artist sort' (`soar`) and 'album artist sort' (`soaa`) MP4

215
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/WebvttPlaybackTest.java
View File

@ -15,6 +15,8 @@
*/
package androidx.media3.exoplayer.e2etest;
import static androidx.media3.test.utils.robolectric.TestPlayerRunHelper.play;
import static androidx.media3.test.utils.robolectric.TestPlayerRunHelper.run;
import static com.google.common.truth.Truth.assertThat;
import android.content.Context;
@ -26,6 +28,10 @@ import androidx.media3.common.C;
import androidx.media3.common.MediaItem;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.Player;
import androidx.media3.common.text.CueGroup;
import androidx.media3.common.util.Clock;
import androidx.media3.common.util.Util;
import androidx.media3.exoplayer.DefaultLoadControl;
import androidx.media3.exoplayer.DefaultRenderersFactory;
import androidx.media3.exoplayer.ExoPlaybackException;
import androidx.media3.exoplayer.ExoPlayer;
@ -39,12 +45,15 @@ import androidx.media3.test.utils.CapturingRenderersFactory;
import androidx.media3.test.utils.DumpFileAsserts;
import androidx.media3.test.utils.FakeClock;
import androidx.media3.test.utils.robolectric.PlaybackOutput;
import androidx.media3.test.utils.robolectric.RobolectricUtil;
import androidx.media3.test.utils.robolectric.ShadowMediaCodecConfig;
import androidx.media3.test.utils.robolectric.TestPlayerRunHelper;
import androidx.test.core.app.ApplicationProvider;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import java.util.ArrayList;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -95,8 +104,10 @@ public class WebvttPlaybackTest {
player.setMediaItem(mediaItem);
player.prepare();
run(player).untilState(Player.STATE_READY);
run(player).untilLoadingIs(false);
player.play();
TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED);
run(player).untilState(Player.STATE_ENDED);
player.release();
surface.release();
@ -104,6 +115,169 @@ public class WebvttPlaybackTest {
applicationContext, playbackOutput, "playbackdumps/webvtt/" + inputFile + ".dump");
}
@Test
public void test_withSeek() throws Exception {
Context applicationContext = ApplicationProvider.getApplicationContext();
CapturingRenderersFactory capturingRenderersFactory =
new CapturingRenderersFactory(applicationContext);
MediaSource.Factory mediaSourceFactory =
new DefaultMediaSourceFactory(applicationContext)
.experimentalParseSubtitlesDuringExtraction(true);
ExoPlayer player =
new ExoPlayer.Builder(applicationContext, capturingRenderersFactory)
.setClock(new FakeClock(/* isAutoAdvancing= */ true))
.setMediaSourceFactory(mediaSourceFactory)
.setLoadControl(
new DefaultLoadControl.Builder()
.setBackBuffer(
/* backBufferDurationMs= */ 10000, /* retainBackBufferFromKeyframe= */ true)
.build())
.build();
Surface surface = new Surface(new SurfaceTexture(/* texName= */ 1));
player.setVideoSurface(surface);
PlaybackOutput playbackOutput = PlaybackOutput.register(player, capturingRenderersFactory);
MediaItem mediaItem =
new MediaItem.Builder()
.setUri("asset:///media/mp4/preroll-5s.mp4")
.setSubtitleConfigurations(
ImmutableList.of(
new MediaItem.SubtitleConfiguration.Builder(
Uri.parse("asset:///media/webvtt/" + inputFile))
.setMimeType(MimeTypes.TEXT_VTT)
.setLanguage("en")
.setSelectionFlags(C.SELECTION_FLAG_DEFAULT)
.build()))
.build();
// Play media fully (with back buffer) to ensure we have all the segment data available.
player.setMediaItem(mediaItem);
player.prepare();
run(player).untilState(Player.STATE_READY);
run(player).untilLoadingIs(false);
player.play();
run(player).untilState(Player.STATE_ENDED);
// Seek back to within first subtitle.
player.seekTo(1000);
player.play();
run(player).untilState(Player.STATE_ENDED);
player.release();
surface.release();
DumpFileAsserts.assertOutput(
applicationContext, playbackOutput, "playbackdumps/webvtt/" + inputFile + ".seek.dump");
}
@Test
public void test_legacyParseInRenderer() throws Exception {
Context applicationContext = ApplicationProvider.getApplicationContext();
CapturingRenderersFactory capturingRenderersFactory =
new CapturingRenderersFactory(applicationContext);
MediaSource.Factory mediaSourceFactory =
new DefaultMediaSourceFactory(applicationContext)
.experimentalParseSubtitlesDuringExtraction(false);
ExoPlayer player =
new ExoPlayer.Builder(applicationContext, capturingRenderersFactory)
.setClock(new FakeClock(/* isAutoAdvancing= */ true))
.setMediaSourceFactory(mediaSourceFactory)
.build();
Surface surface = new Surface(new SurfaceTexture(/* texName= */ 1));
player.setVideoSurface(surface);
PlaybackOutput playbackOutput = PlaybackOutput.register(player, capturingRenderersFactory);
MediaItem mediaItem =
new MediaItem.Builder()
.setUri("asset:///media/mp4/preroll-5s.mp4")
.setSubtitleConfigurations(
ImmutableList.of(
new MediaItem.SubtitleConfiguration.Builder(
Uri.parse("asset:///media/webvtt/" + inputFile))
.setMimeType(MimeTypes.TEXT_VTT)
.setLanguage("en")
.setSelectionFlags(C.SELECTION_FLAG_DEFAULT)
.build()))
.build();
// Carefully play and stall until all expected Cues arrived. This is needed because the legacy
// mode decodes subtitles in a background thread not controlled by our clock and the player also
// doesn't wait for subtitles to be decoded before making progress.
player.setMediaItem(mediaItem);
AtomicBoolean firstCueArrived = createCuesCondition(player, 0, /* cuesEmpty= */ false);
player.prepare();
player.play();
stallPlayerUntilCondition(player, firstCueArrived);
playUntilCuesArrived(player, 1234000, /* cuesEmpty= */ true);
playUntilCuesArrived(player, 2345000, /* cuesEmpty= */ false);
playUntilCuesArrived(player, 3456000, /* cuesEmpty= */ true);
play(player).untilState(Player.STATE_ENDED);
player.release();
surface.release();
DumpFileAsserts.assertOutput(
applicationContext, playbackOutput, "playbackdumps/webvtt/" + inputFile + ".dump");
}
@Test
public void test_legacyParseInRendererWithSeek() throws Exception {
Context applicationContext = ApplicationProvider.getApplicationContext();
CapturingRenderersFactory capturingRenderersFactory =
new CapturingRenderersFactory(applicationContext);
MediaSource.Factory mediaSourceFactory =
new DefaultMediaSourceFactory(applicationContext)
.experimentalParseSubtitlesDuringExtraction(false);
ExoPlayer player =
new ExoPlayer.Builder(applicationContext, capturingRenderersFactory)
.setClock(new FakeClock(/* isAutoAdvancing= */ true))
.setMediaSourceFactory(mediaSourceFactory)
.setLoadControl(
new DefaultLoadControl.Builder()
.setBackBuffer(
/* backBufferDurationMs= */ 10000, /* retainBackBufferFromKeyframe= */ true)
.build())
.build();
Surface surface = new Surface(new SurfaceTexture(/* texName= */ 1));
player.setVideoSurface(surface);
PlaybackOutput playbackOutput = PlaybackOutput.register(player, capturingRenderersFactory);
MediaItem mediaItem =
new MediaItem.Builder()
.setUri("asset:///media/mp4/preroll-5s.mp4")
.setSubtitleConfigurations(
ImmutableList.of(
new MediaItem.SubtitleConfiguration.Builder(
Uri.parse("asset:///media/webvtt/" + inputFile))
.setMimeType(MimeTypes.TEXT_VTT)
.setLanguage("en")
.setSelectionFlags(C.SELECTION_FLAG_DEFAULT)
.build()))
.build();
// Play media fully (with back buffer) to ensure we have all the segment data available.
// Carefully play and stall until all expected Cues arrived. This is needed because the legacy
// mode decodes subtitles in a background thread not controlled by our clock and the player also
// doesn't wait for subtitles to be decoded before making progress.
player.setMediaItem(mediaItem);
AtomicBoolean firstCueArrived = createCuesCondition(player, 0, /* cuesEmpty= */ false);
player.prepare();
player.play();
stallPlayerUntilCondition(player, firstCueArrived);
playUntilCuesArrived(player, 1234000, /* cuesEmpty= */ true);
playUntilCuesArrived(player, 2345000, /* cuesEmpty= */ false);
playUntilCuesArrived(player, 3456000, /* cuesEmpty= */ true);
play(player).untilState(Player.STATE_ENDED);
// Seek back to within first subtitle.
player.pause();
AtomicBoolean newFirstCueArrived = createCuesCondition(player, 0, /* cuesEmpty= */ false);
player.seekTo(1000);
stallPlayerUntilCondition(player, newFirstCueArrived);
playUntilCuesArrived(player, 1234000, /* cuesEmpty= */ true);
playUntilCuesArrived(player, 2345000, /* cuesEmpty= */ false);
playUntilCuesArrived(player, 3456000, /* cuesEmpty= */ true);
play(player).untilState(Player.STATE_ENDED);
player.release();
surface.release();
DumpFileAsserts.assertOutput(
applicationContext, playbackOutput, "playbackdumps/webvtt/" + inputFile + ".seek.dump");
}
@Test
public void textRendererDoesntSupportLegacyDecoding_playbackFails() throws Exception {
Context applicationContext = ApplicationProvider.getApplicationContext();
@ -154,4 +328,43 @@ public class WebvttPlaybackTest {
player.release();
surface.release();
}
private static void playUntilCuesArrived(ExoPlayer player, long cuesTimeUs, boolean cuesEmpty)
throws Exception {
AtomicBoolean cuesFound = createCuesCondition(player, cuesTimeUs, cuesEmpty);
play(player)
.untilBackgroundThreadCondition(
() -> player.getCurrentPosition() >= Util.usToMs(cuesTimeUs));
player.pause();
stallPlayerUntilCondition(player, cuesFound);
}
private static AtomicBoolean createCuesCondition(
ExoPlayer player, long cuesTimeUs, boolean cuesEmpty) {
AtomicBoolean cuesFound = new AtomicBoolean();
player.addListener(
new Player.Listener() {
@Override
public void onCues(CueGroup cueGroup) {
if (cueGroup.presentationTimeUs == cuesTimeUs && cuesEmpty == cueGroup.cues.isEmpty()) {
cuesFound.set(true);
}
}
});
return cuesFound;
}
private static void stallPlayerUntilCondition(ExoPlayer player, AtomicBoolean condition)
throws Exception {
long timeoutTimeMs = Clock.DEFAULT.currentTimeMillis() + RobolectricUtil.DEFAULT_TIMEOUT_MS;
while (!condition.get()) {
// Trigger more work at the current time until the condition is fulfilled.
if (Clock.DEFAULT.currentTimeMillis() >= timeoutTimeMs) {
throw new TimeoutException();
}
player.pause();
player.play();
run(player).untilPendingCommandsAreFullyHandled();
}
}
}

1
libraries/extractor/src/main/java/androidx/media3/extractor/text/SimpleSubtitleDecoder.java
View File

@ -78,6 +78,7 @@ public abstract class SimpleSubtitleDecoder
ByteBuffer inputData = Assertions.checkNotNull(inputBuffer.data);
Subtitle subtitle = decode(inputData.array(), inputData.limit(), reset);
outputBuffer.setContent(inputBuffer.timeUs, subtitle, inputBuffer.subsampleOffsetUs);
outputBuffer.shouldBeSkipped = false; // Skipping is handled by TextRenderer
return null;
} catch (SubtitleDecoderException e) {
return e;

5745
libraries/test_data/src/test/assets/playbackdumps/webvtt/typical.seek.dump Normal file
View File

File diff suppressed because it is too large Load Diff