From f36ab87b387f605dcc65ac89bf339c234ec05d79 Mon Sep 17 00:00:00 2001 From: ibaker Date: Thu, 21 Dec 2023 08:21:13 -0800 Subject: [PATCH] Fix DASH CEA-608 parsing during extraction This is similar to the HLS fix in https://github.com/androidx/media/commit/770ca66fbc97d6ca1ec08f8b110927bd3aaaf781 Similar to HLS, the original problem here was **not** modifying the `Format` for caption tracks embedded into the video stream. I tried just updating the format in both places, but that caused new failures because the new ('transcoded') format was then fed into `FragmentedMp4Extractor` as part of `closedCaptionFormats`, which resulted in the CEA-608 data being emitted from `FragmentedMp4Extractor` with the incorrect `application/x-media3-cues` MIME type (but the bytes were actually CEA-608), meaning the transcoding wrapper passed it through without transcoding and decoding failed (because obviously CEA-608 bytes can't be decoded by `CueDecoder` which is expecting a `Bundle` from `CuesWithTiming.toBundle`. To resolve this we keep track of the 'original' caption formats inside `TrackGroupInfo`, so we can feed them into `FragmentedMp4Extractor`. For all other usages in `DashMediaPeriod` we use the 'transcoded' caption formats. PiperOrigin-RevId: 592866262 --- .../exoplayer/dash/DashMediaPeriod.java | 96 ++++++++++++------- .../dash/e2etest/DashPlaybackTest.java | 43 ++++++++- 2 files changed, 102 insertions(+), 37 deletions(-) diff --git a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaPeriod.java b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaPeriod.java index c4bfbb085f..e2089d8a32 100644 --- a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaPeriod.java +++ b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaPeriod.java @@ -59,6 +59,7 @@ import androidx.media3.exoplayer.upstream.CmcdConfiguration; import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy; import androidx.media3.exoplayer.upstream.LoaderErrorThrower; import androidx.media3.extractor.text.SubtitleParser; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; import java.io.IOException; @@ -689,18 +690,6 @@ import java.util.regex.Pattern; originalFormat .buildUpon() .setCryptoType(drmSessionManager.getCryptoType(originalFormat)); - if (subtitleParserFactory != null && subtitleParserFactory.supportsFormat(originalFormat)) { - updatedFormat - .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES) - .setCueReplacementBehavior( - subtitleParserFactory.getCueReplacementBehavior(originalFormat)) - .setCodecs( - originalFormat.sampleMimeType - + (originalFormat.codecs != null ? " " + originalFormat.codecs : "")) - // Reset this value to the default. All non-default timestamp adjustments are done - // by SubtitleTranscodingExtractor and there are no 'subsamples' after transcoding. - .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE); - } formats[j] = updatedFormat.build(); } @@ -715,6 +704,7 @@ import java.util.regex.Pattern; int closedCaptionTrackGroupIndex = primaryGroupClosedCaptionTrackFormats[i].length != 0 ? trackGroupCount++ : C.INDEX_UNSET; + maybeUpdateFormatsForParsedText(subtitleParserFactory, formats); trackGroups[primaryTrackGroupIndex] = new TrackGroup(trackGroupId, formats); trackGroupInfos[primaryTrackGroupIndex] = TrackGroupInfo.primaryTrack( @@ -736,10 +726,15 @@ import java.util.regex.Pattern; } if (closedCaptionTrackGroupIndex != C.INDEX_UNSET) { String closedCaptionTrackGroupId = trackGroupId + ":cc"; + trackGroupInfos[closedCaptionTrackGroupIndex] = + TrackGroupInfo.embeddedClosedCaptionTrack( + adaptationSetIndices, + primaryTrackGroupIndex, + ImmutableList.copyOf(primaryGroupClosedCaptionTrackFormats[i])); + maybeUpdateFormatsForParsedText( + subtitleParserFactory, primaryGroupClosedCaptionTrackFormats[i]); trackGroups[closedCaptionTrackGroupIndex] = new TrackGroup(closedCaptionTrackGroupId, primaryGroupClosedCaptionTrackFormats[i]); - trackGroupInfos[closedCaptionTrackGroupIndex] = - TrackGroupInfo.embeddedClosedCaptionTrack(adaptationSetIndices, primaryTrackGroupIndex); } } return trackGroupCount; @@ -774,14 +769,12 @@ import java.util.regex.Pattern; trackGroups.get(trackGroupInfo.embeddedEventMessageTrackGroupIndex); embeddedTrackCount++; } - boolean enableClosedCaptionTrack = - trackGroupInfo.embeddedClosedCaptionTrackGroupIndex != C.INDEX_UNSET; - TrackGroup embeddedClosedCaptionTrackGroup = null; - if (enableClosedCaptionTrack) { - embeddedClosedCaptionTrackGroup = - trackGroups.get(trackGroupInfo.embeddedClosedCaptionTrackGroupIndex); - embeddedTrackCount += embeddedClosedCaptionTrackGroup.length; - } + ImmutableList embeddedClosedCaptionOriginalFormats = + trackGroupInfo.embeddedClosedCaptionTrackGroupIndex != C.INDEX_UNSET + ? trackGroupInfos[trackGroupInfo.embeddedClosedCaptionTrackGroupIndex] + .embeddedClosedCaptionTrackOriginalFormats + : ImmutableList.of(); + embeddedTrackCount += embeddedClosedCaptionOriginalFormats.size(); Format[] embeddedTrackFormats = new Format[embeddedTrackCount]; int[] embeddedTrackTypes = new int[embeddedTrackCount]; @@ -792,13 +785,11 @@ import java.util.regex.Pattern; embeddedTrackCount++; } List embeddedClosedCaptionTrackFormats = new ArrayList<>(); - if (enableClosedCaptionTrack) { - for (int i = 0; i < embeddedClosedCaptionTrackGroup.length; i++) { - embeddedTrackFormats[embeddedTrackCount] = embeddedClosedCaptionTrackGroup.getFormat(i); - embeddedTrackTypes[embeddedTrackCount] = C.TRACK_TYPE_TEXT; - embeddedClosedCaptionTrackFormats.add(embeddedTrackFormats[embeddedTrackCount]); - embeddedTrackCount++; - } + for (int i = 0; i < embeddedClosedCaptionOriginalFormats.size(); i++) { + embeddedTrackFormats[embeddedTrackCount] = embeddedClosedCaptionOriginalFormats.get(i); + embeddedTrackTypes[embeddedTrackCount] = C.TRACK_TYPE_TEXT; + embeddedClosedCaptionTrackFormats.add(embeddedTrackFormats[embeddedTrackCount]); + embeddedTrackCount++; } PlayerTrackEmsgHandler trackPlayerEmsgHandler = @@ -932,6 +923,30 @@ import java.util.regex.Pattern; return formats; } + /** + * Modifies the provided {@link Format} array if subtitle/caption parsing is configured to happen + * during extraction. + */ + private static void maybeUpdateFormatsForParsedText( + SubtitleParser.Factory subtitleParserFactory, Format[] formats) { + for (int i = 0; i < formats.length; i++) { + if (subtitleParserFactory == null || !subtitleParserFactory.supportsFormat(formats[i])) { + continue; + } + formats[i] = + formats[i] + .buildUpon() + .setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES) + .setCueReplacementBehavior( + subtitleParserFactory.getCueReplacementBehavior(formats[i])) + .setCodecs( + formats[i].sampleMimeType + + (formats[i].codecs != null ? " " + formats[i].codecs : "")) + .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE) + .build(); + } + } + // We won't assign the array to a variable that erases the generic type, and then write into it. @SuppressWarnings({"unchecked", "rawtypes"}) private static ChunkSampleStream[] newSampleStreamArray(int length) { @@ -973,6 +988,9 @@ import java.util.regex.Pattern; public final int embeddedEventMessageTrackGroupIndex; public final int embeddedClosedCaptionTrackGroupIndex; + /** Only non-empty for track groups representing embedded caption tracks. */ + public final ImmutableList embeddedClosedCaptionTrackOriginalFormats; + public static TrackGroupInfo primaryTrack( int trackType, int[] adaptationSetIndices, @@ -986,7 +1004,8 @@ import java.util.regex.Pattern; primaryTrackGroupIndex, embeddedEventMessageTrackGroupIndex, embeddedClosedCaptionTrackGroupIndex, - /* eventStreamGroupIndex= */ -1); + /* eventStreamGroupIndex= */ -1, + /* embeddedClosedCaptionTrackOriginalFormats= */ ImmutableList.of()); } public static TrackGroupInfo embeddedEmsgTrack( @@ -998,11 +1017,14 @@ import java.util.regex.Pattern; primaryTrackGroupIndex, C.INDEX_UNSET, C.INDEX_UNSET, - /* eventStreamGroupIndex= */ -1); + /* eventStreamGroupIndex= */ -1, + /* embeddedClosedCaptionTrackOriginalFormats= */ ImmutableList.of()); } public static TrackGroupInfo embeddedClosedCaptionTrack( - int[] adaptationSetIndices, int primaryTrackGroupIndex) { + int[] adaptationSetIndices, + int primaryTrackGroupIndex, + ImmutableList originalFormats) { return new TrackGroupInfo( C.TRACK_TYPE_TEXT, CATEGORY_EMBEDDED, @@ -1010,7 +1032,8 @@ import java.util.regex.Pattern; primaryTrackGroupIndex, C.INDEX_UNSET, C.INDEX_UNSET, - /* eventStreamGroupIndex= */ -1); + /* eventStreamGroupIndex= */ -1, + originalFormats); } public static TrackGroupInfo mpdEventTrack(int eventStreamIndex) { @@ -1021,7 +1044,8 @@ import java.util.regex.Pattern; /* primaryTrackGroupIndex= */ -1, C.INDEX_UNSET, C.INDEX_UNSET, - eventStreamIndex); + eventStreamIndex, + /* embeddedClosedCaptionTrackOriginalFormats= */ ImmutableList.of()); } private TrackGroupInfo( @@ -1031,7 +1055,8 @@ import java.util.regex.Pattern; int primaryTrackGroupIndex, int embeddedEventMessageTrackGroupIndex, int embeddedClosedCaptionTrackGroupIndex, - int eventStreamGroupIndex) { + int eventStreamGroupIndex, + ImmutableList embeddedClosedCaptionTrackOriginalFormats) { this.trackType = trackType; this.adaptationSetIndices = adaptationSetIndices; this.trackGroupCategory = trackGroupCategory; @@ -1039,6 +1064,7 @@ import java.util.regex.Pattern; this.embeddedEventMessageTrackGroupIndex = embeddedEventMessageTrackGroupIndex; this.embeddedClosedCaptionTrackGroupIndex = embeddedClosedCaptionTrackGroupIndex; this.eventStreamGroupIndex = eventStreamGroupIndex; + this.embeddedClosedCaptionTrackOriginalFormats = embeddedClosedCaptionTrackOriginalFormats; } } } diff --git a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java index cbdfd3cba0..a6fdeac5fe 100644 --- a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java +++ b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java @@ -167,14 +167,53 @@ public final class DashPlaybackTest { applicationContext, playbackOutput, "playbackdumps/dash/ttml-in-mp4.dump"); } + /** + * This test and {@link #cea608_parseDuringExtraction()} use the same output dump file, to + * demonstrate the flag has no effect on the resulting subtitles. + */ @Test - public void cea608() throws Exception { + public void cea608_parseDuringRendering() throws Exception { Context applicationContext = ApplicationProvider.getApplicationContext(); CapturingRenderersFactory capturingRenderersFactory = new CapturingRenderersFactory(applicationContext); - // TODO(b/181312195): Opt this test into the new subtitle parsing when it works. ExoPlayer player = new ExoPlayer.Builder(applicationContext, capturingRenderersFactory) + .setMediaSourceFactory( + new DashMediaSource.Factory(new DefaultDataSource.Factory(applicationContext)) + .experimentalParseSubtitlesDuringExtraction(false)) + .setClock(new FakeClock(/* isAutoAdvancing= */ true)) + .build(); + player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1))); + PlaybackOutput playbackOutput = PlaybackOutput.register(player, capturingRenderersFactory); + + // Ensure the subtitle track is selected. + DefaultTrackSelector trackSelector = + checkNotNull((DefaultTrackSelector) player.getTrackSelector()); + trackSelector.setParameters(trackSelector.buildUponParameters().setPreferredTextLanguage("en")); + player.setMediaItem(MediaItem.fromUri("asset:///media/dash/cea608/manifest.mpd")); + player.prepare(); + player.play(); + TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED); + player.release(); + + DumpFileAsserts.assertOutput( + applicationContext, playbackOutput, "playbackdumps/dash/cea608.dump"); + } + + /** + * This test and {@link #cea608_parseDuringRendering()} use the same output dump file, to + * demonstrate the flag has no effect on the resulting subtitles. + */ + @Test + public void cea608_parseDuringExtraction() throws Exception { + Context applicationContext = ApplicationProvider.getApplicationContext(); + CapturingRenderersFactory capturingRenderersFactory = + new CapturingRenderersFactory(applicationContext); + ExoPlayer player = + new ExoPlayer.Builder(applicationContext, capturingRenderersFactory) + .setMediaSourceFactory( + new DashMediaSource.Factory(new DefaultDataSource.Factory(applicationContext)) + .experimentalParseSubtitlesDuringExtraction(true)) .setClock(new FakeClock(/* isAutoAdvancing= */ true)) .build(); player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));