From 80bfa819c0309bebb8ade79a8cada7a2c04b018f Mon Sep 17 00:00:00 2001 From: ibaker Date: Wed, 17 Jan 2024 02:49:14 -0800 Subject: [PATCH] Add method to `TextRenderer` to control whether decoding is done or not When we default to 'parse during extraction', we will flip the default of this, to ensure that apps know they are using an incompatible/deprecated flow for subtitle handling. PiperOrigin-RevId: 599109304 --- .../media3/exoplayer/text/TextRenderer.java | 42 +++++++++++++ .../exoplayer/e2etest/WebvttPlaybackTest.java | 62 +++++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/TextRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/TextRenderer.java index 1fb35b81c2..70959aa7c1 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/TextRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/text/TextRenderer.java @@ -125,6 +125,7 @@ public final class TextRenderer extends BaseRenderer implements Callback { private long outputStreamOffsetUs; private long lastRendererPositionUs; private long finalStreamEndPositionUs; + private boolean legacyDecodingEnabled; /** * @param output The output. @@ -163,6 +164,7 @@ public final class TextRenderer extends BaseRenderer implements Callback { finalStreamEndPositionUs = C.TIME_UNSET; outputStreamOffsetUs = C.TIME_UNSET; lastRendererPositionUs = C.TIME_UNSET; + legacyDecodingEnabled = true; } @Override @@ -172,6 +174,10 @@ public final class TextRenderer extends BaseRenderer implements Callback { @Override public @Capabilities int supportsFormat(Format format) { + // TODO: b/289983417 - Return UNSUPPORTED for non-media3-queues once we stop supporting them + // completely. In the meantime, we return SUPPORTED here and then throw later if + // legacyDecodingEnabled is false (when receiving the first Format or sample). This ensures + // apps are aware (via the playback failure) they're using a legacy/deprecated code path. if (isCuesWithTiming(format) || subtitleDecoderFactory.supportsFormat(format)) { return RendererCapabilities.create( format.cryptoType == C.CRYPTO_TYPE_NONE ? C.FORMAT_HANDLED : C.FORMAT_UNSUPPORTED_DRM); @@ -206,6 +212,7 @@ public final class TextRenderer extends BaseRenderer implements Callback { outputStreamOffsetUs = offsetUs; streamFormat = formats[0]; if (!isCuesWithTiming(streamFormat)) { + assertLegacyDecodingEnabledIfRequired(); if (subtitleDecoder != null) { decoderReplacementState = REPLACEMENT_STATE_SIGNAL_END_OF_STREAM; } else { @@ -258,10 +265,30 @@ public final class TextRenderer extends BaseRenderer implements Callback { checkNotNull(cuesResolver); renderFromCuesWithTiming(positionUs); } else { + assertLegacyDecodingEnabledIfRequired(); renderFromSubtitles(positionUs); } } + /** + * Sets whether to decode subtitle data during rendering. + * + *

If this is enabled, then the {@link SubtitleDecoderFactory} passed to the constructor is + * used to decode subtitle data during rendering. + * + *

If this is disabled this text renderer can only handle tracks with MIME type {@link + * MimeTypes#APPLICATION_MEDIA3_CUES} (which have been parsed from their original format during + * extraction), and will throw an exception if passed data of a different type. + * + *

This is enabled by default. + * + *

This method is experimental. It may change behavior, be renamed, or removed in a future + * release. + */ + public void experimentalSetLegacyDecodingEnabled(boolean legacyDecodingEnabled) { + this.legacyDecodingEnabled = legacyDecodingEnabled; + } + @RequiresNonNull("this.cuesResolver") private void renderFromCuesWithTiming(long positionUs) { boolean outputNeedsUpdating = readAndDecodeCuesWithTiming(positionUs); @@ -558,7 +585,22 @@ public final class TextRenderer extends BaseRenderer implements Callback { return positionUs - outputStreamOffsetUs; } + @RequiresNonNull("streamFormat") + private void assertLegacyDecodingEnabledIfRequired() { + checkState( + legacyDecodingEnabled + || Objects.equals(streamFormat.sampleMimeType, MimeTypes.APPLICATION_CEA608) + || Objects.equals(streamFormat.sampleMimeType, MimeTypes.APPLICATION_MP4CEA608) + || Objects.equals(streamFormat.sampleMimeType, MimeTypes.APPLICATION_CEA708), + "Legacy decoding is disabled, can't handle " + + streamFormat.sampleMimeType + + " samples (expected " + + MimeTypes.APPLICATION_MEDIA3_CUES + + ")."); + } + /** Returns whether {@link Format#sampleMimeType} is {@link MimeTypes#APPLICATION_MEDIA3_CUES}. */ + @SideEffectFree private static boolean isCuesWithTiming(Format format) { return Objects.equals(format.sampleMimeType, MimeTypes.APPLICATION_MEDIA3_CUES); } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/WebvttPlaybackTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/WebvttPlaybackTest.java index 8a53df893e..cc422e55f1 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/WebvttPlaybackTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/WebvttPlaybackTest.java @@ -15,17 +15,26 @@ */ package androidx.media3.exoplayer.e2etest; +import static com.google.common.truth.Truth.assertThat; + import android.content.Context; import android.graphics.SurfaceTexture; import android.net.Uri; +import android.os.Looper; import android.view.Surface; import androidx.media3.common.C; import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; import androidx.media3.common.Player; +import androidx.media3.exoplayer.DefaultRenderersFactory; +import androidx.media3.exoplayer.ExoPlaybackException; import androidx.media3.exoplayer.ExoPlayer; +import androidx.media3.exoplayer.Renderer; +import androidx.media3.exoplayer.RenderersFactory; import androidx.media3.exoplayer.source.DefaultMediaSourceFactory; import androidx.media3.exoplayer.source.MediaSource; +import androidx.media3.exoplayer.text.TextOutput; +import androidx.media3.exoplayer.text.TextRenderer; import androidx.media3.test.utils.CapturingRenderersFactory; import androidx.media3.test.utils.DumpFileAsserts; import androidx.media3.test.utils.FakeClock; @@ -34,6 +43,8 @@ import androidx.media3.test.utils.robolectric.ShadowMediaCodecConfig; import androidx.media3.test.utils.robolectric.TestPlayerRunHelper; import androidx.test.core.app.ApplicationProvider; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import java.util.ArrayList; import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; @@ -92,4 +103,55 @@ public class WebvttPlaybackTest { DumpFileAsserts.assertOutput( applicationContext, playbackOutput, "playbackdumps/webvtt/" + inputFile + ".dump"); } + + @Test + public void textRendererDoesntSupportLegacyDecoding_playbackFails() throws Exception { + Context applicationContext = ApplicationProvider.getApplicationContext(); + RenderersFactory renderersFactory = + new DefaultRenderersFactory(applicationContext) { + @Override + protected void buildTextRenderers( + Context context, + TextOutput output, + Looper outputLooper, + @ExtensionRendererMode int extensionRendererMode, + ArrayList out) { + super.buildTextRenderers(context, output, outputLooper, extensionRendererMode, out); + ((TextRenderer) Iterables.getLast(out)).experimentalSetLegacyDecodingEnabled(false); + } + }; + MediaSource.Factory mediaSourceFactory = + new DefaultMediaSourceFactory(applicationContext) + .experimentalParseSubtitlesDuringExtraction(false); + ExoPlayer player = + new ExoPlayer.Builder(applicationContext, renderersFactory) + .setClock(new FakeClock(/* isAutoAdvancing= */ true)) + .setMediaSourceFactory(mediaSourceFactory) + .build(); + Surface surface = new Surface(new SurfaceTexture(/* texName= */ 1)); + player.setVideoSurface(surface); + MediaItem mediaItem = + new MediaItem.Builder() + .setUri("asset:///media/mp4/preroll-5s.mp4") + .setSubtitleConfigurations( + ImmutableList.of( + new MediaItem.SubtitleConfiguration.Builder( + Uri.parse("asset:///media/webvtt/" + inputFile)) + .setMimeType(MimeTypes.TEXT_VTT) + .setLanguage("en") + .setSelectionFlags(C.SELECTION_FLAG_DEFAULT) + .build())) + .build(); + + player.setMediaItem(mediaItem); + player.prepare(); + player.play(); + ExoPlaybackException playbackException = TestPlayerRunHelper.runUntilError(player); + assertThat(playbackException) + .hasCauseThat() + .hasMessageThat() + .contains("Legacy decoding is disabled"); + player.release(); + surface.release(); + } }