diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 87f187075f..e0a6694d68 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -12,6 +12,12 @@ ([#2092](https://github.com/google/ExoPlayer/issues/2092)). * Fix parsing of H265 short term reference picture sets ([#10316](https://github.com/google/ExoPlayer/issues/10316)). +* Metadata: + * `MetadataRenderer` can now be configured to render metadata as soon as + they are available. Create an instance with + `MetadataRenderer(MetadataOutput, Looper, + MetadataDecoderFactory, boolean)` to specify whether the renderer will + output metadata early or in sync with the player position. * RTSP: * Add RTP reader for H263 ([#63](https://github.com/androidx/media/pull/63)). diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/metadata/MetadataRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/metadata/MetadataRenderer.java index 0c7b16f8f3..edaa2d0c5c 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/metadata/MetadataRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/metadata/MetadataRenderer.java @@ -39,7 +39,12 @@ import java.util.ArrayList; import java.util.List; import org.checkerframework.dataflow.qual.SideEffectFree; -/** A renderer for metadata. */ +/** + * A renderer for metadata. + * + *

The renderer can be configured to render metadata as soon as they are available using {@link + * #MetadataRenderer(MetadataOutput, Looper, MetadataDecoderFactory, boolean)}. + */ @UnstableApi public final class MetadataRenderer extends BaseRenderer implements Callback { @@ -50,6 +55,7 @@ public final class MetadataRenderer extends BaseRenderer implements Callback { private final MetadataOutput output; @Nullable private final Handler outputHandler; private final MetadataInputBuffer buffer; + private final boolean outputMetadataEarly; @Nullable private MetadataDecoder decoder; private boolean inputStreamEnded; @@ -59,6 +65,9 @@ public final class MetadataRenderer extends BaseRenderer implements Callback { private long outputStreamOffsetUs; /** + * Creates an instance that uses {@link MetadataDecoderFactory#DEFAULT} to create {@link + * MetadataDecoder} instances. + * * @param output The output. * @param outputLooper The looper associated with the thread on which the output should be called. * If the output makes use of standard Android UI components, then this should normally be the @@ -71,6 +80,8 @@ public final class MetadataRenderer extends BaseRenderer implements Callback { } /** + * Creates an instance. + * * @param output The output. * @param outputLooper The looper associated with the thread on which the output should be called. * If the output makes use of standard Android UI components, then this should normally be the @@ -81,11 +92,34 @@ public final class MetadataRenderer extends BaseRenderer implements Callback { */ public MetadataRenderer( MetadataOutput output, @Nullable Looper outputLooper, MetadataDecoderFactory decoderFactory) { + this(output, outputLooper, decoderFactory, /* outputMetadataEarly= */ false); + } + + /** + * Creates an instance. + * + * @param output The output. + * @param outputLooper The looper associated with the thread on which the output should be called. + * If the output makes use of standard Android UI components, then this should normally be the + * looper associated with the application's main thread, which can be obtained using {@link + * android.app.Activity#getMainLooper()}. Null may be passed if the output should be called + * directly on the player's internal rendering thread. + * @param decoderFactory A factory from which to obtain {@link MetadataDecoder} instances. + * @param outputMetadataEarly Whether the renderer outputs metadata early. When {@code true}, + * {@link #render} will output metadata as soon as they are available to the renderer, + * otherwise {@link #render} will output metadata in sync with the rendering position. + */ + public MetadataRenderer( + MetadataOutput output, + @Nullable Looper outputLooper, + MetadataDecoderFactory decoderFactory, + boolean outputMetadataEarly) { super(C.TRACK_TYPE_METADATA); this.output = Assertions.checkNotNull(output); this.outputHandler = outputLooper == null ? null : Util.createHandler(outputLooper, /* callback= */ this); this.decoderFactory = Assertions.checkNotNull(decoderFactory); + this.outputMetadataEarly = outputMetadataEarly; buffer = new MetadataInputBuffer(); outputStreamOffsetUs = C.TIME_UNSET; } @@ -217,7 +251,8 @@ public final class MetadataRenderer extends BaseRenderer implements Callback { private boolean outputMetadata(long positionUs) { boolean didOutput = false; if (pendingMetadata != null - && pendingMetadata.presentationTimeUs <= getPresentationTimeUs(positionUs)) { + && (outputMetadataEarly + || pendingMetadata.presentationTimeUs <= getPresentationTimeUs(positionUs))) { invokeRenderer(pendingMetadata); pendingMetadata = null; didOutput = true; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/metadata/MetadataRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/metadata/MetadataRendererTest.java index 3e77274924..f4d5b34c6e 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/metadata/MetadataRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/metadata/MetadataRendererTest.java @@ -145,16 +145,87 @@ public class MetadataRendererTest { assertThat(metadata).isEmpty(); } + @Test + public void renderMetadata_withTimelyOutput() throws Exception { + EventMessage emsg = + new EventMessage( + "urn:test-scheme-id", + /* value= */ "", + /* durationMs= */ 1, + /* id= */ 0, + "Test data".getBytes(UTF_8)); + byte[] encodedEmsg = eventMessageEncoder.encode(emsg); + List metadata = new ArrayList<>(); + MetadataRenderer renderer = + new MetadataRenderer(/* output= */ metadata::add, /* outputLooper= */ null); + FakeSampleStream fakeSampleStream = + createFakeSampleStream( + ImmutableList.of( + sample(/* timeUs= */ 100_000, C.BUFFER_FLAG_KEY_FRAME, encodedEmsg), + sample(/* timeUs= */ 1_000_000, C.BUFFER_FLAG_KEY_FRAME, encodedEmsg), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + renderer.replaceStream( + new Format[] {EMSG_FORMAT}, + fakeSampleStream, + /* startPositionUs= */ 0L, + /* offsetUs= */ 0L); + + // Call render() twice, the first call is to read the format and the second call will read the + // metadata. + renderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + renderer.render(/* positionUs= */ 500_000, /* elapsedRealtimeUs= */ 0); + + assertThat(metadata).hasSize(1); + assertThat(metadata.get(0).presentationTimeUs).isEqualTo(100_000); + } + + @Test + public void renderMetadata_withEarlyOutput() throws Exception { + EventMessage emsg = + new EventMessage( + "urn:test-scheme-id", + /* value= */ "", + /* durationMs= */ 1, + /* id= */ 0, + "Test data".getBytes(UTF_8)); + byte[] encodedEmsg = eventMessageEncoder.encode(emsg); + List metadata = new ArrayList<>(); + MetadataRenderer renderer = + new MetadataRenderer( + /* output= */ metadata::add, + /* outputLooper= */ null, + MetadataDecoderFactory.DEFAULT, + /* outputMetadataEarly= */ true); + FakeSampleStream fakeSampleStream = + createFakeSampleStream( + ImmutableList.of( + sample(/* timeUs= */ 100_000, C.BUFFER_FLAG_KEY_FRAME, encodedEmsg), + sample(/* timeUs= */ 1_000_000, C.BUFFER_FLAG_KEY_FRAME, encodedEmsg), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + renderer.replaceStream( + new Format[] {EMSG_FORMAT}, + fakeSampleStream, + /* startPositionUs= */ 0L, + /* offsetUs= */ 0L); + + // Call render() twice, the first call is to read the format and the second call will read the + // metadata. + renderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + renderer.render(/* positionUs= */ 500_000, /* elapsedRealtimeUs= */ 0); + + // The renderer outputs metadata early. + assertThat(metadata).hasSize(2); + assertThat(metadata.get(0).presentationTimeUs).isEqualTo(100_000); + assertThat(metadata.get(1).presentationTimeUs).isEqualTo(1_000_000); + } + private static List runRenderer(byte[] input) throws ExoPlaybackException { List metadata = new ArrayList<>(); MetadataRenderer renderer = new MetadataRenderer(metadata::add, /* outputLooper= */ null); FakeSampleStream fakeSampleStream = - new FakeSampleStream( - new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), - /* mediaSourceEventDispatcher= */ null, - DrmSessionManager.DRM_UNSUPPORTED, - new DrmSessionEventListener.EventDispatcher(), - EMSG_FORMAT, + createFakeSampleStream( ImmutableList.of( sample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME, input), END_OF_STREAM_ITEM)); fakeSampleStream.writeData(/* startPositionUs= */ 0); @@ -169,6 +240,17 @@ public class MetadataRendererTest { return Collections.unmodifiableList(metadata); } + private static FakeSampleStream createFakeSampleStream( + ImmutableList samples) { + return new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + EMSG_FORMAT, + samples); + } + /** * Builds an ID3v2 tag containing a single 'user defined text information frame' (id='TXXX') with * {@code description} and {@code value}. diff --git a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java index 762eefc28f..ce9d2cc779 100644 --- a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java +++ b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/e2etest/DashPlaybackTest.java @@ -23,6 +23,10 @@ import android.view.Surface; import androidx.media3.common.MediaItem; import androidx.media3.common.Player; import androidx.media3.exoplayer.ExoPlayer; +import androidx.media3.exoplayer.Renderer; +import androidx.media3.exoplayer.RenderersFactory; +import androidx.media3.exoplayer.metadata.MetadataDecoderFactory; +import androidx.media3.exoplayer.metadata.MetadataRenderer; import androidx.media3.exoplayer.trackselection.DefaultTrackSelector; import androidx.media3.test.utils.CapturingRenderersFactory; import androidx.media3.test.utils.DumpFileAsserts; @@ -96,4 +100,70 @@ public final class DashPlaybackTest { DumpFileAsserts.assertOutput( applicationContext, playbackOutput, "playbackdumps/dash/emsg.dump"); } + + @Test + public void renderMetadata_withTimelyOutput() throws Exception { + Context applicationContext = ApplicationProvider.getApplicationContext(); + RenderersFactory renderersFactory = + (eventHandler, + videoRendererEventListener, + audioRendererEventListener, + textRendererOutput, + metadataRendererOutput) -> + new Renderer[] {new MetadataRenderer(metadataRendererOutput, eventHandler.getLooper())}; + ExoPlayer player = + new ExoPlayer.Builder(applicationContext, renderersFactory) + .setClock(new FakeClock(/* isAutoAdvancing= */ true)) + .build(); + player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1))); + CapturingRenderersFactory capturingRenderersFactory = + new CapturingRenderersFactory(applicationContext); + PlaybackOutput playbackOutput = PlaybackOutput.register(player, capturingRenderersFactory); + + player.setMediaItem(MediaItem.fromUri("asset:///media/dash/emsg/sample.mpd")); + player.prepare(); + player.play(); + TestPlayerRunHelper.playUntilPosition(player, /* mediaItemIndex= */ 0, /* positionMs= */ 500); + player.release(); + + // Ensure output contains metadata up to the playback position. + DumpFileAsserts.assertOutput( + applicationContext, playbackOutput, "playbackdumps/dash/metadata_from_timely_output.dump"); + } + + @Test + public void renderMetadata_withEarlyOutput() throws Exception { + Context applicationContext = ApplicationProvider.getApplicationContext(); + RenderersFactory renderersFactory = + (eventHandler, + videoRendererEventListener, + audioRendererEventListener, + textRendererOutput, + metadataRendererOutput) -> + new Renderer[] { + new MetadataRenderer( + metadataRendererOutput, + eventHandler.getLooper(), + MetadataDecoderFactory.DEFAULT, + /* outputMetadataEarly= */ true) + }; + ExoPlayer player = + new ExoPlayer.Builder(applicationContext, renderersFactory) + .setClock(new FakeClock(/* isAutoAdvancing= */ true)) + .build(); + player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1))); + CapturingRenderersFactory capturingRenderersFactory = + new CapturingRenderersFactory(applicationContext); + PlaybackOutput playbackOutput = PlaybackOutput.register(player, capturingRenderersFactory); + + player.setMediaItem(MediaItem.fromUri("asset:///media/dash/emsg/sample.mpd")); + player.prepare(); + player.play(); + TestPlayerRunHelper.playUntilPosition(player, /* mediaItemIndex= */ 0, /* positionMs= */ 500); + player.release(); + + // Ensure output contains all metadata irrespective of the playback position. + DumpFileAsserts.assertOutput( + applicationContext, playbackOutput, "playbackdumps/dash/metadata_from_early_output.dump"); + } } diff --git a/libraries/test_data/src/test/assets/playbackdumps/dash/metadata_from_early_output.dump b/libraries/test_data/src/test/assets/playbackdumps/dash/metadata_from_early_output.dump new file mode 100644 index 0000000000..e8261bd2df --- /dev/null +++ b/libraries/test_data/src/test/assets/playbackdumps/dash/metadata_from_early_output.dump @@ -0,0 +1,10 @@ +MetadataOutput: + Metadata[0]: + presentationTimeUs = 100000 + entry[0] = EMSG: scheme=urn:mpeg:dash:event:callback:2015, id=0, durationMs=1000, value=1 + Metadata[1]: + presentationTimeUs = 100000 + entry[0] = EMSG: scheme=urn:mpeg:dash:event:callback:2015, id=1, durationMs=1000, value=1 + Metadata[2]: + presentationTimeUs = 1000000 + entry[0] = EMSG: scheme=urn:mpeg:dash:event:callback:2015, id=2, durationMs=1000, value=1 diff --git a/libraries/test_data/src/test/assets/playbackdumps/dash/metadata_from_timely_output.dump b/libraries/test_data/src/test/assets/playbackdumps/dash/metadata_from_timely_output.dump new file mode 100644 index 0000000000..ebbd67120a --- /dev/null +++ b/libraries/test_data/src/test/assets/playbackdumps/dash/metadata_from_timely_output.dump @@ -0,0 +1,7 @@ +MetadataOutput: + Metadata[0]: + presentationTimeUs = 100000 + entry[0] = EMSG: scheme=urn:mpeg:dash:event:callback:2015, id=0, durationMs=1000, value=1 + Metadata[1]: + presentationTimeUs = 100000 + entry[0] = EMSG: scheme=urn:mpeg:dash:event:callback:2015, id=1, durationMs=1000, value=1