diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java index 55f78781f1..20f95192db 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java @@ -431,6 +431,14 @@ public interface AudioSink { @RequiresApi(23) default void setPreferredDevice(@Nullable AudioDeviceInfo audioDeviceInfo) {} + /** + * Sets the offset that is added to the media timestamp before it is passed as {@code + * presentationTimeUs} in {@link #handleBuffer(ByteBuffer, long, int)}. + * + * @param outputStreamOffsetUs The output stream offset in microseconds. + */ + default void setOutputStreamOffsetUs(long outputStreamOffsetUs) {} + /** * Enables tunneling, if possible. The sink is reset if tunneling was previously disabled. * Enabling tunneling is only possible if the sink is based on a platform {@link AudioTrack}, and diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java index 0e56525f00..0dc2dcacd7 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java @@ -122,6 +122,11 @@ public abstract class DecoderAudioRenderer< * end of stream signal to indicate that it has output any remaining buffers before we release it. */ private static final int REINITIALIZATION_STATE_WAIT_END_OF_STREAM = 2; + /** + * Generally there is zero or one pending output stream offset. We track more offsets to allow for + * pending output streams that have fewer frames than the codec latency. + */ + private static final int MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT = 10; private final EventDispatcher eventDispatcher; private final AudioSink audioSink; @@ -151,6 +156,9 @@ public abstract class DecoderAudioRenderer< private boolean allowPositionDiscontinuity; private boolean inputStreamEnded; private boolean outputStreamEnded; + private long outputStreamOffsetUs; + private final long[] pendingOutputStreamOffsetsUs; + private int pendingOutputStreamOffsetCount; public DecoderAudioRenderer() { this(/* eventHandler= */ null, /* eventListener= */ null); @@ -210,6 +218,8 @@ public abstract class DecoderAudioRenderer< flagsOnlyBuffer = DecoderInputBuffer.newNoDataInstance(); decoderReinitializationState = REINITIALIZATION_STATE_NONE; audioTrackNeedsConfigure = true; + setOutputStreamOffsetUs(C.TIME_UNSET); + pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT]; } /** @@ -394,7 +404,7 @@ public abstract class DecoderAudioRenderer< audioSink.handleDiscontinuity(); } if (outputBuffer.isFirstSample()) { - audioSink.handleDiscontinuity(); + processFirstSampleOfStream(); } } @@ -440,6 +450,27 @@ public abstract class DecoderAudioRenderer< return false; } + private void processFirstSampleOfStream() { + audioSink.handleDiscontinuity(); + if (pendingOutputStreamOffsetCount != 0) { + setOutputStreamOffsetUs(pendingOutputStreamOffsetsUs[0]); + pendingOutputStreamOffsetCount--; + System.arraycopy( + pendingOutputStreamOffsetsUs, + /* srcPos= */ 1, + pendingOutputStreamOffsetsUs, + /* destPos= */ 0, + pendingOutputStreamOffsetCount); + } + } + + private void setOutputStreamOffsetUs(long outputStreamOffsetUs) { + this.outputStreamOffsetUs = outputStreamOffsetUs; + if (outputStreamOffsetUs != C.TIME_UNSET) { + audioSink.setOutputStreamOffsetUs(outputStreamOffsetUs); + } + } + private boolean feedInputBuffer() throws DecoderException, ExoPlaybackException { if (decoder == null || decoderReinitializationState == REINITIALIZATION_STATE_WAIT_END_OF_STREAM @@ -589,6 +620,7 @@ public abstract class DecoderAudioRenderer< protected void onDisabled() { inputFormat = null; audioTrackNeedsConfigure = true; + setOutputStreamOffsetUs(C.TIME_UNSET); try { setSourceDrmSession(null); releaseDecoder(); @@ -603,6 +635,19 @@ public abstract class DecoderAudioRenderer< throws ExoPlaybackException { super.onStreamChanged(formats, startPositionUs, offsetUs); firstStreamSampleRead = false; + if (outputStreamOffsetUs == C.TIME_UNSET) { + setOutputStreamOffsetUs(offsetUs); + } else { + if (pendingOutputStreamOffsetCount == pendingOutputStreamOffsetsUs.length) { + Log.w( + TAG, + "Too many stream changes, so dropping offset: " + + pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1]); + } else { + pendingOutputStreamOffsetCount++; + } + pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1] = offsetUs; + } } @Override diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java index c0c66d3df1..ffd42b41b6 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java @@ -146,6 +146,11 @@ public class ForwardingAudioSink implements AudioSink { sink.setPreferredDevice(audioDeviceInfo); } + @Override + public void setOutputStreamOffsetUs(long outputStreamOffsetUs) { + sink.setOutputStreamOffsetUs(outputStreamOffsetUs); + } + @Override public void enableTunnelingV21() { sink.enableTunnelingV21(); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java index 9e32d6357e..5999adb732 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java @@ -737,6 +737,11 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media } } + @Override + protected void onOutputStreamOffsetUsChanged(long outputStreamOffsetUs) { + audioSink.setOutputStreamOffsetUs(outputStreamOffsetUs); + } + @Override public void handleMessage(@MessageType int messageType, @Nullable Object message) throws ExoPlaybackException { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index aac827d3a0..b6e651419c 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -402,7 +402,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT]; pendingOutputStreamSwitchTimesUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT]; outputStreamStartPositionUs = C.TIME_UNSET; - outputStreamOffsetUs = C.TIME_UNSET; + setOutputStreamOffsetUs(C.TIME_UNSET); // MediaCodec outputs audio buffers in native endian: // https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers // and code called from MediaCodecAudioRenderer.processOutputBuffer expects this endianness. @@ -651,7 +651,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { if (this.outputStreamOffsetUs == C.TIME_UNSET) { checkState(this.outputStreamStartPositionUs == C.TIME_UNSET); this.outputStreamStartPositionUs = startPositionUs; - this.outputStreamOffsetUs = offsetUs; + setOutputStreamOffsetUs(offsetUs); } else { if (pendingOutputStreamOffsetCount == pendingOutputStreamOffsetsUs.length) { Log.w( @@ -688,7 +688,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { } formatQueue.clear(); if (pendingOutputStreamOffsetCount != 0) { - outputStreamOffsetUs = pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1]; + setOutputStreamOffsetUs(pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1]); outputStreamStartPositionUs = pendingOutputStreamStartPositionsUs[pendingOutputStreamOffsetCount - 1]; pendingOutputStreamOffsetCount = 0; @@ -707,7 +707,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { protected void onDisabled() { inputFormat = null; outputStreamStartPositionUs = C.TIME_UNSET; - outputStreamOffsetUs = C.TIME_UNSET; + setOutputStreamOffsetUs(C.TIME_UNSET); pendingOutputStreamOffsetCount = 0; flushOrReleaseCodec(); } @@ -1588,7 +1588,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { while (pendingOutputStreamOffsetCount != 0 && presentationTimeUs >= pendingOutputStreamSwitchTimesUs[0]) { outputStreamStartPositionUs = pendingOutputStreamStartPositionsUs[0]; - outputStreamOffsetUs = pendingOutputStreamOffsetsUs[0]; + setOutputStreamOffsetUs(pendingOutputStreamOffsetsUs[0]); pendingOutputStreamOffsetCount--; System.arraycopy( pendingOutputStreamStartPositionsUs, @@ -1638,6 +1638,17 @@ public abstract class MediaCodecRenderer extends BaseRenderer { DISCARD_REASON_REUSE_NOT_IMPLEMENTED); } + /** + * Called after the output stream offset changes. + * + *

The default implementation is a no-op. + * + * @param outputStreamOffsetUs The output stream offset in microseconds. + */ + protected void onOutputStreamOffsetUsChanged(long outputStreamOffsetUs) { + // Do nothing + } + @Override public boolean isEnded() { return outputStreamEnded; @@ -2046,6 +2057,13 @@ public abstract class MediaCodecRenderer extends BaseRenderer { return outputStreamOffsetUs; } + private void setOutputStreamOffsetUs(long outputStreamOffsetUs) { + this.outputStreamOffsetUs = outputStreamOffsetUs; + if (outputStreamOffsetUs != C.TIME_UNSET) { + onOutputStreamOffsetUsChanged(outputStreamOffsetUs); + } + } + /** Returns whether this renderer supports the given {@link Format Format's} DRM scheme. */ protected static boolean supportsFormatDrm(Format format) { return format.cryptoType == C.CRYPTO_TYPE_NONE || format.cryptoType == C.CRYPTO_TYPE_FRAMEWORK; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java index 36d3d7d603..0c9e4517e3 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java @@ -146,7 +146,8 @@ public class DecoderAudioRendererTest { } @Test - public void firstSampleOfStreamSignalsDiscontinuityToAudioSink() throws Exception { + public void firstSampleOfStreamSignalsDiscontinuityAndSetOutputStreamOffsetToAudioSink() + throws Exception { when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); when(mockAudioSink.isEnded()).thenReturn(true); InOrder inOrderAudioSink = inOrder(mockAudioSink); @@ -177,12 +178,15 @@ public class DecoderAudioRendererTest { audioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); } + inOrderAudioSink.verify(mockAudioSink, times(1)).setOutputStreamOffsetUs(0); inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity(); inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt()); } @Test - public void firstSampleOfReplacementStreamSignalsDiscontinuityToAudioSink() throws Exception { + public void + firstSampleOfReplacementStreamSignalsDiscontinuityAndSetOutputStreamOffsetToAudioSink() + throws Exception { when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); when(mockAudioSink.isEnded()).thenReturn(true); InOrder inOrderAudioSink = inOrder(mockAudioSink); @@ -233,9 +237,11 @@ public class DecoderAudioRendererTest { audioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); } + inOrderAudioSink.verify(mockAudioSink, times(1)).setOutputStreamOffsetUs(0); inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity(); inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt()); inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity(); + inOrderAudioSink.verify(mockAudioSink, times(1)).setOutputStreamOffsetUs(1_000_000); inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt()); } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/MediaCodecAudioRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/MediaCodecAudioRendererTest.java index ef10256391..f7cd34f10c 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/MediaCodecAudioRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/MediaCodecAudioRendererTest.java @@ -24,6 +24,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.robolectric.Shadows.shadowOf; @@ -57,6 +58,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; @@ -324,6 +326,61 @@ public class MediaCodecAudioRendererTest { verify(audioRendererEventListener).onAudioSinkError(error); } + @Test + public void render_callsAudioSinkSetOutputStreamOffset_whenReplaceStream() throws Exception { + FakeSampleStream fakeSampleStream1 = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + AUDIO_AAC, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 1_000), + END_OF_STREAM_ITEM)); + fakeSampleStream1.writeData(/* startPositionUs= */ 0); + FakeSampleStream fakeSampleStream2 = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + AUDIO_AAC, + ImmutableList.of( + oneByteSample(/* timeUs= */ 1_000_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 1_001_000), + END_OF_STREAM_ITEM)); + fakeSampleStream2.writeData(/* startPositionUs= */ 0); + mediaCodecAudioRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {AUDIO_AAC}, + fakeSampleStream1, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + + mediaCodecAudioRenderer.start(); + while (!mediaCodecAudioRenderer.hasReadStreamToEnd()) { + mediaCodecAudioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + } + mediaCodecAudioRenderer.replaceStream( + new Format[] {AUDIO_AAC}, + fakeSampleStream2, + /* startPositionUs= */ 1_000_000, + /* offsetUs= */ 1_000_000); + mediaCodecAudioRenderer.setCurrentStreamFinal(); + while (!mediaCodecAudioRenderer.isEnded()) { + mediaCodecAudioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + } + + InOrder inOrderAudioSink = inOrder(audioSink); + inOrderAudioSink.verify(audioSink).setOutputStreamOffsetUs(0); + inOrderAudioSink.verify(audioSink).setOutputStreamOffsetUs(1_000_000); + } + @Test public void supportsFormat_withEac3JocMediaAndEac3Decoder_returnsTrue() throws Exception { Format mediaFormat =