From 57c82710ea3088a80e3067876cb044085d16c130 Mon Sep 17 00:00:00 2001 From: michaelkatz Date: Tue, 11 Mar 2025 08:15:00 -0700 Subject: [PATCH] Limit dynamic scheduling interval by the audio track buffer size In certain bluetooth playback scenarios, it was found that the delta of audio duration written by the AudioSink from the current playback position was greater than the size of the audio track buffer, causing underruns. The solution is to utilize the audio track buffer size as an upper limit for an audio renderer's getDurationToProgress. PiperOrigin-RevId: 735761604 (cherry picked from commit 2729dbb8a9c9ea29d49efd5ffc30e734b7c86048) --- .../media3/exoplayer/audio/AudioSink.java | 9 + .../exoplayer/audio/DecoderAudioRenderer.java | 16 +- .../exoplayer/audio/DefaultAudioSink.java | 30 ++ .../exoplayer/audio/ForwardingAudioSink.java | 5 + .../audio/MediaCodecAudioRenderer.java | 34 ++- .../audio/DecoderAudioRendererTest.java | 199 +++++++++++++- .../audio/MediaCodecAudioRendererTest.java | 258 +++++++++++++++++- .../transformer/AudioGraphInputAudioSink.java | 5 + 8 files changed, 523 insertions(+), 33 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java index e3612d6a22..fb51f95ca8 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java @@ -591,6 +591,15 @@ public interface AudioSink { */ default void setOutputStreamOffsetUs(long outputStreamOffsetUs) {} + /** + * Returns the size of the underlying {@link AudioTrack} buffer in microseconds. If unsupported or + * the {@link AudioTrack} is not initialized then return {@link C#TIME_UNSET}; + * + *

If the {@link AudioTrack} is configured with a compressed encoding, then the returned + * duration is an estimated minimum based on the encoding's maximum encoded byte rate. + */ + long getAudioTrackBufferSizeUs(); + /** * Enables tunneling, if possible. The sink is reset if tunneling was previously disabled. * Enabling tunneling is only possible if the sink is based on a platform {@link AudioTrack}, and diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java index 639d8c8232..479adb001f 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java @@ -22,6 +22,7 @@ import static androidx.media3.exoplayer.DecoderReuseEvaluation.REUSE_RESULT_NO; import static androidx.media3.exoplayer.source.SampleStream.FLAG_REQUIRE_FORMAT; import static com.google.common.base.MoreObjects.firstNonNull; import static java.lang.Math.max; +import static java.lang.Math.min; import static java.lang.annotation.ElementType.TYPE_USE; import android.media.AudioDeviceInfo; @@ -246,16 +247,23 @@ public abstract class DecoderAudioRenderer< if (nextBufferToWritePresentationTimeUs == C.TIME_UNSET) { return super.getDurationToProgressUs(positionUs, elapsedRealtimeUs); } - long durationUs = + // Compare written, yet-to-play content duration against the audio track buffer size. + long writtenDurationUs = (nextBufferToWritePresentationTimeUs - positionUs); + long audioTrackBufferDurationUs = audioSink.getAudioTrackBufferSizeUs(); + long bufferedDurationUs = + audioTrackBufferDurationUs != C.TIME_UNSET + ? min(audioTrackBufferDurationUs, writtenDurationUs) + : writtenDurationUs; + bufferedDurationUs = (long) - ((nextBufferToWritePresentationTimeUs - positionUs) + (bufferedDurationUs / (getPlaybackParameters() != null ? getPlaybackParameters().speed : 1.0f) / 2); if (isStarted) { // Account for the elapsed time since the start of this iteration of the rendering loop. - durationUs -= Util.msToUs(getClock().elapsedRealtime()) - elapsedRealtimeUs; + bufferedDurationUs -= Util.msToUs(getClock().elapsedRealtime()) - elapsedRealtimeUs; } - return max(DEFAULT_DURATION_TO_PROGRESS_US, durationUs); + return max(DEFAULT_DURATION_TO_PROGRESS_US, bufferedDurationUs); } @Override diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java index 61879db345..557fa1d417 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java @@ -71,6 +71,7 @@ import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import java.math.RoundingMode; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayDeque; @@ -1454,6 +1455,23 @@ public final class DefaultAudioSink implements AudioSink { } } + @Override + public long getAudioTrackBufferSizeUs() { + if (!isAudioTrackInitialized()) { + return C.TIME_UNSET; + } + if (Util.SDK_INT >= 23) { + return Api23.getAudioTrackBufferSizeUs(audioTrack, configuration); + } + long byteRate = + configuration.outputMode == OUTPUT_MODE_PCM + ? (long) configuration.outputSampleRate * configuration.outputPcmFrameSize + : DefaultAudioTrackBufferSizeProvider.getMaximumEncodedRateBytesPerSecond( + configuration.outputEncoding); + return Util.scaleLargeValue( + configuration.bufferSize, C.MICROS_PER_SECOND, byteRate, RoundingMode.DOWN); + } + @Override public void enableTunnelingV21() { Assertions.checkState(externalAudioSessionIdProvided); @@ -2365,6 +2383,18 @@ public final class DefaultAudioSink implements AudioSink { audioTrack.setPreferredDevice( audioDeviceInfo == null ? null : audioDeviceInfo.audioDeviceInfo); } + + public static long getAudioTrackBufferSizeUs( + AudioTrack audioTrack, Configuration configuration) { + return configuration.outputMode == OUTPUT_MODE_PCM + ? configuration.framesToDurationUs(audioTrack.getBufferSizeInFrames()) + : Util.scaleLargeValue( + audioTrack.getBufferSizeInFrames(), + C.MICROS_PER_SECOND, + DefaultAudioTrackBufferSizeProvider.getMaximumEncodedRateBytesPerSecond( + configuration.outputEncoding), + RoundingMode.DOWN); + } } @RequiresApi(31) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java index 4a9dcdacf1..f794ba9e9b 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java @@ -162,6 +162,11 @@ public class ForwardingAudioSink implements AudioSink { sink.setOutputStreamOffsetUs(outputStreamOffsetUs); } + @Override + public long getAudioTrackBufferSizeUs() { + return sink.getAudioTrackBufferSizeUs(); + } + @Override public void enableTunnelingV21() { sink.enableTunnelingV21(); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java index 65c56a197d..e83cf4a377 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java @@ -20,6 +20,7 @@ import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_MA import static androidx.media3.exoplayer.DecoderReuseEvaluation.REUSE_RESULT_NO; import static com.google.common.base.MoreObjects.firstNonNull; import static java.lang.Math.max; +import static java.lang.Math.min; import android.annotation.SuppressLint; import android.content.Context; @@ -518,20 +519,27 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media @Override protected long getDurationToProgressUs( long positionUs, long elapsedRealtimeUs, boolean isOnBufferAvailableListenerRegistered) { - if (nextBufferToWritePresentationTimeUs != C.TIME_UNSET) { - long durationUs = - (long) - ((nextBufferToWritePresentationTimeUs - positionUs) - / (getPlaybackParameters() != null ? getPlaybackParameters().speed : 1.0f) - / 2); - if (isStarted) { - // Account for the elapsed time since the start of this iteration of the rendering loop. - durationUs -= Util.msToUs(getClock().elapsedRealtime()) - elapsedRealtimeUs; - } - return max(DEFAULT_DURATION_TO_PROGRESS_US, durationUs); + if (nextBufferToWritePresentationTimeUs == C.TIME_UNSET) { + return super.getDurationToProgressUs( + positionUs, elapsedRealtimeUs, isOnBufferAvailableListenerRegistered); } - return super.getDurationToProgressUs( - positionUs, elapsedRealtimeUs, isOnBufferAvailableListenerRegistered); + // Compare written, yet-to-play content duration against the audio track buffer size. + long writtenDurationUs = (nextBufferToWritePresentationTimeUs - positionUs); + long audioTrackBufferDurationUs = audioSink.getAudioTrackBufferSizeUs(); + long bufferedDurationUs = + audioTrackBufferDurationUs != C.TIME_UNSET + ? min(audioTrackBufferDurationUs, writtenDurationUs) + : writtenDurationUs; + bufferedDurationUs = + (long) + (bufferedDurationUs + / (getPlaybackParameters() != null ? getPlaybackParameters().speed : 1.0f) + / 2); + if (isStarted) { + // Account for the elapsed time since the start of this iteration of the rendering loop. + bufferedDurationUs -= Util.msToUs(getClock().elapsedRealtime()) - elapsedRealtimeUs; + } + return max(DEFAULT_DURATION_TO_PROGRESS_US, bufferedDurationUs); } @Override diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java index e6c6004fe2..14dd6e2500 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java @@ -55,6 +55,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -226,10 +227,11 @@ public class DecoderAudioRendererTest { } @Test - public void getDurationToProgressUs_withAudioSinkBuffersFull_returnsCalculatedDuration() + public void getDurationToProgressUs_usingWrittenDurationUs_returnsCalculatedDuration() throws Exception { when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); when(mockAudioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT); + when(mockAudioSink.getAudioTrackBufferSizeUs()).thenReturn(C.TIME_UNSET); CountDownLatch latchDecode = new CountDownLatch(4); ForwardingAudioSinkWithCountdownLatch countdownLatchAudioSink = new ForwardingAudioSinkWithCountdownLatch(mockAudioSink, latchDecode); @@ -280,10 +282,11 @@ public class DecoderAudioRendererTest { @Test public void - getDurationToProgressUs_withAudioSinkBuffersFullAndDoublePlaybackSpeed_returnsCalculatedDuration() + getDurationToProgressUs_usingWrittenDurationUsWithDoublePlaybackSpeed_returnsCalculatedDuration() throws Exception { when(mockAudioSink.isEnded()).thenReturn(true); when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); + when(mockAudioSink.getAudioTrackBufferSizeUs()).thenReturn(C.TIME_UNSET); PlaybackParameters playbackParametersWithDoubleSpeed = new PlaybackParameters(/* speed= */ 2.0f); when(mockAudioSink.getPlaybackParameters()).thenReturn(playbackParametersWithDoubleSpeed); @@ -337,11 +340,12 @@ public class DecoderAudioRendererTest { @Test public void - getDurationToProgressUs_withAudioSinkBuffersFullAndPlaybackAdvancement_returnsCalculatedDuration() + getDurationToProgressUs_usingWrittenDurationUsWithPlaybackAdvancement_returnsCalculatedDuration() throws Exception { when(mockAudioSink.isEnded()).thenReturn(true); when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); when(mockAudioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT); + when(mockAudioSink.getAudioTrackBufferSizeUs()).thenReturn(C.TIME_UNSET); FakeClock fakeClock = new FakeClock(/* initialTimeMs= */ 100, /* isAutoAdvancing= */ true); CountDownLatch latchDecode = new CountDownLatch(4); ForwardingAudioSinkWithCountdownLatch countdownLatchAudioSink = @@ -394,14 +398,71 @@ public class DecoderAudioRendererTest { assertThat(durationToProgressUs).isEqualTo(65_000L); } + @Test + public void getDurationToProgressUs_usingAudioTrackBufferDurationUs_returnsCalculatedDuration() + throws Exception { + when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); + when(mockAudioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT); + when(mockAudioSink.getAudioTrackBufferSizeUs()).thenReturn(100_000L); + CountDownLatch latchDecode = new CountDownLatch(4); + ForwardingAudioSinkWithCountdownLatch countdownLatchAudioSink = + new ForwardingAudioSinkWithCountdownLatch(mockAudioSink, latchDecode); + audioRenderer = createAudioRenderer(countdownLatchAudioSink); + audioRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ FORMAT, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 50_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 100_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 150_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 200_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 250_000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + audioRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {FORMAT}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + // Represents audio sink buffers being full when trying to write 150000 us sample. + when(mockAudioSink.handleBuffer( + any(), longThat(presentationTimeUs -> presentationTimeUs == 150000), anyInt())) + .thenReturn(false); + audioRenderer.start(); + while (latchDecode.getCount() != 0) { + audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + } + audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + + long durationToProgressUs = + audioRenderer.getDurationToProgressUs( + /* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + + assertThat(durationToProgressUs).isEqualTo(50_000L); + } + @Test public void - getDurationToProgressUs_afterReadToEndOfStreamWithAudioSinkBuffersFull_returnsCalculatedDuration() + getDurationToProgressUs_usingAudioTrackBufferDurationUsAndDoublePlaybackSpeed_returnsCalculatedDuration() throws Exception { when(mockAudioSink.isEnded()).thenReturn(true); when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); - when(mockAudioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT); - CountDownLatch latchDecode = new CountDownLatch(6); + when(mockAudioSink.getAudioTrackBufferSizeUs()).thenReturn(100_000L); + PlaybackParameters playbackParametersWithDoubleSpeed = + new PlaybackParameters(/* speed= */ 2.0f); + when(mockAudioSink.getPlaybackParameters()).thenReturn(playbackParametersWithDoubleSpeed); + CountDownLatch latchDecode = new CountDownLatch(4); ForwardingAudioSinkWithCountdownLatch countdownLatchAudioSink = new ForwardingAudioSinkWithCountdownLatch(mockAudioSink, latchDecode); audioRenderer = createAudioRenderer(countdownLatchAudioSink); @@ -421,9 +482,9 @@ public class DecoderAudioRendererTest { oneByteSample(/* timeUs= */ 200000, C.BUFFER_FLAG_KEY_FRAME), oneByteSample(/* timeUs= */ 250000, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM)); - // Mock that audio sink is full when trying to write final sample. + // Represents audio sink buffers being full when trying to write 150000 us sample. when(mockAudioSink.handleBuffer( - any(), longThat(presentationTimeUs -> presentationTimeUs == 250000), anyInt())) + any(), longThat(presentationTimeUs -> presentationTimeUs == 150000), anyInt())) .thenReturn(false); fakeSampleStream.writeData(/* startPositionUs= */ 0); audioRenderer.enable( @@ -436,17 +497,135 @@ public class DecoderAudioRendererTest { /* startPositionUs= */ 0, /* offsetUs= */ 0, new MediaSource.MediaPeriodId(new Object())); - // Represents audio sink buffers being full when trying to write 150000 us sample. audioRenderer.start(); while (latchDecode.getCount() != 0) { audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); } + audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); long durationToProgressUs = audioRenderer.getDurationToProgressUs( /* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); - assertThat(durationToProgressUs).isEqualTo(125_000L); + assertThat(durationToProgressUs).isEqualTo(25_000L); + } + + @Test + public void + getDurationToProgressUs_usingAudioTrackBufferDurationUsAndPlaybackAdvancement_returnsCalculatedDuration() + throws Exception { + when(mockAudioSink.isEnded()).thenReturn(true); + when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); + when(mockAudioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT); + when(mockAudioSink.getAudioTrackBufferSizeUs()).thenReturn(100_000L); + FakeClock fakeClock = new FakeClock(/* initialTimeMs= */ 100, /* isAutoAdvancing= */ true); + CountDownLatch latchDecode = new CountDownLatch(4); + ForwardingAudioSinkWithCountdownLatch countdownLatchAudioSink = + new ForwardingAudioSinkWithCountdownLatch(mockAudioSink, latchDecode); + audioRenderer = createAudioRenderer(countdownLatchAudioSink); + audioRenderer.init(/* index= */ 0, PlayerId.UNSET, fakeClock); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ FORMAT, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 50000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 100000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 150000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 200000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 250000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + // Represents audio sink buffers being full when trying to write 150000 us sample. + when(mockAudioSink.handleBuffer( + any(), longThat(presentationTimeUs -> presentationTimeUs == 150000), anyInt())) + .thenReturn(false); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + audioRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {FORMAT}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + audioRenderer.start(); + long rendererPositionElapsedRealtimeUs = SystemClock.elapsedRealtime() * 1000; + while (latchDecode.getCount() != 0) { + audioRenderer.render(/* positionUs= */ 0, rendererPositionElapsedRealtimeUs); + } + audioRenderer.render(/* positionUs= */ 0, rendererPositionElapsedRealtimeUs); + + // Simulate playback progressing between render() and getDurationToProgressUs call + fakeClock.advanceTime(/* timeDiffMs= */ 10); + long durationToProgressUs = + audioRenderer.getDurationToProgressUs( + /* positionUs= */ 0, rendererPositionElapsedRealtimeUs); + + assertThat(durationToProgressUs).isEqualTo(40_000L); + } + + @Test + public void getDurationToProgressUs_afterReadToEndOfStream_returnsCalculatedDuration() + throws Exception { + when(mockAudioSink.isEnded()).thenReturn(true); + when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); + when(mockAudioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT); + when(mockAudioSink.getAudioTrackBufferSizeUs()).thenReturn(100_000L); + AtomicBoolean hasCalledPlayToEndOfStream = new AtomicBoolean(); + ForwardingAudioSink forwardingAudioSink = + new ForwardingAudioSink(mockAudioSink) { + @Override + public void playToEndOfStream() throws WriteException { + super.playToEndOfStream(); + hasCalledPlayToEndOfStream.set(true); + } + }; + audioRenderer = createAudioRenderer(forwardingAudioSink); + audioRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ FORMAT, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 50000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 100000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 150000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 200000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 250000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + audioRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {FORMAT}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + audioRenderer.start(); + audioRenderer.setCurrentStreamFinal(); + while (!hasCalledPlayToEndOfStream.get()) { + audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + } + audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + + long durationToProgressUs = + audioRenderer.getDurationToProgressUs( + /* positionUs= */ 200_000L, SystemClock.elapsedRealtime() * 1000); + + assertThat(durationToProgressUs).isEqualTo(25_000L); } @Test diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/MediaCodecAudioRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/MediaCodecAudioRendererTest.java index d6d819f67f..bcb039aab4 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/MediaCodecAudioRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/MediaCodecAudioRendererTest.java @@ -62,6 +62,7 @@ import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; import java.util.Collections; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -733,7 +734,7 @@ public class MediaCodecAudioRendererTest { } @Test - public void getDurationToProgressUs_withAudioSinkBuffersFull_returnsCalculatedDuration() + public void getDurationToProgressUs_usingWrittenDurationUs_returnsCalculatedDuration() throws Exception { FakeSampleStream fakeSampleStream = new FakeSampleStream( @@ -766,6 +767,178 @@ public class MediaCodecAudioRendererTest { any(), longThat(presentationTimeUs -> presentationTimeUs == 150_000), anyInt())) .thenReturn(false); when(audioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT); + when(audioSink.getAudioTrackBufferSizeUs()).thenReturn(C.TIME_UNSET); + mediaCodecAudioRenderer.start(); + for (int i = 0; i < 10; i++) { + mediaCodecAudioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + maybeIdleAsynchronousMediaCodecAdapterThreads(); + } + + long durationToProgressUs = + mediaCodecAudioRenderer.getDurationToProgressUs( + /* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + + assertThat(durationToProgressUs).isEqualTo(75_000L); + } + + @Test + public void + getDurationToProgressUs_usingWrittenDurationUsWithDoublePlaybackSpeed_returnsCalculatedDuration() + throws Exception { + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ AUDIO_AAC, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 50_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 100_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 150_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 200_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 250_000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + PlaybackParameters playbackParametersWithDoubleSpeed = + new PlaybackParameters(/* speed= */ 2.0f); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecAudioRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {AUDIO_AAC}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ false, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + // Represents audio sink buffers being full when trying to write 150_000 us sample. + when(audioSink.handleBuffer( + any(), longThat(presentationTimeUs -> presentationTimeUs == 150_000), anyInt())) + .thenReturn(false); + when(audioSink.getPlaybackParameters()).thenReturn(playbackParametersWithDoubleSpeed); + when(audioSink.getAudioTrackBufferSizeUs()).thenReturn(C.TIME_UNSET); + mediaCodecAudioRenderer.start(); + for (int i = 0; i < 10; i++) { + mediaCodecAudioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + maybeIdleAsynchronousMediaCodecAdapterThreads(); + } + + long durationToProgressUs = + mediaCodecAudioRenderer.getDurationToProgressUs( + /* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + + assertThat(durationToProgressUs).isEqualTo(37_500L); + } + + @Test + public void + getDurationToProgressUs_usingWrittenDurationUsWithPlaybackAdvancement_returnsCalculatedDuration() + throws Exception { + FakeClock fakeClock = new FakeClock(/* initialTimeMs= */ 100, /* isAutoAdvancing= */ true); + mediaCodecAudioRenderer = + new MediaCodecAudioRenderer( + ApplicationProvider.getApplicationContext(), + new DefaultMediaCodecAdapterFactory( + ApplicationProvider.getApplicationContext(), + () -> { + callbackThread = new HandlerThread("MCARTest:MediaCodecAsyncAdapter"); + return callbackThread; + }, + () -> { + queueingThread = new HandlerThread("MCARTest:MediaCodecQueueingThread"); + return queueingThread; + }), + mediaCodecSelector, + /* enableDecoderFallback= */ false, + /* eventHandler= */ new Handler(Looper.getMainLooper()), + audioRendererEventListener, + audioSink); + mediaCodecAudioRenderer.init(/* index= */ 0, PlayerId.UNSET, fakeClock); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ AUDIO_AAC, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 50_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 100_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 150_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 200_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 250_000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecAudioRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {AUDIO_AAC}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ false, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + // Represents audio sink buffers being full when trying to write 150_000 us sample. + when(audioSink.handleBuffer( + any(), longThat(presentationTimeUs -> presentationTimeUs == 150_000), anyInt())) + .thenReturn(false); + when(audioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT); + when(audioSink.getAudioTrackBufferSizeUs()).thenReturn(C.TIME_UNSET); + mediaCodecAudioRenderer.start(); + for (int i = 0; i < 10; i++) { + mediaCodecAudioRenderer.render(/* positionUs= */ 0, fakeClock.elapsedRealtime() * 1000); + maybeIdleAsynchronousMediaCodecAdapterThreads(); + } + + // Simulate playback progressing between render() and getDurationToProgressUs call + long rendererPositionElapsedRealtimeUs = fakeClock.elapsedRealtime() * 1000; + fakeClock.advanceTime(/* timeDiffMs= */ 10); + long durationToProgressUs = + mediaCodecAudioRenderer.getDurationToProgressUs( + /* positionUs= */ 0, rendererPositionElapsedRealtimeUs); + + assertThat(durationToProgressUs).isEqualTo(65_000L); + } + + @Test + public void getDurationToProgressUs_usingAudioTrackBufferDurationUs_returnsCalculatedDuration() + throws Exception { + when(audioSink.getAudioTrackBufferSizeUs()).thenReturn(100_000L); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ AUDIO_AAC, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 50_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 100_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 150_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 200_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 250_000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecAudioRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {AUDIO_AAC}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ false, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + // Represents audio sink buffers being full when trying to write 150_000 us sample. + when(audioSink.handleBuffer( + any(), longThat(presentationTimeUs -> presentationTimeUs == 150_000), anyInt())) + .thenReturn(false); + when(audioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT); mediaCodecAudioRenderer.start(); for (int i = 0; i < 10; i++) { mediaCodecAudioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); @@ -776,13 +949,14 @@ public class MediaCodecAudioRendererTest { mediaCodecAudioRenderer.getDurationToProgressUs( /* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); - assertThat(durationToProgressUs).isEqualTo(75_000L); + assertThat(durationToProgressUs).isEqualTo(50_000L); } @Test public void - getDurationToProgressUs_withAudioSinkBuffersFullAndDoublePlaybackSpeed_returnsCalculatedDuration() + getDurationToProgressUs_usingAudioTrackBufferDurationUsAndDoublePlaybackSpeed_returnsCalculatedDuration() throws Exception { + when(audioSink.getAudioTrackBufferSizeUs()).thenReturn(100_000L); FakeSampleStream fakeSampleStream = new FakeSampleStream( new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), @@ -826,14 +1000,15 @@ public class MediaCodecAudioRendererTest { mediaCodecAudioRenderer.getDurationToProgressUs( /* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); - assertThat(durationToProgressUs).isEqualTo(37_500L); + assertThat(durationToProgressUs).isEqualTo(25_000L); } @Test public void - getDurationToProgressUs_withAudioSinkBuffersFullAndPlaybackAdvancement_returnsCalculatedDuration() + getDurationToProgressUs_usingAudioTrackBufferDurationUsAndPlaybackAdvancement_returnsCalculatedDuration() throws Exception { FakeClock fakeClock = new FakeClock(/* initialTimeMs= */ 100, /* isAutoAdvancing= */ true); + when(audioSink.getAudioTrackBufferSizeUs()).thenReturn(100_000L); mediaCodecAudioRenderer = new MediaCodecAudioRenderer( ApplicationProvider.getApplicationContext(), @@ -897,7 +1072,78 @@ public class MediaCodecAudioRendererTest { mediaCodecAudioRenderer.getDurationToProgressUs( /* positionUs= */ 0, rendererPositionElapsedRealtimeUs); - assertThat(durationToProgressUs).isEqualTo(65_000L); + assertThat(durationToProgressUs).isEqualTo(40_000L); + } + + @Test + public void getDurationToProgressUs_afterRenderToEndOfStream_returnsCalculatedDuration() + throws Exception { + AtomicBoolean hasCalledRenderToEndOfStream = new AtomicBoolean(); + mediaCodecAudioRenderer = + new MediaCodecAudioRenderer( + ApplicationProvider.getApplicationContext(), + new DefaultMediaCodecAdapterFactory( + ApplicationProvider.getApplicationContext(), + () -> { + callbackThread = new HandlerThread("MCARTest:MediaCodecAsyncAdapter"); + return callbackThread; + }, + () -> { + queueingThread = new HandlerThread("MCARTest:MediaCodecQueueingThread"); + return queueingThread; + }), + mediaCodecSelector, + /* enableDecoderFallback= */ false, + new Handler(Looper.getMainLooper()), + audioRendererEventListener, + audioSink) { + @Override + protected void renderToEndOfStream() throws ExoPlaybackException { + super.renderToEndOfStream(); + hasCalledRenderToEndOfStream.set(true); + } + }; + mediaCodecAudioRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + when(audioSink.getAudioTrackBufferSizeUs()).thenReturn(100_000L); + when(audioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ AUDIO_AAC, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 50_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 100_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 150_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 200_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 250_000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecAudioRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {AUDIO_AAC}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ false, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + mediaCodecAudioRenderer.start(); + mediaCodecAudioRenderer.setCurrentStreamFinal(); + while (!hasCalledRenderToEndOfStream.get()) { + mediaCodecAudioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + maybeIdleAsynchronousMediaCodecAdapterThreads(); + } + + long durationToProgressUs = + mediaCodecAudioRenderer.getDurationToProgressUs( + /* positionUs= */ 200_000L, SystemClock.elapsedRealtime() * 1000); + + assertThat(durationToProgressUs).isEqualTo(25_000L); } @Test diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraphInputAudioSink.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraphInputAudioSink.java index 09eb5246c7..affc2ac8ab 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraphInputAudioSink.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraphInputAudioSink.java @@ -250,6 +250,11 @@ import java.util.Objects; return null; } + @Override + public long getAudioTrackBufferSizeUs() { + return C.TIME_UNSET; + } + @Override public void setPlaybackParameters(PlaybackParameters playbackParameters) {}