loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-05-06 23:20:42 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Provide access to original media timestamps in AudioSink.

Browse Source 
* Add `setOutputStreamOffsetUs(long)` method in `AudioSink`.
* Add private methods `setOutputStreamOffsetUs(long)` method in `MediaCodecRenderer` and `DecoderAudioRenderer`.
* Add protected method `onOutputStreamOffsetUs(long)` method in `MediaCodecRenderer`, in which:
  * `MediaCodecRenderer` itself will be no-op for this method.
  * `MediaCodecAudioRenderer` will propagate this value to its `audioSink`.
* Add logics in `DecoderAudioRenderer` to calculate `outputStreamOffsetUs`.

PiperOrigin-RevId: 479265429
(cherry picked from commit 5bff8623748a1ce9e411cfa9df171143925da89f)
This commit is contained in:
tianyifeng 2022-10-06 09:58:37 +00:00 committed by microkatz
parent 1cfeddc369
commit 243c29a8ad
7 changed files with 152 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java
View File

@ -431,6 +431,14 @@ public interface AudioSink {
@RequiresApi(23)
default void setPreferredDevice(@Nullable AudioDeviceInfo audioDeviceInfo) {}
/**
* Sets the offset that is added to the media timestamp before it is passed as {@code
* presentationTimeUs} in {@link #handleBuffer(ByteBuffer, long, int)}.
*
* @param outputStreamOffsetUs The output stream offset in microseconds.
*/
default void setOutputStreamOffsetUs(long outputStreamOffsetUs) {}
/**
* Enables tunneling, if possible. The sink is reset if tunneling was previously disabled.
* Enabling tunneling is only possible if the sink is based on a platform {@link AudioTrack}, and

47
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java
View File

@ -122,6 +122,11 @@ public abstract class DecoderAudioRenderer<
* end of stream signal to indicate that it has output any remaining buffers before we release it.
*/
private static final int REINITIALIZATION_STATE_WAIT_END_OF_STREAM = 2;
/**
* Generally there is zero or one pending output stream offset. We track more offsets to allow for
* pending output streams that have fewer frames than the codec latency.
*/
private static final int MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT = 10;
private final EventDispatcher eventDispatcher;
private final AudioSink audioSink;
@ -151,6 +156,9 @@ public abstract class DecoderAudioRenderer<
private boolean allowPositionDiscontinuity;
private boolean inputStreamEnded;
private boolean outputStreamEnded;
private long outputStreamOffsetUs;
private final long[] pendingOutputStreamOffsetsUs;
private int pendingOutputStreamOffsetCount;
public DecoderAudioRenderer() {
this(/* eventHandler= */ null, /* eventListener= */ null);
@ -210,6 +218,8 @@ public abstract class DecoderAudioRenderer<
flagsOnlyBuffer = DecoderInputBuffer.newNoDataInstance();
decoderReinitializationState = REINITIALIZATION_STATE_NONE;
audioTrackNeedsConfigure = true;
setOutputStreamOffsetUs(C.TIME_UNSET);
pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT];
}
/**
@ -394,7 +404,7 @@ public abstract class DecoderAudioRenderer<
audioSink.handleDiscontinuity();
}
if (outputBuffer.isFirstSample()) {
audioSink.handleDiscontinuity();
processFirstSampleOfStream();
}
}
@ -440,6 +450,27 @@ public abstract class DecoderAudioRenderer<
return false;
}
private void processFirstSampleOfStream() {
audioSink.handleDiscontinuity();
if (pendingOutputStreamOffsetCount != 0) {
setOutputStreamOffsetUs(pendingOutputStreamOffsetsUs[0]);
pendingOutputStreamOffsetCount--;
System.arraycopy(
pendingOutputStreamOffsetsUs,
/* srcPos= */ 1,
pendingOutputStreamOffsetsUs,
/* destPos= */ 0,
pendingOutputStreamOffsetCount);
}
}
private void setOutputStreamOffsetUs(long outputStreamOffsetUs) {
this.outputStreamOffsetUs = outputStreamOffsetUs;
if (outputStreamOffsetUs != C.TIME_UNSET) {
audioSink.setOutputStreamOffsetUs(outputStreamOffsetUs);
}
}
private boolean feedInputBuffer() throws DecoderException, ExoPlaybackException {
if (decoder == null
|| decoderReinitializationState == REINITIALIZATION_STATE_WAIT_END_OF_STREAM
@ -589,6 +620,7 @@ public abstract class DecoderAudioRenderer<
protected void onDisabled() {
inputFormat = null;
audioTrackNeedsConfigure = true;
setOutputStreamOffsetUs(C.TIME_UNSET);
try {
setSourceDrmSession(null);
releaseDecoder();
@ -603,6 +635,19 @@ public abstract class DecoderAudioRenderer<
throws ExoPlaybackException {
super.onStreamChanged(formats, startPositionUs, offsetUs);
firstStreamSampleRead = false;
if (outputStreamOffsetUs == C.TIME_UNSET) {
setOutputStreamOffsetUs(offsetUs);
} else {
if (pendingOutputStreamOffsetCount == pendingOutputStreamOffsetsUs.length) {
Log.w(
TAG,
"Too many stream changes, so dropping offset: "
+ pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1]);
} else {
pendingOutputStreamOffsetCount++;
}
pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1] = offsetUs;
}
}
@Override

5
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java
View File

@ -146,6 +146,11 @@ public class ForwardingAudioSink implements AudioSink {
sink.setPreferredDevice(audioDeviceInfo);
}
@Override
public void setOutputStreamOffsetUs(long outputStreamOffsetUs) {
sink.setOutputStreamOffsetUs(outputStreamOffsetUs);
}
@Override
public void enableTunnelingV21() {
sink.enableTunnelingV21();

5
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java
View File

@ -737,6 +737,11 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
}
}
@Override
protected void onOutputStreamOffsetUsChanged(long outputStreamOffsetUs) {
audioSink.setOutputStreamOffsetUs(outputStreamOffsetUs);
}
@Override
public void handleMessage(@MessageType int messageType, @Nullable Object message)
throws ExoPlaybackException {

28
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java
View File

@ -402,7 +402,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT];
pendingOutputStreamSwitchTimesUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT];
outputStreamStartPositionUs = C.TIME_UNSET;
outputStreamOffsetUs = C.TIME_UNSET;
setOutputStreamOffsetUs(C.TIME_UNSET);
// MediaCodec outputs audio buffers in native endian:
// https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers
// and code called from MediaCodecAudioRenderer.processOutputBuffer expects this endianness.
@ -651,7 +651,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
if (this.outputStreamOffsetUs == C.TIME_UNSET) {
checkState(this.outputStreamStartPositionUs == C.TIME_UNSET);
this.outputStreamStartPositionUs = startPositionUs;
this.outputStreamOffsetUs = offsetUs;
setOutputStreamOffsetUs(offsetUs);
} else {
if (pendingOutputStreamOffsetCount == pendingOutputStreamOffsetsUs.length) {
Log.w(
@ -688,7 +688,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
}
formatQueue.clear();
if (pendingOutputStreamOffsetCount != 0) {
outputStreamOffsetUs = pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1];
setOutputStreamOffsetUs(pendingOutputStreamOffsetsUs[pendingOutputStreamOffsetCount - 1]);
outputStreamStartPositionUs =
pendingOutputStreamStartPositionsUs[pendingOutputStreamOffsetCount - 1];
pendingOutputStreamOffsetCount = 0;
@ -707,7 +707,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
protected void onDisabled() {
inputFormat = null;
outputStreamStartPositionUs = C.TIME_UNSET;
outputStreamOffsetUs = C.TIME_UNSET;
setOutputStreamOffsetUs(C.TIME_UNSET);
pendingOutputStreamOffsetCount = 0;
flushOrReleaseCodec();
}
@ -1588,7 +1588,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
while (pendingOutputStreamOffsetCount != 0
&& presentationTimeUs >= pendingOutputStreamSwitchTimesUs[0]) {
outputStreamStartPositionUs = pendingOutputStreamStartPositionsUs[0];
outputStreamOffsetUs = pendingOutputStreamOffsetsUs[0];
setOutputStreamOffsetUs(pendingOutputStreamOffsetsUs[0]);
pendingOutputStreamOffsetCount--;
System.arraycopy(
pendingOutputStreamStartPositionsUs,
@ -1638,6 +1638,17 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
DISCARD_REASON_REUSE_NOT_IMPLEMENTED);
}
/**
* Called after the output stream offset changes.
*
* <p>The default implementation is a no-op.
*
* @param outputStreamOffsetUs The output stream offset in microseconds.
*/
protected void onOutputStreamOffsetUsChanged(long outputStreamOffsetUs) {
// Do nothing
}
@Override
public boolean isEnded() {
return outputStreamEnded;
@ -2046,6 +2057,13 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
return outputStreamOffsetUs;
}
private void setOutputStreamOffsetUs(long outputStreamOffsetUs) {
this.outputStreamOffsetUs = outputStreamOffsetUs;
if (outputStreamOffsetUs != C.TIME_UNSET) {
onOutputStreamOffsetUsChanged(outputStreamOffsetUs);
}
}
/** Returns whether this renderer supports the given {@link Format Format's} DRM scheme. */
protected static boolean supportsFormatDrm(Format format) {
return format.cryptoType == C.CRYPTO_TYPE_NONE || format.cryptoType == C.CRYPTO_TYPE_FRAMEWORK;

10
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java
View File

@ -146,7 +146,8 @@ public class DecoderAudioRendererTest {
}
@Test
public void firstSampleOfStreamSignalsDiscontinuityToAudioSink() throws Exception {
public void firstSampleOfStreamSignalsDiscontinuityAndSetOutputStreamOffsetToAudioSink()
throws Exception {
when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true);
when(mockAudioSink.isEnded()).thenReturn(true);
InOrder inOrderAudioSink = inOrder(mockAudioSink);
@ -177,12 +178,15 @@ public class DecoderAudioRendererTest {
audioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
}
inOrderAudioSink.verify(mockAudioSink, times(1)).setOutputStreamOffsetUs(0);
inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity();
inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt());
}
@Test
public void firstSampleOfReplacementStreamSignalsDiscontinuityToAudioSink() throws Exception {
public void
firstSampleOfReplacementStreamSignalsDiscontinuityAndSetOutputStreamOffsetToAudioSink()
throws Exception {
when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true);
when(mockAudioSink.isEnded()).thenReturn(true);
InOrder inOrderAudioSink = inOrder(mockAudioSink);
@ -233,9 +237,11 @@ public class DecoderAudioRendererTest {
audioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
}
inOrderAudioSink.verify(mockAudioSink, times(1)).setOutputStreamOffsetUs(0);
inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity();
inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt());
inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity();
inOrderAudioSink.verify(mockAudioSink, times(1)).setOutputStreamOffsetUs(1_000_000);
inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt());
}

57
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/MediaCodecAudioRendererTest.java
View File

@ -24,6 +24,7 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.robolectric.Shadows.shadowOf;
@ -57,6 +58,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnit;
import org.mockito.junit.MockitoRule;
@ -324,6 +326,61 @@ public class MediaCodecAudioRendererTest {
verify(audioRendererEventListener).onAudioSinkError(error);
}
@Test
public void render_callsAudioSinkSetOutputStreamOffset_whenReplaceStream() throws Exception {
FakeSampleStream fakeSampleStream1 =
new FakeSampleStream(
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
/* mediaSourceEventDispatcher= */ null,
DrmSessionManager.DRM_UNSUPPORTED,
new DrmSessionEventListener.EventDispatcher(),
AUDIO_AAC,
ImmutableList.of(
oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 1_000),
END_OF_STREAM_ITEM));
fakeSampleStream1.writeData(/* startPositionUs= */ 0);
FakeSampleStream fakeSampleStream2 =
new FakeSampleStream(
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
/* mediaSourceEventDispatcher= */ null,
DrmSessionManager.DRM_UNSUPPORTED,
new DrmSessionEventListener.EventDispatcher(),
AUDIO_AAC,
ImmutableList.of(
oneByteSample(/* timeUs= */ 1_000_000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 1_001_000),
END_OF_STREAM_ITEM));
fakeSampleStream2.writeData(/* startPositionUs= */ 0);
mediaCodecAudioRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {AUDIO_AAC},
fakeSampleStream1,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 0,
/* offsetUs= */ 0);
mediaCodecAudioRenderer.start();
while (!mediaCodecAudioRenderer.hasReadStreamToEnd()) {
mediaCodecAudioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
}
mediaCodecAudioRenderer.replaceStream(
new Format[] {AUDIO_AAC},
fakeSampleStream2,
/* startPositionUs= */ 1_000_000,
/* offsetUs= */ 1_000_000);
mediaCodecAudioRenderer.setCurrentStreamFinal();
while (!mediaCodecAudioRenderer.isEnded()) {
mediaCodecAudioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0);
}
InOrder inOrderAudioSink = inOrder(audioSink);
inOrderAudioSink.verify(audioSink).setOutputStreamOffsetUs(0);
inOrderAudioSink.verify(audioSink).setOutputStreamOffsetUs(1_000_000);
}
@Test
public void supportsFormat_withEac3JocMediaAndEac3Decoder_returnsTrue() throws Exception {
Format mediaFormat =