From 3760f514a9a4ab823b8416a1710bdd92993c2754 Mon Sep 17 00:00:00 2001 From: olly Date: Fri, 13 May 2016 04:59:12 -0700 Subject: [PATCH] Flesh out ExoPlayer extensions. 1. AudioDecoderTrackRenderer now reports decoder initialization and AudioTrack underruns. 2. AudioDecoderTrackRenderer can now render more than one output buffer per call to doSomeWork, to be consistent with MediaCodecAudioTrackRenderer. This may also prevent audio underruns in the case that audio is fragmented into many small buffers. 3. AudioDecoderTrackRenderer now has an overridable method that receives the audio session id, to be consistent with MediaCodecAudioTrackRenderer. 4. Fix unsafe event notification in LibvpxVideoTrackRenderer. 5. Vpx and AudioDecoder extensions now increment the CodecCounter inputBufferCount field correctly. 6. Decoders now have names :). ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=122250009 --- .../exoplayer/ext/flac/FlacDecoder.java | 5 ++ .../exoplayer/ext/opus/OpusDecoder.java | 5 ++ .../ext/vp9/LibvpxVideoTrackRenderer.java | 37 ++++---- .../android/exoplayer/ext/vp9/VpxDecoder.java | 5 ++ .../extensions/AudioDecoderTrackRenderer.java | 84 ++++++++++++++++--- .../android/exoplayer/extensions/Decoder.java | 7 ++ .../exoplayer/text/SimpleSubtitleParser.java | 10 ++- .../exoplayer/text/eia608/Eia608Parser.java | 5 ++ .../exoplayer/text/subrip/SubripParser.java | 1 + .../exoplayer/text/ttml/TtmlParser.java | 1 + .../exoplayer/text/tx3g/Tx3gParser.java | 4 + .../text/webvtt/Mp4WebvttParser.java | 1 + .../exoplayer/text/webvtt/WebvttParser.java | 1 + 13 files changed, 139 insertions(+), 27 deletions(-) diff --git a/extensions/flac/src/main/java/com/google/android/exoplayer/ext/flac/FlacDecoder.java b/extensions/flac/src/main/java/com/google/android/exoplayer/ext/flac/FlacDecoder.java index 50a7c6bedd..d94d5cdad3 100644 --- a/extensions/flac/src/main/java/com/google/android/exoplayer/ext/flac/FlacDecoder.java +++ b/extensions/flac/src/main/java/com/google/android/exoplayer/ext/flac/FlacDecoder.java @@ -61,6 +61,11 @@ import java.util.List; maxOutputBufferSize = streamInfo.maxDecodedFrameSize(); } + @Override + public String getName() { + return "libflac"; + } + @Override public DecoderInputBuffer createInputBuffer() { return new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL); diff --git a/extensions/opus/src/main/java/com/google/android/exoplayer/ext/opus/OpusDecoder.java b/extensions/opus/src/main/java/com/google/android/exoplayer/ext/opus/OpusDecoder.java index 7490b763ba..8f5a70a7f5 100644 --- a/extensions/opus/src/main/java/com/google/android/exoplayer/ext/opus/OpusDecoder.java +++ b/extensions/opus/src/main/java/com/google/android/exoplayer/ext/opus/OpusDecoder.java @@ -132,6 +132,11 @@ import java.util.List; setInitialInputBufferSize(initialInputBufferSize); } + @Override + public String getName() { + return "libopus" + getLibopusVersion(); + } + @Override public DecoderInputBuffer createInputBuffer() { return new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/LibvpxVideoTrackRenderer.java b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/LibvpxVideoTrackRenderer.java index 403ed47eda..670599908a 100644 --- a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/LibvpxVideoTrackRenderer.java +++ b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/LibvpxVideoTrackRenderer.java @@ -151,13 +151,15 @@ public final class LibvpxVideoTrackRenderer extends TrackRenderer { try { if (decoder == null) { // If we don't have a decoder yet, we need to instantiate one. - long startElapsedRealtimeMs = SystemClock.elapsedRealtime(); + long codecInitializingTimestamp = SystemClock.elapsedRealtime(); decoder = new VpxDecoder(NUM_BUFFERS, NUM_BUFFERS, INITIAL_INPUT_BUFFER_SIZE); decoder.setOutputMode(outputMode); - notifyDecoderInitialized(startElapsedRealtimeMs, SystemClock.elapsedRealtime()); + long codecInitializedTimestamp = SystemClock.elapsedRealtime(); + notifyDecoderInitialized(decoder.getName(), codecInitializedTimestamp, + codecInitializedTimestamp - codecInitializingTimestamp); codecCounters.codecInitCount++; } - while (processOutputBuffer(positionUs)) {} + while (drainOutputBuffer(positionUs)) {} while (feedInputBuffer()) {} } catch (VpxDecoderException e) { throw ExoPlaybackException.createForRenderer(e, getIndex()); @@ -165,8 +167,7 @@ public final class LibvpxVideoTrackRenderer extends TrackRenderer { codecCounters.ensureUpdated(); } - private boolean processOutputBuffer(long positionUs) - throws VpxDecoderException { + private boolean drainOutputBuffer(long positionUs) throws VpxDecoderException { if (outputStreamEnded) { return false; } @@ -226,7 +227,7 @@ public final class LibvpxVideoTrackRenderer extends TrackRenderer { private void renderBuffer() { codecCounters.renderedOutputBufferCount++; - notifyIfVideoSizeChanged(outputBuffer); + notifyIfVideoSizeChanged(outputBuffer.width, outputBuffer.height); if (outputBuffer.mode == VpxDecoder.OUTPUT_MODE_RGB && surface != null) { renderRgbFrame(outputBuffer, scaleToFit); if (!drawnToSurface) { @@ -280,9 +281,13 @@ public final class LibvpxVideoTrackRenderer extends TrackRenderer { } if (inputBuffer.isEndOfStream()) { inputStreamEnded = true; + decoder.queueInputBuffer(inputBuffer); + inputBuffer = null; + return false; } inputBuffer.flip(); decoder.queueInputBuffer(inputBuffer); + codecCounters.inputBufferCount++; inputBuffer = null; return true; } @@ -398,16 +403,16 @@ public final class LibvpxVideoTrackRenderer extends TrackRenderer { } } - private void notifyIfVideoSizeChanged(final VpxOutputBuffer outputBuffer) { - if (previousWidth == -1 || previousHeight == -1 - || previousWidth != outputBuffer.width || previousHeight != outputBuffer.height) { - previousWidth = outputBuffer.width; - previousHeight = outputBuffer.height; + private void notifyIfVideoSizeChanged(final int width, final int height) { + if (previousWidth == -1 || previousHeight == -1 || previousWidth != width + || previousHeight != height) { + previousWidth = width; + previousHeight = height; if (eventHandler != null && eventListener != null) { eventHandler.post(new Runnable() { @Override public void run() { - eventListener.onVideoSizeChanged(outputBuffer.width, outputBuffer.height, 0, 1); + eventListener.onVideoSizeChanged(width, height, 0, 1); } }); } @@ -441,14 +446,14 @@ public final class LibvpxVideoTrackRenderer extends TrackRenderer { } } - private void notifyDecoderInitialized( - final long startElapsedRealtimeMs, final long finishElapsedRealtimeMs) { + private void notifyDecoderInitialized(final String decoderName, + final long initializedTimestamp, final long initializationDuration) { if (eventHandler != null && eventListener != null) { eventHandler.post(new Runnable() { @Override public void run() { - eventListener.onDecoderInitialized("libvpx" + getLibvpxVersion(), - finishElapsedRealtimeMs, finishElapsedRealtimeMs - startElapsedRealtimeMs); + eventListener.onDecoderInitialized(decoderName, initializedTimestamp, + initializationDuration); } }); } diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxDecoder.java b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxDecoder.java index d5dfc3f0da..acaf6fc3a7 100644 --- a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxDecoder.java +++ b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxDecoder.java @@ -74,6 +74,11 @@ import java.nio.ByteBuffer; setInitialInputBufferSize(initialInputBufferSize); } + @Override + public String getName() { + return "libvpx" + getLibvpxVersion(); + } + /** * Sets the output mode for frames rendered by the decoder. * diff --git a/library/src/main/java/com/google/android/exoplayer/extensions/AudioDecoderTrackRenderer.java b/library/src/main/java/com/google/android/exoplayer/extensions/AudioDecoderTrackRenderer.java index 679b0e2b9e..b4629ca03b 100644 --- a/library/src/main/java/com/google/android/exoplayer/extensions/AudioDecoderTrackRenderer.java +++ b/library/src/main/java/com/google/android/exoplayer/extensions/AudioDecoderTrackRenderer.java @@ -30,6 +30,7 @@ import com.google.android.exoplayer.audio.AudioTrack; import com.google.android.exoplayer.util.MimeTypes; import android.os.Handler; +import android.os.SystemClock; /** * Decodes and renders audio using a {@link SimpleDecoder}. @@ -63,6 +64,9 @@ public abstract class AudioDecoderTrackRenderer extends TrackRenderer implements private final AudioTrack audioTrack; private int audioSessionId; + private boolean audioTrackHasData; + private long lastFeedElapsedRealtimeMs; + public AudioDecoderTrackRenderer() { this(null, null); } @@ -101,16 +105,20 @@ public abstract class AudioDecoderTrackRenderer extends TrackRenderer implements // If we don't have a decoder yet, we need to instantiate one. if (decoder == null) { try { + long codecInitializingTimestamp = SystemClock.elapsedRealtime(); decoder = createDecoder(inputFormat); + long codecInitializedTimestamp = SystemClock.elapsedRealtime(); + notifyDecoderInitialized(decoder.getName(), codecInitializedTimestamp, + codecInitializedTimestamp - codecInitializingTimestamp); + codecCounters.codecInitCount++; } catch (AudioDecoderException e) { throw ExoPlaybackException.createForRenderer(e, getIndex()); } - codecCounters.codecInitCount++; } // Rendering loop. try { - renderBuffer(); + while (drainOutputBuffer()) {} while (feedInputBuffer()) {} } catch (AudioTrack.InitializationException e) { notifyAudioTrackInitializationError(e); @@ -145,16 +153,16 @@ public abstract class AudioDecoderTrackRenderer extends TrackRenderer implements null, null, null); } - private void renderBuffer() throws AudioDecoderException, AudioTrack.InitializationException, - AudioTrack.WriteException { + private boolean drainOutputBuffer() throws AudioDecoderException, + AudioTrack.InitializationException, AudioTrack.WriteException { if (outputStreamEnded) { - return; + return false; } if (outputBuffer == null) { outputBuffer = decoder.dequeueOutputBuffer(); if (outputBuffer == null) { - return; + return false; } } @@ -163,7 +171,7 @@ public abstract class AudioDecoderTrackRenderer extends TrackRenderer implements audioTrack.handleEndOfStream(); outputBuffer.release(); outputBuffer = null; - return; + return false; } if (!audioTrack.isInitialized()) { @@ -174,14 +182,26 @@ public abstract class AudioDecoderTrackRenderer extends TrackRenderer implements audioTrack.initialize(audioSessionId); } else { audioSessionId = audioTrack.initialize(); + onAudioSessionId(audioSessionId); } + audioTrackHasData = false; if (getState() == TrackRenderer.STATE_STARTED) { audioTrack.play(); } + } else { + // Check for AudioTrack underrun. + boolean audioTrackHadData = audioTrackHasData; + audioTrackHasData = audioTrack.hasPendingData(); + if (audioTrackHadData && !audioTrackHasData && getState() == TrackRenderer.STATE_STARTED) { + long elapsedSinceLastFeedMs = SystemClock.elapsedRealtime() - lastFeedElapsedRealtimeMs; + long bufferSizeUs = audioTrack.getBufferSizeUs(); + long bufferSizeMs = bufferSizeUs == C.UNSET_TIME_US ? -1 : bufferSizeUs / 1000; + notifyAudioTrackUnderrun(audioTrack.getBufferSize(), bufferSizeMs, elapsedSinceLastFeedMs); + } } - int handleBufferResult; - handleBufferResult = audioTrack.handleBuffer(outputBuffer.data, outputBuffer.timestampUs); + int handleBufferResult = audioTrack.handleBuffer(outputBuffer.data, outputBuffer.timestampUs); + lastFeedElapsedRealtimeMs = SystemClock.elapsedRealtime(); // If we are out of sync, allow currentPositionUs to jump backwards. if ((handleBufferResult & AudioTrack.RESULT_POSITION_DISCONTINUITY) != 0) { @@ -193,7 +213,10 @@ public abstract class AudioDecoderTrackRenderer extends TrackRenderer implements codecCounters.renderedOutputBufferCount++; outputBuffer.release(); outputBuffer = null; + return true; } + + return false; } private boolean feedInputBuffer() throws AudioDecoderException { @@ -218,10 +241,13 @@ public abstract class AudioDecoderTrackRenderer extends TrackRenderer implements } if (inputBuffer.isEndOfStream()) { inputStreamEnded = true; + decoder.queueInputBuffer(inputBuffer); + inputBuffer = null; + return false; } - inputBuffer.flip(); decoder.queueInputBuffer(inputBuffer); + codecCounters.inputBufferCount++; inputBuffer = null; return true; } @@ -269,6 +295,19 @@ public abstract class AudioDecoderTrackRenderer extends TrackRenderer implements } } + /** + * Invoked when the audio session id becomes known. Once the id is known it will not change + * (and hence this method will not be invoked again) unless the renderer is disabled and then + * subsequently re-enabled. + *

+ * The default implementation is a no-op. + * + * @param audioSessionId The audio session id. + */ + protected void onAudioSessionId(int audioSessionId) { + // Do nothing. + } + @Override protected void onEnabled(Format[] formats, boolean joining) throws ExoPlaybackException { notifyAudioCodecCounters(); @@ -320,6 +359,19 @@ public abstract class AudioDecoderTrackRenderer extends TrackRenderer implements } } + private void notifyDecoderInitialized(final String decoderName, + final long initializedTimestamp, final long initializationDuration) { + if (eventHandler != null && eventListener != null) { + eventHandler.post(new Runnable() { + @Override + public void run() { + eventListener.onDecoderInitialized(decoderName, initializedTimestamp, + initializationDuration); + } + }); + } + } + private void notifyAudioTrackInitializationError(final AudioTrack.InitializationException e) { if (eventHandler != null && eventListener != null) { eventHandler.post(new Runnable() { @@ -342,6 +394,18 @@ public abstract class AudioDecoderTrackRenderer extends TrackRenderer implements } } + private void notifyAudioTrackUnderrun(final int bufferSize, final long bufferSizeMs, + final long elapsedSinceLastFeedMs) { + if (eventHandler != null && eventListener != null) { + eventHandler.post(new Runnable() { + @Override + public void run() { + eventListener.onAudioTrackUnderrun(bufferSize, bufferSizeMs, elapsedSinceLastFeedMs); + } + }); + } + } + private void notifyAudioCodecCounters() { if (eventHandler != null && eventListener != null) { eventHandler.post(new Runnable() { diff --git a/library/src/main/java/com/google/android/exoplayer/extensions/Decoder.java b/library/src/main/java/com/google/android/exoplayer/extensions/Decoder.java index 4d0218c328..38eb4f07f5 100644 --- a/library/src/main/java/com/google/android/exoplayer/extensions/Decoder.java +++ b/library/src/main/java/com/google/android/exoplayer/extensions/Decoder.java @@ -24,6 +24,13 @@ package com.google.android.exoplayer.extensions; */ public interface Decoder { + /** + * Returns the name of the decoder. + * + * @return The name of the decoder. + */ + String getName(); + /** * Dequeues the next input buffer to be filled and queued to the decoder. * diff --git a/library/src/main/java/com/google/android/exoplayer/text/SimpleSubtitleParser.java b/library/src/main/java/com/google/android/exoplayer/text/SimpleSubtitleParser.java index b8dd1106d4..5dbfc37058 100644 --- a/library/src/main/java/com/google/android/exoplayer/text/SimpleSubtitleParser.java +++ b/library/src/main/java/com/google/android/exoplayer/text/SimpleSubtitleParser.java @@ -27,11 +27,19 @@ public abstract class SimpleSubtitleParser extends SimpleDecoder implements SubtitleParser { - protected SimpleSubtitleParser() { + private final String name; + + protected SimpleSubtitleParser(String name) { super(new SubtitleInputBuffer[2], new SubtitleOutputBuffer[2]); + this.name = name; setInitialInputBufferSize(1024); } + @Override + public final String getName() { + return name; + } + @Override public void setPositionUs(long timeUs) { // Do nothing diff --git a/library/src/main/java/com/google/android/exoplayer/text/eia608/Eia608Parser.java b/library/src/main/java/com/google/android/exoplayer/text/eia608/Eia608Parser.java index 7c0bb83b08..db1ea3373f 100644 --- a/library/src/main/java/com/google/android/exoplayer/text/eia608/Eia608Parser.java +++ b/library/src/main/java/com/google/android/exoplayer/text/eia608/Eia608Parser.java @@ -205,6 +205,11 @@ public final class Eia608Parser implements SubtitleParser { captionRowCount = DEFAULT_CAPTIONS_ROW_COUNT; } + @Override + public String getName() { + return "Eia608Parser"; + } + @Override public void setPositionUs(long positionUs) { playbackPositionUs = positionUs; diff --git a/library/src/main/java/com/google/android/exoplayer/text/subrip/SubripParser.java b/library/src/main/java/com/google/android/exoplayer/text/subrip/SubripParser.java index f39d49b5f4..9e752deb3d 100644 --- a/library/src/main/java/com/google/android/exoplayer/text/subrip/SubripParser.java +++ b/library/src/main/java/com/google/android/exoplayer/text/subrip/SubripParser.java @@ -43,6 +43,7 @@ public final class SubripParser extends SimpleSubtitleParser { private final StringBuilder textBuilder; public SubripParser() { + super("SubripParser"); textBuilder = new StringBuilder(); } diff --git a/library/src/main/java/com/google/android/exoplayer/text/ttml/TtmlParser.java b/library/src/main/java/com/google/android/exoplayer/text/ttml/TtmlParser.java index e0907d0364..527bd533ae 100644 --- a/library/src/main/java/com/google/android/exoplayer/text/ttml/TtmlParser.java +++ b/library/src/main/java/com/google/android/exoplayer/text/ttml/TtmlParser.java @@ -87,6 +87,7 @@ public final class TtmlParser extends SimpleSubtitleParser { private final XmlPullParserFactory xmlParserFactory; public TtmlParser() { + super("TtmlParser"); try { xmlParserFactory = XmlPullParserFactory.newInstance(); xmlParserFactory.setNamespaceAware(true); diff --git a/library/src/main/java/com/google/android/exoplayer/text/tx3g/Tx3gParser.java b/library/src/main/java/com/google/android/exoplayer/text/tx3g/Tx3gParser.java index b2259ab085..f6a1436940 100644 --- a/library/src/main/java/com/google/android/exoplayer/text/tx3g/Tx3gParser.java +++ b/library/src/main/java/com/google/android/exoplayer/text/tx3g/Tx3gParser.java @@ -26,6 +26,10 @@ import com.google.android.exoplayer.text.Subtitle; */ public final class Tx3gParser extends SimpleSubtitleParser { + public Tx3gParser() { + super("Tx3gParser"); + } + @Override protected Subtitle decode(byte[] bytes, int length) { String cueText = new String(bytes, 0, length); diff --git a/library/src/main/java/com/google/android/exoplayer/text/webvtt/Mp4WebvttParser.java b/library/src/main/java/com/google/android/exoplayer/text/webvtt/Mp4WebvttParser.java index 82da06ae84..1690507961 100644 --- a/library/src/main/java/com/google/android/exoplayer/text/webvtt/Mp4WebvttParser.java +++ b/library/src/main/java/com/google/android/exoplayer/text/webvtt/Mp4WebvttParser.java @@ -40,6 +40,7 @@ public final class Mp4WebvttParser extends SimpleSubtitleParser { private final WebvttCue.Builder builder; public Mp4WebvttParser() { + super("Mp4WebvttParser"); sampleData = new ParsableByteArray(); builder = new WebvttCue.Builder(); } diff --git a/library/src/main/java/com/google/android/exoplayer/text/webvtt/WebvttParser.java b/library/src/main/java/com/google/android/exoplayer/text/webvtt/WebvttParser.java index 6dfd59d7f4..dd0e7337d2 100644 --- a/library/src/main/java/com/google/android/exoplayer/text/webvtt/WebvttParser.java +++ b/library/src/main/java/com/google/android/exoplayer/text/webvtt/WebvttParser.java @@ -46,6 +46,7 @@ public final class WebvttParser extends SimpleSubtitleParser { private final List definedStyles; public WebvttParser() { + super("WebvttParser"); cueParser = new WebvttCueParser(); parsableWebvttData = new ParsableByteArray(); webvttCueBuilder = new WebvttCue.Builder();