From 6febc88dce52501d40d5dda2c83289584f8d5a09 Mon Sep 17 00:00:00 2001 From: olly Date: Tue, 2 Jul 2019 13:42:05 +0100 Subject: [PATCH] FLV extractor fixes 1. Only output video starting from a keyframe 2. When calculating the timestamp offset to adjust live streams to start at t=0, use the timestamp of the first tag from which a sample is actually output, rather than just the first audio/video tag. The test streams in the referenced GitHub issue start with a video tag whose packet type is AVC_PACKET_TYPE_SEQUENCE_HEADER (i.e. does not contain a sample) and whose timestamp is set to 0 (i.e. isn't set). The timestamp is set correctly on tags that from which a sample is actually output. Issue: #6111 PiperOrigin-RevId: 256147747 --- RELEASENOTES.md | 2 ++ .../extractor/flv/AudioTagPayloadReader.java | 8 ++++-- .../extractor/flv/FlvExtractor.java | 26 ++++++++++++------- .../extractor/flv/ScriptTagPayloadReader.java | 7 ++--- .../extractor/flv/TagPayloadReader.java | 16 ++++++------ .../extractor/flv/VideoTagPayloadReader.java | 18 ++++++++++--- 6 files changed, 51 insertions(+), 26 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 62e985f98b..d76ca54b7b 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -20,6 +20,8 @@ * Wrap decoder exceptions in a new `DecoderException` class and report as renderer error. * SmoothStreaming: Parse text stream `Subtype` into `Format.roleFlags`. +* FLV: Fix bug that caused playback of some live streams to not start + ([#6111](https://github.com/google/ExoPlayer/issues/6111)). ### 2.10.2 ### diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/AudioTagPayloadReader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/AudioTagPayloadReader.java index ec5ad88aeb..b10f2bf80b 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/AudioTagPayloadReader.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/AudioTagPayloadReader.java @@ -86,11 +86,12 @@ import java.util.Collections; } @Override - protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserException { + protected boolean parsePayload(ParsableByteArray data, long timeUs) throws ParserException { if (audioFormat == AUDIO_FORMAT_MP3) { int sampleSize = data.bytesLeft(); output.sampleData(data, sampleSize); output.sampleMetadata(timeUs, C.BUFFER_FLAG_KEY_FRAME, sampleSize, 0, null); + return true; } else { int packetType = data.readUnsignedByte(); if (packetType == AAC_PACKET_TYPE_SEQUENCE_HEADER && !hasOutputFormat) { @@ -104,12 +105,15 @@ import java.util.Collections; Collections.singletonList(audioSpecificConfig), null, 0, null); output.format(format); hasOutputFormat = true; + return false; } else if (audioFormat != AUDIO_FORMAT_AAC || packetType == AAC_PACKET_TYPE_AAC_RAW) { int sampleSize = data.bytesLeft(); output.sampleData(data, sampleSize); output.sampleMetadata(timeUs, C.BUFFER_FLAG_KEY_FRAME, sampleSize, 0, null); + return true; + } else { + return false; } } } - } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/FlvExtractor.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/FlvExtractor.java index 15b36157fb..f6835558f2 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/FlvExtractor.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/FlvExtractor.java @@ -73,6 +73,7 @@ public final class FlvExtractor implements Extractor { private ExtractorOutput extractorOutput; private @States int state; + private boolean outputFirstSample; private long mediaTagTimestampOffsetUs; private int bytesToNextTagHeader; private int tagType; @@ -89,7 +90,6 @@ public final class FlvExtractor implements Extractor { tagData = new ParsableByteArray(); metadataReader = new ScriptTagPayloadReader(); state = STATE_READING_FLV_HEADER; - mediaTagTimestampOffsetUs = C.TIME_UNSET; } @Override @@ -131,7 +131,7 @@ public final class FlvExtractor implements Extractor { @Override public void seek(long position, long timeUs) { state = STATE_READING_FLV_HEADER; - mediaTagTimestampOffsetUs = C.TIME_UNSET; + outputFirstSample = false; bytesToNextTagHeader = 0; } @@ -252,14 +252,16 @@ public final class FlvExtractor implements Extractor { */ private boolean readTagData(ExtractorInput input) throws IOException, InterruptedException { boolean wasConsumed = true; + boolean wasSampleOutput = false; + long timestampUs = getCurrentTimestampUs(); if (tagType == TAG_TYPE_AUDIO && audioReader != null) { ensureReadyForMediaOutput(); - audioReader.consume(prepareTagData(input), mediaTagTimestampOffsetUs + tagTimestampUs); + wasSampleOutput = audioReader.consume(prepareTagData(input), timestampUs); } else if (tagType == TAG_TYPE_VIDEO && videoReader != null) { ensureReadyForMediaOutput(); - videoReader.consume(prepareTagData(input), mediaTagTimestampOffsetUs + tagTimestampUs); + wasSampleOutput = videoReader.consume(prepareTagData(input), timestampUs); } else if (tagType == TAG_TYPE_SCRIPT_DATA && !outputSeekMap) { - metadataReader.consume(prepareTagData(input), tagTimestampUs); + wasSampleOutput = metadataReader.consume(prepareTagData(input), timestampUs); long durationUs = metadataReader.getDurationUs(); if (durationUs != C.TIME_UNSET) { extractorOutput.seekMap(new SeekMap.Unseekable(durationUs)); @@ -269,6 +271,11 @@ public final class FlvExtractor implements Extractor { input.skipFully(tagDataSize); wasConsumed = false; } + if (!outputFirstSample && wasSampleOutput) { + outputFirstSample = true; + mediaTagTimestampOffsetUs = + metadataReader.getDurationUs() == C.TIME_UNSET ? -tagTimestampUs : 0; + } bytesToNextTagHeader = 4; // There's a 4 byte previous tag size before the next header. state = STATE_SKIPPING_TO_TAG_HEADER; return wasConsumed; @@ -291,10 +298,11 @@ public final class FlvExtractor implements Extractor { extractorOutput.seekMap(new SeekMap.Unseekable(C.TIME_UNSET)); outputSeekMap = true; } - if (mediaTagTimestampOffsetUs == C.TIME_UNSET) { - mediaTagTimestampOffsetUs = - metadataReader.getDurationUs() == C.TIME_UNSET ? -tagTimestampUs : 0; - } } + private long getCurrentTimestampUs() { + return outputFirstSample + ? (mediaTagTimestampOffsetUs + tagTimestampUs) + : (metadataReader.getDurationUs() == C.TIME_UNSET ? 0 : tagTimestampUs); + } } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/ScriptTagPayloadReader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/ScriptTagPayloadReader.java index 2dec85ffcc..eb1cc8f336 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/ScriptTagPayloadReader.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/ScriptTagPayloadReader.java @@ -63,7 +63,7 @@ import java.util.Map; } @Override - protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserException { + protected boolean parsePayload(ParsableByteArray data, long timeUs) throws ParserException { int nameType = readAmfType(data); if (nameType != AMF_TYPE_STRING) { // Should never happen. @@ -72,12 +72,12 @@ import java.util.Map; String name = readAmfString(data); if (!NAME_METADATA.equals(name)) { // We're only interested in metadata. - return; + return false; } int type = readAmfType(data); if (type != AMF_TYPE_ECMA_ARRAY) { // We're not interested in this metadata. - return; + return false; } // Set the duration to the value contained in the metadata, if present. Map metadata = readAmfEcmaArray(data); @@ -87,6 +87,7 @@ import java.util.Map; durationUs = (long) (durationSeconds * C.MICROS_PER_SECOND); } } + return false; } private static int readAmfType(ParsableByteArray data) { diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/TagPayloadReader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/TagPayloadReader.java index e8652d653f..48914b7c2c 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/TagPayloadReader.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/TagPayloadReader.java @@ -58,12 +58,11 @@ import com.google.android.exoplayer2.util.ParsableByteArray; * * @param data The payload data to consume. * @param timeUs The timestamp associated with the payload. + * @return Whether a sample was output. * @throws ParserException If an error occurs parsing the data. */ - public final void consume(ParsableByteArray data, long timeUs) throws ParserException { - if (parseHeader(data)) { - parsePayload(data, timeUs); - } + public final boolean consume(ParsableByteArray data, long timeUs) throws ParserException { + return parseHeader(data) && parsePayload(data, timeUs); } /** @@ -78,10 +77,11 @@ import com.google.android.exoplayer2.util.ParsableByteArray; /** * Parses tag payload. * - * @param data Buffer where tag payload is stored - * @param timeUs Time position of the frame + * @param data Buffer where tag payload is stored. + * @param timeUs Time position of the frame. + * @return Whether a sample was output. * @throws ParserException If an error occurs parsing the payload. */ - protected abstract void parsePayload(ParsableByteArray data, long timeUs) throws ParserException; - + protected abstract boolean parsePayload(ParsableByteArray data, long timeUs) + throws ParserException; } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/VideoTagPayloadReader.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/VideoTagPayloadReader.java index 92db91e20b..5ddaafb4a8 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/VideoTagPayloadReader.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/extractor/flv/VideoTagPayloadReader.java @@ -47,6 +47,7 @@ import com.google.android.exoplayer2.video.AvcConfig; // State variables. private boolean hasOutputFormat; + private boolean hasOutputKeyframe; private int frameType; /** @@ -60,7 +61,7 @@ import com.google.android.exoplayer2.video.AvcConfig; @Override public void seek() { - // Do nothing. + hasOutputKeyframe = false; } @Override @@ -77,7 +78,7 @@ import com.google.android.exoplayer2.video.AvcConfig; } @Override - protected void parsePayload(ParsableByteArray data, long timeUs) throws ParserException { + protected boolean parsePayload(ParsableByteArray data, long timeUs) throws ParserException { int packetType = data.readUnsignedByte(); int compositionTimeMs = data.readInt24(); @@ -94,7 +95,12 @@ import com.google.android.exoplayer2.video.AvcConfig; avcConfig.initializationData, Format.NO_VALUE, avcConfig.pixelWidthAspectRatio, null); output.format(format); hasOutputFormat = true; + return false; } else if (packetType == AVC_PACKET_TYPE_AVC_NALU && hasOutputFormat) { + boolean isKeyframe = frameType == VIDEO_FRAME_KEYFRAME; + if (!hasOutputKeyframe && !isKeyframe) { + return false; + } // TODO: Deduplicate with Mp4Extractor. // Zero the top three bytes of the array that we'll use to decode nal unit lengths, in case // they're only 1 or 2 bytes long. @@ -123,8 +129,12 @@ import com.google.android.exoplayer2.video.AvcConfig; output.sampleData(data, bytesToWrite); bytesWritten += bytesToWrite; } - output.sampleMetadata(timeUs, frameType == VIDEO_FRAME_KEYFRAME ? C.BUFFER_FLAG_KEY_FRAME : 0, - bytesWritten, 0, null); + output.sampleMetadata( + timeUs, isKeyframe ? C.BUFFER_FLAG_KEY_FRAME : 0, bytesWritten, 0, null); + hasOutputKeyframe = true; + return true; + } else { + return false; } }