From 04c56c44cf01d6b393d1ecfbadc87f7c293d5e37 Mon Sep 17 00:00:00 2001 From: aquilescanta Date: Wed, 28 Oct 2020 23:02:54 +0000 Subject: [PATCH] Publish components that depend on MediaParser This change will be followed up by: - Changes adding APIs to enable the use of MediaParser in each of the supported media sources. - Changes removing TODOs related to the change of the stable SDK to API 30. PiperOrigin-RevId: 339556777 --- .../source/MediaParserExtractorAdapter.java | 121 +++ .../source/ProgressiveMediaPeriod.java | 4 +- .../chunk/MediaParserChunkExtractor.java | 169 +++++ .../mediaparser/InputReaderAdapterV30.java | 87 +++ .../source/mediaparser/MediaParserUtil.java | 60 ++ .../mediaparser/OutputConsumerAdapterV30.java | 691 ++++++++++++++++++ .../source/mediaparser/package-info.java | 19 + .../source/dash/DefaultDashChunkSource.java | 1 - .../exoplayer2/source/hls/HlsMediaSource.java | 2 - .../MediaParserHlsMediaChunkExtractor.java | 281 +++++++ 10 files changed, 1429 insertions(+), 6 deletions(-) create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/source/MediaParserExtractorAdapter.java create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/source/chunk/MediaParserChunkExtractor.java create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/InputReaderAdapterV30.java create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/MediaParserUtil.java create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/OutputConsumerAdapterV30.java create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/package-info.java create mode 100644 library/hls/src/main/java/com/google/android/exoplayer2/source/hls/MediaParserHlsMediaChunkExtractor.java diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/MediaParserExtractorAdapter.java b/library/core/src/main/java/com/google/android/exoplayer2/source/MediaParserExtractorAdapter.java new file mode 100644 index 0000000000..6cb20c9fbe --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/source/MediaParserExtractorAdapter.java @@ -0,0 +1,121 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.source; + +import static com.google.android.exoplayer2.source.mediaparser.MediaParserUtil.PARAMETER_EAGERLY_EXPOSE_TRACK_TYPE; +import static com.google.android.exoplayer2.source.mediaparser.MediaParserUtil.PARAMETER_INCLUDE_SUPPLEMENTAL_DATA; +import static com.google.android.exoplayer2.source.mediaparser.MediaParserUtil.PARAMETER_IN_BAND_CRYPTO_INFO; + +import android.annotation.SuppressLint; +import android.media.MediaParser; +import android.media.MediaParser.SeekPoint; +import android.net.Uri; +import android.util.Pair; +import androidx.annotation.RequiresApi; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.extractor.Extractor; +import com.google.android.exoplayer2.extractor.ExtractorOutput; +import com.google.android.exoplayer2.extractor.PositionHolder; +import com.google.android.exoplayer2.source.mediaparser.InputReaderAdapterV30; +import com.google.android.exoplayer2.source.mediaparser.OutputConsumerAdapterV30; +import com.google.android.exoplayer2.upstream.DataReader; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** {@link ProgressiveMediaExtractor} implemented on top of the platform's {@link MediaParser}. */ +@RequiresApi(30) +/* package */ final class MediaParserExtractorAdapter implements ProgressiveMediaExtractor { + + private final OutputConsumerAdapterV30 outputConsumerAdapter; + private final InputReaderAdapterV30 inputReaderAdapter; + private final MediaParser mediaParser; + private String parserName; + + @SuppressLint("WrongConstant") + public MediaParserExtractorAdapter() { + // TODO: Add support for injecting the desired extractor list. + outputConsumerAdapter = new OutputConsumerAdapterV30(); + inputReaderAdapter = new InputReaderAdapterV30(); + mediaParser = MediaParser.create(outputConsumerAdapter); + mediaParser.setParameter(PARAMETER_EAGERLY_EXPOSE_TRACK_TYPE, true); + mediaParser.setParameter(PARAMETER_IN_BAND_CRYPTO_INFO, true); + mediaParser.setParameter(PARAMETER_INCLUDE_SUPPLEMENTAL_DATA, true); + parserName = MediaParser.PARSER_NAME_UNKNOWN; + } + + @Override + public void init( + DataReader dataReader, + Uri uri, + Map> responseHeaders, + long position, + long length, + ExtractorOutput output) + throws IOException { + outputConsumerAdapter.setExtractorOutput(output); + inputReaderAdapter.setDataReader(dataReader, length); + inputReaderAdapter.setCurrentPosition(position); + String currentParserName = mediaParser.getParserName(); + if (MediaParser.PARSER_NAME_UNKNOWN.equals(currentParserName)) { + // We need to sniff. + mediaParser.advance(inputReaderAdapter); + parserName = mediaParser.getParserName(); + outputConsumerAdapter.setSelectedParserName(parserName); + } else if (!currentParserName.equals(parserName)) { + // The parser was created by name. + parserName = mediaParser.getParserName(); + outputConsumerAdapter.setSelectedParserName(parserName); + } else { + // The parser implementation has already been selected. Do nothing. + } + } + + @Override + public void release() { + mediaParser.release(); + } + + @Override + public void disableSeekingOnMp3Streams() { + if (MediaParser.PARSER_NAME_MP3.equals(parserName)) { + outputConsumerAdapter.disableSeeking(); + } + } + + @Override + public long getCurrentInputPosition() { + return inputReaderAdapter.getPosition(); + } + + @Override + public void seek(long position, long seekTimeUs) { + inputReaderAdapter.setCurrentPosition(position); + Pair seekPoints = outputConsumerAdapter.getSeekPoints(seekTimeUs); + mediaParser.seek(seekPoints.second.position == position ? seekPoints.second : seekPoints.first); + } + + @Override + public int read(PositionHolder positionHolder) throws IOException { + boolean shouldContinue = mediaParser.advance(inputReaderAdapter); + positionHolder.position = inputReaderAdapter.getAndResetSeekPosition(); + return !shouldContinue + ? Extractor.RESULT_END_OF_INPUT + : positionHolder.position != C.POSITION_UNSET + ? Extractor.RESULT_SEEK + : Extractor.RESULT_CONTINUE; + } +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/ProgressiveMediaPeriod.java b/library/core/src/main/java/com/google/android/exoplayer2/source/ProgressiveMediaPeriod.java index 121eeb940d..f261b7ce05 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/source/ProgressiveMediaPeriod.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/source/ProgressiveMediaPeriod.java @@ -188,9 +188,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.customCacheKey = customCacheKey; this.continueLoadingCheckIntervalBytes = continueLoadingCheckIntervalBytes; loader = new Loader("Loader:ProgressiveMediaPeriod"); - ProgressiveMediaExtractor progressiveMediaExtractor = - new BundledExtractorsAdapter(extractorsFactory); - this.progressiveMediaExtractor = progressiveMediaExtractor; + this.progressiveMediaExtractor = new BundledExtractorsAdapter(extractorsFactory); loadCondition = new ConditionVariable(); maybeFinishPrepareRunnable = this::maybeFinishPrepare; onContinueLoadingRequestedRunnable = diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/chunk/MediaParserChunkExtractor.java b/library/core/src/main/java/com/google/android/exoplayer2/source/chunk/MediaParserChunkExtractor.java new file mode 100644 index 0000000000..7c440b46d7 --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/source/chunk/MediaParserChunkExtractor.java @@ -0,0 +1,169 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.source.chunk; + +import static com.google.android.exoplayer2.source.mediaparser.MediaParserUtil.PARAMETER_EAGERLY_EXPOSE_TRACK_TYPE; +import static com.google.android.exoplayer2.source.mediaparser.MediaParserUtil.PARAMETER_EXPOSE_CAPTION_FORMATS; +import static com.google.android.exoplayer2.source.mediaparser.MediaParserUtil.PARAMETER_EXPOSE_CHUNK_INDEX_AS_MEDIA_FORMAT; +import static com.google.android.exoplayer2.source.mediaparser.MediaParserUtil.PARAMETER_EXPOSE_DUMMY_SEEK_MAP; +import static com.google.android.exoplayer2.source.mediaparser.MediaParserUtil.PARAMETER_INCLUDE_SUPPLEMENTAL_DATA; +import static com.google.android.exoplayer2.source.mediaparser.MediaParserUtil.PARAMETER_IN_BAND_CRYPTO_INFO; +import static com.google.android.exoplayer2.source.mediaparser.MediaParserUtil.PARAMETER_OVERRIDE_IN_BAND_CAPTION_DECLARATIONS; + +import android.annotation.SuppressLint; +import android.media.MediaFormat; +import android.media.MediaParser; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.extractor.ChunkIndex; +import com.google.android.exoplayer2.extractor.DummyTrackOutput; +import com.google.android.exoplayer2.extractor.ExtractorInput; +import com.google.android.exoplayer2.extractor.ExtractorOutput; +import com.google.android.exoplayer2.extractor.SeekMap; +import com.google.android.exoplayer2.extractor.TrackOutput; +import com.google.android.exoplayer2.source.mediaparser.InputReaderAdapterV30; +import com.google.android.exoplayer2.source.mediaparser.MediaParserUtil; +import com.google.android.exoplayer2.source.mediaparser.OutputConsumerAdapterV30; +import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.MimeTypes; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** {@link ChunkExtractor} implemented on top of the platform's {@link MediaParser}. */ +@RequiresApi(30) +public final class MediaParserChunkExtractor implements ChunkExtractor { + + private final OutputConsumerAdapterV30 outputConsumerAdapter; + private final InputReaderAdapterV30 inputReaderAdapter; + private final MediaParser mediaParser; + private final TrackOutputProviderAdapter trackOutputProviderAdapter; + private final DummyTrackOutput dummyTrackOutput; + private long pendingSeekUs; + @Nullable private TrackOutputProvider trackOutputProvider; + @Nullable private Format[] sampleFormats; + + /** + * Creates a new instance. + * + * @param primaryTrackType The type of the primary track, or {@link C#TRACK_TYPE_NONE} if there is + * no primary track. Must be one of the {@link C C.TRACK_TYPE_*} constants. + * @param manifestFormat The chunks {@link Format} as obtained from the manifest. + * @param closedCaptionFormats A list containing the {@link Format Formats} of the closed-caption + * tracks in the chunks. + */ + @SuppressLint("WrongConstant") + public MediaParserChunkExtractor( + int primaryTrackType, Format manifestFormat, List closedCaptionFormats) { + outputConsumerAdapter = + new OutputConsumerAdapterV30( + manifestFormat, primaryTrackType, /* expectDummySeekMap= */ true); + inputReaderAdapter = new InputReaderAdapterV30(); + String mimeType = Assertions.checkNotNull(manifestFormat.containerMimeType); + String parserName = + MimeTypes.isMatroska(mimeType) + ? MediaParser.PARSER_NAME_MATROSKA + : MediaParser.PARSER_NAME_FMP4; + outputConsumerAdapter.setSelectedParserName(parserName); + mediaParser = MediaParser.createByName(parserName, outputConsumerAdapter); + mediaParser.setParameter(MediaParser.PARAMETER_MATROSKA_DISABLE_CUES_SEEKING, true); + mediaParser.setParameter(PARAMETER_IN_BAND_CRYPTO_INFO, true); + mediaParser.setParameter(PARAMETER_INCLUDE_SUPPLEMENTAL_DATA, true); + mediaParser.setParameter(PARAMETER_EAGERLY_EXPOSE_TRACK_TYPE, true); + mediaParser.setParameter(PARAMETER_EXPOSE_DUMMY_SEEK_MAP, true); + mediaParser.setParameter(PARAMETER_EXPOSE_CHUNK_INDEX_AS_MEDIA_FORMAT, true); + mediaParser.setParameter(PARAMETER_OVERRIDE_IN_BAND_CAPTION_DECLARATIONS, true); + ArrayList closedCaptionMediaFormats = new ArrayList<>(); + for (int i = 0; i < closedCaptionFormats.size(); i++) { + closedCaptionMediaFormats.add( + MediaParserUtil.toCaptionsMediaFormat(closedCaptionFormats.get(i))); + } + mediaParser.setParameter(PARAMETER_EXPOSE_CAPTION_FORMATS, closedCaptionMediaFormats); + outputConsumerAdapter.setMuxedCaptionFormats(closedCaptionFormats); + trackOutputProviderAdapter = new TrackOutputProviderAdapter(); + dummyTrackOutput = new DummyTrackOutput(); + pendingSeekUs = C.TIME_UNSET; + } + + // ChunkExtractor implementation. + + @Override + public void init( + @Nullable TrackOutputProvider trackOutputProvider, long startTimeUs, long endTimeUs) { + this.trackOutputProvider = trackOutputProvider; + outputConsumerAdapter.setSampleTimestampUpperLimitFilterUs(endTimeUs); + outputConsumerAdapter.setExtractorOutput(trackOutputProviderAdapter); + pendingSeekUs = startTimeUs; + } + + @Override + public void release() { + mediaParser.release(); + } + + @Override + public boolean read(ExtractorInput extractorInput) throws IOException { + maybeExecutePendingSeek(); + inputReaderAdapter.setDataReader(extractorInput, extractorInput.getLength()); + return mediaParser.advance(inputReaderAdapter); + } + + @Nullable + @Override + public ChunkIndex getChunkIndex() { + return outputConsumerAdapter.getChunkIndex(); + } + + @Nullable + @Override + public Format[] getSampleFormats() { + return sampleFormats; + } + + // Internal methods. + + private void maybeExecutePendingSeek() { + @Nullable MediaParser.SeekMap dummySeekMap = outputConsumerAdapter.getDummySeekMap(); + if (pendingSeekUs != C.TIME_UNSET && dummySeekMap != null) { + mediaParser.seek(dummySeekMap.getSeekPoints(pendingSeekUs).first); + pendingSeekUs = C.TIME_UNSET; + } + } + + // Internal classes. + + private class TrackOutputProviderAdapter implements ExtractorOutput { + + @Override + public TrackOutput track(int id, int type) { + return trackOutputProvider != null ? trackOutputProvider.track(id, type) : dummyTrackOutput; + } + + @Override + public void endTracks() { + // Imitate BundledChunkExtractor behavior, which captures a sample format snapshot when + // endTracks is called. + sampleFormats = outputConsumerAdapter.getSampleFormats(); + } + + @Override + public void seekMap(SeekMap seekMap) { + // Do nothing. + } + } +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/InputReaderAdapterV30.java b/library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/InputReaderAdapterV30.java new file mode 100644 index 0000000000..3a55645e96 --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/InputReaderAdapterV30.java @@ -0,0 +1,87 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.source.mediaparser; + +import android.annotation.SuppressLint; +import android.media.MediaParser; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.upstream.DataReader; +import com.google.android.exoplayer2.util.Util; +import java.io.IOException; + +/** {@link MediaParser.SeekableInputReader} implementation wrapping a {@link DataReader}. */ +@RequiresApi(30) +@SuppressLint("Override") // TODO: Remove once the SDK becomes stable. +public final class InputReaderAdapterV30 implements MediaParser.SeekableInputReader { + + @Nullable private DataReader dataReader; + private long resourceLength; + private long currentPosition; + private long lastSeekPosition; + + /** + * Sets the wrapped {@link DataReader}. + * + * @param dataReader The {@link DataReader} to wrap. + * @param length The length of the resource from which {@code dataReader} reads. + */ + public void setDataReader(DataReader dataReader, long length) { + this.dataReader = dataReader; + resourceLength = length; + lastSeekPosition = C.POSITION_UNSET; + } + + /** Sets the absolute position in the resource from which the wrapped {@link DataReader} reads. */ + public void setCurrentPosition(long position) { + currentPosition = position; + } + + /** + * Returns the last value passed to {@link #seekToPosition(long)} and sets the stored value to + * {@link C#POSITION_UNSET}. + */ + public long getAndResetSeekPosition() { + long lastSeekPosition = this.lastSeekPosition; + this.lastSeekPosition = C.POSITION_UNSET; + return lastSeekPosition; + } + + // SeekableInputReader implementation. + + @Override + public void seekToPosition(long position) { + lastSeekPosition = position; + } + + @Override + public int read(byte[] bytes, int offset, int readLength) throws IOException { + int bytesRead = Util.castNonNull(dataReader).read(bytes, offset, readLength); + currentPosition += bytesRead; + return bytesRead; + } + + @Override + public long getPosition() { + return currentPosition; + } + + @Override + public long getLength() { + return resourceLength; + } +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/MediaParserUtil.java b/library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/MediaParserUtil.java new file mode 100644 index 0000000000..db37535a15 --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/MediaParserUtil.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.source.mediaparser; + +import android.media.MediaFormat; +import android.media.MediaParser; +import com.google.android.exoplayer2.Format; + +/** + * Miscellaneous constants and utility methods related to the {@link MediaParser} integration. + * + *

For documentation on constants, please see the {@link MediaParser} documentation. + */ +public final class MediaParserUtil { + + public static final String PARAMETER_IN_BAND_CRYPTO_INFO = + "android.media.mediaparser.inBandCryptoInfo"; + public static final String PARAMETER_INCLUDE_SUPPLEMENTAL_DATA = + "android.media.mediaparser.includeSupplementalData"; + public static final String PARAMETER_EAGERLY_EXPOSE_TRACK_TYPE = + "android.media.mediaparser.eagerlyExposeTrackType"; + public static final String PARAMETER_EXPOSE_DUMMY_SEEK_MAP = + "android.media.mediaparser.exposeDummySeekMap"; + public static final String PARAMETER_EXPOSE_CHUNK_INDEX_AS_MEDIA_FORMAT = + "android.media.mediaParser.exposeChunkIndexAsMediaFormat"; + public static final String PARAMETER_OVERRIDE_IN_BAND_CAPTION_DECLARATIONS = + "android.media.mediaParser.overrideInBandCaptionDeclarations"; + public static final String PARAMETER_EXPOSE_CAPTION_FORMATS = + "android.media.mediaParser.exposeCaptionFormats"; + public static final String PARAMETER_IGNORE_TIMESTAMP_OFFSET = + "android.media.mediaparser.ignoreTimestampOffset"; + + private MediaParserUtil() {} + + /** + * Returns a {@link MediaFormat} with equivalent {@link MediaFormat#KEY_MIME} and {@link + * MediaFormat#KEY_CAPTION_SERVICE_NUMBER} to the given {@link Format}. + */ + public static MediaFormat toCaptionsMediaFormat(Format format) { + MediaFormat mediaFormat = new MediaFormat(); + mediaFormat.setString(MediaFormat.KEY_MIME, format.sampleMimeType); + if (format.accessibilityChannel != Format.NO_VALUE) { + mediaFormat.setInteger(MediaFormat.KEY_CAPTION_SERVICE_NUMBER, format.accessibilityChannel); + } + return mediaFormat; + } +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/OutputConsumerAdapterV30.java b/library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/OutputConsumerAdapterV30.java new file mode 100644 index 0000000000..f3bed012ec --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/OutputConsumerAdapterV30.java @@ -0,0 +1,691 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.source.mediaparser; + +import static android.media.MediaParser.PARSER_NAME_AC3; +import static android.media.MediaParser.PARSER_NAME_AC4; +import static android.media.MediaParser.PARSER_NAME_ADTS; +import static android.media.MediaParser.PARSER_NAME_AMR; +import static android.media.MediaParser.PARSER_NAME_FLAC; +import static android.media.MediaParser.PARSER_NAME_FLV; +import static android.media.MediaParser.PARSER_NAME_FMP4; +import static android.media.MediaParser.PARSER_NAME_MATROSKA; +import static android.media.MediaParser.PARSER_NAME_MP3; +import static android.media.MediaParser.PARSER_NAME_MP4; +import static android.media.MediaParser.PARSER_NAME_OGG; +import static android.media.MediaParser.PARSER_NAME_PS; +import static android.media.MediaParser.PARSER_NAME_TS; +import static android.media.MediaParser.PARSER_NAME_WAV; + +import android.annotation.SuppressLint; +import android.media.DrmInitData.SchemeInitData; +import android.media.MediaCodec; +import android.media.MediaCodec.CryptoInfo; +import android.media.MediaFormat; +import android.media.MediaParser; +import android.media.MediaParser.TrackData; +import android.util.Pair; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.C.SelectionFlags; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.drm.DrmInitData; +import com.google.android.exoplayer2.drm.DrmInitData.SchemeData; +import com.google.android.exoplayer2.extractor.ChunkIndex; +import com.google.android.exoplayer2.extractor.DummyExtractorOutput; +import com.google.android.exoplayer2.extractor.ExtractorOutput; +import com.google.android.exoplayer2.extractor.SeekMap; +import com.google.android.exoplayer2.extractor.SeekPoint; +import com.google.android.exoplayer2.extractor.TrackOutput; +import com.google.android.exoplayer2.extractor.TrackOutput.CryptoData; +import com.google.android.exoplayer2.upstream.DataReader; +import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.Log; +import com.google.android.exoplayer2.util.MimeTypes; +import com.google.android.exoplayer2.util.TimestampAdjuster; +import com.google.android.exoplayer2.util.Util; +import com.google.android.exoplayer2.video.ColorInfo; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; +import java.nio.LongBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.checkerframework.checker.nullness.compatqual.NullableType; + +/** + * {@link MediaParser.OutputConsumer} implementation that redirects output to an {@link + * ExtractorOutput}. + */ +@RequiresApi(30) +@SuppressLint("Override") // TODO: Remove once the SDK becomes stable. +public final class OutputConsumerAdapterV30 implements MediaParser.OutputConsumer { + + private static final String TAG = "OutputConsumerAdapterV30"; + + private static final Pair SEEK_POINT_PAIR_START = + Pair.create(MediaParser.SeekPoint.START, MediaParser.SeekPoint.START); + private static final String MEDIA_FORMAT_KEY_TRACK_TYPE = "track-type-string"; + private static final String MEDIA_FORMAT_KEY_CHUNK_INDEX_SIZES = "chunk-index-int-sizes"; + private static final String MEDIA_FORMAT_KEY_CHUNK_INDEX_OFFSETS = "chunk-index-long-offsets"; + private static final String MEDIA_FORMAT_KEY_CHUNK_INDEX_DURATIONS = + "chunk-index-long-us-durations"; + private static final String MEDIA_FORMAT_KEY_CHUNK_INDEX_TIMES = "chunk-index-long-us-times"; + private static final Pattern REGEX_CRYPTO_INFO_PATTERN = + Pattern.compile("pattern \\(encrypt: (\\d+), skip: (\\d+)\\)"); + + private final ArrayList<@NullableType TrackOutput> trackOutputs; + private final ArrayList<@NullableType Format> trackFormats; + private final ArrayList<@NullableType CryptoInfo> lastReceivedCryptoInfos; + private final ArrayList<@NullableType CryptoData> lastOutputCryptoDatas; + private final DataReaderAdapter scratchDataReaderAdapter; + private final boolean expectDummySeekMap; + private final int primaryTrackType; + @Nullable private final Format primaryTrackManifestFormat; + + private ExtractorOutput extractorOutput; + @Nullable private MediaParser.SeekMap dummySeekMap; + @Nullable private MediaParser.SeekMap lastSeekMap; + @Nullable private String containerMimeType; + @Nullable private ChunkIndex lastChunkIndex; + @Nullable private TimestampAdjuster timestampAdjuster; + private List muxedCaptionFormats; + private int primaryTrackIndex; + private long sampleTimestampUpperLimitFilterUs; + private boolean tracksFoundCalled; + private boolean tracksEnded; + private boolean seekingDisabled; + + /** + * Equivalent to {@link #OutputConsumerAdapterV30(Format, int, boolean) + * OutputConsumerAdapterV30(primaryTrackManifestFormat= null, primaryTrackType= C.TRACK_TYPE_NONE, + * expectDummySeekMap= false)} + */ + public OutputConsumerAdapterV30() { + this( + /* primaryTrackManifestFormat= */ null, + /* primaryTrackType= */ C.TRACK_TYPE_NONE, + /* expectDummySeekMap= */ false); + } + + /** + * Creates a new instance. + * + * @param primaryTrackManifestFormat The manifest-obtained format of the primary track, or null if + * not applicable. + * @param primaryTrackType The type of the primary track, or {@link C#TRACK_TYPE_NONE} if there is + * no primary track. Must be one of the {@link C C.TRACK_TYPE_*} constants. + * @param expectDummySeekMap Whether the output consumer should expect an initial dummy seek map + * which should be exposed through {@link #getDummySeekMap()}. + */ + public OutputConsumerAdapterV30( + @Nullable Format primaryTrackManifestFormat, + int primaryTrackType, + boolean expectDummySeekMap) { + this.expectDummySeekMap = expectDummySeekMap; + this.primaryTrackManifestFormat = primaryTrackManifestFormat; + this.primaryTrackType = primaryTrackType; + trackOutputs = new ArrayList<>(); + trackFormats = new ArrayList<>(); + lastReceivedCryptoInfos = new ArrayList<>(); + lastOutputCryptoDatas = new ArrayList<>(); + scratchDataReaderAdapter = new DataReaderAdapter(); + extractorOutput = new DummyExtractorOutput(); + sampleTimestampUpperLimitFilterUs = C.TIME_UNSET; + muxedCaptionFormats = ImmutableList.of(); + } + + /** + * Sets an upper limit for sample timestamp filtering. + * + *

When set, samples with timestamps greater than {@code sampleTimestampUpperLimitFilterUs} + * will be discarded. + * + * @param sampleTimestampUpperLimitFilterUs The maximum allowed sample timestamp, or {@link + * C#TIME_UNSET} to remove filtering. + */ + public void setSampleTimestampUpperLimitFilterUs(long sampleTimestampUpperLimitFilterUs) { + this.sampleTimestampUpperLimitFilterUs = sampleTimestampUpperLimitFilterUs; + } + + /** Sets a {@link TimestampAdjuster} for adjusting the timestamps of the output samples. */ + public void setTimestampAdjuster(TimestampAdjuster timestampAdjuster) { + this.timestampAdjuster = timestampAdjuster; + } + + /** + * Sets the {@link ExtractorOutput} to which {@link MediaParser MediaParser's} output is directed. + */ + public void setExtractorOutput(ExtractorOutput extractorOutput) { + this.extractorOutput = extractorOutput; + } + + /** Sets {@link Format} information associated to the caption tracks multiplexed in the media. */ + public void setMuxedCaptionFormats(List muxedCaptionFormats) { + this.muxedCaptionFormats = muxedCaptionFormats; + } + + /** Overrides future received {@link SeekMap SeekMaps} with non-seekable instances. */ + public void disableSeeking() { + seekingDisabled = true; + } + + /** + * Returns a dummy {@link MediaParser.SeekMap}, or null if not available. + * + *

the dummy {@link MediaParser.SeekMap} returns a single {@link MediaParser.SeekPoint} whose + * {@link MediaParser.SeekPoint#timeMicros} matches the requested timestamp, and {@link + * MediaParser.SeekPoint#position} is 0. + */ + @Nullable + public MediaParser.SeekMap getDummySeekMap() { + return dummySeekMap; + } + + /** Returns the most recently output {@link ChunkIndex}, or null if none has been output. */ + @Nullable + public ChunkIndex getChunkIndex() { + return lastChunkIndex; + } + + /** + * Returns the {@link MediaParser.SeekPoint} instances corresponding to the given timestamp. + * + * @param seekTimeUs The timestamp in microseconds to retrieve {@link MediaParser.SeekPoint} + * instances for. + * @return The {@link MediaParser.SeekPoint} instances corresponding to the given timestamp. + */ + public Pair getSeekPoints(long seekTimeUs) { + return lastSeekMap != null ? lastSeekMap.getSeekPoints(seekTimeUs) : SEEK_POINT_PAIR_START; + } + + /** + * Defines the container mime type to propagate through {@link TrackOutput#format}. + * + * @param parserName The name of the selected parser. + */ + public void setSelectedParserName(String parserName) { + containerMimeType = getMimeType(parserName); + } + + /** + * Returns the last output format for each track, or null if not all the tracks have been + * identified. + */ + @Nullable + public Format[] getSampleFormats() { + if (!tracksFoundCalled) { + return null; + } + Format[] sampleFormats = new Format[trackFormats.size()]; + for (int i = 0; i < trackFormats.size(); i++) { + sampleFormats[i] = Assertions.checkNotNull(trackFormats.get(i)); + } + return sampleFormats; + } + + // MediaParser.OutputConsumer implementation. + + @Override + public void onTrackCountFound(int numberOfTracks) { + tracksFoundCalled = true; + maybeEndTracks(); + } + + @Override + public void onSeekMapFound(MediaParser.SeekMap seekMap) { + if (expectDummySeekMap && dummySeekMap == null) { + // This is a dummy seek map. + dummySeekMap = seekMap; + } else { + lastSeekMap = seekMap; + long durationUs = seekMap.getDurationMicros(); + extractorOutput.seekMap( + seekingDisabled + ? new SeekMap.Unseekable( + durationUs != MediaParser.SeekMap.UNKNOWN_DURATION ? durationUs : C.TIME_UNSET) + : new SeekMapAdapter(seekMap)); + } + } + + @Override + public void onTrackDataFound(int trackIndex, TrackData trackData) { + if (maybeObtainChunkIndex(trackData.mediaFormat)) { + // The MediaFormat contains a chunk index. It does not contain anything else. + return; + } + + ensureSpaceForTrackIndex(trackIndex); + @Nullable TrackOutput trackOutput = trackOutputs.get(trackIndex); + if (trackOutput == null) { + @Nullable + String trackTypeString = trackData.mediaFormat.getString(MEDIA_FORMAT_KEY_TRACK_TYPE); + int trackType = + toTrackTypeConstant( + trackTypeString != null + ? trackTypeString + : trackData.mediaFormat.getString(MediaFormat.KEY_MIME)); + if (trackType == primaryTrackType) { + primaryTrackIndex = trackIndex; + } + trackOutput = extractorOutput.track(trackIndex, trackType); + trackOutputs.set(trackIndex, trackOutput); + if (trackTypeString != null) { + // The MediaFormat includes the track type string, so it cannot include any other keys, as + // per the android.media.mediaparser.eagerlyExposeTrackType parameter documentation. + return; + } + } + Format format = toExoPlayerFormat(trackData); + trackOutput.format( + primaryTrackManifestFormat != null && trackIndex == primaryTrackIndex + ? format.withManifestFormatInfo(primaryTrackManifestFormat) + : format); + trackFormats.set(trackIndex, format); + maybeEndTracks(); + } + + @Override + public void onSampleDataFound(int trackIndex, MediaParser.InputReader sampleData) + throws IOException { + ensureSpaceForTrackIndex(trackIndex); + scratchDataReaderAdapter.input = sampleData; + TrackOutput trackOutput = trackOutputs.get(trackIndex); + if (trackOutput == null) { + trackOutput = extractorOutput.track(trackIndex, C.TRACK_TYPE_UNKNOWN); + trackOutputs.set(trackIndex, trackOutput); + } + trackOutput.sampleData( + scratchDataReaderAdapter, (int) sampleData.getLength(), /* allowEndOfInput= */ true); + } + + @Override + public void onSampleCompleted( + int trackIndex, + long timeUs, + int flags, + int size, + int offset, + @Nullable MediaCodec.CryptoInfo cryptoInfo) { + if (sampleTimestampUpperLimitFilterUs != C.TIME_UNSET + && timeUs >= sampleTimestampUpperLimitFilterUs) { + // Ignore this sample. + return; + } else if (timestampAdjuster != null) { + timeUs = timestampAdjuster.adjustSampleTimestamp(timeUs); + } + Assertions.checkNotNull(trackOutputs.get(trackIndex)) + .sampleMetadata(timeUs, flags, size, offset, toExoPlayerCryptoData(trackIndex, cryptoInfo)); + } + + // Private methods. + + private boolean maybeObtainChunkIndex(MediaFormat mediaFormat) { + @Nullable + ByteBuffer chunkIndexSizesByteBuffer = + mediaFormat.getByteBuffer(MEDIA_FORMAT_KEY_CHUNK_INDEX_SIZES); + if (chunkIndexSizesByteBuffer == null) { + return false; + } + IntBuffer chunkIndexSizes = chunkIndexSizesByteBuffer.asIntBuffer(); + LongBuffer chunkIndexOffsets = + Assertions.checkNotNull(mediaFormat.getByteBuffer(MEDIA_FORMAT_KEY_CHUNK_INDEX_OFFSETS)) + .asLongBuffer(); + LongBuffer chunkIndexDurationsUs = + Assertions.checkNotNull(mediaFormat.getByteBuffer(MEDIA_FORMAT_KEY_CHUNK_INDEX_DURATIONS)) + .asLongBuffer(); + LongBuffer chunkIndexTimesUs = + Assertions.checkNotNull(mediaFormat.getByteBuffer(MEDIA_FORMAT_KEY_CHUNK_INDEX_TIMES)) + .asLongBuffer(); + int[] sizes = new int[chunkIndexSizes.remaining()]; + long[] offsets = new long[chunkIndexOffsets.remaining()]; + long[] durationsUs = new long[chunkIndexDurationsUs.remaining()]; + long[] timesUs = new long[chunkIndexTimesUs.remaining()]; + chunkIndexSizes.get(sizes); + chunkIndexOffsets.get(offsets); + chunkIndexDurationsUs.get(durationsUs); + chunkIndexTimesUs.get(timesUs); + lastChunkIndex = new ChunkIndex(sizes, offsets, durationsUs, timesUs); + extractorOutput.seekMap(lastChunkIndex); + return true; + } + + private void ensureSpaceForTrackIndex(int trackIndex) { + for (int i = trackOutputs.size(); i <= trackIndex; i++) { + trackOutputs.add(null); + trackFormats.add(null); + lastReceivedCryptoInfos.add(null); + lastOutputCryptoDatas.add(null); + } + } + + @Nullable + private CryptoData toExoPlayerCryptoData(int trackIndex, @Nullable CryptoInfo cryptoInfo) { + if (cryptoInfo == null) { + return null; + } + + @Nullable CryptoInfo lastReceivedCryptoInfo = lastReceivedCryptoInfos.get(trackIndex); + CryptoData cryptoDataToOutput; + // MediaParser keeps identity and value equality aligned for efficient comparison. + if (lastReceivedCryptoInfo == cryptoInfo) { + // They match, we can reuse the last one we created. + cryptoDataToOutput = Assertions.checkNotNull(lastOutputCryptoDatas.get(trackIndex)); + } else { + // They don't match, we create a new CryptoData. + + // TODO: Access pattern encryption info directly once the Android SDK makes it visible. + // See [Internal ref: b/154248283]. + int encryptedBlocks; + int clearBlocks; + try { + Matcher matcher = REGEX_CRYPTO_INFO_PATTERN.matcher(cryptoInfo.toString()); + matcher.find(); + encryptedBlocks = Integer.parseInt(Util.castNonNull(matcher.group(1))); + clearBlocks = Integer.parseInt(Util.castNonNull(matcher.group(2))); + } catch (RuntimeException e) { + // Should never happen. + Log.e(TAG, "Unexpected error while parsing CryptoInfo: " + cryptoInfo, e); + // Assume no-pattern encryption. + encryptedBlocks = 0; + clearBlocks = 0; + } + cryptoDataToOutput = + new CryptoData(cryptoInfo.mode, cryptoInfo.key, encryptedBlocks, clearBlocks); + lastReceivedCryptoInfos.set(trackIndex, cryptoInfo); + lastOutputCryptoDatas.set(trackIndex, cryptoDataToOutput); + } + return cryptoDataToOutput; + } + + private void maybeEndTracks() { + if (!tracksFoundCalled || tracksEnded) { + return; + } + int size = trackOutputs.size(); + for (int i = 0; i < size; i++) { + if (trackOutputs.get(i) == null) { + return; + } + } + extractorOutput.endTracks(); + tracksEnded = true; + } + + private static int toTrackTypeConstant(@Nullable String string) { + if (string == null) { + return C.TRACK_TYPE_UNKNOWN; + } + switch (string) { + case "audio": + return C.TRACK_TYPE_AUDIO; + case "video": + return C.TRACK_TYPE_VIDEO; + case "text": + return C.TRACK_TYPE_TEXT; + case "metadata": + return C.TRACK_TYPE_METADATA; + case "unknown": + return C.TRACK_TYPE_UNKNOWN; + default: + // Must be a MIME type. + return MimeTypes.getTrackType(string); + } + } + + private Format toExoPlayerFormat(TrackData trackData) { + // TODO: Consider adding support for the following: + // format.id + // format.stereoMode + // format.projectionData + MediaFormat mediaFormat = trackData.mediaFormat; + @Nullable String mediaFormatMimeType = mediaFormat.getString(MediaFormat.KEY_MIME); + int mediaFormatAccessibilityChannel = + mediaFormat.getInteger( + MediaFormat.KEY_CAPTION_SERVICE_NUMBER, /* defaultValue= */ Format.NO_VALUE); + Format.Builder formatBuilder = + new Format.Builder() + .setDrmInitData( + toExoPlayerDrmInitData( + mediaFormat.getString("crypto-mode-fourcc"), trackData.drmInitData)) + .setContainerMimeType(containerMimeType) + .setPeakBitrate( + mediaFormat.getInteger( + MediaFormat.KEY_BIT_RATE, /* defaultValue= */ Format.NO_VALUE)) + .setChannelCount( + mediaFormat.getInteger( + MediaFormat.KEY_CHANNEL_COUNT, /* defaultValue= */ Format.NO_VALUE)) + .setColorInfo(getColorInfo(mediaFormat)) + .setSampleMimeType(mediaFormatMimeType) + .setCodecs(mediaFormat.getString(MediaFormat.KEY_CODECS_STRING)) + .setFrameRate( + mediaFormat.getFloat( + MediaFormat.KEY_FRAME_RATE, /* defaultValue= */ Format.NO_VALUE)) + .setWidth( + mediaFormat.getInteger(MediaFormat.KEY_WIDTH, /* defaultValue= */ Format.NO_VALUE)) + .setHeight( + mediaFormat.getInteger(MediaFormat.KEY_HEIGHT, /* defaultValue= */ Format.NO_VALUE)) + .setInitializationData(getInitializationData(mediaFormat)) + .setLanguage(mediaFormat.getString(MediaFormat.KEY_LANGUAGE)) + .setMaxInputSize( + mediaFormat.getInteger( + MediaFormat.KEY_MAX_INPUT_SIZE, /* defaultValue= */ Format.NO_VALUE)) + .setPcmEncoding( + mediaFormat.getInteger("exo-pcm-encoding", /* defaultValue= */ Format.NO_VALUE)) + .setRotationDegrees( + mediaFormat.getInteger(MediaFormat.KEY_ROTATION, /* defaultValue= */ 0)) + .setSampleRate( + mediaFormat.getInteger( + MediaFormat.KEY_SAMPLE_RATE, /* defaultValue= */ Format.NO_VALUE)) + .setSelectionFlags(getSelectionFlags(mediaFormat)) + .setEncoderDelay( + mediaFormat.getInteger(MediaFormat.KEY_ENCODER_DELAY, /* defaultValue= */ 0)) + .setEncoderPadding( + mediaFormat.getInteger(MediaFormat.KEY_ENCODER_PADDING, /* defaultValue= */ 0)) + .setPixelWidthHeightRatio( + mediaFormat.getFloat("pixel-width-height-ratio-float", /* defaultValue= */ 1f)) + .setSubsampleOffsetUs( + mediaFormat.getLong( + "subsample-offset-us-long", /* defaultValue= */ Format.OFFSET_SAMPLE_RELATIVE)) + .setAccessibilityChannel(mediaFormatAccessibilityChannel); + for (int i = 0; i < muxedCaptionFormats.size(); i++) { + Format muxedCaptionFormat = muxedCaptionFormats.get(i); + if (Util.areEqual(muxedCaptionFormat.sampleMimeType, mediaFormatMimeType) + && muxedCaptionFormat.accessibilityChannel == mediaFormatAccessibilityChannel) { + // The track's format matches this muxedCaptionFormat, so we apply the manifest format + // information to the track. + formatBuilder + .setLanguage(muxedCaptionFormat.language) + .setRoleFlags(muxedCaptionFormat.roleFlags) + .setSelectionFlags(muxedCaptionFormat.selectionFlags) + .setLabel(muxedCaptionFormat.label) + .setMetadata(muxedCaptionFormat.metadata); + break; + } + } + return formatBuilder.build(); + } + + @Nullable + private static DrmInitData toExoPlayerDrmInitData( + @Nullable String schemeType, @Nullable android.media.DrmInitData drmInitData) { + if (drmInitData == null) { + return null; + } + SchemeData[] schemeDatas = new SchemeData[drmInitData.getSchemeInitDataCount()]; + for (int i = 0; i < schemeDatas.length; i++) { + SchemeInitData schemeInitData = drmInitData.getSchemeInitDataAt(i); + schemeDatas[i] = + new SchemeData(schemeInitData.uuid, schemeInitData.mimeType, schemeInitData.data); + } + return new DrmInitData(schemeType, schemeDatas); + } + + @SelectionFlags + private static int getSelectionFlags(MediaFormat mediaFormat) { + int selectionFlags = 0; + selectionFlags |= + getFlag( + mediaFormat, + /* key= */ MediaFormat.KEY_IS_AUTOSELECT, + /* returnValueIfPresent= */ C.SELECTION_FLAG_AUTOSELECT); + selectionFlags |= + getFlag( + mediaFormat, + /* key= */ MediaFormat.KEY_IS_DEFAULT, + /* returnValueIfPresent= */ C.SELECTION_FLAG_DEFAULT); + selectionFlags |= + getFlag( + mediaFormat, + /* key= */ MediaFormat.KEY_IS_FORCED_SUBTITLE, + /* returnValueIfPresent= */ C.SELECTION_FLAG_FORCED); + return selectionFlags; + } + + private static int getFlag(MediaFormat mediaFormat, String key, int returnValueIfPresent) { + return mediaFormat.getInteger(key, /* defaultValue= */ 0) != 0 ? returnValueIfPresent : 0; + } + + private static List getInitializationData(MediaFormat mediaFormat) { + ArrayList initData = new ArrayList<>(); + int i = 0; + while (true) { + @Nullable ByteBuffer byteBuffer = mediaFormat.getByteBuffer("csd-" + i++); + if (byteBuffer == null) { + break; + } + initData.add(getArray(byteBuffer)); + } + return initData; + } + + @Nullable + private static ColorInfo getColorInfo(MediaFormat mediaFormat) { + @Nullable + ByteBuffer hdrStaticInfoByteBuffer = mediaFormat.getByteBuffer(MediaFormat.KEY_HDR_STATIC_INFO); + @Nullable + byte[] hdrStaticInfo = + hdrStaticInfoByteBuffer != null ? getArray(hdrStaticInfoByteBuffer) : null; + int colorTransfer = + mediaFormat.getInteger(MediaFormat.KEY_COLOR_TRANSFER, /* defaultValue= */ Format.NO_VALUE); + int colorRange = + mediaFormat.getInteger(MediaFormat.KEY_COLOR_RANGE, /* defaultValue= */ Format.NO_VALUE); + int colorStandard = + mediaFormat.getInteger(MediaFormat.KEY_COLOR_STANDARD, /* defaultValue= */ Format.NO_VALUE); + + if (hdrStaticInfo != null + || colorTransfer != Format.NO_VALUE + || colorRange != Format.NO_VALUE + || colorStandard != Format.NO_VALUE) { + return new ColorInfo(colorStandard, colorRange, colorTransfer, hdrStaticInfo); + } + return null; + } + + private static byte[] getArray(ByteBuffer byteBuffer) { + byte[] array = new byte[byteBuffer.remaining()]; + byteBuffer.get(array); + return array; + } + + private static String getMimeType(String parserName) { + switch (parserName) { + case PARSER_NAME_MATROSKA: + return MimeTypes.VIDEO_WEBM; + case PARSER_NAME_FMP4: + case PARSER_NAME_MP4: + return MimeTypes.VIDEO_MP4; + case PARSER_NAME_MP3: + return MimeTypes.AUDIO_MPEG; + case PARSER_NAME_ADTS: + return MimeTypes.AUDIO_AAC; + case PARSER_NAME_AC3: + return MimeTypes.AUDIO_AC3; + case PARSER_NAME_TS: + return MimeTypes.VIDEO_MP2T; + case PARSER_NAME_FLV: + return MimeTypes.VIDEO_FLV; + case PARSER_NAME_OGG: + return MimeTypes.AUDIO_OGG; + case PARSER_NAME_PS: + return MimeTypes.VIDEO_PS; + case PARSER_NAME_WAV: + return MimeTypes.AUDIO_RAW; + case PARSER_NAME_AMR: + return MimeTypes.AUDIO_AMR; + case PARSER_NAME_AC4: + return MimeTypes.AUDIO_AC4; + case PARSER_NAME_FLAC: + return MimeTypes.AUDIO_FLAC; + default: + throw new IllegalArgumentException("Illegal parser name: " + parserName); + } + } + + private static final class SeekMapAdapter implements SeekMap { + + private final MediaParser.SeekMap adaptedSeekMap; + + public SeekMapAdapter(MediaParser.SeekMap adaptedSeekMap) { + this.adaptedSeekMap = adaptedSeekMap; + } + + @Override + public boolean isSeekable() { + return adaptedSeekMap.isSeekable(); + } + + @Override + public long getDurationUs() { + long durationMicros = adaptedSeekMap.getDurationMicros(); + return durationMicros != MediaParser.SeekMap.UNKNOWN_DURATION ? durationMicros : C.TIME_UNSET; + } + + @Override + @SuppressWarnings("ReferenceEquality") + public SeekPoints getSeekPoints(long timeUs) { + Pair seekPoints = + adaptedSeekMap.getSeekPoints(timeUs); + SeekPoints exoPlayerSeekPoints; + if (seekPoints.first == seekPoints.second) { + exoPlayerSeekPoints = new SeekPoints(asExoPlayerSeekPoint(seekPoints.first)); + } else { + exoPlayerSeekPoints = + new SeekPoints( + asExoPlayerSeekPoint(seekPoints.first), asExoPlayerSeekPoint(seekPoints.second)); + } + return exoPlayerSeekPoints; + } + + private static SeekPoint asExoPlayerSeekPoint(MediaParser.SeekPoint seekPoint) { + return new SeekPoint(seekPoint.timeMicros, seekPoint.position); + } + } + + private static final class DataReaderAdapter implements DataReader { + + @Nullable public MediaParser.InputReader input; + + @Override + public int read(byte[] target, int offset, int length) throws IOException { + return Util.castNonNull(input).read(target, offset, length); + } + } +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/package-info.java b/library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/package-info.java new file mode 100644 index 0000000000..3eedf0c7a4 --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/source/mediaparser/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@NonNullApi +package com.google.android.exoplayer2.source.mediaparser; + +import com.google.android.exoplayer2.util.NonNullApi; diff --git a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java index 12a5bf8512..ad679ebe7f 100644 --- a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java +++ b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java @@ -804,7 +804,6 @@ public class DefaultDashChunkSource implements DashChunkSource { List closedCaptionFormats, @Nullable TrackOutput playerEmsgTrackOutput) { String containerMimeType = representation.format.containerMimeType; - Extractor extractor; if (MimeTypes.isText(containerMimeType)) { if (MimeTypes.APPLICATION_RAWCC.equals(containerMimeType)) { diff --git a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsMediaSource.java b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsMediaSource.java index 3a7a8de791..8d3b633dc7 100644 --- a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsMediaSource.java +++ b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/HlsMediaSource.java @@ -52,7 +52,6 @@ import com.google.android.exoplayer2.upstream.HttpDataSource; import com.google.android.exoplayer2.upstream.LoadErrorHandlingPolicy; import com.google.android.exoplayer2.upstream.TransferListener; import com.google.android.exoplayer2.util.MimeTypes; -import com.google.android.exoplayer2.util.Util; import java.io.IOException; import java.lang.annotation.Documented; import java.lang.annotation.Retention; @@ -546,5 +545,4 @@ public final class HlsMediaSource extends BaseMediaSource } refreshSourceInfo(timeline); } - } diff --git a/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/MediaParserHlsMediaChunkExtractor.java b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/MediaParserHlsMediaChunkExtractor.java new file mode 100644 index 0000000000..06de8544f2 --- /dev/null +++ b/library/hls/src/main/java/com/google/android/exoplayer2/source/hls/MediaParserHlsMediaChunkExtractor.java @@ -0,0 +1,281 @@ +/* + * Copyright 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.source.hls; + +import static android.media.MediaParser.PARAMETER_TS_IGNORE_AAC_STREAM; +import static android.media.MediaParser.PARAMETER_TS_IGNORE_AVC_STREAM; +import static android.media.MediaParser.PARAMETER_TS_IGNORE_SPLICE_INFO_STREAM; +import static android.media.MediaParser.PARAMETER_TS_MODE; +import static com.google.android.exoplayer2.source.mediaparser.MediaParserUtil.PARAMETER_EAGERLY_EXPOSE_TRACK_TYPE; +import static com.google.android.exoplayer2.source.mediaparser.MediaParserUtil.PARAMETER_EXPOSE_CAPTION_FORMATS; +import static com.google.android.exoplayer2.source.mediaparser.MediaParserUtil.PARAMETER_IGNORE_TIMESTAMP_OFFSET; +import static com.google.android.exoplayer2.source.mediaparser.MediaParserUtil.PARAMETER_IN_BAND_CRYPTO_INFO; +import static com.google.android.exoplayer2.source.mediaparser.MediaParserUtil.PARAMETER_OVERRIDE_IN_BAND_CAPTION_DECLARATIONS; + +import android.annotation.SuppressLint; +import android.media.MediaFormat; +import android.media.MediaParser; +import android.media.MediaParser.OutputConsumer; +import android.text.TextUtils; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.extractor.ExtractorInput; +import com.google.android.exoplayer2.extractor.ExtractorOutput; +import com.google.android.exoplayer2.source.mediaparser.InputReaderAdapterV30; +import com.google.android.exoplayer2.source.mediaparser.MediaParserUtil; +import com.google.android.exoplayer2.source.mediaparser.OutputConsumerAdapterV30; +import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.FileTypes; +import com.google.android.exoplayer2.util.MimeTypes; +import com.google.common.collect.ImmutableList; +import java.io.IOException; + +/** {@link HlsMediaChunkExtractor} implemented on top of the platform's {@link MediaParser}. */ +@RequiresApi(30) +public final class MediaParserHlsMediaChunkExtractor implements HlsMediaChunkExtractor { + + /** + * {@link HlsExtractorFactory} implementation that produces {@link + * MediaParserHlsMediaChunkExtractor} for all container formats except WebVTT, for which a {@link + * BundledHlsMediaChunkExtractor} is returned. + */ + public static final HlsExtractorFactory FACTORY = + (uri, + format, + muxedCaptionFormats, + timestampAdjuster, + responseHeaders, + sniffingExtractorInput) -> { + if (FileTypes.inferFileTypeFromMimeType(format.sampleMimeType) == FileTypes.WEBVTT) { + // The segment contains WebVTT. MediaParser does not support WebVTT parsing, so we use the + // bundled extractor. + return new BundledHlsMediaChunkExtractor( + new WebvttExtractor(format.language, timestampAdjuster), format, timestampAdjuster); + } + + boolean overrideInBandCaptionDeclarations = muxedCaptionFormats != null; + ImmutableList.Builder muxedCaptionMediaFormatsBuilder = + ImmutableList.builder(); + if (muxedCaptionFormats != null) { + // The manifest contains captions declarations. We use those to determine which captions + // will be exposed by MediaParser. + for (int i = 0; i < muxedCaptionFormats.size(); i++) { + muxedCaptionMediaFormatsBuilder.add( + MediaParserUtil.toCaptionsMediaFormat(muxedCaptionFormats.get(i))); + } + } else { + // The manifest does not declare any captions in the stream. Imitate the default HLS + // extractor factory and declare a 608 track by default. + muxedCaptionMediaFormatsBuilder.add( + MediaParserUtil.toCaptionsMediaFormat( + new Format.Builder().setSampleMimeType(MimeTypes.APPLICATION_CEA608).build())); + } + + ImmutableList muxedCaptionMediaFormats = + muxedCaptionMediaFormatsBuilder.build(); + + // TODO: Factor out code for optimizing the sniffing order across both factories. + OutputConsumerAdapterV30 outputConsumerAdapter = new OutputConsumerAdapterV30(); + outputConsumerAdapter.setMuxedCaptionFormats( + muxedCaptionFormats != null ? muxedCaptionFormats : ImmutableList.of()); + outputConsumerAdapter.setTimestampAdjuster(timestampAdjuster); + MediaParser mediaParser = + createMediaParserInstance( + outputConsumerAdapter, + format, + overrideInBandCaptionDeclarations, + muxedCaptionMediaFormats, + MediaParser.PARSER_NAME_FMP4, + MediaParser.PARSER_NAME_AC3, + MediaParser.PARSER_NAME_AC4, + MediaParser.PARSER_NAME_ADTS, + MediaParser.PARSER_NAME_MP3, + MediaParser.PARSER_NAME_TS); + + PeekingInputReader peekingInputReader = new PeekingInputReader(sniffingExtractorInput); + // The chunk extractor constructor requires an instance with a known parser name, so we + // advance once for MediaParser to sniff the content. + mediaParser.advance(peekingInputReader); + outputConsumerAdapter.setSelectedParserName(mediaParser.getParserName()); + + return new MediaParserHlsMediaChunkExtractor( + mediaParser, + outputConsumerAdapter, + format, + overrideInBandCaptionDeclarations, + muxedCaptionMediaFormats, + /* leadingBytesToSkip= */ peekingInputReader.totalPeekedBytes); + }; + + private final OutputConsumerAdapterV30 outputConsumerAdapter; + private final InputReaderAdapterV30 inputReaderAdapter; + private final MediaParser mediaParser; + private final Format format; + private final boolean overrideInBandCaptionDeclarations; + private final ImmutableList muxedCaptionMediaFormats; + private int pendingSkipBytes; + + /** + * Creates a new instance. + * + * @param mediaParser The {@link MediaParser} instance to use for extraction of segments. The + * provided instance must have completed sniffing, or must have been created by name. + * @param outputConsumerAdapter The {@link OutputConsumerAdapterV30} with which {@code + * mediaParser} was created. + * @param format The {@link Format} associated with the segment. + * @param overrideInBandCaptionDeclarations Whether to ignore any in-band caption track + * declarations in favor of using the {@code muxedCaptionMediaFormats} instead. If false, + * caption declarations found in the extracted media will be used, causing {@code + * muxedCaptionMediaFormats} to be ignored instead. + * @param muxedCaptionMediaFormats The list of in-band caption {@link MediaFormat MediaFormats} + * that {@link MediaParser} should expose. + * @param leadingBytesToSkip The number of bytes to skip from the start of the input before + * starting extraction. + */ + public MediaParserHlsMediaChunkExtractor( + MediaParser mediaParser, + OutputConsumerAdapterV30 outputConsumerAdapter, + Format format, + boolean overrideInBandCaptionDeclarations, + ImmutableList muxedCaptionMediaFormats, + int leadingBytesToSkip) { + this.mediaParser = mediaParser; + this.outputConsumerAdapter = outputConsumerAdapter; + this.overrideInBandCaptionDeclarations = overrideInBandCaptionDeclarations; + this.muxedCaptionMediaFormats = muxedCaptionMediaFormats; + this.format = format; + pendingSkipBytes = leadingBytesToSkip; + inputReaderAdapter = new InputReaderAdapterV30(); + } + + // ChunkExtractor implementation. + + @Override + public void init(ExtractorOutput extractorOutput) { + outputConsumerAdapter.setExtractorOutput(extractorOutput); + } + + @Override + public boolean read(ExtractorInput extractorInput) throws IOException { + extractorInput.skipFully(pendingSkipBytes); + pendingSkipBytes = 0; + inputReaderAdapter.setDataReader(extractorInput, extractorInput.getLength()); + return mediaParser.advance(inputReaderAdapter); + } + + @Override + public boolean isPackedAudioExtractor() { + String parserName = mediaParser.getParserName(); + return MediaParser.PARSER_NAME_AC3.equals(parserName) + || MediaParser.PARSER_NAME_AC4.equals(parserName) + || MediaParser.PARSER_NAME_ADTS.equals(parserName) + || MediaParser.PARSER_NAME_MP3.equals(parserName); + } + + @Override + public boolean isReusable() { + String parserName = mediaParser.getParserName(); + return MediaParser.PARSER_NAME_FMP4.equals(parserName) + || MediaParser.PARSER_NAME_TS.equals(parserName); + } + + @Override + public HlsMediaChunkExtractor recreate() { + Assertions.checkState(!isReusable()); + return new MediaParserHlsMediaChunkExtractor( + createMediaParserInstance( + outputConsumerAdapter, + format, + overrideInBandCaptionDeclarations, + muxedCaptionMediaFormats, + mediaParser.getParserName()), + outputConsumerAdapter, + format, + overrideInBandCaptionDeclarations, + muxedCaptionMediaFormats, + /* leadingBytesToSkip= */ 0); + } + + // Allow constants that are not part of the public MediaParser API. + @SuppressLint({"WrongConstant"}) + private static MediaParser createMediaParserInstance( + OutputConsumer outputConsumer, + Format format, + boolean overrideInBandCaptionDeclarations, + ImmutableList muxedCaptionMediaFormats, + String... parserNames) { + MediaParser mediaParser = + parserNames.length == 1 + ? MediaParser.createByName(parserNames[0], outputConsumer) + : MediaParser.create(outputConsumer, parserNames); + mediaParser.setParameter(PARAMETER_EXPOSE_CAPTION_FORMATS, muxedCaptionMediaFormats); + mediaParser.setParameter( + PARAMETER_OVERRIDE_IN_BAND_CAPTION_DECLARATIONS, overrideInBandCaptionDeclarations); + mediaParser.setParameter(PARAMETER_IN_BAND_CRYPTO_INFO, true); + mediaParser.setParameter(PARAMETER_EAGERLY_EXPOSE_TRACK_TYPE, true); + mediaParser.setParameter(PARAMETER_IGNORE_TIMESTAMP_OFFSET, true); + mediaParser.setParameter(PARAMETER_TS_IGNORE_SPLICE_INFO_STREAM, true); + mediaParser.setParameter(PARAMETER_TS_MODE, "hls"); + @Nullable String codecs = format.codecs; + if (!TextUtils.isEmpty(codecs)) { + // Sometimes AAC and H264 streams are declared in TS chunks even though they don't really + // exist. If we know from the codec attribute that they don't exist, then we can + // explicitly ignore them even if they're declared. + if (!MimeTypes.AUDIO_AAC.equals(MimeTypes.getAudioMediaMimeType(codecs))) { + mediaParser.setParameter(PARAMETER_TS_IGNORE_AAC_STREAM, true); + } + if (!MimeTypes.VIDEO_H264.equals(MimeTypes.getVideoMediaMimeType(codecs))) { + mediaParser.setParameter(PARAMETER_TS_IGNORE_AVC_STREAM, true); + } + } + return mediaParser; + } + + private static final class PeekingInputReader implements MediaParser.SeekableInputReader { + + private final ExtractorInput extractorInput; + private int totalPeekedBytes; + + private PeekingInputReader(ExtractorInput extractorInput) { + this.extractorInput = extractorInput; + } + + @Override + public int read(@NonNull byte[] buffer, int offset, int readLength) throws IOException { + int peekedBytes = extractorInput.peek(buffer, offset, readLength); + totalPeekedBytes += peekedBytes; + return peekedBytes; + } + + @Override + public long getPosition() { + return extractorInput.getPeekPosition(); + } + + @Override + public long getLength() { + return extractorInput.getLength(); + } + + @Override + public void seekToPosition(long position) { + // Seeking is not allowed when sniffing the content. + throw new UnsupportedOperationException(); + } + } +}