Enable ID3-in-EMSG for HLS streams

This supports both chunkless & traditional preparation

PiperOrigin-RevId: 273938344
This commit is contained in:
ibaker 2019-10-10 13:10:41 +01:00 committed by Oliver Woodman
parent 6934c6fb33
commit 3b9a9c9b7e
7 changed files with 285 additions and 34 deletions

View File

@ -7,6 +7,8 @@
([#6203](https://github.com/google/ExoPlayer/issues/6203)).
* DASH: Support `Label` elements
([#6297](https://github.com/google/ExoPlayer/issues/6297)).
* HLS: Add support for ID3 in EMSG when using FMP4 streams
([spec](https://aomediacodec.github.io/av1-id3/)).
* Metadata: Expose the raw ICY metadata through `IcyInfo`
([#6476](https://github.com/google/ExoPlayer/issues/6476)).
* UI

View File

@ -29,6 +29,7 @@ import com.google.android.exoplayer2.extractor.ts.Ac4Extractor;
import com.google.android.exoplayer2.extractor.ts.AdtsExtractor;
import com.google.android.exoplayer2.extractor.ts.DefaultTsPayloadReaderFactory;
import com.google.android.exoplayer2.extractor.ts.TsExtractor;
import com.google.android.exoplayer2.metadata.Metadata;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.TimestampAdjuster;
import java.io.EOFException;
@ -158,7 +159,7 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
if (!(extractorByFileExtension instanceof FragmentedMp4Extractor)) {
FragmentedMp4Extractor fragmentedMp4Extractor =
createFragmentedMp4Extractor(timestampAdjuster, drmInitData, muxedCaptionFormats);
createFragmentedMp4Extractor(timestampAdjuster, format, drmInitData, muxedCaptionFormats);
if (sniffQuietly(fragmentedMp4Extractor, extractorInput)) {
return buildResult(fragmentedMp4Extractor);
}
@ -208,7 +209,8 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
|| lastPathSegment.startsWith(M4_FILE_EXTENSION_PREFIX, lastPathSegment.length() - 4)
|| lastPathSegment.startsWith(MP4_FILE_EXTENSION_PREFIX, lastPathSegment.length() - 5)
|| lastPathSegment.startsWith(CMF_FILE_EXTENSION_PREFIX, lastPathSegment.length() - 5)) {
return createFragmentedMp4Extractor(timestampAdjuster, drmInitData, muxedCaptionFormats);
return createFragmentedMp4Extractor(
timestampAdjuster, format, drmInitData, muxedCaptionFormats);
} else {
// For any other file extension, we assume TS format.
return createTsExtractor(
@ -267,10 +269,21 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
private static FragmentedMp4Extractor createFragmentedMp4Extractor(
TimestampAdjuster timestampAdjuster,
Format format,
DrmInitData drmInitData,
@Nullable List<Format> muxedCaptionFormats) {
boolean isVariant = false;
for (int i = 0; i < format.metadata.length(); i++) {
Metadata.Entry entry = format.metadata.get(i);
if (entry instanceof HlsTrackMetadataEntry) {
isVariant = !((HlsTrackMetadataEntry) entry).variantInfos.isEmpty();
break;
}
}
// Only enable the EMSG TrackOutput if this is the 'variant' track (i.e. the main one) to avoid
// creating a separate EMSG track for every audio track in a video stream.
return new FragmentedMp4Extractor(
/* flags= */ 0,
/* flags= */ isVariant ? FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK : 0,
timestampAdjuster,
/* sideloadedTrack= */ null,
drmInitData,

View File

@ -71,6 +71,7 @@ public final class HlsMediaPeriod implements MediaPeriod, HlsSampleStreamWrapper
private final TimestampAdjusterProvider timestampAdjusterProvider;
private final CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
private final boolean allowChunklessPreparation;
private final @HlsMetadataType int metadataType;
private final boolean useSessionKeys;
private @Nullable Callback callback;
@ -110,6 +111,7 @@ public final class HlsMediaPeriod implements MediaPeriod, HlsSampleStreamWrapper
Allocator allocator,
CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory,
boolean allowChunklessPreparation,
@HlsMetadataType int metadataType,
boolean useSessionKeys) {
this.extractorFactory = extractorFactory;
this.playlistTracker = playlistTracker;
@ -120,6 +122,7 @@ public final class HlsMediaPeriod implements MediaPeriod, HlsSampleStreamWrapper
this.allocator = allocator;
this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
this.allowChunklessPreparation = allowChunklessPreparation;
this.metadataType = metadataType;
this.useSessionKeys = useSessionKeys;
compositeSequenceableLoader =
compositeSequenceableLoaderFactory.createCompositeSequenceableLoader();
@ -736,7 +739,8 @@ public final class HlsMediaPeriod implements MediaPeriod, HlsSampleStreamWrapper
positionUs,
muxedAudioFormat,
loadErrorHandlingPolicy,
eventDispatcher);
eventDispatcher,
metadataType);
}
private static Map<String, DrmInitData> deriveOverridingDrmInitData(

View File

@ -67,6 +67,7 @@ public final class HlsMediaSource extends BaseMediaSource
private CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
private LoadErrorHandlingPolicy loadErrorHandlingPolicy;
private boolean allowChunklessPreparation;
@HlsMetadataType private int metadataType;
private boolean useSessionKeys;
private boolean isCreateCalled;
@Nullable private Object tag;
@ -95,6 +96,7 @@ public final class HlsMediaSource extends BaseMediaSource
extractorFactory = HlsExtractorFactory.DEFAULT;
loadErrorHandlingPolicy = new DefaultLoadErrorHandlingPolicy();
compositeSequenceableLoaderFactory = new DefaultCompositeSequenceableLoaderFactory();
metadataType = HlsMetadataType.ID3;
}
/**
@ -237,6 +239,31 @@ public final class HlsMediaSource extends BaseMediaSource
return this;
}
/**
* Sets the type of metadata to extract from the HLS source (defaults to {@link
* HlsMetadataType#ID3}).
*
* <p>HLS supports in-band ID3 in both TS and fMP4 streams, but in the fMP4 case the data is
* wrapped in an EMSG box [<a href="https://aomediacodec.github.io/av1-id3/">spec</a>].
*
* <p>If this is set to {@link HlsMetadataType#ID3} then raw ID3 metadata of will be extracted
* from TS sources. From fMP4 streams EMSGs containing metadata of this type (in the variant
* stream only) will be unwrapped to expose the inner data. All other in-band metadata will be
* dropped.
*
* <p>If this is set to {@link HlsMetadataType#EMSG} then all EMSG data from the fMP4 variant
* stream will be extracted. No metadata will be extracted from TS streams, since they don't
* support EMSG.
*
* @param metadataType The type of metadata to extract.
* @return This factory, for convenience.
*/
public Factory setMetadataType(@HlsMetadataType int metadataType) {
Assertions.checkState(!isCreateCalled);
this.metadataType = metadataType;
return this;
}
/**
* Sets whether to use #EXT-X-SESSION-KEY tags provided in the master playlist. If enabled, it's
* assumed that any single session key declared in the master playlist can be used to obtain all
@ -272,6 +299,7 @@ public final class HlsMediaSource extends BaseMediaSource
playlistTrackerFactory.createTracker(
hlsDataSourceFactory, loadErrorHandlingPolicy, playlistParserFactory),
allowChunklessPreparation,
metadataType,
useSessionKeys,
tag);
}
@ -305,6 +333,7 @@ public final class HlsMediaSource extends BaseMediaSource
private final CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
private final LoadErrorHandlingPolicy loadErrorHandlingPolicy;
private final boolean allowChunklessPreparation;
private final @HlsMetadataType int metadataType;
private final boolean useSessionKeys;
private final HlsPlaylistTracker playlistTracker;
private final @Nullable Object tag;
@ -319,6 +348,7 @@ public final class HlsMediaSource extends BaseMediaSource
LoadErrorHandlingPolicy loadErrorHandlingPolicy,
HlsPlaylistTracker playlistTracker,
boolean allowChunklessPreparation,
@HlsMetadataType int metadataType,
boolean useSessionKeys,
@Nullable Object tag) {
this.manifestUri = manifestUri;
@ -328,6 +358,7 @@ public final class HlsMediaSource extends BaseMediaSource
this.loadErrorHandlingPolicy = loadErrorHandlingPolicy;
this.playlistTracker = playlistTracker;
this.allowChunklessPreparation = allowChunklessPreparation;
this.metadataType = metadataType;
this.useSessionKeys = useSessionKeys;
this.tag = tag;
}
@ -363,6 +394,7 @@ public final class HlsMediaSource extends BaseMediaSource
allocator,
compositeSequenceableLoaderFactory,
allowChunklessPreparation,
metadataType,
useSessionKeys);
}

View File

@ -0,0 +1,34 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.google.android.exoplayer2.source.hls;
import static java.lang.annotation.RetentionPolicy.SOURCE;
import androidx.annotation.IntDef;
import java.lang.annotation.Retention;
/**
* The types of metadata that can be extracted from HLS streams.
*
* <p>See {@link HlsMediaSource.Factory#setMetadataType(int)}.
*/
@Retention(SOURCE)
@IntDef({HlsMetadataType.ID3, HlsMetadataType.EMSG})
public @interface HlsMetadataType {
int ID3 = 1;
int EMSG = 3;
}

View File

@ -25,10 +25,13 @@ import com.google.android.exoplayer2.FormatHolder;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.drm.DrmInitData;
import com.google.android.exoplayer2.extractor.DummyTrackOutput;
import com.google.android.exoplayer2.extractor.ExtractorInput;
import com.google.android.exoplayer2.extractor.ExtractorOutput;
import com.google.android.exoplayer2.extractor.SeekMap;
import com.google.android.exoplayer2.extractor.TrackOutput;
import com.google.android.exoplayer2.metadata.Metadata;
import com.google.android.exoplayer2.metadata.emsg.EventMessage;
import com.google.android.exoplayer2.metadata.emsg.EventMessageDecoder;
import com.google.android.exoplayer2.metadata.id3.PrivFrame;
import com.google.android.exoplayer2.source.MediaSourceEventListener.EventDispatcher;
import com.google.android.exoplayer2.source.SampleQueue;
@ -47,7 +50,9 @@ import com.google.android.exoplayer2.upstream.Loader.LoadErrorAction;
import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.ParsableByteArray;
import com.google.android.exoplayer2.util.Util;
import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@ -56,6 +61,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* Loads {@link HlsMediaChunk}s obtained from a {@link HlsChunkSource}, and provides
@ -92,7 +98,8 @@ import java.util.Set;
private static final Set<Integer> MAPPABLE_TYPES =
Collections.unmodifiableSet(
new HashSet<>(Arrays.asList(C.TRACK_TYPE_AUDIO, C.TRACK_TYPE_VIDEO)));
new HashSet<>(
Arrays.asList(C.TRACK_TYPE_AUDIO, C.TRACK_TYPE_VIDEO, C.TRACK_TYPE_METADATA)));
private final int trackType;
private final Callback callback;
@ -102,6 +109,7 @@ import java.util.Set;
private final LoadErrorHandlingPolicy loadErrorHandlingPolicy;
private final Loader loader;
private final EventDispatcher eventDispatcher;
private final @HlsMetadataType int metadataType;
private final HlsChunkSource.HlsChunkHolder nextChunkHolder;
private final ArrayList<HlsMediaChunk> mediaChunks;
private final List<HlsMediaChunk> readOnlyMediaChunks;
@ -115,6 +123,7 @@ import java.util.Set;
private int[] sampleQueueTrackIds;
private Set<Integer> sampleQueueMappingDoneByType;
private SparseIntArray sampleQueueIndicesByType;
private TrackOutput emsgUnwrappingTrackOutput;
private int primarySampleQueueType;
private int primarySampleQueueIndex;
private boolean sampleQueuesBuilt;
@ -124,7 +133,7 @@ import java.util.Set;
private Format downstreamTrackFormat;
private boolean released;
// Tracks are complicated in HLS. See documentation of buildTracks for details.
// Tracks are complicated in HLS. See documentation of buildTracksFromSampleStreams for details.
// Indexed by track (as exposed by this source).
private TrackGroupArray trackGroups;
private TrackGroupArray optionalTrackGroups;
@ -169,7 +178,8 @@ import java.util.Set;
long positionUs,
Format muxedAudioFormat,
LoadErrorHandlingPolicy loadErrorHandlingPolicy,
EventDispatcher eventDispatcher) {
EventDispatcher eventDispatcher,
@HlsMetadataType int metadataType) {
this.trackType = trackType;
this.callback = callback;
this.chunkSource = chunkSource;
@ -178,6 +188,7 @@ import java.util.Set;
this.muxedAudioFormat = muxedAudioFormat;
this.loadErrorHandlingPolicy = loadErrorHandlingPolicy;
this.eventDispatcher = eventDispatcher;
this.metadataType = metadataType;
loader = new Loader("Loader:HlsSampleStreamWrapper");
nextChunkHolder = new HlsChunkSource.HlsChunkHolder();
sampleQueueTrackIds = new int[0];
@ -788,42 +799,34 @@ import java.util.Set;
@Override
public TrackOutput track(int id, int type) {
@Nullable TrackOutput trackOutput = null;
if (MAPPABLE_TYPES.contains(type)) {
// Track types in MAPPABLE_TYPES are handled manually to ignore IDs.
@Nullable TrackOutput mappedTrackOutput = getMappedTrackOutput(id, type);
if (mappedTrackOutput != null) {
return mappedTrackOutput;
}
} else /* sparse track */ {
trackOutput = getMappedTrackOutput(id, type);
} else /* non-mappable type track */ {
for (int i = 0; i < sampleQueues.length; i++) {
if (sampleQueueTrackIds[i] == id) {
return sampleQueues[i];
trackOutput = sampleQueues[i];
break;
}
}
}
if (tracksEnded) {
return createDummyTrackOutput(id, type);
if (trackOutput == null) {
if (tracksEnded) {
return createDummyTrackOutput(id, type);
} else {
// The relevant SampleQueue hasn't been constructed yet - so construct it.
trackOutput = createSampleQueue(id, type);
}
}
int trackCount = sampleQueues.length;
SampleQueue trackOutput = new PrivTimestampStrippingSampleQueue(allocator);
trackOutput.setSampleOffsetUs(sampleOffsetUs);
trackOutput.sourceId(chunkUid);
trackOutput.setUpstreamFormatChangeListener(this);
sampleQueueTrackIds = Arrays.copyOf(sampleQueueTrackIds, trackCount + 1);
sampleQueueTrackIds[trackCount] = id;
sampleQueues = Arrays.copyOf(sampleQueues, trackCount + 1);
sampleQueues[trackCount] = trackOutput;
sampleQueueIsAudioVideoFlags = Arrays.copyOf(sampleQueueIsAudioVideoFlags, trackCount + 1);
sampleQueueIsAudioVideoFlags[trackCount] = type == C.TRACK_TYPE_AUDIO
|| type == C.TRACK_TYPE_VIDEO;
haveAudioVideoSampleQueues |= sampleQueueIsAudioVideoFlags[trackCount];
sampleQueueMappingDoneByType.add(type);
sampleQueueIndicesByType.append(type, trackCount);
if (getTrackTypeScore(type) > getTrackTypeScore(primarySampleQueueType)) {
primarySampleQueueIndex = trackCount;
primarySampleQueueType = type;
if (type == C.TRACK_TYPE_METADATA) {
if (emsgUnwrappingTrackOutput == null) {
emsgUnwrappingTrackOutput = new EmsgUnwrappingTrackOutput(trackOutput, metadataType);
}
return emsgUnwrappingTrackOutput;
}
sampleQueuesEnabledStates = Arrays.copyOf(sampleQueuesEnabledStates, trackCount + 1);
return trackOutput;
}
@ -858,6 +861,31 @@ import java.util.Set;
: createDummyTrackOutput(id, type);
}
private SampleQueue createSampleQueue(int id, int type) {
int trackCount = sampleQueues.length;
SampleQueue trackOutput = new PrivTimestampStrippingSampleQueue(allocator);
trackOutput.setSampleOffsetUs(sampleOffsetUs);
trackOutput.sourceId(chunkUid);
trackOutput.setUpstreamFormatChangeListener(this);
sampleQueueTrackIds = Arrays.copyOf(sampleQueueTrackIds, trackCount + 1);
sampleQueueTrackIds[trackCount] = id;
sampleQueues = Arrays.copyOf(sampleQueues, trackCount + 1);
sampleQueues[trackCount] = trackOutput;
sampleQueueIsAudioVideoFlags = Arrays.copyOf(sampleQueueIsAudioVideoFlags, trackCount + 1);
sampleQueueIsAudioVideoFlags[trackCount] =
type == C.TRACK_TYPE_AUDIO || type == C.TRACK_TYPE_VIDEO;
haveAudioVideoSampleQueues |= sampleQueueIsAudioVideoFlags[trackCount];
sampleQueueMappingDoneByType.add(type);
sampleQueueIndicesByType.append(type, trackCount);
if (getTrackTypeScore(type) > getTrackTypeScore(primarySampleQueueType)) {
primarySampleQueueIndex = trackCount;
primarySampleQueueType = type;
}
sampleQueuesEnabledStates = Arrays.copyOf(sampleQueuesEnabledStates, trackCount + 1);
return trackOutput;
}
@Override
public void endTracks() {
tracksEnded = true;
@ -1221,4 +1249,141 @@ import java.util.Set;
return new Metadata(newMetadataEntries);
}
}
private static class EmsgUnwrappingTrackOutput implements TrackOutput {
private static final String TAG = "EmsgUnwrappingTrackOutput";
// TODO(ibaker): Create a Formats util class with common constants like this.
private static final Format ID3_FORMAT =
Format.createSampleFormat(
/* id= */ null, MimeTypes.APPLICATION_ID3, Format.OFFSET_SAMPLE_RELATIVE);
private static final Format EMSG_FORMAT =
Format.createSampleFormat(
/* id= */ null, MimeTypes.APPLICATION_EMSG, Format.OFFSET_SAMPLE_RELATIVE);
private final EventMessageDecoder emsgDecoder;
private final TrackOutput delegate;
private final Format delegateFormat;
@MonotonicNonNull private Format format;
private byte[] buffer;
private int bufferPosition;
public EmsgUnwrappingTrackOutput(TrackOutput delegate, @HlsMetadataType int metadataType) {
this.emsgDecoder = new EventMessageDecoder();
this.delegate = delegate;
switch (metadataType) {
case HlsMetadataType.ID3:
delegateFormat = ID3_FORMAT;
break;
case HlsMetadataType.EMSG:
delegateFormat = EMSG_FORMAT;
break;
default:
throw new IllegalArgumentException("Unknown metadataType: " + metadataType);
}
this.buffer = new byte[0];
this.bufferPosition = 0;
}
@Override
public void format(Format format) {
this.format = format;
delegate.format(delegateFormat);
}
@Override
public int sampleData(ExtractorInput input, int length, boolean allowEndOfInput)
throws IOException, InterruptedException {
ensureBufferCapacity(bufferPosition + length);
int numBytesRead = input.read(buffer, bufferPosition, length);
if (numBytesRead == C.RESULT_END_OF_INPUT) {
if (allowEndOfInput) {
return C.RESULT_END_OF_INPUT;
} else {
throw new EOFException();
}
}
bufferPosition += numBytesRead;
return numBytesRead;
}
@Override
public void sampleData(ParsableByteArray buffer, int length) {
ensureBufferCapacity(bufferPosition + length);
buffer.readBytes(this.buffer, bufferPosition, length);
bufferPosition += length;
}
@Override
public void sampleMetadata(
long timeUs,
@C.BufferFlags int flags,
int size,
int offset,
@Nullable CryptoData cryptoData) {
Assertions.checkState(format != null);
ParsableByteArray sample = getSampleAndTrimBuffer(size, offset);
ParsableByteArray sampleForDelegate;
if (Util.areEqual(format.sampleMimeType, delegateFormat.sampleMimeType)) {
// Incoming format matches delegate track's format, so pass straight through.
sampleForDelegate = sample;
} else if (MimeTypes.APPLICATION_EMSG.equals(format.sampleMimeType)) {
// Incoming sample is EMSG, and delegate track is not expecting EMSG, so try unwrapping.
EventMessage emsg = emsgDecoder.decode(sample);
if (!emsgContainsExpectedWrappedFormat(emsg)) {
Log.w(
TAG,
String.format(
"Ignoring EMSG. Expected it to contain wrapped %s but actual wrapped format: %s",
delegateFormat.sampleMimeType, emsg.getWrappedMetadataFormat()));
return;
}
sampleForDelegate =
new ParsableByteArray(Assertions.checkNotNull(emsg.getWrappedMetadataBytes()));
} else {
Log.w(TAG, "Ignoring sample for unsupported format: " + format.sampleMimeType);
return;
}
int sampleSize = sampleForDelegate.bytesLeft();
delegate.sampleData(sampleForDelegate, sampleSize);
delegate.sampleMetadata(timeUs, flags, sampleSize, offset, cryptoData);
}
private boolean emsgContainsExpectedWrappedFormat(EventMessage emsg) {
@Nullable Format wrappedMetadataFormat = emsg.getWrappedMetadataFormat();
return wrappedMetadataFormat != null
&& Util.areEqual(delegateFormat.sampleMimeType, wrappedMetadataFormat.sampleMimeType);
}
private void ensureBufferCapacity(int requiredLength) {
if (buffer.length < requiredLength) {
buffer = Arrays.copyOf(buffer, requiredLength + requiredLength / 2);
}
}
/**
* Removes a complete sample from the {@link #buffer} field & reshuffles the tail data skipped
* by {@code offset} to the head of the array.
*
* @param size see {@code size} param of {@link #sampleMetadata}.
* @param offset see {@code offset} param of {@link #sampleMetadata}.
* @return A {@link ParsableByteArray} containing the sample removed from {@link #buffer}.
*/
private ParsableByteArray getSampleAndTrimBuffer(int size, int offset) {
int sampleEnd = bufferPosition - offset;
int sampleStart = sampleEnd - size;
byte[] sampleBytes = Arrays.copyOfRange(buffer, sampleStart, sampleEnd);
ParsableByteArray sample = new ParsableByteArray(sampleBytes);
System.arraycopy(buffer, sampleEnd, buffer, 0, offset);
bufferPosition = offset;
return sample;
}
}
}

View File

@ -91,6 +91,7 @@ public final class HlsMediaPeriodTest {
mock(Allocator.class),
mock(CompositeSequenceableLoaderFactory.class),
/* allowChunklessPreparation =*/ true,
HlsMetadataType.ID3,
/* useSessionKeys= */ false);
};