Enable ID3-in-EMSG for HLS streams

This supports both chunkless & traditional preparation

PiperOrigin-RevId: 273938344
This commit is contained in:
ibaker 2019-10-10 13:10:41 +01:00 committed by Ian Baker
parent a96a82588e
commit 60566721d4
7 changed files with 287 additions and 37 deletions

View File

@ -105,6 +105,8 @@
* Fail more explicitly when local-file Uris contain invalid parts (e.g.
fragment) ([#6470](https://github.com/google/ExoPlayer/issues/6470)).
* Add `MediaPeriod.isLoading` to improve `Player.isLoading` state.
* Add support for ID3-in-EMSG in HLS streams
([spec](https://aomediacodec.github.io/av1-id3/)).
### 2.10.5 (2019-09-20) ###

View File

@ -29,6 +29,7 @@ import com.google.android.exoplayer2.extractor.ts.Ac4Extractor;
import com.google.android.exoplayer2.extractor.ts.AdtsExtractor;
import com.google.android.exoplayer2.extractor.ts.DefaultTsPayloadReaderFactory;
import com.google.android.exoplayer2.extractor.ts.TsExtractor;
import com.google.android.exoplayer2.metadata.Metadata;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.TimestampAdjuster;
import java.io.EOFException;
@ -158,7 +159,7 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
if (!(extractorByFileExtension instanceof FragmentedMp4Extractor)) {
FragmentedMp4Extractor fragmentedMp4Extractor =
createFragmentedMp4Extractor(timestampAdjuster, drmInitData, muxedCaptionFormats);
createFragmentedMp4Extractor(timestampAdjuster, format, drmInitData, muxedCaptionFormats);
if (sniffQuietly(fragmentedMp4Extractor, extractorInput)) {
return buildResult(fragmentedMp4Extractor);
}
@ -208,7 +209,8 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
|| lastPathSegment.startsWith(M4_FILE_EXTENSION_PREFIX, lastPathSegment.length() - 4)
|| lastPathSegment.startsWith(MP4_FILE_EXTENSION_PREFIX, lastPathSegment.length() - 5)
|| lastPathSegment.startsWith(CMF_FILE_EXTENSION_PREFIX, lastPathSegment.length() - 5)) {
return createFragmentedMp4Extractor(timestampAdjuster, drmInitData, muxedCaptionFormats);
return createFragmentedMp4Extractor(
timestampAdjuster, format, drmInitData, muxedCaptionFormats);
} else {
// For any other file extension, we assume TS format.
return createTsExtractor(
@ -267,10 +269,21 @@ public final class DefaultHlsExtractorFactory implements HlsExtractorFactory {
private static FragmentedMp4Extractor createFragmentedMp4Extractor(
TimestampAdjuster timestampAdjuster,
Format format,
DrmInitData drmInitData,
@Nullable List<Format> muxedCaptionFormats) {
boolean isVariant = false;
for (int i = 0; i < format.metadata.length(); i++) {
Metadata.Entry entry = format.metadata.get(i);
if (entry instanceof HlsTrackMetadataEntry) {
isVariant = !((HlsTrackMetadataEntry) entry).variantInfos.isEmpty();
break;
}
}
// Only enable the EMSG TrackOutput if this is the 'variant' track (i.e. the main one) to avoid
// creating a separate EMSG track for every audio track in a video stream.
return new FragmentedMp4Extractor(
/* flags= */ 0,
/* flags= */ isVariant ? FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK : 0,
timestampAdjuster,
/* sideloadedTrack= */ null,
drmInitData,

View File

@ -75,6 +75,7 @@ public final class HlsMediaPeriod implements MediaPeriod, HlsSampleStreamWrapper
private final TimestampAdjusterProvider timestampAdjusterProvider;
private final CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
private final boolean allowChunklessPreparation;
private final @HlsMetadataType int metadataType;
private final boolean useSessionKeys;
@Nullable private Callback callback;
@ -117,6 +118,7 @@ public final class HlsMediaPeriod implements MediaPeriod, HlsSampleStreamWrapper
Allocator allocator,
CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory,
boolean allowChunklessPreparation,
@HlsMetadataType int metadataType,
boolean useSessionKeys) {
this.extractorFactory = extractorFactory;
this.playlistTracker = playlistTracker;
@ -128,6 +130,7 @@ public final class HlsMediaPeriod implements MediaPeriod, HlsSampleStreamWrapper
this.allocator = allocator;
this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
this.allowChunklessPreparation = allowChunklessPreparation;
this.metadataType = metadataType;
this.useSessionKeys = useSessionKeys;
compositeSequenceableLoader =
compositeSequenceableLoaderFactory.createCompositeSequenceableLoader();
@ -755,7 +758,8 @@ public final class HlsMediaPeriod implements MediaPeriod, HlsSampleStreamWrapper
muxedAudioFormat,
drmSessionManager,
loadErrorHandlingPolicy,
eventDispatcher);
eventDispatcher,
metadataType);
}
private static Map<String, DrmInitData> deriveOverridingDrmInitData(

View File

@ -70,6 +70,7 @@ public final class HlsMediaSource extends BaseMediaSource
private DrmSessionManager<?> drmSessionManager;
private LoadErrorHandlingPolicy loadErrorHandlingPolicy;
private boolean allowChunklessPreparation;
@HlsMetadataType private int metadataType;
private boolean useSessionKeys;
private boolean isCreateCalled;
@Nullable private Object tag;
@ -99,6 +100,7 @@ public final class HlsMediaSource extends BaseMediaSource
drmSessionManager = DrmSessionManager.getDummyDrmSessionManager();
loadErrorHandlingPolicy = new DefaultLoadErrorHandlingPolicy();
compositeSequenceableLoaderFactory = new DefaultCompositeSequenceableLoaderFactory();
metadataType = HlsMetadataType.ID3;
}
/**
@ -242,6 +244,31 @@ public final class HlsMediaSource extends BaseMediaSource
return this;
}
/**
* Sets the type of metadata to extract from the HLS source (defaults to {@link
* HlsMetadataType#ID3}).
*
* <p>HLS supports in-band ID3 in both TS and fMP4 streams, but in the fMP4 case the data is
* wrapped in an EMSG box [<a href="https://aomediacodec.github.io/av1-id3/">spec</a>].
*
* <p>If this is set to {@link HlsMetadataType#ID3} then raw ID3 metadata of will be extracted
* from TS sources. From fMP4 streams EMSGs containing metadata of this type (in the variant
* stream only) will be unwrapped to expose the inner data. All other in-band metadata will be
* dropped.
*
* <p>If this is set to {@link HlsMetadataType#EMSG} then all EMSG data from the fMP4 variant
* stream will be extracted. No metadata will be extracted from TS streams, since they don't
* support EMSG.
*
* @param metadataType The type of metadata to extract.
* @return This factory, for convenience.
*/
public Factory setMetadataType(@HlsMetadataType int metadataType) {
Assertions.checkState(!isCreateCalled);
this.metadataType = metadataType;
return this;
}
/**
* Sets whether to use #EXT-X-SESSION-KEY tags provided in the master playlist. If enabled, it's
* assumed that any single session key declared in the master playlist can be used to obtain all
@ -294,6 +321,7 @@ public final class HlsMediaSource extends BaseMediaSource
playlistTrackerFactory.createTracker(
hlsDataSourceFactory, loadErrorHandlingPolicy, playlistParserFactory),
allowChunklessPreparation,
metadataType,
useSessionKeys,
tag);
}
@ -319,6 +347,7 @@ public final class HlsMediaSource extends BaseMediaSource
private final DrmSessionManager<?> drmSessionManager;
private final LoadErrorHandlingPolicy loadErrorHandlingPolicy;
private final boolean allowChunklessPreparation;
private final @HlsMetadataType int metadataType;
private final boolean useSessionKeys;
private final HlsPlaylistTracker playlistTracker;
@Nullable private final Object tag;
@ -334,6 +363,7 @@ public final class HlsMediaSource extends BaseMediaSource
LoadErrorHandlingPolicy loadErrorHandlingPolicy,
HlsPlaylistTracker playlistTracker,
boolean allowChunklessPreparation,
@HlsMetadataType int metadataType,
boolean useSessionKeys,
@Nullable Object tag) {
this.manifestUri = manifestUri;
@ -344,6 +374,7 @@ public final class HlsMediaSource extends BaseMediaSource
this.loadErrorHandlingPolicy = loadErrorHandlingPolicy;
this.playlistTracker = playlistTracker;
this.allowChunklessPreparation = allowChunklessPreparation;
this.metadataType = metadataType;
this.useSessionKeys = useSessionKeys;
this.tag = tag;
}
@ -381,6 +412,7 @@ public final class HlsMediaSource extends BaseMediaSource
allocator,
compositeSequenceableLoaderFactory,
allowChunklessPreparation,
metadataType,
useSessionKeys);
}

View File

@ -0,0 +1,34 @@
/*
* Copyright (C) 2019 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.google.android.exoplayer2.source.hls;
import static java.lang.annotation.RetentionPolicy.SOURCE;
import androidx.annotation.IntDef;
import java.lang.annotation.Retention;
/**
* The types of metadata that can be extracted from HLS streams.
*
* <p>See {@link HlsMediaSource.Factory#setMetadataType(int)}.
*/
@Retention(SOURCE)
@IntDef({HlsMetadataType.ID3, HlsMetadataType.EMSG})
public @interface HlsMetadataType {
int ID3 = 1;
int EMSG = 3;
}

View File

@ -28,10 +28,13 @@ import com.google.android.exoplayer2.drm.DrmInitData;
import com.google.android.exoplayer2.drm.DrmSession;
import com.google.android.exoplayer2.drm.DrmSessionManager;
import com.google.android.exoplayer2.extractor.DummyTrackOutput;
import com.google.android.exoplayer2.extractor.ExtractorInput;
import com.google.android.exoplayer2.extractor.ExtractorOutput;
import com.google.android.exoplayer2.extractor.SeekMap;
import com.google.android.exoplayer2.extractor.TrackOutput;
import com.google.android.exoplayer2.metadata.Metadata;
import com.google.android.exoplayer2.metadata.emsg.EventMessage;
import com.google.android.exoplayer2.metadata.emsg.EventMessageDecoder;
import com.google.android.exoplayer2.metadata.id3.PrivFrame;
import com.google.android.exoplayer2.source.DecryptableSampleQueueReader;
import com.google.android.exoplayer2.source.MediaSourceEventListener.EventDispatcher;
@ -51,7 +54,9 @@ import com.google.android.exoplayer2.upstream.Loader.LoadErrorAction;
import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.ParsableByteArray;
import com.google.android.exoplayer2.util.Util;
import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@ -60,6 +65,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* Loads {@link HlsMediaChunk}s obtained from a {@link HlsChunkSource}, and provides
@ -96,7 +102,8 @@ import java.util.Set;
private static final Set<Integer> MAPPABLE_TYPES =
Collections.unmodifiableSet(
new HashSet<>(Arrays.asList(C.TRACK_TYPE_AUDIO, C.TRACK_TYPE_VIDEO)));
new HashSet<>(
Arrays.asList(C.TRACK_TYPE_AUDIO, C.TRACK_TYPE_VIDEO, C.TRACK_TYPE_METADATA)));
private final int trackType;
private final Callback callback;
@ -107,6 +114,7 @@ import java.util.Set;
private final LoadErrorHandlingPolicy loadErrorHandlingPolicy;
private final Loader loader;
private final EventDispatcher eventDispatcher;
private final @HlsMetadataType int metadataType;
private final HlsChunkSource.HlsChunkHolder nextChunkHolder;
private final ArrayList<HlsMediaChunk> mediaChunks;
private final List<HlsMediaChunk> readOnlyMediaChunks;
@ -121,6 +129,7 @@ import java.util.Set;
private int[] sampleQueueTrackIds;
private Set<Integer> sampleQueueMappingDoneByType;
private SparseIntArray sampleQueueIndicesByType;
private TrackOutput emsgUnwrappingTrackOutput;
private int primarySampleQueueType;
private int primarySampleQueueIndex;
private boolean sampleQueuesBuilt;
@ -130,7 +139,7 @@ import java.util.Set;
private Format downstreamTrackFormat;
private boolean released;
// Tracks are complicated in HLS. See documentation of buildTracks for details.
// Tracks are complicated in HLS. See documentation of buildTracksFromSampleStreams for details.
// Indexed by track (as exposed by this source).
private TrackGroupArray trackGroups;
private Set<TrackGroup> optionalTrackGroups;
@ -178,7 +187,8 @@ import java.util.Set;
Format muxedAudioFormat,
DrmSessionManager<?> drmSessionManager,
LoadErrorHandlingPolicy loadErrorHandlingPolicy,
EventDispatcher eventDispatcher) {
EventDispatcher eventDispatcher,
@HlsMetadataType int metadataType) {
this.trackType = trackType;
this.callback = callback;
this.chunkSource = chunkSource;
@ -188,6 +198,7 @@ import java.util.Set;
this.drmSessionManager = drmSessionManager;
this.loadErrorHandlingPolicy = loadErrorHandlingPolicy;
this.eventDispatcher = eventDispatcher;
this.metadataType = metadataType;
loader = new Loader("Loader:HlsSampleStreamWrapper");
nextChunkHolder = new HlsChunkSource.HlsChunkHolder();
sampleQueueTrackIds = new int[0];
@ -819,46 +830,34 @@ import java.util.Set;
@Override
public TrackOutput track(int id, int type) {
@Nullable TrackOutput trackOutput = null;
if (MAPPABLE_TYPES.contains(type)) {
// Track types in MAPPABLE_TYPES are handled manually to ignore IDs.
@Nullable TrackOutput mappedTrackOutput = getMappedTrackOutput(id, type);
if (mappedTrackOutput != null) {
return mappedTrackOutput;
}
} else /* sparse track */ {
trackOutput = getMappedTrackOutput(id, type);
} else /* non-mappable type track */ {
for (int i = 0; i < sampleQueues.length; i++) {
if (sampleQueueTrackIds[i] == id) {
return sampleQueues[i];
trackOutput = sampleQueues[i];
break;
}
}
}
if (tracksEnded) {
return createDummyTrackOutput(id, type);
if (trackOutput == null) {
if (tracksEnded) {
return createDummyTrackOutput(id, type);
} else {
// The relevant SampleQueue hasn't been constructed yet - so construct it.
trackOutput = createSampleQueue(id, type);
}
}
int trackCount = sampleQueues.length;
SampleQueue trackOutput = new FormatAdjustingSampleQueue(allocator, overridingDrmInitData);
trackOutput.setSampleOffsetUs(sampleOffsetUs);
trackOutput.sourceId(chunkUid);
trackOutput.setUpstreamFormatChangeListener(this);
sampleQueueTrackIds = Arrays.copyOf(sampleQueueTrackIds, trackCount + 1);
sampleQueueTrackIds[trackCount] = id;
sampleQueues = Arrays.copyOf(sampleQueues, trackCount + 1);
sampleQueues[trackCount] = trackOutput;
sampleQueueReaders = Arrays.copyOf(sampleQueueReaders, trackCount + 1);
sampleQueueReaders[trackCount] =
new DecryptableSampleQueueReader(sampleQueues[trackCount], drmSessionManager);
sampleQueueIsAudioVideoFlags = Arrays.copyOf(sampleQueueIsAudioVideoFlags, trackCount + 1);
sampleQueueIsAudioVideoFlags[trackCount] = type == C.TRACK_TYPE_AUDIO
|| type == C.TRACK_TYPE_VIDEO;
haveAudioVideoSampleQueues |= sampleQueueIsAudioVideoFlags[trackCount];
sampleQueueMappingDoneByType.add(type);
sampleQueueIndicesByType.append(type, trackCount);
if (getTrackTypeScore(type) > getTrackTypeScore(primarySampleQueueType)) {
primarySampleQueueIndex = trackCount;
primarySampleQueueType = type;
if (type == C.TRACK_TYPE_METADATA) {
if (emsgUnwrappingTrackOutput == null) {
emsgUnwrappingTrackOutput = new EmsgUnwrappingTrackOutput(trackOutput, metadataType);
}
return emsgUnwrappingTrackOutput;
}
sampleQueuesEnabledStates = Arrays.copyOf(sampleQueuesEnabledStates, trackCount + 1);
return trackOutput;
}
@ -893,6 +892,34 @@ import java.util.Set;
: createDummyTrackOutput(id, type);
}
private SampleQueue createSampleQueue(int id, int type) {
int trackCount = sampleQueues.length;
SampleQueue trackOutput = new FormatAdjustingSampleQueue(allocator, overridingDrmInitData);
trackOutput.setSampleOffsetUs(sampleOffsetUs);
trackOutput.sourceId(chunkUid);
trackOutput.setUpstreamFormatChangeListener(this);
sampleQueueTrackIds = Arrays.copyOf(sampleQueueTrackIds, trackCount + 1);
sampleQueueTrackIds[trackCount] = id;
sampleQueues = Arrays.copyOf(sampleQueues, trackCount + 1);
sampleQueues[trackCount] = trackOutput;
sampleQueueReaders = Arrays.copyOf(sampleQueueReaders, trackCount + 1);
sampleQueueReaders[trackCount] =
new DecryptableSampleQueueReader(sampleQueues[trackCount], drmSessionManager);
sampleQueueIsAudioVideoFlags = Arrays.copyOf(sampleQueueIsAudioVideoFlags, trackCount + 1);
sampleQueueIsAudioVideoFlags[trackCount] =
type == C.TRACK_TYPE_AUDIO || type == C.TRACK_TYPE_VIDEO;
haveAudioVideoSampleQueues |= sampleQueueIsAudioVideoFlags[trackCount];
sampleQueueMappingDoneByType.add(type);
sampleQueueIndicesByType.append(type, trackCount);
if (getTrackTypeScore(type) > getTrackTypeScore(primarySampleQueueType)) {
primarySampleQueueIndex = trackCount;
primarySampleQueueType = type;
}
sampleQueuesEnabledStates = Arrays.copyOf(sampleQueuesEnabledStates, trackCount + 1);
return trackOutput;
}
@Override
public void endTracks() {
tracksEnded = true;
@ -1285,4 +1312,141 @@ import java.util.Set;
return new Metadata(newMetadataEntries);
}
}
private static class EmsgUnwrappingTrackOutput implements TrackOutput {
private static final String TAG = "EmsgUnwrappingTrackOutput";
// TODO(ibaker): Create a Formats util class with common constants like this.
private static final Format ID3_FORMAT =
Format.createSampleFormat(
/* id= */ null, MimeTypes.APPLICATION_ID3, Format.OFFSET_SAMPLE_RELATIVE);
private static final Format EMSG_FORMAT =
Format.createSampleFormat(
/* id= */ null, MimeTypes.APPLICATION_EMSG, Format.OFFSET_SAMPLE_RELATIVE);
private final EventMessageDecoder emsgDecoder;
private final TrackOutput delegate;
private final Format delegateFormat;
@MonotonicNonNull private Format format;
private byte[] buffer;
private int bufferPosition;
public EmsgUnwrappingTrackOutput(TrackOutput delegate, @HlsMetadataType int metadataType) {
this.emsgDecoder = new EventMessageDecoder();
this.delegate = delegate;
switch (metadataType) {
case HlsMetadataType.ID3:
delegateFormat = ID3_FORMAT;
break;
case HlsMetadataType.EMSG:
delegateFormat = EMSG_FORMAT;
break;
default:
throw new IllegalArgumentException("Unknown metadataType: " + metadataType);
}
this.buffer = new byte[0];
this.bufferPosition = 0;
}
@Override
public void format(Format format) {
this.format = format;
delegate.format(delegateFormat);
}
@Override
public int sampleData(ExtractorInput input, int length, boolean allowEndOfInput)
throws IOException, InterruptedException {
ensureBufferCapacity(bufferPosition + length);
int numBytesRead = input.read(buffer, bufferPosition, length);
if (numBytesRead == C.RESULT_END_OF_INPUT) {
if (allowEndOfInput) {
return C.RESULT_END_OF_INPUT;
} else {
throw new EOFException();
}
}
bufferPosition += numBytesRead;
return numBytesRead;
}
@Override
public void sampleData(ParsableByteArray buffer, int length) {
ensureBufferCapacity(bufferPosition + length);
buffer.readBytes(this.buffer, bufferPosition, length);
bufferPosition += length;
}
@Override
public void sampleMetadata(
long timeUs,
@C.BufferFlags int flags,
int size,
int offset,
@Nullable CryptoData cryptoData) {
Assertions.checkState(format != null);
ParsableByteArray sample = getSampleAndTrimBuffer(size, offset);
ParsableByteArray sampleForDelegate;
if (Util.areEqual(format.sampleMimeType, delegateFormat.sampleMimeType)) {
// Incoming format matches delegate track's format, so pass straight through.
sampleForDelegate = sample;
} else if (MimeTypes.APPLICATION_EMSG.equals(format.sampleMimeType)) {
// Incoming sample is EMSG, and delegate track is not expecting EMSG, so try unwrapping.
EventMessage emsg = emsgDecoder.decode(sample);
if (!emsgContainsExpectedWrappedFormat(emsg)) {
Log.w(
TAG,
String.format(
"Ignoring EMSG. Expected it to contain wrapped %s but actual wrapped format: %s",
delegateFormat.sampleMimeType, emsg.getWrappedMetadataFormat()));
return;
}
sampleForDelegate =
new ParsableByteArray(Assertions.checkNotNull(emsg.getWrappedMetadataBytes()));
} else {
Log.w(TAG, "Ignoring sample for unsupported format: " + format.sampleMimeType);
return;
}
int sampleSize = sampleForDelegate.bytesLeft();
delegate.sampleData(sampleForDelegate, sampleSize);
delegate.sampleMetadata(timeUs, flags, sampleSize, offset, cryptoData);
}
private boolean emsgContainsExpectedWrappedFormat(EventMessage emsg) {
@Nullable Format wrappedMetadataFormat = emsg.getWrappedMetadataFormat();
return wrappedMetadataFormat != null
&& Util.areEqual(delegateFormat.sampleMimeType, wrappedMetadataFormat.sampleMimeType);
}
private void ensureBufferCapacity(int requiredLength) {
if (buffer.length < requiredLength) {
buffer = Arrays.copyOf(buffer, requiredLength + requiredLength / 2);
}
}
/**
* Removes a complete sample from the {@link #buffer} field & reshuffles the tail data skipped
* by {@code offset} to the head of the array.
*
* @param size see {@code size} param of {@link #sampleMetadata}.
* @param offset see {@code offset} param of {@link #sampleMetadata}.
* @return A {@link ParsableByteArray} containing the sample removed from {@link #buffer}.
*/
private ParsableByteArray getSampleAndTrimBuffer(int size, int offset) {
int sampleEnd = bufferPosition - offset;
int sampleStart = sampleEnd - size;
byte[] sampleBytes = Arrays.copyOfRange(buffer, sampleStart, sampleEnd);
ParsableByteArray sample = new ParsableByteArray(sampleBytes);
System.arraycopy(buffer, sampleEnd, buffer, 0, offset);
bufferPosition = offset;
return sample;
}
}
}

View File

@ -92,6 +92,7 @@ public final class HlsMediaPeriodTest {
mock(Allocator.class),
mock(CompositeSequenceableLoaderFactory.class),
/* allowChunklessPreparation =*/ true,
HlsMetadataType.ID3,
/* useSessionKeys= */ false);
};