From d2d6227188453eefe32e4538f7b6babf186aea66 Mon Sep 17 00:00:00 2001 From: tonihei Date: Fri, 14 Jan 2022 14:33:15 +0000 Subject: [PATCH 01/12] De-duplicate track selection code. We currently run (almost) the same code for all track types. De-duplicate this by using a single method that takes functional interfaces for track-type dependent logic. This has the benefit that all track-type dependent logic is contained within their subclasses and the generic logic doesn't need to make any assumption about the eligibility of tracks for selection or adaptation, and doesn't need to access Parameters. Make this change for audio and text only for now. Video can be updated in a subsequent change. PiperOrigin-RevId: 421811411 --- .../trackselection/DefaultTrackSelector.java | 572 ++++++++++-------- 1 file changed, 312 insertions(+), 260 deletions(-) diff --git a/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java b/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java index 21e05f33bb..f25c4232a2 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java @@ -15,6 +15,9 @@ */ package com.google.android.exoplayer2.trackselection; +import static java.util.Collections.max; + +import android.annotation.SuppressLint; import android.content.Context; import android.graphics.Point; import android.os.Bundle; @@ -52,6 +55,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -1327,6 +1331,26 @@ public class DefaultTrackSelector extends MappingTrackSelector { } } + /** + * The extent to which tracks are eligible for selection. One of {@link + * #SELECTION_ELIGIBILITY_NO}, {@link #SELECTION_ELIGIBILITY_FIXED} or {@link + * #SELECTION_ELIGIBILITY_ADAPTIVE}. + */ + @Documented + @Retention(RetentionPolicy.SOURCE) + @IntDef({SELECTION_ELIGIBILITY_NO, SELECTION_ELIGIBILITY_FIXED, SELECTION_ELIGIBILITY_ADAPTIVE}) + protected @interface SelectionEligibility {} + + /** Track is not eligible for selection. */ + protected static final int SELECTION_ELIGIBILITY_NO = 0; + /** Track is eligible for a fixed selection with one track. */ + protected static final int SELECTION_ELIGIBILITY_FIXED = 1; + /** + * Track is eligible for both a fixed selection and as part of an adaptive selection with multiple + * tracks. + */ + protected static final int SELECTION_ELIGIBILITY_ADAPTIVE = 2; + /** * If a dimension (i.e. width or height) of a video is greater or equal to this fraction of the * corresponding viewport dimension, then the video is considered as filling the viewport (in that @@ -1626,7 +1650,6 @@ public class DefaultTrackSelector extends MappingTrackSelector { ExoTrackSelection.@NullableType Definition[] definitions = new ExoTrackSelection.Definition[rendererCount]; - boolean seenVideoRendererWithMappedTracks = false; boolean selectedVideoTracks = false; for (int i = 0; i < rendererCount; i++) { if (C.TRACK_TYPE_VIDEO == mappedTrackInfo.getRendererType(i)) { @@ -1640,78 +1663,40 @@ public class DefaultTrackSelector extends MappingTrackSelector { /* enableAdaptiveTrackSelection= */ true); selectedVideoTracks = definitions[i] != null; } - seenVideoRendererWithMappedTracks |= mappedTrackInfo.getTrackGroups(i).length > 0; } } - @Nullable AudioTrackScore selectedAudioTrackScore = null; - @Nullable String selectedAudioLanguage = null; - int selectedAudioRendererIndex = C.INDEX_UNSET; - for (int i = 0; i < rendererCount; i++) { - if (C.TRACK_TYPE_AUDIO == mappedTrackInfo.getRendererType(i)) { - boolean enableAdaptiveTrackSelection = - params.allowMultipleAdaptiveSelections || !seenVideoRendererWithMappedTracks; - @Nullable - Pair audioSelection = - selectAudioTrack( - mappedTrackInfo.getTrackGroups(i), - rendererFormatSupports[i], - rendererMixedMimeTypeAdaptationSupports[i], - params, - enableAdaptiveTrackSelection); - if (audioSelection != null - && (selectedAudioTrackScore == null - || audioSelection.second.compareTo(selectedAudioTrackScore) > 0)) { - if (selectedAudioRendererIndex != C.INDEX_UNSET) { - // We've already made a selection for another audio renderer, but it had a lower - // score. Clear the selection for that renderer. - definitions[selectedAudioRendererIndex] = null; - } - ExoTrackSelection.Definition definition = audioSelection.first; - definitions[i] = definition; - // We assume that audio tracks in the same group have matching language. - selectedAudioLanguage = definition.group.getFormat(definition.tracks[0]).language; - selectedAudioTrackScore = audioSelection.second; - selectedAudioRendererIndex = i; - } - } + @Nullable + Pair selectedAudio = + selectAudioTrack( + mappedTrackInfo, + rendererFormatSupports, + rendererMixedMimeTypeAdaptationSupports, + params); + if (selectedAudio != null) { + definitions[selectedAudio.second] = selectedAudio.first; + } + + @Nullable + String selectedAudioLanguage = + selectedAudio == null + ? null + : selectedAudio.first.group.getFormat(selectedAudio.first.tracks[0]).language; + @Nullable + Pair selectedText = + selectTextTrack(mappedTrackInfo, rendererFormatSupports, params, selectedAudioLanguage); + if (selectedText != null) { + definitions[selectedText.second] = selectedText.first; } - @Nullable TextTrackScore selectedTextTrackScore = null; - int selectedTextRendererIndex = C.INDEX_UNSET; for (int i = 0; i < rendererCount; i++) { int trackType = mappedTrackInfo.getRendererType(i); - switch (trackType) { - case C.TRACK_TYPE_VIDEO: - case C.TRACK_TYPE_AUDIO: - // Already done. Do nothing. - break; - case C.TRACK_TYPE_TEXT: - @Nullable - Pair textSelection = - selectTextTrack( - mappedTrackInfo.getTrackGroups(i), - rendererFormatSupports[i], - params, - selectedAudioLanguage); - if (textSelection != null - && (selectedTextTrackScore == null - || textSelection.second.compareTo(selectedTextTrackScore) > 0)) { - if (selectedTextRendererIndex != C.INDEX_UNSET) { - // We've already made a selection for another text renderer, but it had a lower score. - // Clear the selection for that renderer. - definitions[selectedTextRendererIndex] = null; - } - definitions[i] = textSelection.first; - selectedTextTrackScore = textSelection.second; - selectedTextRendererIndex = i; - } - break; - default: - definitions[i] = - selectOtherTrack( - trackType, mappedTrackInfo.getTrackGroups(i), rendererFormatSupports[i], params); - break; + if (trackType != C.TRACK_TYPE_VIDEO + && trackType != C.TRACK_TYPE_AUDIO + && trackType != C.TRACK_TYPE_TEXT) { + definitions[i] = + selectOtherTrack( + trackType, mappedTrackInfo.getTrackGroups(i), rendererFormatSupports[i], params); } } @@ -2025,187 +2010,80 @@ public class DefaultTrackSelector extends MappingTrackSelector { /** * Called by {@link #selectAllTracks(MappedTrackInfo, int[][][], int[], Parameters)} to create a - * {@link ExoTrackSelection} for an audio renderer. + * {@link ExoTrackSelection.Definition} for an audio track selection. * - * @param groups The {@link TrackGroupArray} mapped to the renderer. - * @param formatSupport The {@link Capabilities} for each mapped track, indexed by track group and - * track (in that order). - * @param mixedMimeTypeAdaptationSupports The {@link AdaptiveSupport} for mixed MIME type + * @param mappedTrackInfo Mapped track information. + * @param rendererFormatSupports The {@link Capabilities} for each mapped track, indexed by + * renderer, track group and track (in that order). + * @param rendererMixedMimeTypeAdaptationSupports The {@link AdaptiveSupport} for mixed MIME type * adaptation for the renderer. * @param params The selector's current constraint parameters. - * @param enableAdaptiveTrackSelection Whether adaptive track selection is allowed. - * @return The {@link ExoTrackSelection.Definition} and corresponding {@link AudioTrackScore}, or - * null if no selection was made. + * @return A pair of the selected {@link ExoTrackSelection.Definition} and the corresponding + * renderer index, or null if no selection was made. * @throws ExoPlaybackException If an error occurs while selecting the tracks. */ - @SuppressWarnings("unused") + @SuppressLint("WrongConstant") // Lint doesn't understand arrays of IntDefs. @Nullable - protected Pair selectAudioTrack( - TrackGroupArray groups, - @Capabilities int[][] formatSupport, - @AdaptiveSupport int mixedMimeTypeAdaptationSupports, - Parameters params, - boolean enableAdaptiveTrackSelection) + protected Pair selectAudioTrack( + MappedTrackInfo mappedTrackInfo, + @Capabilities int[][][] rendererFormatSupports, + @AdaptiveSupport int[] rendererMixedMimeTypeAdaptationSupports, + Parameters params) throws ExoPlaybackException { - int selectedTrackIndex = C.INDEX_UNSET; - int selectedGroupIndex = C.INDEX_UNSET; - @Nullable AudioTrackScore selectedTrackScore = null; - for (int groupIndex = 0; groupIndex < groups.length; groupIndex++) { - TrackGroup trackGroup = groups.get(groupIndex); - @Capabilities int[] trackFormatSupport = formatSupport[groupIndex]; - for (int trackIndex = 0; trackIndex < trackGroup.length; trackIndex++) { - if (isSupported( - trackFormatSupport[trackIndex], params.exceedRendererCapabilitiesIfNecessary)) { - Format format = trackGroup.getFormat(trackIndex); - AudioTrackScore trackScore = - new AudioTrackScore(format, params, trackFormatSupport[trackIndex]); - if (!trackScore.isWithinConstraints && !params.exceedAudioConstraintsIfNecessary) { - // Track should not be selected. - continue; - } - if (selectedTrackScore == null || trackScore.compareTo(selectedTrackScore) > 0) { - selectedGroupIndex = groupIndex; - selectedTrackIndex = trackIndex; - selectedTrackScore = trackScore; - } - } + boolean hasVideoRendererWithMappedTracks = false; + for (int i = 0; i < mappedTrackInfo.getRendererCount(); i++) { + if (C.TRACK_TYPE_VIDEO == mappedTrackInfo.getRendererType(i) + && mappedTrackInfo.getTrackGroups(i).length > 0) { + hasVideoRendererWithMappedTracks = true; + break; } } - - if (selectedGroupIndex == C.INDEX_UNSET) { - return null; - } - - TrackGroup selectedGroup = groups.get(selectedGroupIndex); - - ExoTrackSelection.Definition definition = null; - if (!params.forceHighestSupportedBitrate - && !params.forceLowestBitrate - && enableAdaptiveTrackSelection) { - // If the group of the track with the highest score allows it, try to enable adaptation. - int[] adaptiveTracks = - getAdaptiveAudioTracks( - selectedGroup, - formatSupport[selectedGroupIndex], - selectedTrackIndex, - params.maxAudioBitrate, - params.allowAudioMixedMimeTypeAdaptiveness, - params.allowAudioMixedSampleRateAdaptiveness, - params.allowAudioMixedChannelCountAdaptiveness); - if (adaptiveTracks.length > 1) { - definition = new ExoTrackSelection.Definition(selectedGroup, adaptiveTracks); - } - } - if (definition == null) { - // We didn't make an adaptive selection, so make a fixed one instead. - definition = new ExoTrackSelection.Definition(selectedGroup, selectedTrackIndex); - } - - return Pair.create(definition, Assertions.checkNotNull(selectedTrackScore)); - } - - private static int[] getAdaptiveAudioTracks( - TrackGroup group, - @Capabilities int[] formatSupport, - int primaryTrackIndex, - int maxAudioBitrate, - boolean allowMixedMimeTypeAdaptiveness, - boolean allowMixedSampleRateAdaptiveness, - boolean allowAudioMixedChannelCountAdaptiveness) { - Format primaryFormat = group.getFormat(primaryTrackIndex); - int[] adaptiveIndices = new int[group.length]; - int count = 0; - for (int i = 0; i < group.length; i++) { - if (i == primaryTrackIndex - || isSupportedAdaptiveAudioTrack( - group.getFormat(i), - formatSupport[i], - primaryFormat, - maxAudioBitrate, - allowMixedMimeTypeAdaptiveness, - allowMixedSampleRateAdaptiveness, - allowAudioMixedChannelCountAdaptiveness)) { - adaptiveIndices[count++] = i; - } - } - return Arrays.copyOf(adaptiveIndices, count); - } - - private static boolean isSupportedAdaptiveAudioTrack( - Format format, - @Capabilities int formatSupport, - Format primaryFormat, - int maxAudioBitrate, - boolean allowMixedMimeTypeAdaptiveness, - boolean allowMixedSampleRateAdaptiveness, - boolean allowAudioMixedChannelCountAdaptiveness) { - return isSupported(formatSupport, /* allowExceedsCapabilities= */ false) - && format.bitrate != Format.NO_VALUE - && format.bitrate <= maxAudioBitrate - && (allowAudioMixedChannelCountAdaptiveness - || (format.channelCount != Format.NO_VALUE - && format.channelCount == primaryFormat.channelCount)) - && (allowMixedMimeTypeAdaptiveness - || (format.sampleMimeType != null - && TextUtils.equals(format.sampleMimeType, primaryFormat.sampleMimeType))) - && (allowMixedSampleRateAdaptiveness - || (format.sampleRate != Format.NO_VALUE - && format.sampleRate == primaryFormat.sampleRate)); + boolean hasVideoRendererWithMappedTracksFinal = hasVideoRendererWithMappedTracks; + return selectTracksForType( + C.TRACK_TYPE_AUDIO, + mappedTrackInfo, + rendererFormatSupports, + (rendererIndex, group, support) -> + AudioTrackInfo.createForTrackGroup( + rendererIndex, group, params, support, hasVideoRendererWithMappedTracksFinal), + AudioTrackInfo::compareSelections); } // Text track selection implementation. /** * Called by {@link #selectAllTracks(MappedTrackInfo, int[][][], int[], Parameters)} to create a - * {@link ExoTrackSelection} for a text renderer. + * {@link ExoTrackSelection.Definition} for a text track selection. * - * @param groups The {@link TrackGroupArray} mapped to the renderer. - * @param formatSupport The {@link Capabilities} for each mapped track, indexed by track group and - * track (in that order). + * @param mappedTrackInfo Mapped track information. + * @param rendererFormatSupports The {@link Capabilities} for each mapped track, indexed by + * renderer, track group and track (in that order). * @param params The selector's current constraint parameters. * @param selectedAudioLanguage The language of the selected audio track. May be null if the - * selected text track declares no language or no text track was selected. - * @return The {@link ExoTrackSelection.Definition} and corresponding {@link TextTrackScore}, or - * null if no selection was made. + * selected audio track declares no language or no audio track was selected. + * @return A pair of the selected {@link ExoTrackSelection.Definition} and the corresponding + * renderer index, or null if no selection was made. * @throws ExoPlaybackException If an error occurs while selecting the tracks. */ + @SuppressLint("WrongConstant") // Lint doesn't understand arrays of IntDefs. @Nullable - protected Pair selectTextTrack( - TrackGroupArray groups, - @Capabilities int[][] formatSupport, + protected Pair selectTextTrack( + MappedTrackInfo mappedTrackInfo, + @Capabilities int[][][] rendererFormatSupports, Parameters params, @Nullable String selectedAudioLanguage) throws ExoPlaybackException { - @Nullable TrackGroup selectedGroup = null; - int selectedTrackIndex = C.INDEX_UNSET; - @Nullable TextTrackScore selectedTrackScore = null; - for (int groupIndex = 0; groupIndex < groups.length; groupIndex++) { - TrackGroup trackGroup = groups.get(groupIndex); - @Capabilities int[] trackFormatSupport = formatSupport[groupIndex]; - for (int trackIndex = 0; trackIndex < trackGroup.length; trackIndex++) { - if (isSupported( - trackFormatSupport[trackIndex], params.exceedRendererCapabilitiesIfNecessary)) { - Format format = trackGroup.getFormat(trackIndex); - TextTrackScore trackScore = - new TextTrackScore( - format, params, trackFormatSupport[trackIndex], selectedAudioLanguage); - if (trackScore.isWithinConstraints - && (selectedTrackScore == null || trackScore.compareTo(selectedTrackScore) > 0)) { - selectedGroup = trackGroup; - selectedTrackIndex = trackIndex; - selectedTrackScore = trackScore; - } - } - } - } - return selectedGroup == null - ? null - : Pair.create( - new ExoTrackSelection.Definition(selectedGroup, selectedTrackIndex), - Assertions.checkNotNull(selectedTrackScore)); + return selectTracksForType( + C.TRACK_TYPE_TEXT, + mappedTrackInfo, + rendererFormatSupports, + (rendererIndex, group, support) -> + TextTrackInfo.createForTrackGroup( + rendererIndex, group, params, support, selectedAudioLanguage), + TextTrackInfo::compareSelections); } - // General track selection methods. + // Generic track selection methods. /** * Called by {@link #selectAllTracks(MappedTrackInfo, int[][][], int[], Parameters)} to create a @@ -2247,6 +2125,64 @@ public class DefaultTrackSelector extends MappingTrackSelector { : new ExoTrackSelection.Definition(selectedGroup, selectedTrackIndex); } + @Nullable + private Pair selectTracksForType( + @C.TrackType int trackType, + MappedTrackInfo mappedTrackInfo, + @Capabilities int[][][] formatSupport, + TrackInfo.Factory trackInfoFactory, + Comparator> selectionComparator) { + ArrayList> possibleSelections = new ArrayList<>(); + int rendererCount = mappedTrackInfo.getRendererCount(); + for (int rendererIndex = 0; rendererIndex < rendererCount; rendererIndex++) { + if (trackType == mappedTrackInfo.getRendererType(rendererIndex)) { + TrackGroupArray groups = mappedTrackInfo.getTrackGroups(rendererIndex); + for (int groupIndex = 0; groupIndex < groups.length; groupIndex++) { + TrackGroup trackGroup = groups.get(groupIndex); + @Capabilities int[] groupSupport = formatSupport[rendererIndex][groupIndex]; + List trackInfos = trackInfoFactory.create(rendererIndex, trackGroup, groupSupport); + boolean[] usedTrackInSelection = new boolean[trackGroup.length]; + for (int trackIndex = 0; trackIndex < trackGroup.length; trackIndex++) { + T trackInfo = trackInfos.get(trackIndex); + @SelectionEligibility int eligibility = trackInfo.getSelectionEligibility(); + if (usedTrackInSelection[trackIndex] || eligibility == SELECTION_ELIGIBILITY_NO) { + continue; + } + List selection; + if (eligibility == SELECTION_ELIGIBILITY_FIXED) { + selection = ImmutableList.of(trackInfo); + } else { + selection = new ArrayList<>(); + selection.add(trackInfo); + for (int i = trackIndex + 1; i < trackGroup.length; i++) { + T otherTrackInfo = trackInfos.get(i); + if (otherTrackInfo.getSelectionEligibility() == SELECTION_ELIGIBILITY_ADAPTIVE) { + if (trackInfo.isCompatibleForAdaptationWith(otherTrackInfo)) { + selection.add(otherTrackInfo); + usedTrackInSelection[i] = true; + } + } + } + } + possibleSelections.add(selection); + } + } + } + } + if (possibleSelections.isEmpty()) { + return null; + } + List bestSelection = max(possibleSelections, selectionComparator); + int[] trackIndices = new int[bestSelection.size()]; + for (int i = 0; i < bestSelection.size(); i++) { + trackIndices[i] = bestSelection.get(i).trackIndex; + } + T firstTrackInfo = bestSelection.get(0); + return Pair.create( + new ExoTrackSelection.Definition(firstTrackInfo.trackGroup, trackIndices), + firstTrackInfo.rendererIndex); + } + // Utility methods. /** @@ -2484,6 +2420,36 @@ public class DefaultTrackSelector extends MappingTrackSelector { return Integer.bitCount(trackRoleFlags & preferredRoleFlags); } + /** Base class for track selection information of a {@link Format}. */ + private abstract static class TrackInfo { + /** Factory for {@link TrackInfo} implementations for a given {@link TrackGroup}. */ + public interface Factory { + List create(int rendererIndex, TrackGroup trackGroup, @Capabilities int[] formatSupports); + } + + public final int rendererIndex; + public final TrackGroup trackGroup; + public final int trackIndex; + public final Format format; + + public TrackInfo(int rendererIndex, TrackGroup trackGroup, int trackIndex) { + this.rendererIndex = rendererIndex; + this.trackGroup = trackGroup; + this.trackIndex = trackIndex; + this.format = trackGroup.getFormat(trackIndex); + } + + /** Returns to what extent the track is {@link SelectionEligibility eligible for selection}. */ + @SelectionEligibility + public abstract int getSelectionEligibility(); + + /** + * Returns whether this track is compatible for an adaptive selection with the specified other + * track. + */ + public abstract boolean isCompatibleForAdaptationWith(TrackInfo otherTrack); + } + /** Represents how well a video track matches the selection {@link Parameters}. */ protected static final class VideoTrackScore implements Comparable { @@ -2580,15 +2546,31 @@ public class DefaultTrackSelector extends MappingTrackSelector { } } - /** Represents how well an audio track matches the selection {@link Parameters}. */ - protected static final class AudioTrackScore implements Comparable { + private static final class AudioTrackInfo extends TrackInfo + implements Comparable { - /** - * Whether the provided format is within the parameter constraints. If {@code false}, the format - * should not be selected. - */ - public final boolean isWithinConstraints; + public static ImmutableList createForTrackGroup( + int rendererIndex, + TrackGroup trackGroup, + Parameters params, + @Capabilities int[] formatSupport, + boolean hasMappedVideoTracks) { + ImmutableList.Builder listBuilder = ImmutableList.builder(); + for (int i = 0; i < trackGroup.length; i++) { + listBuilder.add( + new AudioTrackInfo( + rendererIndex, + trackGroup, + /* trackIndex= */ i, + params, + formatSupport[i], + hasMappedVideoTracks)); + } + return listBuilder.build(); + } + @SelectionEligibility private final int selectionEligibility; + private final boolean isWithinConstraints; @Nullable private final String language; private final Parameters parameters; private final boolean isWithinRendererCapabilities; @@ -2604,7 +2586,14 @@ public class DefaultTrackSelector extends MappingTrackSelector { private final int bitrate; private final int preferredMimeTypeMatchIndex; - public AudioTrackScore(Format format, Parameters parameters, @Capabilities int formatSupport) { + public AudioTrackInfo( + int rendererIndex, + TrackGroup trackGroup, + int trackIndex, + Parameters parameters, + @Capabilities int formatSupport, + boolean hasMappedVideoTracks) { + super(rendererIndex, trackGroup, trackIndex); this.parameters = parameters; this.language = normalizeUndeterminedLanguageToNull(format.language); isWithinRendererCapabilities = @@ -2660,17 +2649,30 @@ public class DefaultTrackSelector extends MappingTrackSelector { } } preferredMimeTypeMatchIndex = bestMimeTypeMatchIndex; + selectionEligibility = evaluateSelectionEligibility(formatSupport, hasMappedVideoTracks); } - /** - * Compares this score with another. - * - * @param other The other score to compare to. - * @return A positive integer if this score is better than the other. Zero if they are equal. A - * negative integer if this score is worse than the other. - */ @Override - public int compareTo(AudioTrackScore other) { + @SelectionEligibility + public int getSelectionEligibility() { + return selectionEligibility; + } + + @Override + public boolean isCompatibleForAdaptationWith(TrackInfo otherTrack) { + return (parameters.allowAudioMixedChannelCountAdaptiveness + || (format.channelCount != Format.NO_VALUE + && format.channelCount == otherTrack.format.channelCount)) + && (parameters.allowAudioMixedMimeTypeAdaptiveness + || (format.sampleMimeType != null + && TextUtils.equals(format.sampleMimeType, otherTrack.format.sampleMimeType))) + && (parameters.allowAudioMixedSampleRateAdaptiveness + || (format.sampleRate != Format.NO_VALUE + && format.sampleRate == otherTrack.format.sampleRate)); + } + + @Override + public int compareTo(AudioTrackInfo other) { // If the formats are within constraints and renderer capabilities then prefer higher values // of channel count, sample rate and bit rate in that order. Otherwise, prefer lower values. Ordering qualityOrdering = @@ -2714,17 +2716,55 @@ public class DefaultTrackSelector extends MappingTrackSelector { Util.areEqual(this.language, other.language) ? qualityOrdering : NO_ORDER) .result(); } + + @SelectionEligibility + private int evaluateSelectionEligibility( + @Capabilities int rendererSupport, boolean hasMappedVideoTracks) { + if (!isSupported(rendererSupport, parameters.exceedRendererCapabilitiesIfNecessary)) { + return SELECTION_ELIGIBILITY_NO; + } + if (!isWithinConstraints && !parameters.exceedAudioConstraintsIfNecessary) { + return SELECTION_ELIGIBILITY_NO; + } + return isSupported(rendererSupport, /* allowExceedsCapabilities= */ false) + && isWithinConstraints + && format.bitrate != Format.NO_VALUE + && !parameters.forceHighestSupportedBitrate + && !parameters.forceLowestBitrate + && (parameters.allowMultipleAdaptiveSelections || !hasMappedVideoTracks) + ? SELECTION_ELIGIBILITY_ADAPTIVE + : SELECTION_ELIGIBILITY_FIXED; + } + + public static int compareSelections(List infos1, List infos2) { + // Compare best tracks of each selection with each other. + return max(infos1).compareTo(max(infos2)); + } } - /** Represents how well a text track matches the selection {@link Parameters}. */ - protected static final class TextTrackScore implements Comparable { + private static final class TextTrackInfo extends TrackInfo implements Comparable { - /** - * Whether the provided format is within the parameter constraints. If {@code false}, the format - * should not be selected. - */ - public final boolean isWithinConstraints; + public static ImmutableList createForTrackGroup( + int rendererIndex, + TrackGroup trackGroup, + Parameters params, + @Capabilities int[] formatSupport, + @Nullable String selectedAudioLanguage) { + ImmutableList.Builder listBuilder = ImmutableList.builder(); + for (int i = 0; i < trackGroup.length; i++) { + listBuilder.add( + new TextTrackInfo( + rendererIndex, + trackGroup, + /* trackIndex= */ i, + params, + formatSupport[i], + selectedAudioLanguage)); + } + return listBuilder.build(); + } + @SelectionEligibility private final int selectionEligibility; private final boolean isWithinRendererCapabilities; private final boolean isDefault; private final boolean isForced; @@ -2734,11 +2774,14 @@ public class DefaultTrackSelector extends MappingTrackSelector { private final int selectedAudioLanguageScore; private final boolean hasCaptionRoleFlags; - public TextTrackScore( - Format format, + public TextTrackInfo( + int rendererIndex, + TrackGroup trackGroup, + int trackIndex, Parameters parameters, @Capabilities int trackFormatSupport, @Nullable String selectedAudioLanguage) { + super(rendererIndex, trackGroup, trackIndex); isWithinRendererCapabilities = isSupported(trackFormatSupport, /* allowExceedsCapabilities= */ false); int maskedSelectionFlags = @@ -2773,22 +2816,31 @@ public class DefaultTrackSelector extends MappingTrackSelector { normalizeUndeterminedLanguageToNull(selectedAudioLanguage) == null; selectedAudioLanguageScore = getFormatLanguageScore(format, selectedAudioLanguage, selectedAudioLanguageUndetermined); - isWithinConstraints = + boolean isWithinConstraints = preferredLanguageScore > 0 || (parameters.preferredTextLanguages.isEmpty() && preferredRoleFlagsScore > 0) || isDefault || (isForced && selectedAudioLanguageScore > 0); + selectionEligibility = + isSupported(trackFormatSupport, parameters.exceedRendererCapabilitiesIfNecessary) + && isWithinConstraints + ? SELECTION_ELIGIBILITY_FIXED + : SELECTION_ELIGIBILITY_NO; } - /** - * Compares this score with another. - * - * @param other The other score to compare to. - * @return A positive integer if this score is better than the other. Zero if they are equal. A - * negative integer if this score is worse than the other. - */ @Override - public int compareTo(TextTrackScore other) { + @SelectionEligibility + public int getSelectionEligibility() { + return selectionEligibility; + } + + @Override + public boolean isCompatibleForAdaptationWith(TrackInfo otherTrack) { + return false; + } + + @Override + public int compareTo(TextTrackInfo other) { ComparisonChain chain = ComparisonChain.start() .compareFalseFirst( @@ -2815,13 +2867,13 @@ public class DefaultTrackSelector extends MappingTrackSelector { } return chain.result(); } + + public static int compareSelections(List infos1, List infos2) { + return infos1.get(0).compareTo(infos2.get(0)); + } } - /** - * Represents how well any other track (non video, audio or text) matches the selection {@link - * Parameters}. - */ - protected static final class OtherTrackScore implements Comparable { + private static final class OtherTrackScore implements Comparable { private final boolean isDefault; private final boolean isWithinRendererCapabilities; From 308eaf55c653b849c0402badc5e6b04bd3a324db Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Fri, 14 Jan 2022 15:01:20 +0000 Subject: [PATCH 02/12] GL: Update BitmapOverlayVideoProcessor naming conventions. To be more readable and consistent with Transformer GL. Tested by running gl-demo with no crash. PiperOrigin-RevId: 421815519 --- ...itmap_overlay_video_processor_fragment.glsl | 18 +++++++++--------- .../bitmap_overlay_video_processor_vertex.glsl | 12 ++++++------ .../gldemo/BitmapOverlayVideoProcessor.java | 14 +++++++------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/demos/gl/src/main/assets/bitmap_overlay_video_processor_fragment.glsl b/demos/gl/src/main/assets/bitmap_overlay_video_processor_fragment.glsl index 17fec0601d..1c39979a50 100644 --- a/demos/gl/src/main/assets/bitmap_overlay_video_processor_fragment.glsl +++ b/demos/gl/src/main/assets/bitmap_overlay_video_processor_fragment.glsl @@ -15,19 +15,19 @@ #extension GL_OES_EGL_image_external : require precision mediump float; // External texture containing video decoder output. -uniform samplerExternalOES tex_sampler_0; +uniform samplerExternalOES uTexSampler0; // Texture containing the overlap bitmap. -uniform sampler2D tex_sampler_1; +uniform sampler2D uTexSampler1; // Horizontal scaling factor for the overlap bitmap. -uniform float scaleX; +uniform float uScaleX; // Vertical scaling factory for the overlap bitmap. -uniform float scaleY; -varying vec2 v_texcoord; +uniform float uScaleY; +varying vec2 vTexCoords; void main() { - vec4 videoColor = texture2D(tex_sampler_0, v_texcoord); - vec4 overlayColor = texture2D(tex_sampler_1, - vec2(v_texcoord.x * scaleX, - v_texcoord.y * scaleY)); + vec4 videoColor = texture2D(uTexSampler0, vTexCoords); + vec4 overlayColor = texture2D(uTexSampler1, + vec2(vTexCoords.x * uScaleX, + vTexCoords.y * uScaleY)); // Blend the video decoder output and the overlay bitmap. gl_FragColor = videoColor * (1.0 - overlayColor.a) + overlayColor * overlayColor.a; diff --git a/demos/gl/src/main/assets/bitmap_overlay_video_processor_vertex.glsl b/demos/gl/src/main/assets/bitmap_overlay_video_processor_vertex.glsl index 1cb01b8293..b10aa6880e 100644 --- a/demos/gl/src/main/assets/bitmap_overlay_video_processor_vertex.glsl +++ b/demos/gl/src/main/assets/bitmap_overlay_video_processor_vertex.glsl @@ -11,11 +11,11 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -attribute vec4 a_position; -attribute vec4 a_texcoord; -uniform mat4 tex_transform; -varying vec2 v_texcoord; +attribute vec4 aFramePosition; +attribute vec4 aTexCoords; +uniform mat4 uTexTransform; +varying vec2 vTexCoords; void main() { - gl_Position = a_position; - v_texcoord = (tex_transform * a_texcoord).xy; + gl_Position = aFramePosition; + vTexCoords = (uTexTransform * aTexCoords).xy; } diff --git a/demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/BitmapOverlayVideoProcessor.java b/demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/BitmapOverlayVideoProcessor.java index 1294990ec5..8a5d135dee 100644 --- a/demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/BitmapOverlayVideoProcessor.java +++ b/demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/BitmapOverlayVideoProcessor.java @@ -86,9 +86,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throw new IllegalStateException(e); } program.setBufferAttribute( - "a_position", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); + "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); program.setBufferAttribute( - "a_texcoord", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); + "aTexCoords", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); GLES20.glGenTextures(1, textures, 0); GLES20.glBindTexture(GL10.GL_TEXTURE_2D, textures[0]); GLES20.glTexParameterf(GL10.GL_TEXTURE_2D, GL10.GL_TEXTURE_MIN_FILTER, GL10.GL_NEAREST); @@ -118,11 +118,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // Run the shader program. GlUtil.Program program = checkNotNull(this.program); - program.setSamplerTexIdUniform("tex_sampler_0", frameTexture, /* unit= */ 0); - program.setSamplerTexIdUniform("tex_sampler_1", textures[0], /* unit= */ 1); - program.setFloatUniform("scaleX", bitmapScaleX); - program.setFloatUniform("scaleY", bitmapScaleY); - program.setFloatsUniform("tex_transform", transformMatrix); + program.setSamplerTexIdUniform("uTexSampler0", frameTexture, /* unit= */ 0); + program.setSamplerTexIdUniform("uTexSampler1", textures[0], /* unit= */ 1); + program.setFloatUniform("uScaleX", bitmapScaleX); + program.setFloatUniform("uScaleY", bitmapScaleY); + program.setFloatsUniform("uTexTransform", transformMatrix); program.bindAttributesAndUniforms(); GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); From f747fed874482417bac71051167ca546af8b7e5b Mon Sep 17 00:00:00 2001 From: hschlueter Date: Fri, 14 Jan 2022 17:07:43 +0000 Subject: [PATCH 03/12] Add FallbackListener. The app will be notified about fallback using a callback on Transformer.Listener. Fallback may be applied separately for the audio and video options, so an intermediate internal FallbackListener is needed to accumulate and merge the track-specific changes to the TransformationRequest. PiperOrigin-RevId: 421839991 --- .../transformer/FallbackListener.java | 104 ++++++++++++++ .../transformer/TransformationRequest.java | 33 ++--- .../exoplayer2/transformer/Transformer.java | 14 ++ .../transformer/FallbackListenerTest.java | 134 ++++++++++++++++++ 4 files changed, 267 insertions(+), 18 deletions(-) create mode 100644 library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FallbackListener.java create mode 100644 library/transformer/src/test/java/com/google/android/exoplayer2/transformer/FallbackListenerTest.java diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FallbackListener.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FallbackListener.java new file mode 100644 index 0000000000..d335f0f63f --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FallbackListener.java @@ -0,0 +1,104 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.exoplayer2.transformer; + +import static com.google.android.exoplayer2.util.Assertions.checkState; + +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.MediaItem; +import com.google.android.exoplayer2.util.ListenerSet; +import com.google.android.exoplayer2.util.Util; + +/** + * Listener for fallback {@link TransformationRequest TransformationRequests} from the audio and + * video renderers. + */ +/* package */ final class FallbackListener { + + private final MediaItem mediaItem; + private final TransformationRequest originalTransformationRequest; + private final ListenerSet transformerListeners; + + private TransformationRequest fallbackTransformationRequest; + private int trackCount; + + /** + * Creates a new instance. + * + * @param mediaItem The {@link MediaItem} to transform. + * @param transformerListeners The {@link Transformer.Listener listeners} to forward events to. + * @param originalTransformationRequest The original {@link TransformationRequest}. + */ + public FallbackListener( + MediaItem mediaItem, + ListenerSet transformerListeners, + TransformationRequest originalTransformationRequest) { + this.mediaItem = mediaItem; + this.transformerListeners = transformerListeners; + this.originalTransformationRequest = originalTransformationRequest; + this.fallbackTransformationRequest = originalTransformationRequest; + } + + /** + * Registers an output track. + * + *

All tracks must be registered before a transformation request is {@link + * #onTransformationRequestFinalized(TransformationRequest) finalized}. + */ + public void registerTrack() { + trackCount++; + } + + /** + * Updates the fallback {@link TransformationRequest}. + * + *

Should be called with the final {@link TransformationRequest} for each track after all + * fallback has been applied. Calls {@link Transformer.Listener#onFallbackApplied(MediaItem, + * TransformationRequest, TransformationRequest)} once this method has been called for each track. + * + * @param transformationRequest The final {@link TransformationRequest} for a track. + * @throws IllegalStateException If called for more tracks than registered using {@link + * #registerTrack()}. + */ + public void onTransformationRequestFinalized(TransformationRequest transformationRequest) { + checkState(trackCount-- > 0); + + TransformationRequest.Builder fallbackRequestBuilder = + fallbackTransformationRequest.buildUpon(); + if (!Util.areEqual( + transformationRequest.audioMimeType, originalTransformationRequest.audioMimeType)) { + fallbackRequestBuilder.setAudioMimeType(transformationRequest.audioMimeType); + } + if (!Util.areEqual( + transformationRequest.videoMimeType, originalTransformationRequest.videoMimeType)) { + fallbackRequestBuilder.setVideoMimeType(transformationRequest.videoMimeType); + } + if (transformationRequest.outputHeight != originalTransformationRequest.outputHeight) { + fallbackRequestBuilder.setResolution(transformationRequest.outputHeight); + } + fallbackTransformationRequest = fallbackRequestBuilder.build(); + + if (trackCount == 0 && !originalTransformationRequest.equals(fallbackTransformationRequest)) { + transformerListeners.queueEvent( + /* eventFlag= */ C.INDEX_UNSET, + listener -> + listener.onFallbackApplied( + mediaItem, originalTransformationRequest, fallbackTransformationRequest)); + transformerListeners.flushEvents(); + } + } +} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java index 22a7afabb1..0dcf388cf5 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java @@ -23,6 +23,7 @@ import com.google.android.exoplayer2.extractor.mp4.Mp4Extractor; import com.google.android.exoplayer2.source.MediaSource; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; +import com.google.common.collect.ImmutableSet; /** A media transformation request. */ public final class TransformationRequest { @@ -30,6 +31,9 @@ public final class TransformationRequest { /** A builder for {@link TransformationRequest} instances. */ public static final class Builder { + private static final ImmutableSet SUPPORTED_OUTPUT_HEIGHTS = + ImmutableSet.of(144, 240, 360, 480, 720, 1080, 1440, 2160); + private Matrix transformationMatrix; private boolean flattenForSlowMotion; private int outputHeight; @@ -113,8 +117,9 @@ public final class TransformationRequest { } /** - * Sets the output resolution using the output height. The default value is the same height as - * the input. Output width will scale to preserve the input video's aspect ratio. + * Sets the output resolution using the output height. The default value {@link C#LENGTH_UNSET} + * corresponds to using the same height as the input. Output width will scale to preserve the + * input video's aspect ratio. * *

For now, only "popular" heights like 144, 240, 360, 480, 720, 1080, 1440, or 2160 are * supported, to ensure compatibility on different devices. @@ -128,24 +133,16 @@ public final class TransformationRequest { // TODO(b/201293185): Restructure to input a Presentation class. // TODO(b/201293185): Check encoder codec capabilities in order to allow arbitrary // resolutions and reasonable fallbacks. - if (outputHeight != 144 - && outputHeight != 240 - && outputHeight != 360 - && outputHeight != 480 - && outputHeight != 720 - && outputHeight != 1080 - && outputHeight != 1440 - && outputHeight != 2160) { - throw new IllegalArgumentException( - "Please use a height of 144, 240, 360, 480, 720, 1080, 1440, or 2160."); + if (outputHeight != C.LENGTH_UNSET && !SUPPORTED_OUTPUT_HEIGHTS.contains(outputHeight)) { + throw new IllegalArgumentException("Unsupported outputHeight: " + outputHeight); } this.outputHeight = outputHeight; return this; } /** - * Sets the video MIME type of the output. The default value is to use the same MIME type as the - * input. Supported values are: + * Sets the video MIME type of the output. The default value is {@code null} which corresponds + * to using the same MIME type as the input. Supported MIME types are: * *

    *
  • {@link MimeTypes#VIDEO_H263} @@ -157,7 +154,7 @@ public final class TransformationRequest { * @param videoMimeType The MIME type of the video samples in the output. * @return This builder. */ - public Builder setVideoMimeType(String videoMimeType) { + public Builder setVideoMimeType(@Nullable String videoMimeType) { // TODO(b/209469847): Validate videoMimeType here once deprecated // Transformer.Builder#setOuputMimeType(String) has been removed. this.videoMimeType = videoMimeType; @@ -165,8 +162,8 @@ public final class TransformationRequest { } /** - * Sets the audio MIME type of the output. The default value is to use the same MIME type as the - * input. Supported values are: + * Sets the audio MIME type of the output. The default value is {@code null} which corresponds + * to using the same MIME type as the input. Supported MIME types are: * *
      *
    • {@link MimeTypes#AUDIO_AAC} @@ -177,7 +174,7 @@ public final class TransformationRequest { * @param audioMimeType The MIME type of the audio samples in the output. * @return This builder. */ - public Builder setAudioMimeType(String audioMimeType) { + public Builder setAudioMimeType(@Nullable String audioMimeType) { // TODO(b/209469847): Validate audioMimeType here once deprecated // Transformer.Builder#setOuputMimeType(String) has been removed. this.audioMimeType = audioMimeType; diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java index 52f0e96f61..bd9eac24ac 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java @@ -462,6 +462,20 @@ public final class Transformer { */ default void onTransformationError( MediaItem inputMediaItem, TransformationException exception) {} + + /** + * Called when fallback to an alternative {@link TransformationRequest} is necessary to comply + * with muxer or device constraints. + * + * @param inputMediaItem The {@link MediaItem} for which the transformation is requested. + * @param originalTransformationRequest The unsupported {@link TransformationRequest} used when + * building {@link Transformer}. + * @param fallbackTransformationRequest The alternative {@link TransformationRequest}. + */ + default void onFallbackApplied( + MediaItem inputMediaItem, + TransformationRequest originalTransformationRequest, + TransformationRequest fallbackTransformationRequest) {} } /** Provider for views to show diagnostic information during transformation, for debugging. */ diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/FallbackListenerTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/FallbackListenerTest.java new file mode 100644 index 0000000000..f7f2894b45 --- /dev/null +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/FallbackListenerTest.java @@ -0,0 +1,134 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.transformer; + +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import android.net.Uri; +import android.os.Looper; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.MediaItem; +import com.google.android.exoplayer2.util.Clock; +import com.google.android.exoplayer2.util.ListenerSet; +import com.google.android.exoplayer2.util.MimeTypes; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link FallbackListener}. */ +@RunWith(AndroidJUnit4.class) +public class FallbackListenerTest { + + private static final MediaItem PLACEHOLDER_MEDIA_ITEM = MediaItem.fromUri(Uri.EMPTY); + + @Test + public void onTransformationRequestFinalized_withoutTrackRegistration_throwsException() { + TransformationRequest transformationRequest = new TransformationRequest.Builder().build(); + FallbackListener fallbackListener = + new FallbackListener(PLACEHOLDER_MEDIA_ITEM, createListenerSet(), transformationRequest); + + assertThrows( + IllegalStateException.class, + () -> fallbackListener.onTransformationRequestFinalized(transformationRequest)); + } + + @Test + public void onTransformationRequestFinalized_afterTrackRegistration_completesSuccessfully() { + TransformationRequest transformationRequest = new TransformationRequest.Builder().build(); + FallbackListener fallbackListener = + new FallbackListener(PLACEHOLDER_MEDIA_ITEM, createListenerSet(), transformationRequest); + + fallbackListener.registerTrack(); + fallbackListener.onTransformationRequestFinalized(transformationRequest); + } + + @Test + public void onTransformationRequestFinalized_withUnchangedRequest_doesNotCallback() { + TransformationRequest originalRequest = + new TransformationRequest.Builder().setAudioMimeType(MimeTypes.AUDIO_AAC).build(); + TransformationRequest unchangedRequest = originalRequest.buildUpon().build(); + Transformer.Listener mockListener = mock(Transformer.Listener.class); + FallbackListener fallbackListener = + new FallbackListener( + PLACEHOLDER_MEDIA_ITEM, createListenerSet(mockListener), originalRequest); + + fallbackListener.registerTrack(); + fallbackListener.onTransformationRequestFinalized(unchangedRequest); + + verify(mockListener, never()).onFallbackApplied(any(), any(), any()); + } + + @Test + public void onTransformationRequestFinalized_withDifferentRequest_callsCallback() { + TransformationRequest originalRequest = + new TransformationRequest.Builder().setAudioMimeType(MimeTypes.AUDIO_AAC).build(); + TransformationRequest audioFallbackRequest = + new TransformationRequest.Builder().setAudioMimeType(MimeTypes.AUDIO_AMR_WB).build(); + Transformer.Listener mockListener = mock(Transformer.Listener.class); + FallbackListener fallbackListener = + new FallbackListener( + PLACEHOLDER_MEDIA_ITEM, createListenerSet(mockListener), originalRequest); + + fallbackListener.registerTrack(); + fallbackListener.onTransformationRequestFinalized(audioFallbackRequest); + + verify(mockListener, times(1)) + .onFallbackApplied(PLACEHOLDER_MEDIA_ITEM, originalRequest, audioFallbackRequest); + } + + @Test + public void + onTransformationRequestFinalized_forMultipleTracks_callsCallbackOnceWithMergedRequest() { + TransformationRequest originalRequest = + new TransformationRequest.Builder().setAudioMimeType(MimeTypes.AUDIO_AAC).build(); + TransformationRequest audioFallbackRequest = + originalRequest.buildUpon().setAudioMimeType(MimeTypes.AUDIO_AMR_WB).build(); + TransformationRequest videoFallbackRequest = + originalRequest.buildUpon().setVideoMimeType(MimeTypes.VIDEO_H264).build(); + TransformationRequest mergedFallbackRequest = + new TransformationRequest.Builder() + .setAudioMimeType(MimeTypes.AUDIO_AMR_WB) + .setVideoMimeType(MimeTypes.VIDEO_H264) + .build(); + Transformer.Listener mockListener = mock(Transformer.Listener.class); + FallbackListener fallbackListener = + new FallbackListener( + PLACEHOLDER_MEDIA_ITEM, createListenerSet(mockListener), originalRequest); + + fallbackListener.registerTrack(); + fallbackListener.registerTrack(); + fallbackListener.onTransformationRequestFinalized(audioFallbackRequest); + fallbackListener.onTransformationRequestFinalized(videoFallbackRequest); + + verify(mockListener, times(1)) + .onFallbackApplied(PLACEHOLDER_MEDIA_ITEM, originalRequest, mergedFallbackRequest); + } + + private static ListenerSet createListenerSet( + Transformer.Listener transformerListener) { + ListenerSet listenerSet = createListenerSet(); + listenerSet.add(transformerListener); + return listenerSet; + } + + private static ListenerSet createListenerSet() { + return new ListenerSet<>(Looper.myLooper(), Clock.DEFAULT, (listener, flags) -> {}); + } +} From c4a2579b43edb2172491077977513f3147319791 Mon Sep 17 00:00:00 2001 From: bachinger Date: Fri, 14 Jan 2022 17:18:17 +0000 Subject: [PATCH 04/12] Inherit parent properties for manifests with dvb profile only Issue: google/ExoPlayer#9856 PiperOrigin-RevId: 421842579 --- RELEASENOTES.md | 4 + .../source/dash/BaseUrlExclusionList.java | 4 +- .../source/dash/manifest/BaseUrl.java | 10 +- .../dash/manifest/DashManifestParser.java | 92 ++++++++++++++----- .../source/dash/manifest/Representation.java | 2 +- .../source/dash/BaseUrlExclusionListTest.java | 31 ++++++- .../dash/manifest/DashManifestParserTest.java | 39 ++++++++ .../sample_mpd_availabilityTimeOffset_baseUrl | 2 +- .../media/mpd/sample_mpd_multiple_baseUrls | 2 +- ...mpd_relative_baseUrls_dvb_profile_declared | 17 ++++ ...relative_baseUrls_dvb_profile_not_declared | 16 ++++ .../mpd/sample_mpd_vod_location_fallback | 2 +- 12 files changed, 186 insertions(+), 35 deletions(-) create mode 100644 testdata/src/test/assets/media/mpd/sample_mpd_relative_baseUrls_dvb_profile_declared create mode 100644 testdata/src/test/assets/media/mpd/sample_mpd_relative_baseUrls_dvb_profile_not_declared diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 1704847087..a46f07da76 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -65,6 +65,10 @@ * Support the `forced-subtitle` track role ([#9727](https://github.com/google/ExoPlayer/issues/9727)). * Stop interpreting the `main` track role as `C.SELECTION_FLAG_DEFAULT`. + * Fix bug when base URLs have been assigned the same service location and + priority in manifests that do not declare the dvb namespace. This + prevents the exclusion logic to exclude base URL when they actually + should be used as a fallback base URL. * HLS: * Use chunkless preparation by default to improve start up time. If your renditions contain muxed closed-caption tracks that are *not* declared diff --git a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/BaseUrlExclusionList.java b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/BaseUrlExclusionList.java index e2110a7c10..1cf34b8f42 100644 --- a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/BaseUrlExclusionList.java +++ b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/BaseUrlExclusionList.java @@ -66,7 +66,9 @@ public final class BaseUrlExclusionList { public void exclude(BaseUrl baseUrlToExclude, long exclusionDurationMs) { long excludeUntilMs = SystemClock.elapsedRealtime() + exclusionDurationMs; addExclusion(baseUrlToExclude.serviceLocation, excludeUntilMs, excludedServiceLocations); - addExclusion(baseUrlToExclude.priority, excludeUntilMs, excludedPriorities); + if (baseUrlToExclude.priority != BaseUrl.PRIORITY_UNSET) { + addExclusion(baseUrlToExclude.priority, excludeUntilMs, excludedPriorities); + } } /** diff --git a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/BaseUrl.java b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/BaseUrl.java index 1792afd905..5cf79e7fc4 100644 --- a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/BaseUrl.java +++ b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/BaseUrl.java @@ -21,10 +21,12 @@ import com.google.common.base.Objects; /** A base URL, as defined by ISO 23009-1, 2nd edition, 5.6. and ETSI TS 103 285 V1.2.1, 10.8.2.1 */ public final class BaseUrl { - /** The default priority. */ - public static final int DEFAULT_PRIORITY = 1; /** The default weight. */ public static final int DEFAULT_WEIGHT = 1; + /** The default priority. */ + public static final int DEFAULT_DVB_PRIORITY = 1; + /** Constant representing an unset priority in a manifest that does not declare a DVB profile. */ + public static final int PRIORITY_UNSET = Integer.MIN_VALUE; /** The URL. */ public final String url; @@ -36,11 +38,11 @@ public final class BaseUrl { public final int weight; /** - * Creates an instance with {@link #DEFAULT_PRIORITY default priority}, {@link #DEFAULT_WEIGHT + * Creates an instance with {@link #PRIORITY_UNSET an unset priority}, {@link #DEFAULT_WEIGHT * default weight} and using the URL as the service location. */ public BaseUrl(String url) { - this(url, /* serviceLocation= */ url, DEFAULT_PRIORITY, DEFAULT_WEIGHT); + this(url, /* serviceLocation= */ url, PRIORITY_UNSET, DEFAULT_WEIGHT); } /** Creates an instance. */ diff --git a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/DashManifestParser.java b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/DashManifestParser.java index e0d7e2c43a..867018d33c 100644 --- a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/DashManifestParser.java +++ b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/DashManifestParser.java @@ -15,6 +15,10 @@ */ package com.google.android.exoplayer2.source.dash.manifest; +import static com.google.android.exoplayer2.source.dash.manifest.BaseUrl.DEFAULT_DVB_PRIORITY; +import static com.google.android.exoplayer2.source.dash.manifest.BaseUrl.DEFAULT_WEIGHT; +import static com.google.android.exoplayer2.source.dash.manifest.BaseUrl.PRIORITY_UNSET; + import android.net.Uri; import android.text.TextUtils; import android.util.Base64; @@ -103,14 +107,16 @@ public class DashManifestParser extends DefaultHandler "inputStream does not contain a valid media presentation description", /* cause= */ null); } - return parseMediaPresentationDescription(xpp, new BaseUrl(uri.toString())); + return parseMediaPresentationDescription(xpp, uri); } catch (XmlPullParserException e) { throw ParserException.createForMalformedManifest(/* message= */ null, /* cause= */ e); } } - protected DashManifest parseMediaPresentationDescription( - XmlPullParser xpp, BaseUrl documentBaseUrl) throws XmlPullParserException, IOException { + protected DashManifest parseMediaPresentationDescription(XmlPullParser xpp, Uri documentBaseUri) + throws XmlPullParserException, IOException { + boolean dvbProfileDeclared = + isDvbProfileDeclared(parseProfiles(xpp, "profiles", new String[0])); long availabilityStartTime = parseDateTime(xpp, "availabilityStartTime", C.TIME_UNSET); long durationMs = parseDuration(xpp, "mediaPresentationDuration", C.TIME_UNSET); long minBufferTimeMs = parseDuration(xpp, "minBufferTime", C.TIME_UNSET); @@ -128,6 +134,12 @@ public class DashManifestParser extends DefaultHandler Uri location = null; ServiceDescriptionElement serviceDescription = null; long baseUrlAvailabilityTimeOffsetUs = dynamic ? 0 : C.TIME_UNSET; + BaseUrl documentBaseUrl = + new BaseUrl( + documentBaseUri.toString(), + /* serviceLocation= */ documentBaseUri.toString(), + dvbProfileDeclared ? DEFAULT_DVB_PRIORITY : PRIORITY_UNSET, + DEFAULT_WEIGHT); ArrayList parentBaseUrls = Lists.newArrayList(documentBaseUrl); List periods = new ArrayList<>(); @@ -143,7 +155,7 @@ public class DashManifestParser extends DefaultHandler parseAvailabilityTimeOffsetUs(xpp, baseUrlAvailabilityTimeOffsetUs); seenFirstBaseUrl = true; } - baseUrls.addAll(parseBaseUrl(xpp, parentBaseUrls)); + baseUrls.addAll(parseBaseUrl(xpp, parentBaseUrls, dvbProfileDeclared)); } else if (XmlPullParserUtil.isStartTag(xpp, "ProgramInformation")) { programInformation = parseProgramInformation(xpp); } else if (XmlPullParserUtil.isStartTag(xpp, "UTCTiming")) { @@ -160,7 +172,8 @@ public class DashManifestParser extends DefaultHandler nextPeriodStartMs, baseUrlAvailabilityTimeOffsetUs, availabilityStartTime, - timeShiftBufferDepthMs); + timeShiftBufferDepthMs, + dvbProfileDeclared); Period period = periodWithDurationMs.first; if (period.startMs == C.TIME_UNSET) { if (dynamic) { @@ -280,7 +293,8 @@ public class DashManifestParser extends DefaultHandler long defaultStartMs, long baseUrlAvailabilityTimeOffsetUs, long availabilityStartTimeMs, - long timeShiftBufferDepthMs) + long timeShiftBufferDepthMs, + boolean dvbProfileDeclared) throws XmlPullParserException, IOException { @Nullable String id = xpp.getAttributeValue(null, "id"); long startMs = parseDuration(xpp, "start", defaultStartMs); @@ -302,7 +316,7 @@ public class DashManifestParser extends DefaultHandler parseAvailabilityTimeOffsetUs(xpp, baseUrlAvailabilityTimeOffsetUs); seenFirstBaseUrl = true; } - baseUrls.addAll(parseBaseUrl(xpp, parentBaseUrls)); + baseUrls.addAll(parseBaseUrl(xpp, parentBaseUrls, dvbProfileDeclared)); } else if (XmlPullParserUtil.isStartTag(xpp, "AdaptationSet")) { adaptationSets.add( parseAdaptationSet( @@ -313,7 +327,8 @@ public class DashManifestParser extends DefaultHandler baseUrlAvailabilityTimeOffsetUs, segmentBaseAvailabilityTimeOffsetUs, periodStartUnixTimeMs, - timeShiftBufferDepthMs)); + timeShiftBufferDepthMs, + dvbProfileDeclared)); } else if (XmlPullParserUtil.isStartTag(xpp, "EventStream")) { eventStreams.add(parseEventStream(xpp)); } else if (XmlPullParserUtil.isStartTag(xpp, "SegmentBase")) { @@ -373,7 +388,8 @@ public class DashManifestParser extends DefaultHandler long baseUrlAvailabilityTimeOffsetUs, long segmentBaseAvailabilityTimeOffsetUs, long periodStartUnixTimeMs, - long timeShiftBufferDepthMs) + long timeShiftBufferDepthMs, + boolean dvbProfileDeclared) throws XmlPullParserException, IOException { int id = parseInt(xpp, "id", AdaptationSet.ID_UNSET); @C.TrackType int contentType = parseContentType(xpp); @@ -406,7 +422,7 @@ public class DashManifestParser extends DefaultHandler parseAvailabilityTimeOffsetUs(xpp, baseUrlAvailabilityTimeOffsetUs); seenFirstBaseUrl = true; } - baseUrls.addAll(parseBaseUrl(xpp, parentBaseUrls)); + baseUrls.addAll(parseBaseUrl(xpp, parentBaseUrls, dvbProfileDeclared)); } else if (XmlPullParserUtil.isStartTag(xpp, "ContentProtection")) { Pair contentProtection = parseContentProtection(xpp); if (contentProtection.first != null) { @@ -450,7 +466,8 @@ public class DashManifestParser extends DefaultHandler periodDurationMs, baseUrlAvailabilityTimeOffsetUs, segmentBaseAvailabilityTimeOffsetUs, - timeShiftBufferDepthMs); + timeShiftBufferDepthMs, + dvbProfileDeclared); contentType = checkContentTypeConsistency( contentType, MimeTypes.getTrackType(representationInfo.format.sampleMimeType)); @@ -650,7 +667,8 @@ public class DashManifestParser extends DefaultHandler long periodDurationMs, long baseUrlAvailabilityTimeOffsetUs, long segmentBaseAvailabilityTimeOffsetUs, - long timeShiftBufferDepthMs) + long timeShiftBufferDepthMs, + boolean dvbProfileDeclared) throws XmlPullParserException, IOException { String id = xpp.getAttributeValue(null, "id"); int bandwidth = parseInt(xpp, "bandwidth", Format.NO_VALUE); @@ -679,7 +697,7 @@ public class DashManifestParser extends DefaultHandler parseAvailabilityTimeOffsetUs(xpp, baseUrlAvailabilityTimeOffsetUs); seenFirstBaseUrl = true; } - baseUrls.addAll(parseBaseUrl(xpp, parentBaseUrls)); + baseUrls.addAll(parseBaseUrl(xpp, parentBaseUrls, dvbProfileDeclared)); } else if (XmlPullParserUtil.isStartTag(xpp, "AudioChannelConfiguration")) { audioChannels = parseAudioChannelConfiguration(xpp); } else if (XmlPullParserUtil.isStartTag(xpp, "SegmentBase")) { @@ -1371,35 +1389,42 @@ public class DashManifestParser extends DefaultHandler * * @param xpp The parser from which to read. * @param parentBaseUrls The parent base URLs for resolving the parsed URLs. + * @param dvbProfileDeclared Whether the dvb profile is declared. * @throws XmlPullParserException If an error occurs parsing the element. * @throws IOException If an error occurs reading the element. * @return The list of parsed and resolved URLs. */ - protected List parseBaseUrl(XmlPullParser xpp, List parentBaseUrls) + protected List parseBaseUrl( + XmlPullParser xpp, List parentBaseUrls, boolean dvbProfileDeclared) throws XmlPullParserException, IOException { @Nullable String priorityValue = xpp.getAttributeValue(null, "dvb:priority"); int priority = - priorityValue != null ? Integer.parseInt(priorityValue) : BaseUrl.DEFAULT_PRIORITY; + priorityValue != null + ? Integer.parseInt(priorityValue) + : (dvbProfileDeclared ? DEFAULT_DVB_PRIORITY : PRIORITY_UNSET); @Nullable String weightValue = xpp.getAttributeValue(null, "dvb:weight"); - int weight = weightValue != null ? Integer.parseInt(weightValue) : BaseUrl.DEFAULT_WEIGHT; + int weight = weightValue != null ? Integer.parseInt(weightValue) : DEFAULT_WEIGHT; @Nullable String serviceLocation = xpp.getAttributeValue(null, "serviceLocation"); String baseUrl = parseText(xpp, "BaseURL"); - if (serviceLocation == null) { - serviceLocation = baseUrl; - } if (UriUtil.isAbsolute(baseUrl)) { + if (serviceLocation == null) { + serviceLocation = baseUrl; + } return Lists.newArrayList(new BaseUrl(baseUrl, serviceLocation, priority, weight)); } List baseUrls = new ArrayList<>(); for (int i = 0; i < parentBaseUrls.size(); i++) { BaseUrl parentBaseUrl = parentBaseUrls.get(i); - priority = parentBaseUrl.priority; - weight = parentBaseUrl.weight; - serviceLocation = parentBaseUrl.serviceLocation; - baseUrls.add( - new BaseUrl( - UriUtil.resolve(parentBaseUrl.url, baseUrl), serviceLocation, priority, weight)); + String resolvedBaseUri = UriUtil.resolve(parentBaseUrl.url, baseUrl); + String resolvedServiceLocation = serviceLocation == null ? resolvedBaseUri : serviceLocation; + if (dvbProfileDeclared) { + // Inherit parent properties only if dvb profile is declared. + priority = parentBaseUrl.priority; + weight = parentBaseUrl.weight; + resolvedServiceLocation = parentBaseUrl.serviceLocation; + } + baseUrls.add(new BaseUrl(resolvedBaseUri, resolvedServiceLocation, priority, weight)); } return baseUrls; } @@ -1581,6 +1606,14 @@ public class DashManifestParser extends DefaultHandler } } + protected String[] parseProfiles(XmlPullParser xpp, String attributeName, String[] defaultValue) { + @Nullable String attributeValue = xpp.getAttributeValue(/* namespace= */ null, attributeName); + if (attributeValue == null) { + return defaultValue; + } + return attributeValue.split(","); + } + // Utility methods. /** @@ -1907,6 +1940,15 @@ public class DashManifestParser extends DefaultHandler return availabilityTimeOffsetUs; } + private boolean isDvbProfileDeclared(String[] profiles) { + for (String profile : profiles) { + if (profile.startsWith("urn:dvb:dash:profile:dvb-dash:")) { + return true; + } + } + return false; + } + /** A parsed Representation element. */ protected static final class RepresentationInfo { diff --git a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/Representation.java b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/Representation.java index 1e02e9cd62..62787cee4c 100644 --- a/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/Representation.java +++ b/library/dash/src/main/java/com/google/android/exoplayer2/source/dash/manifest/Representation.java @@ -213,7 +213,7 @@ public abstract class Representation { new RangedUri(null, initializationStart, initializationEnd - initializationStart + 1); SingleSegmentBase segmentBase = new SingleSegmentBase(rangedUri, 1, 0, indexStart, indexEnd - indexStart + 1); - List baseUrls = ImmutableList.of(new BaseUrl(uri)); + ImmutableList baseUrls = ImmutableList.of(new BaseUrl(uri)); return new SingleSegmentRepresentation( revisionId, format, diff --git a/library/dash/src/test/java/com/google/android/exoplayer2/source/dash/BaseUrlExclusionListTest.java b/library/dash/src/test/java/com/google/android/exoplayer2/source/dash/BaseUrlExclusionListTest.java index cb75903c81..22391608be 100644 --- a/library/dash/src/test/java/com/google/android/exoplayer2/source/dash/BaseUrlExclusionListTest.java +++ b/library/dash/src/test/java/com/google/android/exoplayer2/source/dash/BaseUrlExclusionListTest.java @@ -15,6 +15,8 @@ */ package com.google.android.exoplayer2.source.dash; +import static com.google.android.exoplayer2.source.dash.manifest.BaseUrl.DEFAULT_DVB_PRIORITY; +import static com.google.android.exoplayer2.source.dash.manifest.BaseUrl.DEFAULT_WEIGHT; import static com.google.android.exoplayer2.upstream.DefaultLoadErrorHandlingPolicy.DEFAULT_LOCATION_EXCLUSION_MS; import static com.google.common.truth.Truth.assertThat; import static org.mockito.ArgumentMatchers.anyInt; @@ -173,6 +175,32 @@ public class BaseUrlExclusionListTest { assertThat(baseUrlExclusionList.getPriorityCountAfterExclusion(baseUrls)).isEqualTo(2); } + @Test + public void selectBaseUrl_priorityUnset_isNotExcluded() { + BaseUrlExclusionList baseUrlExclusionList = new BaseUrlExclusionList(); + ImmutableList baseUrls = + ImmutableList.of( + new BaseUrl( + /* url= */ "a-1", + /* serviceLocation= */ "a", + BaseUrl.PRIORITY_UNSET, + /* weight= */ 1), + new BaseUrl( + /* url= */ "a-2", + /* serviceLocation= */ "a", + BaseUrl.PRIORITY_UNSET, + /* weight= */ 1), + new BaseUrl( + /* url= */ "b", + /* serviceLocation= */ "b", + BaseUrl.PRIORITY_UNSET, + /* weight= */ 1)); + + baseUrlExclusionList.exclude(baseUrls.get(0), 10_000); + + assertThat(baseUrlExclusionList.selectBaseUrl(baseUrls).serviceLocation).isEqualTo("b"); + } + @Test public void selectBaseUrl_emptyBaseUrlList_selectionIsNull() { BaseUrlExclusionList baseUrlExclusionList = new BaseUrlExclusionList(); @@ -183,7 +211,8 @@ public class BaseUrlExclusionListTest { @Test public void reset_dropsAllExclusions() { BaseUrlExclusionList baseUrlExclusionList = new BaseUrlExclusionList(); - List baseUrls = ImmutableList.of(new BaseUrl("a")); + ImmutableList baseUrls = + ImmutableList.of(new BaseUrl("a", "a", DEFAULT_DVB_PRIORITY, DEFAULT_WEIGHT)); baseUrlExclusionList.exclude(baseUrls.get(0), 5000); baseUrlExclusionList.reset(); diff --git a/library/dash/src/test/java/com/google/android/exoplayer2/source/dash/manifest/DashManifestParserTest.java b/library/dash/src/test/java/com/google/android/exoplayer2/source/dash/manifest/DashManifestParserTest.java index 6516f9d71a..3a2ce9a309 100644 --- a/library/dash/src/test/java/com/google/android/exoplayer2/source/dash/manifest/DashManifestParserTest.java +++ b/library/dash/src/test/java/com/google/android/exoplayer2/source/dash/manifest/DashManifestParserTest.java @@ -61,6 +61,10 @@ public class DashManifestParserTest { "media/mpd/sample_mpd_availabilityTimeOffset_baseUrl"; private static final String SAMPLE_MPD_MULTIPLE_BASE_URLS = "media/mpd/sample_mpd_multiple_baseUrls"; + private static final String SAMPLE_MPD_RELATIVE_BASE_URLS_DVB_PROFILE_NOT_DECLARED = + "media/mpd/sample_mpd_relative_baseUrls_dvb_profile_not_declared"; + private static final String SAMPLE_MPD_RELATIVE_BASE_URLS_DVB_PROFILE_DECLARED = + "media/mpd/sample_mpd_relative_baseUrls_dvb_profile_declared"; private static final String SAMPLE_MPD_AVAILABILITY_TIME_OFFSET_SEGMENT_TEMPLATE = "media/mpd/sample_mpd_availabilityTimeOffset_segmentTemplate"; private static final String SAMPLE_MPD_AVAILABILITY_TIME_OFFSET_SEGMENT_LIST = @@ -748,6 +752,41 @@ public class DashManifestParserTest { assertThat(textBaseUrls.get(0).serviceLocation).isEqualTo("e"); } + @Test + public void baseUrl_relativeBaseUrlsNoDvbNamespace_hasDifferentPrioritiesAndServiceLocation() + throws IOException { + DashManifestParser parser = new DashManifestParser(); + DashManifest manifest = + parser.parse( + Uri.parse("https://example.com/test.mpd"), + TestUtil.getInputStream( + ApplicationProvider.getApplicationContext(), + SAMPLE_MPD_RELATIVE_BASE_URLS_DVB_PROFILE_NOT_DECLARED)); + + ImmutableList baseUrls = + manifest.getPeriod(0).adaptationSets.get(0).representations.get(0).baseUrls; + assertThat(baseUrls.get(0).priority).isEqualTo(BaseUrl.PRIORITY_UNSET); + assertThat(baseUrls.get(1).priority).isEqualTo(BaseUrl.PRIORITY_UNSET); + assertThat(baseUrls.get(0).serviceLocation).isNotEqualTo(baseUrls.get(1).serviceLocation); + } + + @Test + public void baseUrl_relativeBaseUrlsWithDvbNamespace_inheritsPrioritiesAndServiceLocation() + throws IOException { + DashManifestParser parser = new DashManifestParser(); + DashManifest manifest = + parser.parse( + Uri.parse("https://example.com/test.mpd"), + TestUtil.getInputStream( + ApplicationProvider.getApplicationContext(), + SAMPLE_MPD_RELATIVE_BASE_URLS_DVB_PROFILE_DECLARED)); + + ImmutableList baseUrls = + manifest.getPeriod(0).adaptationSets.get(0).representations.get(0).baseUrls; + assertThat(baseUrls.get(0).priority).isEqualTo(baseUrls.get(1).priority); + assertThat(baseUrls.get(0).serviceLocation).isEqualTo(baseUrls.get(1).serviceLocation); + } + @Test public void serviceDescriptionElement_allValuesSet() throws IOException { DashManifestParser parser = new DashManifestParser(); diff --git a/testdata/src/test/assets/media/mpd/sample_mpd_availabilityTimeOffset_baseUrl b/testdata/src/test/assets/media/mpd/sample_mpd_availabilityTimeOffset_baseUrl index 365416825c..7db98f166a 100644 --- a/testdata/src/test/assets/media/mpd/sample_mpd_availabilityTimeOffset_baseUrl +++ b/testdata/src/test/assets/media/mpd/sample_mpd_availabilityTimeOffset_baseUrl @@ -3,7 +3,7 @@ xmlns="urn:mpeg:DASH:schema:MPD:2011" xsi:schemaLocation="urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd" xmlns:dvb="urn:dvb:dash:dash-extensions:2014-1" - profiles="urn:mpeg:dash:profile:isoff-main:2011" + profiles="urn:mpeg:dash:profile:isoff-main:2011,urn:dvb:dash:profile:dvb-dash:2014" type="dynamic" availabilityStartTime="2016-10-14T17:00:17"> http://video.com/baseUrl diff --git a/testdata/src/test/assets/media/mpd/sample_mpd_multiple_baseUrls b/testdata/src/test/assets/media/mpd/sample_mpd_multiple_baseUrls index 228e64a03b..cb09376176 100644 --- a/testdata/src/test/assets/media/mpd/sample_mpd_multiple_baseUrls +++ b/testdata/src/test/assets/media/mpd/sample_mpd_multiple_baseUrls @@ -3,7 +3,7 @@ xmlns="urn:mpeg:DASH:schema:MPD:2011" xsi:schemaLocation="urn:mpeg:DASH:schema:MPD:2011 DASH-MPD.xsd" xmlns:dvb="urn:dvb:dash:dash-extensions:2014-1" - profiles="urn:mpeg:dash:profile:isoff-main:2011" + profiles="urn:mpeg:dash:profile:isoff-main:2011,urn:dvb:dash:profile:dvb-dash:2014" type="dynamic" availabilityStartTime="2016-10-14T17:00:17"> http://video.com/baseUrl/a/ diff --git a/testdata/src/test/assets/media/mpd/sample_mpd_relative_baseUrls_dvb_profile_declared b/testdata/src/test/assets/media/mpd/sample_mpd_relative_baseUrls_dvb_profile_declared new file mode 100644 index 0000000000..270df26127 --- /dev/null +++ b/testdata/src/test/assets/media/mpd/sample_mpd_relative_baseUrls_dvb_profile_declared @@ -0,0 +1,17 @@ + + + + //anotherhost.com/some/url/1/ + //anotherhost.com/some/url/2/ + + + + + + diff --git a/testdata/src/test/assets/media/mpd/sample_mpd_relative_baseUrls_dvb_profile_not_declared b/testdata/src/test/assets/media/mpd/sample_mpd_relative_baseUrls_dvb_profile_not_declared new file mode 100644 index 0000000000..c622ba56a3 --- /dev/null +++ b/testdata/src/test/assets/media/mpd/sample_mpd_relative_baseUrls_dvb_profile_not_declared @@ -0,0 +1,16 @@ + + + + //anotherhost.com/some/url/1/ + //anotherhost.com/some/url/2/ + + + + + + diff --git a/testdata/src/test/assets/media/mpd/sample_mpd_vod_location_fallback b/testdata/src/test/assets/media/mpd/sample_mpd_vod_location_fallback index 32207dde45..acc7886b83 100644 --- a/testdata/src/test/assets/media/mpd/sample_mpd_vod_location_fallback +++ b/testdata/src/test/assets/media/mpd/sample_mpd_vod_location_fallback @@ -4,7 +4,7 @@ xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011" xmlns:dvb="urn:dvb:dash:dash-extensions:2014-1" minBufferTime="PT1S" - profiles="urn:mpeg:dash:profile:isoff-main:2011" + profiles="urn:mpeg:dash:profile:isoff-main:2011,urn:dvb:dash:profile:dvb-dash:2014" type="static" mediaPresentationDuration="PT904S"> http://video.com/baseUrl/a/ From d0c13733e4586bfafb1ede462329f2c802fbadf4 Mon Sep 17 00:00:00 2001 From: olly Date: Fri, 14 Jan 2022 18:10:39 +0000 Subject: [PATCH 05/12] Don't sample VSYNC when not started PiperOrigin-RevId: 421855453 --- .../video/MediaCodecVideoRenderer.java | 2 -- .../video/VideoFrameReleaseHelper.java | 18 +++++------------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index 7f53747ea3..b29b0256a4 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -490,7 +490,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { releaseCodec(); } eventDispatcher.enabled(decoderCounters); - frameReleaseHelper.onEnabled(); mayRenderFirstFrameAfterEnableIfNotStarted = mayRenderStartOfStream; renderedFirstFrameAfterEnable = false; } @@ -558,7 +557,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { clearReportedVideoSize(); clearRenderedFirstFrame(); haveReportedFirstFrameRenderedForCurrentSurface = false; - frameReleaseHelper.onDisabled(); tunnelingOnFrameRenderedListener = null; try { super.onDisabled(); diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameReleaseHelper.java b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameReleaseHelper.java index 33a445f52f..1875eea81a 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameReleaseHelper.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameReleaseHelper.java @@ -149,18 +149,14 @@ public final class VideoFrameReleaseHelper { updateSurfacePlaybackFrameRate(/* forceUpdate= */ true); } - /** Called when the renderer is enabled. */ - public void onEnabled() { - if (displayHelper != null) { - checkNotNull(vsyncSampler).addObserver(); - displayHelper.register(this::updateDefaultDisplayRefreshRateParams); - } - } - /** Called when the renderer is started. */ public void onStarted() { started = true; resetAdjustment(); + if (displayHelper != null) { + checkNotNull(vsyncSampler).addObserver(); + displayHelper.register(this::updateDefaultDisplayRefreshRateParams); + } updateSurfacePlaybackFrameRate(/* forceUpdate= */ false); } @@ -227,15 +223,11 @@ public final class VideoFrameReleaseHelper { /** Called when the renderer is stopped. */ public void onStopped() { started = false; - clearSurfaceFrameRate(); - } - - /** Called when the renderer is disabled. */ - public void onDisabled() { if (displayHelper != null) { displayHelper.unregister(); checkNotNull(vsyncSampler).removeObserver(); } + clearSurfaceFrameRate(); } // Frame release time adjustment. From 3f47da1fd6ffa5d8412969b0b3c62d205f313ba5 Mon Sep 17 00:00:00 2001 From: tonihei Date: Mon, 17 Jan 2022 09:18:42 +0000 Subject: [PATCH 06/12] Reset readingPositionUs in BaseRenderer.enable This does currently only happen by chance in replaceStream (called from enable) if the stream previosly played read until C.TIME_END_OF_SOURCE. enable already makes all changes done in resetPosition (except resetting the reading position), so it's less error-prone and makes the intention clearer if the same code is called from both enable and resetPosition. The effect of this bug was quite limited because the numerical value of readingPositionUs was only relevant for periods with changing durations and server-side inserted ads. PiperOrigin-RevId: 422300690 --- .../java/com/google/android/exoplayer2/BaseRenderer.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/library/core/src/main/java/com/google/android/exoplayer2/BaseRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/BaseRenderer.java index 683db1ecbc..ac35a5148a 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/BaseRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/BaseRenderer.java @@ -99,10 +99,9 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities { Assertions.checkState(state == STATE_DISABLED); this.configuration = configuration; state = STATE_ENABLED; - lastResetPositionUs = positionUs; onEnabled(joining, mayRenderStartOfStream); replaceStream(formats, stream, startPositionUs, offsetUs); - onPositionReset(positionUs, joining); + resetPosition(positionUs, joining); } @Override @@ -159,10 +158,14 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities { @Override public final void resetPosition(long positionUs) throws ExoPlaybackException { + resetPosition(positionUs, /* joining= */ false); + } + + private void resetPosition(long positionUs, boolean joining) throws ExoPlaybackException { streamIsFinal = false; lastResetPositionUs = positionUs; readingPositionUs = positionUs; - onPositionReset(positionUs, false); + onPositionReset(positionUs, joining); } @Override From 327ec97e582bbde9be2c0d28415cb68ae0ed32ad Mon Sep 17 00:00:00 2001 From: tonihei Date: Mon, 17 Jan 2022 10:07:40 +0000 Subject: [PATCH 07/12] Reorder adaptive video track preferences. This change moves the video track selection to the generic selection method introcuced for audio and text. This ensures we can apply the same criteria for fixed and adaptive video track selections. Implicitly, this reorders the preferences for adaptive tracks to give non-quality preferences (like preferred MIME type or preferred role flags) a higher priority than number of tracks in the selection. Issue: google/ExoPlayer#9519 PiperOrigin-RevId: 422310902 --- RELEASENOTES.md | 3 + .../trackselection/DefaultTrackSelector.java | 520 ++++++------------ .../DefaultTrackSelectorTest.java | 38 +- 3 files changed, 186 insertions(+), 375 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index a46f07da76..8e1c8106ad 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -31,6 +31,9 @@ * Disable automatic speed adjustment for live streams that neither have low-latency features nor a user request setting the speed ((#9329)[https://github.com/google/ExoPlayer/issues/9329]). + * Update video track selection logic to take preferred MIME types and role + flags into account when selecting multiple video tracks for adaptation + ((#9519)[https://github.com/google/ExoPlayer/issues/9519]). * Android 12 compatibility: * Upgrade the Cast extension to depend on `com.google.android.gms:play-services-cast-framework:20.1.0`. Earlier diff --git a/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java b/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java index f25c4232a2..37ad7e002b 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java @@ -57,7 +57,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -1358,7 +1357,6 @@ public class DefaultTrackSelector extends MappingTrackSelector { */ private static final float FRACTION_TO_CONSIDER_FULLSCREEN = 0.98f; - private static final int[] NO_TRACKS = new int[0]; /** Ordering of two format values. A known value is considered greater than Format#NO_VALUE. */ private static final Ordering FORMAT_VALUE_ORDERING = Ordering.from( @@ -1650,20 +1648,15 @@ public class DefaultTrackSelector extends MappingTrackSelector { ExoTrackSelection.@NullableType Definition[] definitions = new ExoTrackSelection.Definition[rendererCount]; - boolean selectedVideoTracks = false; - for (int i = 0; i < rendererCount; i++) { - if (C.TRACK_TYPE_VIDEO == mappedTrackInfo.getRendererType(i)) { - if (!selectedVideoTracks) { - definitions[i] = - selectVideoTrack( - mappedTrackInfo.getTrackGroups(i), - rendererFormatSupports[i], - rendererMixedMimeTypeAdaptationSupports[i], - params, - /* enableAdaptiveTrackSelection= */ true); - selectedVideoTracks = definitions[i] != null; - } - } + @Nullable + Pair selectedVideo = + selectVideoTrack( + mappedTrackInfo, + rendererFormatSupports, + rendererMixedMimeTypeAdaptationSupports, + params); + if (selectedVideo != null) { + definitions[selectedVideo.second] = selectedVideo.first; } @Nullable @@ -1707,303 +1700,34 @@ public class DefaultTrackSelector extends MappingTrackSelector { /** * Called by {@link #selectAllTracks(MappedTrackInfo, int[][][], int[], Parameters)} to create a - * {@link ExoTrackSelection} for a video renderer. + * {@link ExoTrackSelection.Definition} for a video track selection. * - * @param groups The {@link TrackGroupArray} mapped to the renderer. - * @param formatSupport The {@link Capabilities} for each mapped track, indexed by track group and - * track (in that order). - * @param mixedMimeTypeAdaptationSupports The {@link AdaptiveSupport} for mixed MIME type - * adaptation for the renderer. + * @param mappedTrackInfo Mapped track information. + * @param rendererFormatSupports The {@link Capabilities} for each mapped track, indexed by + * renderer, track group and track (in that order). + * @param mixedMimeTypeSupports The {@link AdaptiveSupport} for mixed MIME type adaptation for the + * renderer. * @param params The selector's current constraint parameters. - * @param enableAdaptiveTrackSelection Whether adaptive track selection is allowed. - * @return The {@link ExoTrackSelection.Definition} for the renderer, or null if no selection was - * made. + * @return A pair of the selected {@link ExoTrackSelection.Definition} and the corresponding + * renderer index, or null if no selection was made. * @throws ExoPlaybackException If an error occurs while selecting the tracks. */ + @SuppressLint("WrongConstant") // Lint doesn't understand arrays of IntDefs. @Nullable - protected ExoTrackSelection.Definition selectVideoTrack( - TrackGroupArray groups, - @Capabilities int[][] formatSupport, - @AdaptiveSupport int mixedMimeTypeAdaptationSupports, - Parameters params, - boolean enableAdaptiveTrackSelection) + protected Pair selectVideoTrack( + MappedTrackInfo mappedTrackInfo, + @Capabilities int[][][] rendererFormatSupports, + @AdaptiveSupport int[] mixedMimeTypeSupports, + Parameters params) throws ExoPlaybackException { - ExoTrackSelection.Definition definition = null; - if (!params.forceHighestSupportedBitrate - && !params.forceLowestBitrate - && enableAdaptiveTrackSelection) { - definition = - selectAdaptiveVideoTrack(groups, formatSupport, mixedMimeTypeAdaptationSupports, params); - } - if (definition == null) { - definition = selectFixedVideoTrack(groups, formatSupport, params); - } - return definition; - } - - @Nullable - private static ExoTrackSelection.Definition selectAdaptiveVideoTrack( - TrackGroupArray groups, - @Capabilities int[][] formatSupport, - @AdaptiveSupport int mixedMimeTypeAdaptationSupports, - Parameters params) { - int requiredAdaptiveSupport = - params.allowVideoNonSeamlessAdaptiveness - ? (RendererCapabilities.ADAPTIVE_NOT_SEAMLESS | RendererCapabilities.ADAPTIVE_SEAMLESS) - : RendererCapabilities.ADAPTIVE_SEAMLESS; - boolean allowMixedMimeTypes = - params.allowVideoMixedMimeTypeAdaptiveness - && (mixedMimeTypeAdaptationSupports & requiredAdaptiveSupport) != 0; - for (int i = 0; i < groups.length; i++) { - TrackGroup group = groups.get(i); - int[] adaptiveTracks = - getAdaptiveVideoTracksForGroup( - group, - formatSupport[i], - allowMixedMimeTypes, - requiredAdaptiveSupport, - params.maxVideoWidth, - params.maxVideoHeight, - params.maxVideoFrameRate, - params.maxVideoBitrate, - params.minVideoWidth, - params.minVideoHeight, - params.minVideoFrameRate, - params.minVideoBitrate, - params.viewportWidth, - params.viewportHeight, - params.viewportOrientationMayChange); - if (adaptiveTracks.length > 0) { - return new ExoTrackSelection.Definition(group, adaptiveTracks); - } - } - return null; - } - - private static int[] getAdaptiveVideoTracksForGroup( - TrackGroup group, - @Capabilities int[] formatSupport, - boolean allowMixedMimeTypes, - int requiredAdaptiveSupport, - int maxVideoWidth, - int maxVideoHeight, - int maxVideoFrameRate, - int maxVideoBitrate, - int minVideoWidth, - int minVideoHeight, - int minVideoFrameRate, - int minVideoBitrate, - int viewportWidth, - int viewportHeight, - boolean viewportOrientationMayChange) { - if (group.length < 2) { - return NO_TRACKS; - } - - List selectedTrackIndices = - getViewportFilteredTrackIndices( - group, viewportWidth, viewportHeight, viewportOrientationMayChange); - if (selectedTrackIndices.size() < 2) { - return NO_TRACKS; - } - - String selectedMimeType = null; - if (!allowMixedMimeTypes) { - // Select the mime type for which we have the most adaptive tracks. - HashSet<@NullableType String> seenMimeTypes = new HashSet<>(); - int selectedMimeTypeTrackCount = 0; - for (int i = 0; i < selectedTrackIndices.size(); i++) { - int trackIndex = selectedTrackIndices.get(i); - String sampleMimeType = group.getFormat(trackIndex).sampleMimeType; - if (seenMimeTypes.add(sampleMimeType)) { - int countForMimeType = - getAdaptiveVideoTrackCountForMimeType( - group, - formatSupport, - requiredAdaptiveSupport, - sampleMimeType, - maxVideoWidth, - maxVideoHeight, - maxVideoFrameRate, - maxVideoBitrate, - minVideoWidth, - minVideoHeight, - minVideoFrameRate, - minVideoBitrate, - selectedTrackIndices); - if (countForMimeType > selectedMimeTypeTrackCount) { - selectedMimeType = sampleMimeType; - selectedMimeTypeTrackCount = countForMimeType; - } - } - } - } - - // Filter by the selected mime type. - filterAdaptiveVideoTrackCountForMimeType( - group, - formatSupport, - requiredAdaptiveSupport, - selectedMimeType, - maxVideoWidth, - maxVideoHeight, - maxVideoFrameRate, - maxVideoBitrate, - minVideoWidth, - minVideoHeight, - minVideoFrameRate, - minVideoBitrate, - selectedTrackIndices); - - return selectedTrackIndices.size() < 2 ? NO_TRACKS : Ints.toArray(selectedTrackIndices); - } - - private static int getAdaptiveVideoTrackCountForMimeType( - TrackGroup group, - @Capabilities int[] formatSupport, - int requiredAdaptiveSupport, - @Nullable String mimeType, - int maxVideoWidth, - int maxVideoHeight, - int maxVideoFrameRate, - int maxVideoBitrate, - int minVideoWidth, - int minVideoHeight, - int minVideoFrameRate, - int minVideoBitrate, - List selectedTrackIndices) { - int adaptiveTrackCount = 0; - for (int i = 0; i < selectedTrackIndices.size(); i++) { - int trackIndex = selectedTrackIndices.get(i); - if (isSupportedAdaptiveVideoTrack( - group.getFormat(trackIndex), - mimeType, - formatSupport[trackIndex], - requiredAdaptiveSupport, - maxVideoWidth, - maxVideoHeight, - maxVideoFrameRate, - maxVideoBitrate, - minVideoWidth, - minVideoHeight, - minVideoFrameRate, - minVideoBitrate)) { - adaptiveTrackCount++; - } - } - return adaptiveTrackCount; - } - - private static void filterAdaptiveVideoTrackCountForMimeType( - TrackGroup group, - @Capabilities int[] formatSupport, - int requiredAdaptiveSupport, - @Nullable String mimeType, - int maxVideoWidth, - int maxVideoHeight, - int maxVideoFrameRate, - int maxVideoBitrate, - int minVideoWidth, - int minVideoHeight, - int minVideoFrameRate, - int minVideoBitrate, - List selectedTrackIndices) { - for (int i = selectedTrackIndices.size() - 1; i >= 0; i--) { - int trackIndex = selectedTrackIndices.get(i); - if (!isSupportedAdaptiveVideoTrack( - group.getFormat(trackIndex), - mimeType, - formatSupport[trackIndex], - requiredAdaptiveSupport, - maxVideoWidth, - maxVideoHeight, - maxVideoFrameRate, - maxVideoBitrate, - minVideoWidth, - minVideoHeight, - minVideoFrameRate, - minVideoBitrate)) { - selectedTrackIndices.remove(i); - } - } - } - - private static boolean isSupportedAdaptiveVideoTrack( - Format format, - @Nullable String mimeType, - @Capabilities int formatSupport, - int requiredAdaptiveSupport, - int maxVideoWidth, - int maxVideoHeight, - int maxVideoFrameRate, - int maxVideoBitrate, - int minVideoWidth, - int minVideoHeight, - int minVideoFrameRate, - int minVideoBitrate) { - if ((format.roleFlags & C.ROLE_FLAG_TRICK_PLAY) != 0) { - // Ignore trick-play tracks for now. - return false; - } - return isSupported(formatSupport, /* allowExceedsCapabilities= */ false) - && ((formatSupport & requiredAdaptiveSupport) != 0) - && (mimeType == null || Util.areEqual(format.sampleMimeType, mimeType)) - && (format.width == Format.NO_VALUE - || (minVideoWidth <= format.width && format.width <= maxVideoWidth)) - && (format.height == Format.NO_VALUE - || (minVideoHeight <= format.height && format.height <= maxVideoHeight)) - && (format.frameRate == Format.NO_VALUE - || (minVideoFrameRate <= format.frameRate && format.frameRate <= maxVideoFrameRate)) - && format.bitrate != Format.NO_VALUE - && minVideoBitrate <= format.bitrate - && format.bitrate <= maxVideoBitrate; - } - - @Nullable - private static ExoTrackSelection.Definition selectFixedVideoTrack( - TrackGroupArray groups, @Capabilities int[][] formatSupport, Parameters params) { - int selectedTrackIndex = C.INDEX_UNSET; - @Nullable TrackGroup selectedGroup = null; - @Nullable VideoTrackScore selectedTrackScore = null; - for (int groupIndex = 0; groupIndex < groups.length; groupIndex++) { - TrackGroup trackGroup = groups.get(groupIndex); - List viewportFilteredTrackIndices = - getViewportFilteredTrackIndices( - trackGroup, - params.viewportWidth, - params.viewportHeight, - params.viewportOrientationMayChange); - @Capabilities int[] trackFormatSupport = formatSupport[groupIndex]; - for (int trackIndex = 0; trackIndex < trackGroup.length; trackIndex++) { - Format format = trackGroup.getFormat(trackIndex); - if ((format.roleFlags & C.ROLE_FLAG_TRICK_PLAY) != 0) { - // Ignore trick-play tracks for now. - continue; - } - if (isSupported( - trackFormatSupport[trackIndex], params.exceedRendererCapabilitiesIfNecessary)) { - VideoTrackScore trackScore = - new VideoTrackScore( - format, - params, - trackFormatSupport[trackIndex], - viewportFilteredTrackIndices.contains(trackIndex)); - if (!trackScore.isWithinMaxConstraints && !params.exceedVideoConstraintsIfNecessary) { - // Track should not be selected. - continue; - } - if (selectedTrackScore == null || trackScore.compareTo(selectedTrackScore) > 0) { - selectedGroup = trackGroup; - selectedTrackIndex = trackIndex; - selectedTrackScore = trackScore; - } - } - } - } - - return selectedGroup == null - ? null - : new ExoTrackSelection.Definition(selectedGroup, selectedTrackIndex); + return selectTracksForType( + C.TRACK_TYPE_VIDEO, + mappedTrackInfo, + rendererFormatSupports, + (rendererIndex, group, support) -> + VideoTrackInfo.createForTrackGroup( + rendererIndex, group, params, support, mixedMimeTypeSupports[rendererIndex]), + VideoTrackInfo::compareSelections); } // Audio track selection implementation. @@ -2338,25 +2062,16 @@ public class DefaultTrackSelector extends MappingTrackSelector { return 0; } - private static List getViewportFilteredTrackIndices( + private static int getMaxVideoPixelsToRetainForViewport( TrackGroup group, int viewportWidth, int viewportHeight, boolean orientationMayChange) { - // Initially include all indices. - ArrayList selectedTrackIndices = new ArrayList<>(group.length); - for (int i = 0; i < group.length; i++) { - selectedTrackIndices.add(i); - } - if (viewportWidth == Integer.MAX_VALUE || viewportHeight == Integer.MAX_VALUE) { - // Viewport dimensions not set. Return the full set of indices. - return selectedTrackIndices; + return Integer.MAX_VALUE; } - int maxVideoPixelsToRetain = Integer.MAX_VALUE; for (int i = 0; i < group.length; i++) { Format format = group.getFormat(i); // Keep track of the number of pixels of the selected format whose resolution is the // smallest to exceed the maximum size at which it can be displayed within the viewport. - // We'll discard formats of higher resolution. if (format.width > 0 && format.height > 0) { Point maxVideoSizeInViewport = getMaxVideoSizeInViewport( @@ -2369,21 +2084,7 @@ public class DefaultTrackSelector extends MappingTrackSelector { } } } - - // Filter out formats that exceed maxVideoPixelsToRetain. These formats have an unnecessarily - // high resolution given the size at which the video will be displayed within the viewport. Also - // filter out formats with unknown dimensions, since we have some whose dimensions are known. - if (maxVideoPixelsToRetain != Integer.MAX_VALUE) { - for (int i = selectedTrackIndices.size() - 1; i >= 0; i--) { - Format format = group.getFormat(selectedTrackIndices.get(i)); - int pixelCount = format.getPixelCount(); - if (pixelCount == Format.NO_VALUE || pixelCount > maxVideoPixelsToRetain) { - selectedTrackIndices.remove(i); - } - } - } - - return selectedTrackIndices; + return maxVideoPixelsToRetain; } /** @@ -2450,15 +2151,40 @@ public class DefaultTrackSelector extends MappingTrackSelector { public abstract boolean isCompatibleForAdaptationWith(TrackInfo otherTrack); } - /** Represents how well a video track matches the selection {@link Parameters}. */ - protected static final class VideoTrackScore implements Comparable { + private static final class VideoTrackInfo extends TrackInfo { - /** - * Whether the provided format is within the parameter maximum constraints. If {@code false}, - * the format should not be selected. - */ - public final boolean isWithinMaxConstraints; + public static ImmutableList createForTrackGroup( + int rendererIndex, + TrackGroup trackGroup, + Parameters params, + @Capabilities int[] formatSupport, + @AdaptiveSupport int mixedMimeTypeAdaptionSupport) { + int maxPixelsToRetainForViewport = + getMaxVideoPixelsToRetainForViewport( + trackGroup, + params.viewportWidth, + params.viewportHeight, + params.viewportOrientationMayChange); + ImmutableList.Builder listBuilder = ImmutableList.builder(); + for (int i = 0; i < trackGroup.length; i++) { + int pixelCount = trackGroup.getFormat(i).getPixelCount(); + boolean isSuitableForViewport = + maxPixelsToRetainForViewport == Integer.MAX_VALUE + || (pixelCount != Format.NO_VALUE && pixelCount <= maxPixelsToRetainForViewport); + listBuilder.add( + new VideoTrackInfo( + rendererIndex, + trackGroup, + /* trackIndex= */ i, + params, + formatSupport[i], + mixedMimeTypeAdaptionSupport, + isSuitableForViewport)); + } + return listBuilder.build(); + } + private final boolean isWithinMaxConstraints; private final Parameters parameters; private final boolean isWithinMinConstraints; private final boolean isWithinRendererCapabilities; @@ -2467,13 +2193,28 @@ public class DefaultTrackSelector extends MappingTrackSelector { private final int preferredMimeTypeMatchIndex; private final int preferredRoleFlagsScore; private final boolean hasMainOrNoRoleFlag; + private final boolean allowMixedMimeTypes; + @SelectionEligibility private final int selectionEligibility; - public VideoTrackScore( - Format format, + public VideoTrackInfo( + int rendererIndex, + TrackGroup trackGroup, + int trackIndex, Parameters parameters, @Capabilities int formatSupport, + @AdaptiveSupport int mixedMimeTypeAdaptationSupport, boolean isSuitableForViewport) { + super(rendererIndex, trackGroup, trackIndex); this.parameters = parameters; + @SuppressLint("WrongConstant") + int requiredAdaptiveSupport = + parameters.allowVideoNonSeamlessAdaptiveness + ? (RendererCapabilities.ADAPTIVE_NOT_SEAMLESS + | RendererCapabilities.ADAPTIVE_SEAMLESS) + : RendererCapabilities.ADAPTIVE_SEAMLESS; + allowMixedMimeTypes = + parameters.allowVideoMixedMimeTypeAdaptiveness + && (mixedMimeTypeAdaptationSupport & requiredAdaptiveSupport) != 0; isWithinMaxConstraints = isSuitableForViewport && (format.width == Format.NO_VALUE || format.width <= parameters.maxVideoWidth) @@ -2506,10 +2247,63 @@ public class DefaultTrackSelector extends MappingTrackSelector { } } preferredMimeTypeMatchIndex = bestMimeTypeMatchIndex; + selectionEligibility = evaluateSelectionEligibility(formatSupport, requiredAdaptiveSupport); } @Override - public int compareTo(VideoTrackScore other) { + @SelectionEligibility + public int getSelectionEligibility() { + return selectionEligibility; + } + + @Override + public boolean isCompatibleForAdaptationWith(TrackInfo otherTrack) { + return allowMixedMimeTypes + || Util.areEqual(format.sampleMimeType, otherTrack.format.sampleMimeType); + } + + @SelectionEligibility + private int evaluateSelectionEligibility( + @Capabilities int rendererSupport, @AdaptiveSupport int requiredAdaptiveSupport) { + if ((format.roleFlags & C.ROLE_FLAG_TRICK_PLAY) != 0) { + // Ignore trick-play tracks for now. + return SELECTION_ELIGIBILITY_NO; + } + if (!isSupported(rendererSupport, parameters.exceedRendererCapabilitiesIfNecessary)) { + return SELECTION_ELIGIBILITY_NO; + } + if (!isWithinMaxConstraints && !parameters.exceedVideoConstraintsIfNecessary) { + return SELECTION_ELIGIBILITY_NO; + } + return isSupported(rendererSupport, /* allowExceedsCapabilities= */ false) + && isWithinMinConstraints + && isWithinMaxConstraints + && format.bitrate != Format.NO_VALUE + && !parameters.forceHighestSupportedBitrate + && !parameters.forceLowestBitrate + && ((rendererSupport & requiredAdaptiveSupport) != 0) + ? SELECTION_ELIGIBILITY_ADAPTIVE + : SELECTION_ELIGIBILITY_FIXED; + } + + private static int compareNonQualityPreferences(VideoTrackInfo info1, VideoTrackInfo info2) { + return ComparisonChain.start() + .compareFalseFirst(info1.isWithinRendererCapabilities, info2.isWithinRendererCapabilities) + // 1. Compare match with specific content preferences set by the parameters. + .compare(info1.preferredRoleFlagsScore, info2.preferredRoleFlagsScore) + // 2. Compare match with implicit content preferences set by the media. + .compareFalseFirst(info1.hasMainOrNoRoleFlag, info2.hasMainOrNoRoleFlag) + // 3. Compare match with technical preferences set by the parameters. + .compareFalseFirst(info1.isWithinMaxConstraints, info2.isWithinMaxConstraints) + .compareFalseFirst(info1.isWithinMinConstraints, info2.isWithinMinConstraints) + .compare( + info1.preferredMimeTypeMatchIndex, + info2.preferredMimeTypeMatchIndex, + Ordering.natural().reverse()) + .result(); + } + + private static int compareQualityPreferences(VideoTrackInfo info1, VideoTrackInfo info2) { // The preferred ordering by video quality depends on the constraints: // - Not within renderer capabilities: Prefer lower quality because it's more likely to play. // - Within min and max constraints: Prefer higher quality. @@ -2519,29 +2313,33 @@ public class DefaultTrackSelector extends MappingTrackSelector { // satisfying the violated max constraints. // - Outside min and max constraints: Arbitrarily prefer lower quality. Ordering qualityOrdering = - isWithinMaxConstraints && isWithinRendererCapabilities + info1.isWithinMaxConstraints && info1.isWithinRendererCapabilities ? FORMAT_VALUE_ORDERING : FORMAT_VALUE_ORDERING.reverse(); return ComparisonChain.start() - .compareFalseFirst(this.isWithinRendererCapabilities, other.isWithinRendererCapabilities) - // 1. Compare match with specific content preferences set by the parameters. - .compare(this.preferredRoleFlagsScore, other.preferredRoleFlagsScore) - // 2. Compare match with implicit content preferences set by the media. - .compareFalseFirst(this.hasMainOrNoRoleFlag, other.hasMainOrNoRoleFlag) - // 3. Compare match with technical preferences set by the parameters. - .compareFalseFirst(this.isWithinMaxConstraints, other.isWithinMaxConstraints) - .compareFalseFirst(this.isWithinMinConstraints, other.isWithinMinConstraints) .compare( - this.preferredMimeTypeMatchIndex, - other.preferredMimeTypeMatchIndex, - Ordering.natural().reverse()) - // 4. Compare technical quality. + info1.bitrate, + info2.bitrate, + info1.parameters.forceLowestBitrate ? FORMAT_VALUE_ORDERING.reverse() : NO_ORDER) + .compare(info1.pixelCount, info2.pixelCount, qualityOrdering) + .compare(info1.bitrate, info2.bitrate, qualityOrdering) + .result(); + } + + public static int compareSelections(List infos1, List infos2) { + return ComparisonChain.start() + // Compare non-quality preferences of the best individual track with each other. .compare( - this.bitrate, - other.bitrate, - parameters.forceLowestBitrate ? FORMAT_VALUE_ORDERING.reverse() : NO_ORDER) - .compare(this.pixelCount, other.pixelCount, qualityOrdering) - .compare(this.bitrate, other.bitrate, qualityOrdering) + max(infos1, VideoTrackInfo::compareNonQualityPreferences), + max(infos2, VideoTrackInfo::compareNonQualityPreferences), + VideoTrackInfo::compareNonQualityPreferences) + // Prefer selections with more formats (all non-quality preferences being equal). + .compare(infos1.size(), infos2.size()) + // Prefer selections with the best individual track quality. + .compare( + max(infos1, VideoTrackInfo::compareQualityPreferences), + max(infos2, VideoTrackInfo::compareQualityPreferences), + VideoTrackInfo::compareQualityPreferences) .result(); } } diff --git a/library/core/src/test/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelectorTest.java b/library/core/src/test/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelectorTest.java index 40a8e3d93c..6f01cd7e1e 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelectorTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelectorTest.java @@ -1859,11 +1859,17 @@ public final class DefaultTrackSelectorTest { throws Exception { Format formatAv1 = new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_AV1).build(); Format formatVp9 = new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_VP9).build(); - Format formatH264 = new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_H264).build(); - TrackGroupArray trackGroups = wrapFormats(formatAv1, formatVp9, formatH264); + Format formatH264Low = + new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_H264).setAverageBitrate(400).build(); + Format formatH264High = + new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_H264).setAverageBitrate(800).build(); + // Use an adaptive group to check that MIME type has a higher priority than number of tracks. + TrackGroup adaptiveGroup = new TrackGroup(formatH264Low, formatH264High); + TrackGroupArray trackGroups = + new TrackGroupArray(new TrackGroup(formatAv1), new TrackGroup(formatVp9), adaptiveGroup); trackSelector.setParameters( - trackSelector.buildUponParameters().setPreferredVideoMimeType(MimeTypes.VIDEO_VP9)); + defaultParameters.buildUpon().setPreferredVideoMimeType(MimeTypes.VIDEO_VP9)); TrackSelectorResult result = trackSelector.selectTracks( new RendererCapabilities[] {VIDEO_CAPABILITIES}, trackGroups, periodId, TIMELINE); @@ -1871,8 +1877,8 @@ public final class DefaultTrackSelectorTest { assertFixedSelection(result.selections[0], trackGroups, formatVp9); trackSelector.setParameters( - trackSelector - .buildUponParameters() + defaultParameters + .buildUpon() .setPreferredVideoMimeTypes(MimeTypes.VIDEO_VP9, MimeTypes.VIDEO_AV1)); result = trackSelector.selectTracks( @@ -1881,23 +1887,22 @@ public final class DefaultTrackSelectorTest { assertFixedSelection(result.selections[0], trackGroups, formatVp9); trackSelector.setParameters( - trackSelector - .buildUponParameters() + defaultParameters + .buildUpon() .setPreferredVideoMimeTypes(MimeTypes.VIDEO_DIVX, MimeTypes.VIDEO_H264)); result = trackSelector.selectTracks( new RendererCapabilities[] {VIDEO_CAPABILITIES}, trackGroups, periodId, TIMELINE); assertThat(result.length).isEqualTo(1); - assertFixedSelection(result.selections[0], trackGroups, formatH264); + assertAdaptiveSelection(result.selections[0], adaptiveGroup, /* expectedTracks...= */ 1, 0); - // Select first in the list if no preference is specified. - trackSelector.setParameters( - trackSelector.buildUponParameters().setPreferredVideoMimeType(null)); + // Select default (=most tracks) if no preference is specified. + trackSelector.setParameters(defaultParameters.buildUpon().setPreferredVideoMimeType(null)); result = trackSelector.selectTracks( new RendererCapabilities[] {VIDEO_CAPABILITIES}, trackGroups, periodId, TIMELINE); assertThat(result.length).isEqualTo(1); - assertFixedSelection(result.selections[0], trackGroups, formatAv1); + assertAdaptiveSelection(result.selections[0], adaptiveGroup, /* expectedTracks...= */ 1, 0); } /** @@ -1907,13 +1912,18 @@ public final class DefaultTrackSelectorTest { @Test public void selectTracks_withPreferredVideoRoleFlags_selectPreferredTrack() throws Exception { Format.Builder formatBuilder = VIDEO_FORMAT.buildUpon(); - Format noRoleFlags = formatBuilder.build(); + Format noRoleFlagsLow = formatBuilder.setAverageBitrate(4000).build(); + Format noRoleFlagsHigh = formatBuilder.setAverageBitrate(8000).build(); Format lessRoleFlags = formatBuilder.setRoleFlags(C.ROLE_FLAG_CAPTION).build(); Format moreRoleFlags = formatBuilder .setRoleFlags(C.ROLE_FLAG_CAPTION | C.ROLE_FLAG_COMMENTARY | C.ROLE_FLAG_DUB) .build(); - TrackGroupArray trackGroups = wrapFormats(noRoleFlags, moreRoleFlags, lessRoleFlags); + // Use an adaptive group to check that role flags have higher priority than number of tracks. + TrackGroup adaptiveNoRoleFlagsGroup = new TrackGroup(noRoleFlagsLow, noRoleFlagsHigh); + TrackGroupArray trackGroups = + new TrackGroupArray( + adaptiveNoRoleFlagsGroup, new TrackGroup(moreRoleFlags), new TrackGroup(lessRoleFlags)); trackSelector.setParameters( defaultParameters From ba58be2c41c8f6ed7f1bc7bdb7eb3903b527e639 Mon Sep 17 00:00:00 2001 From: ibaker Date: Mon, 17 Jan 2022 11:09:39 +0000 Subject: [PATCH 08/12] Add an anchor tag for the IMA compat URL in ImaAdsLoader javadoc The naked URL is not hyperlinked by the javadoc compiler, meaning a user is forced to awkwardly copy it into the address bar of their browser: https://exoplayer.dev/doc/reference/com/google/android/exoplayer2/ext/ima/ImaAdsLoader.html #minor-release PiperOrigin-RevId: 422320571 --- .../exoplayer2/ext/ima/ImaAdsLoader.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaAdsLoader.java b/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaAdsLoader.java index f9564154ca..fe0df4e399 100644 --- a/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaAdsLoader.java +++ b/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaAdsLoader.java @@ -71,14 +71,15 @@ import java.util.Set; * #setPlayer(Player)}. If the ads loader is no longer required, it must be released by calling * {@link #release()}. * - *

      See https://developers.google.com/interactive-media-ads/docs/sdks/android/compatibility for - * information on compatible ad tag formats. Pass the ad tag URI when setting media item playback - * properties (if using the media item API) or as a {@link DataSpec} when constructing the {@link - * AdsMediaSource} (if using media sources directly). For the latter case, please note that this - * implementation delegates loading of the data spec to the IMA SDK, so range and headers - * specifications will be ignored in ad tag URIs. Literal ads responses can be encoded as data - * scheme data specs, for example, by constructing the data spec using a URI generated via {@link - * Util#getDataUriForString(String, String)}. + *

      See IMA's + * Support and compatibility page for information on compatible ad tag formats. Pass the ad tag + * URI when setting media item playback properties (if using the media item API) or as a {@link + * DataSpec} when constructing the {@link AdsMediaSource} (if using media sources directly). For the + * latter case, please note that this implementation delegates loading of the data spec to the IMA + * SDK, so range and headers specifications will be ignored in ad tag URIs. Literal ads responses + * can be encoded as data scheme data specs, for example, by constructing the data spec using a URI + * generated via {@link Util#getDataUriForString(String, String)}. * *

      The IMA SDK can report obstructions to the ad view for accurate viewability measurement. This * means that any overlay views that obstruct the ad overlay but are essential for playback need to From 2e7ca0b7b8ca1c5b0894d94a872a97b7cbbaf2b1 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Mon, 17 Jan 2022 11:37:05 +0000 Subject: [PATCH 09/12] Add javadoc to TransformationRequest's public fields. PiperOrigin-RevId: 422325859 --- .../transformer/TransformationRequest.java | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java index 0dcf388cf5..a870365754 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java @@ -130,6 +130,8 @@ public final class TransformationRequest { * @return This builder. */ public Builder setResolution(int outputHeight) { + // TODO(b/209781577): Define outputHeight in the javadoc as height can be ambiguous for videos + // where rotationDegrees is set in the Format. // TODO(b/201293185): Restructure to input a Presentation class. // TODO(b/201293185): Check encoder codec capabilities in order to allow arbitrary // resolutions and reasonable fallbacks. @@ -188,10 +190,37 @@ public final class TransformationRequest { } } + /** + * A {@link Matrix transformation matrix} to apply to video frames. + * + * @see Builder#setTransformationMatrix(Matrix) + */ public final Matrix transformationMatrix; + /** + * Whether the input should be flattened for media containing slow motion markers. + * + * @see Builder#setFlattenForSlowMotion(boolean) + */ public final boolean flattenForSlowMotion; + /** + * The requested height of the output video, or {@link C#LENGTH_UNSET} if inferred from the input. + * + * @see Builder#setResolution(int) + */ public final int outputHeight; + /** + * The requested output audio sample {@link MimeTypes MIME type}, or {@code null} if inferred from + * the input. + * + * @see Builder#setAudioMimeType(String) + */ @Nullable public final String audioMimeType; + /** + * The requested output video sample {@link MimeTypes MIME type}, or {@code null} if inferred from + * the input. + * + * @see Builder#setVideoMimeType(String) + */ @Nullable public final String videoMimeType; private TransformationRequest( From 4145273bc4ad88cd44578312ba9311118054c3b6 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Mon, 17 Jan 2022 12:23:05 +0000 Subject: [PATCH 10/12] Revise TransformationRequest MIME type validation. PiperOrigin-RevId: 422333929 --- .../transformer/TransformationRequest.java | 23 +++++++--- .../TransformationRequestBuilderTest.java | 43 +++++++++++++++++++ .../transformer/TransformerBuilderTest.java | 4 -- 3 files changed, 59 insertions(+), 11 deletions(-) create mode 100644 library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformationRequestBuilderTest.java diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java index a870365754..f623365fce 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java @@ -16,6 +16,8 @@ package com.google.android.exoplayer2.transformer; +import static com.google.android.exoplayer2.util.Assertions.checkArgument; + import android.graphics.Matrix; import androidx.annotation.Nullable; import com.google.android.exoplayer2.C; @@ -128,6 +130,7 @@ public final class TransformationRequest { * * @param outputHeight The output height in pixels. * @return This builder. + * @throws IllegalArgumentException If the {@code outputHeight} is not supported. */ public Builder setResolution(int outputHeight) { // TODO(b/209781577): Define outputHeight in the javadoc as height can be ambiguous for videos @@ -135,9 +138,9 @@ public final class TransformationRequest { // TODO(b/201293185): Restructure to input a Presentation class. // TODO(b/201293185): Check encoder codec capabilities in order to allow arbitrary // resolutions and reasonable fallbacks. - if (outputHeight != C.LENGTH_UNSET && !SUPPORTED_OUTPUT_HEIGHTS.contains(outputHeight)) { - throw new IllegalArgumentException("Unsupported outputHeight: " + outputHeight); - } + checkArgument( + outputHeight == C.LENGTH_UNSET || SUPPORTED_OUTPUT_HEIGHTS.contains(outputHeight), + "Unsupported outputHeight: " + outputHeight); this.outputHeight = outputHeight; return this; } @@ -155,10 +158,13 @@ public final class TransformationRequest { * * @param videoMimeType The MIME type of the video samples in the output. * @return This builder. + * @throws IllegalArgumentException If the {@code videoMimeType} is non-null but not a video + * {@link MimeTypes MIME type}. */ public Builder setVideoMimeType(@Nullable String videoMimeType) { - // TODO(b/209469847): Validate videoMimeType here once deprecated - // Transformer.Builder#setOuputMimeType(String) has been removed. + checkArgument( + videoMimeType == null || MimeTypes.isVideo(videoMimeType), + "Not a video MIME type: " + videoMimeType); this.videoMimeType = videoMimeType; return this; } @@ -175,10 +181,13 @@ public final class TransformationRequest { * * @param audioMimeType The MIME type of the audio samples in the output. * @return This builder. + * @throws IllegalArgumentException If the {@code audioMimeType} is non-null but not an audio + * {@link MimeTypes MIME type}. */ public Builder setAudioMimeType(@Nullable String audioMimeType) { - // TODO(b/209469847): Validate audioMimeType here once deprecated - // Transformer.Builder#setOuputMimeType(String) has been removed. + checkArgument( + audioMimeType == null || MimeTypes.isAudio(audioMimeType), + "Not an audio MIME type: " + audioMimeType); this.audioMimeType = audioMimeType; return this; } diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformationRequestBuilderTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformationRequestBuilderTest.java new file mode 100644 index 0000000000..b55125c98a --- /dev/null +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformationRequestBuilderTest.java @@ -0,0 +1,43 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.exoplayer2.transformer; + +import static org.junit.Assert.assertThrows; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.util.MimeTypes; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit test for {@link TransformationRequest.Builder}. */ +@RunWith(AndroidJUnit4.class) +public class TransformationRequestBuilderTest { + + @Test + public void setAudioMimeType_withVideoMimeType_throws() { + assertThrows( + IllegalArgumentException.class, + () -> new TransformationRequest.Builder().setAudioMimeType(MimeTypes.VIDEO_H264)); + } + + @Test + public void setVideoMimeType_withAudioMimeType_throws() { + assertThrows( + IllegalArgumentException.class, + () -> new TransformationRequest.Builder().setVideoMimeType(MimeTypes.AUDIO_AAC)); + } +} diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerBuilderTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerBuilderTest.java index bff87ff35e..e96f136fdd 100644 --- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerBuilderTest.java +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerBuilderTest.java @@ -52,8 +52,6 @@ public class TransformerBuilderTest { () -> new Transformer.Builder(context).setRemoveAudio(true).setRemoveVideo(true).build()); } - // TODO(b/209469847): Move this test to TransformationRequestBuilderTest once deprecated - // Transformer.Builder#setOuputMimeType(String) has been removed. @Test public void build_withUnsupportedAudioMimeType_throws() { Context context = ApplicationProvider.getApplicationContext(); @@ -68,8 +66,6 @@ public class TransformerBuilderTest { .build()); } - // TODO(b/209469847): Move this test to TransformationRequestBuilderTest once deprecated - // Transformer.Builder#setOuputMimeType(String) has been removed. @Test public void build_withUnsupportedVideoMimeType_throws() { Context context = ApplicationProvider.getApplicationContext(); From b3981be8b91992976d78c8803349db9680b648a7 Mon Sep 17 00:00:00 2001 From: tonihei Date: Mon, 17 Jan 2022 13:38:11 +0000 Subject: [PATCH 11/12] Limit adaptive selections to same level of decoder support Adaptive video and audio selections will be limited to formats with the same level of DecoderSupport and HardwareAccelatationSupport, unless specifically allowed by new flags. If different levels of decoder support are available, prefer primary over fallback decoders and hardware-accelerated over software decoders (in this order). For video, also prefer more efficient codecs, if both are supported by hardware-accelerated primary decoders. Issue: google/ExoPlayer#4835 Issue: google/ExoPlayer#9565 PiperOrigin-RevId: 422345048 --- RELEASENOTES.md | 6 + .../trackselection/DefaultTrackSelector.java | 193 ++++++++++++-- .../DefaultTrackSelectorTest.java | 239 ++++++++++++++++++ 3 files changed, 410 insertions(+), 28 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 8e1c8106ad..245a591074 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -34,6 +34,12 @@ * Update video track selection logic to take preferred MIME types and role flags into account when selecting multiple video tracks for adaptation ((#9519)[https://github.com/google/ExoPlayer/issues/9519]). + * Update video and audio track selection logic to only choose formats for + adaptive selections that have the same level of decoder and hardware + support ((#9565)[https://github.com/google/ExoPlayer/issues/9565]). + * Update video track selection logic to prefer more efficient codecs if + multiple codecs are supported by primary, hardware-accelerated decoders + ((#4835)[https://github.com/google/ExoPlayer/issues/4835]). * Android 12 compatibility: * Upgrade the Cast extension to depend on `com.google.android.gms:play-services-cast-framework:20.1.0`. Earlier diff --git a/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java b/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java index 37ad7e002b..f10f0b765a 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java @@ -45,6 +45,7 @@ import com.google.android.exoplayer2.source.TrackGroupArray; import com.google.android.exoplayer2.trackselection.TrackSelectionOverrides.TrackSelectionOverride; import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.BundleableUtil; +import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; import com.google.common.collect.ComparisonChain; import com.google.common.collect.ImmutableList; @@ -107,11 +108,13 @@ public class DefaultTrackSelector extends MappingTrackSelector { private boolean exceedVideoConstraintsIfNecessary; private boolean allowVideoMixedMimeTypeAdaptiveness; private boolean allowVideoNonSeamlessAdaptiveness; + private boolean allowVideoMixedDecoderSupportAdaptiveness; // Audio private boolean exceedAudioConstraintsIfNecessary; private boolean allowAudioMixedMimeTypeAdaptiveness; private boolean allowAudioMixedSampleRateAdaptiveness; private boolean allowAudioMixedChannelCountAdaptiveness; + private boolean allowAudioMixedDecoderSupportAdaptiveness; // Text @C.SelectionFlags private int disabledTextTrackSelectionFlags; // General @@ -160,12 +163,16 @@ public class DefaultTrackSelector extends MappingTrackSelector { exceedVideoConstraintsIfNecessary = initialValues.exceedVideoConstraintsIfNecessary; allowVideoMixedMimeTypeAdaptiveness = initialValues.allowVideoMixedMimeTypeAdaptiveness; allowVideoNonSeamlessAdaptiveness = initialValues.allowVideoNonSeamlessAdaptiveness; + allowVideoMixedDecoderSupportAdaptiveness = + initialValues.allowVideoMixedDecoderSupportAdaptiveness; // Audio exceedAudioConstraintsIfNecessary = initialValues.exceedAudioConstraintsIfNecessary; allowAudioMixedMimeTypeAdaptiveness = initialValues.allowAudioMixedMimeTypeAdaptiveness; allowAudioMixedSampleRateAdaptiveness = initialValues.allowAudioMixedSampleRateAdaptiveness; allowAudioMixedChannelCountAdaptiveness = initialValues.allowAudioMixedChannelCountAdaptiveness; + allowAudioMixedDecoderSupportAdaptiveness = + initialValues.allowAudioMixedDecoderSupportAdaptiveness; // General exceedRendererCapabilitiesIfNecessary = initialValues.exceedRendererCapabilitiesIfNecessary; tunnelingEnabled = initialValues.tunnelingEnabled; @@ -192,6 +199,11 @@ public class DefaultTrackSelector extends MappingTrackSelector { bundle.getBoolean( Parameters.keyForField(Parameters.FIELD_ALLOW_VIDEO_NON_SEAMLESS_ADAPTIVENESS), defaultValue.allowVideoNonSeamlessAdaptiveness)); + setAllowVideoMixedDecoderSupportAdaptiveness( + bundle.getBoolean( + Parameters.keyForField( + Parameters.FIELD_ALLOW_VIDEO_MIXED_DECODER_SUPPORT_ADAPTIVENESS), + defaultValue.allowVideoMixedDecoderSupportAdaptiveness)); // Audio setExceedAudioConstraintsIfNecessary( bundle.getBoolean( @@ -209,6 +221,11 @@ public class DefaultTrackSelector extends MappingTrackSelector { bundle.getBoolean( Parameters.keyForField(Parameters.FIELD_ALLOW_AUDIO_MIXED_CHANNEL_COUNT_ADAPTIVENESS), defaultValue.allowAudioMixedChannelCountAdaptiveness)); + setAllowAudioMixedDecoderSupportAdaptiveness( + bundle.getBoolean( + Parameters.keyForField( + Parameters.FIELD_ALLOW_AUDIO_MIXED_DECODER_SUPPORT_ADAPTIVENESS), + defaultValue.allowAudioMixedDecoderSupportAdaptiveness)); // Text setDisabledTextTrackSelectionFlags( bundle.getInt( @@ -340,6 +357,21 @@ public class DefaultTrackSelector extends MappingTrackSelector { return this; } + /** + * Sets whether to allow adaptive video selections with mixed levels of {@link + * RendererCapabilities.DecoderSupport} and {@link + * RendererCapabilities.HardwareAccelerationSupport}. + * + * @param allowVideoMixedDecoderSupportAdaptiveness Whether to allow adaptive video selections + * with mixed levels of decoder and hardware acceleration support. + * @return This builder. + */ + public ParametersBuilder setAllowVideoMixedDecoderSupportAdaptiveness( + boolean allowVideoMixedDecoderSupportAdaptiveness) { + this.allowVideoMixedDecoderSupportAdaptiveness = allowVideoMixedDecoderSupportAdaptiveness; + return this; + } + @Override public ParametersBuilder setViewportSizeToPhysicalDisplaySize( Context context, boolean viewportOrientationMayChange) { @@ -470,6 +502,21 @@ public class DefaultTrackSelector extends MappingTrackSelector { return this; } + /** + * Sets whether to allow adaptive audio selections with mixed levels of {@link + * RendererCapabilities.DecoderSupport} and {@link + * RendererCapabilities.HardwareAccelerationSupport}. + * + * @param allowAudioMixedDecoderSupportAdaptiveness Whether to allow adaptive audio selections + * with mixed levels of decoder and hardware acceleration support. + * @return This builder. + */ + public ParametersBuilder setAllowAudioMixedDecoderSupportAdaptiveness( + boolean allowAudioMixedDecoderSupportAdaptiveness) { + this.allowAudioMixedDecoderSupportAdaptiveness = allowAudioMixedDecoderSupportAdaptiveness; + return this; + } + @Override public ParametersBuilder setPreferredAudioMimeType(@Nullable String mimeType) { super.setPreferredAudioMimeType(mimeType); @@ -740,11 +787,13 @@ public class DefaultTrackSelector extends MappingTrackSelector { exceedVideoConstraintsIfNecessary = true; allowVideoMixedMimeTypeAdaptiveness = false; allowVideoNonSeamlessAdaptiveness = true; + allowVideoMixedDecoderSupportAdaptiveness = false; // Audio exceedAudioConstraintsIfNecessary = true; allowAudioMixedMimeTypeAdaptiveness = false; allowAudioMixedSampleRateAdaptiveness = false; allowAudioMixedChannelCountAdaptiveness = false; + allowAudioMixedDecoderSupportAdaptiveness = false; // Text disabledTextTrackSelectionFlags = 0; // General @@ -864,6 +913,12 @@ public class DefaultTrackSelector extends MappingTrackSelector { * The default value is {@code true}. */ public final boolean allowVideoNonSeamlessAdaptiveness; + /** + * Whether to allow adaptive video selections with mixed levels of {@link + * RendererCapabilities.DecoderSupport} and {@link + * RendererCapabilities.HardwareAccelerationSupport}. + */ + public final boolean allowVideoMixedDecoderSupportAdaptiveness; /** * Whether to exceed the {@link #maxAudioChannelCount} and {@link #maxAudioBitrate} constraints * when no selection can be made otherwise. The default value is {@code true}. @@ -885,6 +940,12 @@ public class DefaultTrackSelector extends MappingTrackSelector { * false}. */ public final boolean allowAudioMixedChannelCountAdaptiveness; + /** + * Whether to allow adaptive audio selections with mixed levels of {@link + * RendererCapabilities.DecoderSupport} and {@link + * RendererCapabilities.HardwareAccelerationSupport}. + */ + public final boolean allowAudioMixedDecoderSupportAdaptiveness; /** * Whether to exceed renderer capabilities when no selection can be made otherwise. * @@ -918,11 +979,13 @@ public class DefaultTrackSelector extends MappingTrackSelector { exceedVideoConstraintsIfNecessary = builder.exceedVideoConstraintsIfNecessary; allowVideoMixedMimeTypeAdaptiveness = builder.allowVideoMixedMimeTypeAdaptiveness; allowVideoNonSeamlessAdaptiveness = builder.allowVideoNonSeamlessAdaptiveness; + allowVideoMixedDecoderSupportAdaptiveness = builder.allowVideoMixedDecoderSupportAdaptiveness; // Audio exceedAudioConstraintsIfNecessary = builder.exceedAudioConstraintsIfNecessary; allowAudioMixedMimeTypeAdaptiveness = builder.allowAudioMixedMimeTypeAdaptiveness; allowAudioMixedSampleRateAdaptiveness = builder.allowAudioMixedSampleRateAdaptiveness; allowAudioMixedChannelCountAdaptiveness = builder.allowAudioMixedChannelCountAdaptiveness; + allowAudioMixedDecoderSupportAdaptiveness = builder.allowAudioMixedDecoderSupportAdaptiveness; // Text disabledTextTrackSelectionFlags = builder.disabledTextTrackSelectionFlags; // General @@ -1000,12 +1063,16 @@ public class DefaultTrackSelector extends MappingTrackSelector { && exceedVideoConstraintsIfNecessary == other.exceedVideoConstraintsIfNecessary && allowVideoMixedMimeTypeAdaptiveness == other.allowVideoMixedMimeTypeAdaptiveness && allowVideoNonSeamlessAdaptiveness == other.allowVideoNonSeamlessAdaptiveness + && allowVideoMixedDecoderSupportAdaptiveness + == other.allowVideoMixedDecoderSupportAdaptiveness // Audio && exceedAudioConstraintsIfNecessary == other.exceedAudioConstraintsIfNecessary && allowAudioMixedMimeTypeAdaptiveness == other.allowAudioMixedMimeTypeAdaptiveness && allowAudioMixedSampleRateAdaptiveness == other.allowAudioMixedSampleRateAdaptiveness && allowAudioMixedChannelCountAdaptiveness == other.allowAudioMixedChannelCountAdaptiveness + && allowAudioMixedDecoderSupportAdaptiveness + == other.allowAudioMixedDecoderSupportAdaptiveness // Text && disabledTextTrackSelectionFlags == other.disabledTextTrackSelectionFlags // General @@ -1025,11 +1092,13 @@ public class DefaultTrackSelector extends MappingTrackSelector { result = 31 * result + (exceedVideoConstraintsIfNecessary ? 1 : 0); result = 31 * result + (allowVideoMixedMimeTypeAdaptiveness ? 1 : 0); result = 31 * result + (allowVideoNonSeamlessAdaptiveness ? 1 : 0); + result = 31 * result + (allowVideoMixedDecoderSupportAdaptiveness ? 1 : 0); // Audio result = 31 * result + (exceedAudioConstraintsIfNecessary ? 1 : 0); result = 31 * result + (allowAudioMixedMimeTypeAdaptiveness ? 1 : 0); result = 31 * result + (allowAudioMixedSampleRateAdaptiveness ? 1 : 0); result = 31 * result + (allowAudioMixedChannelCountAdaptiveness ? 1 : 0); + result = 31 * result + (allowAudioMixedDecoderSupportAdaptiveness ? 1 : 0); // Text result = 31 * result + disabledTextTrackSelectionFlags; // General @@ -1060,6 +1129,8 @@ public class DefaultTrackSelector extends MappingTrackSelector { FIELD_SELECTION_OVERRIDES_TRACK_GROUP_ARRAYS, FIELD_SELECTION_OVERRIDES, FIELD_RENDERER_DISABLED_INDICES, + FIELD_ALLOW_VIDEO_MIXED_DECODER_SUPPORT_ADAPTIVENESS, + FIELD_ALLOW_AUDIO_MIXED_DECODER_SUPPORT_ADAPTIVENESS }) private @interface FieldNumber {} @@ -1079,6 +1150,8 @@ public class DefaultTrackSelector extends MappingTrackSelector { private static final int FIELD_SELECTION_OVERRIDES_TRACK_GROUP_ARRAYS = 1012; private static final int FIELD_SELECTION_OVERRIDES = 1013; private static final int FIELD_RENDERER_DISABLED_INDICES = 1014; + private static final int FIELD_ALLOW_VIDEO_MIXED_DECODER_SUPPORT_ADAPTIVENESS = 1015; + private static final int FIELD_ALLOW_AUDIO_MIXED_DECODER_SUPPORT_ADAPTIVENESS = 1016; @Override public Bundle toBundle() { @@ -1094,6 +1167,9 @@ public class DefaultTrackSelector extends MappingTrackSelector { bundle.putBoolean( keyForField(FIELD_ALLOW_VIDEO_NON_SEAMLESS_ADAPTIVENESS), allowVideoNonSeamlessAdaptiveness); + bundle.putBoolean( + keyForField(FIELD_ALLOW_VIDEO_MIXED_DECODER_SUPPORT_ADAPTIVENESS), + allowVideoMixedDecoderSupportAdaptiveness); // Audio bundle.putBoolean( keyForField(FIELD_EXCEED_AUDIO_CONSTRAINTS_IF_NCESSARY), @@ -1107,6 +1183,9 @@ public class DefaultTrackSelector extends MappingTrackSelector { bundle.putBoolean( keyForField(FIELD_ALLOW_AUDIO_MIXED_CHANNEL_COUNT_ADAPTIVENESS), allowAudioMixedChannelCountAdaptiveness); + bundle.putBoolean( + keyForField(FIELD_ALLOW_AUDIO_MIXED_DECODER_SUPPORT_ADAPTIVENESS), + allowAudioMixedDecoderSupportAdaptiveness); // Text bundle.putInt( keyForField(FIELD_DISABLED_TEXT_TRACK_SELECTION_FLAGS), disabledTextTrackSelectionFlags); @@ -1850,7 +1929,7 @@ public class DefaultTrackSelector extends MappingTrackSelector { } @Nullable - private Pair selectTracksForType( + private > Pair selectTracksForType( @C.TrackType int trackType, MappedTrackInfo mappedTrackInfo, @Capabilities int[][][] formatSupport, @@ -2121,10 +2200,32 @@ public class DefaultTrackSelector extends MappingTrackSelector { return Integer.bitCount(trackRoleFlags & preferredRoleFlags); } + /** + * Returns preference score for primary, hardware-accelerated video codecs, with higher score + * being preferred. + */ + private static int getVideoCodecPreferenceScore(@Nullable String mimeType) { + if (mimeType == null) { + return 0; + } + switch (mimeType) { + case MimeTypes.VIDEO_AV1: + return 4; + case MimeTypes.VIDEO_H265: + return 3; + case MimeTypes.VIDEO_VP9: + return 2; + case MimeTypes.VIDEO_H264: + return 1; + default: + return 0; + } + } + /** Base class for track selection information of a {@link Format}. */ - private abstract static class TrackInfo { + private abstract static class TrackInfo> { /** Factory for {@link TrackInfo} implementations for a given {@link TrackGroup}. */ - public interface Factory { + public interface Factory> { List create(int rendererIndex, TrackGroup trackGroup, @Capabilities int[] formatSupports); } @@ -2148,10 +2249,10 @@ public class DefaultTrackSelector extends MappingTrackSelector { * Returns whether this track is compatible for an adaptive selection with the specified other * track. */ - public abstract boolean isCompatibleForAdaptationWith(TrackInfo otherTrack); + public abstract boolean isCompatibleForAdaptationWith(T otherTrack); } - private static final class VideoTrackInfo extends TrackInfo { + private static final class VideoTrackInfo extends TrackInfo { public static ImmutableList createForTrackGroup( int rendererIndex, @@ -2195,6 +2296,9 @@ public class DefaultTrackSelector extends MappingTrackSelector { private final boolean hasMainOrNoRoleFlag; private final boolean allowMixedMimeTypes; @SelectionEligibility private final int selectionEligibility; + private final boolean usesPrimaryDecoder; + private final boolean usesHardwareAcceleration; + private final int codecPreferenceScore; public VideoTrackInfo( int rendererIndex, @@ -2247,6 +2351,13 @@ public class DefaultTrackSelector extends MappingTrackSelector { } } preferredMimeTypeMatchIndex = bestMimeTypeMatchIndex; + usesPrimaryDecoder = + RendererCapabilities.getDecoderSupport(formatSupport) + == RendererCapabilities.DECODER_SUPPORT_PRIMARY; + usesHardwareAcceleration = + RendererCapabilities.getHardwareAccelerationSupport(formatSupport) + == RendererCapabilities.HARDWARE_ACCELERATION_SUPPORTED; + codecPreferenceScore = getVideoCodecPreferenceScore(format.sampleMimeType); selectionEligibility = evaluateSelectionEligibility(formatSupport, requiredAdaptiveSupport); } @@ -2257,9 +2368,12 @@ public class DefaultTrackSelector extends MappingTrackSelector { } @Override - public boolean isCompatibleForAdaptationWith(TrackInfo otherTrack) { - return allowMixedMimeTypes - || Util.areEqual(format.sampleMimeType, otherTrack.format.sampleMimeType); + public boolean isCompatibleForAdaptationWith(VideoTrackInfo otherTrack) { + return (allowMixedMimeTypes + || Util.areEqual(format.sampleMimeType, otherTrack.format.sampleMimeType)) + && (parameters.allowVideoMixedDecoderSupportAdaptiveness + || (this.usesPrimaryDecoder == otherTrack.usesPrimaryDecoder + && this.usesHardwareAcceleration == otherTrack.usesHardwareAcceleration)); } @SelectionEligibility @@ -2287,20 +2401,28 @@ public class DefaultTrackSelector extends MappingTrackSelector { } private static int compareNonQualityPreferences(VideoTrackInfo info1, VideoTrackInfo info2) { - return ComparisonChain.start() - .compareFalseFirst(info1.isWithinRendererCapabilities, info2.isWithinRendererCapabilities) - // 1. Compare match with specific content preferences set by the parameters. - .compare(info1.preferredRoleFlagsScore, info2.preferredRoleFlagsScore) - // 2. Compare match with implicit content preferences set by the media. - .compareFalseFirst(info1.hasMainOrNoRoleFlag, info2.hasMainOrNoRoleFlag) - // 3. Compare match with technical preferences set by the parameters. - .compareFalseFirst(info1.isWithinMaxConstraints, info2.isWithinMaxConstraints) - .compareFalseFirst(info1.isWithinMinConstraints, info2.isWithinMinConstraints) - .compare( - info1.preferredMimeTypeMatchIndex, - info2.preferredMimeTypeMatchIndex, - Ordering.natural().reverse()) - .result(); + ComparisonChain chain = + ComparisonChain.start() + .compareFalseFirst( + info1.isWithinRendererCapabilities, info2.isWithinRendererCapabilities) + // 1. Compare match with specific content preferences set by the parameters. + .compare(info1.preferredRoleFlagsScore, info2.preferredRoleFlagsScore) + // 2. Compare match with implicit content preferences set by the media. + .compareFalseFirst(info1.hasMainOrNoRoleFlag, info2.hasMainOrNoRoleFlag) + // 3. Compare match with technical preferences set by the parameters. + .compareFalseFirst(info1.isWithinMaxConstraints, info2.isWithinMaxConstraints) + .compareFalseFirst(info1.isWithinMinConstraints, info2.isWithinMinConstraints) + .compare( + info1.preferredMimeTypeMatchIndex, + info2.preferredMimeTypeMatchIndex, + Ordering.natural().reverse()) + // 4. Compare match with renderer capability preferences. + .compareFalseFirst(info1.usesPrimaryDecoder, info2.usesPrimaryDecoder) + .compareFalseFirst(info1.usesHardwareAcceleration, info2.usesHardwareAcceleration); + if (info1.usesPrimaryDecoder && info1.usesHardwareAcceleration) { + chain = chain.compare(info1.codecPreferenceScore, info2.codecPreferenceScore); + } + return chain.result(); } private static int compareQualityPreferences(VideoTrackInfo info1, VideoTrackInfo info2) { @@ -2344,7 +2466,7 @@ public class DefaultTrackSelector extends MappingTrackSelector { } } - private static final class AudioTrackInfo extends TrackInfo + private static final class AudioTrackInfo extends TrackInfo implements Comparable { public static ImmutableList createForTrackGroup( @@ -2383,6 +2505,8 @@ public class DefaultTrackSelector extends MappingTrackSelector { private final int sampleRate; private final int bitrate; private final int preferredMimeTypeMatchIndex; + private final boolean usesPrimaryDecoder; + private final boolean usesHardwareAcceleration; public AudioTrackInfo( int rendererIndex, @@ -2447,6 +2571,12 @@ public class DefaultTrackSelector extends MappingTrackSelector { } } preferredMimeTypeMatchIndex = bestMimeTypeMatchIndex; + usesPrimaryDecoder = + RendererCapabilities.getDecoderSupport(formatSupport) + == RendererCapabilities.DECODER_SUPPORT_PRIMARY; + usesHardwareAcceleration = + RendererCapabilities.getHardwareAccelerationSupport(formatSupport) + == RendererCapabilities.HARDWARE_ACCELERATION_SUPPORTED; selectionEligibility = evaluateSelectionEligibility(formatSupport, hasMappedVideoTracks); } @@ -2457,7 +2587,7 @@ public class DefaultTrackSelector extends MappingTrackSelector { } @Override - public boolean isCompatibleForAdaptationWith(TrackInfo otherTrack) { + public boolean isCompatibleForAdaptationWith(AudioTrackInfo otherTrack) { return (parameters.allowAudioMixedChannelCountAdaptiveness || (format.channelCount != Format.NO_VALUE && format.channelCount == otherTrack.format.channelCount)) @@ -2466,7 +2596,10 @@ public class DefaultTrackSelector extends MappingTrackSelector { && TextUtils.equals(format.sampleMimeType, otherTrack.format.sampleMimeType))) && (parameters.allowAudioMixedSampleRateAdaptiveness || (format.sampleRate != Format.NO_VALUE - && format.sampleRate == otherTrack.format.sampleRate)); + && format.sampleRate == otherTrack.format.sampleRate)) + && (parameters.allowAudioMixedDecoderSupportAdaptiveness + || (this.usesPrimaryDecoder == otherTrack.usesPrimaryDecoder + && this.usesHardwareAcceleration == otherTrack.usesHardwareAcceleration)); } @Override @@ -2504,7 +2637,10 @@ public class DefaultTrackSelector extends MappingTrackSelector { this.bitrate, other.bitrate, parameters.forceLowestBitrate ? FORMAT_VALUE_ORDERING.reverse() : NO_ORDER) - // 4. Compare technical quality. + // 4. Compare match with renderer capability preferences. + .compareFalseFirst(this.usesPrimaryDecoder, other.usesPrimaryDecoder) + .compareFalseFirst(this.usesHardwareAcceleration, other.usesHardwareAcceleration) + // 5. Compare technical quality. .compare(this.channelCount, other.channelCount, qualityOrdering) .compare(this.sampleRate, other.sampleRate, qualityOrdering) .compare( @@ -2540,7 +2676,8 @@ public class DefaultTrackSelector extends MappingTrackSelector { } } - private static final class TextTrackInfo extends TrackInfo implements Comparable { + private static final class TextTrackInfo extends TrackInfo + implements Comparable { public static ImmutableList createForTrackGroup( int rendererIndex, @@ -2633,7 +2770,7 @@ public class DefaultTrackSelector extends MappingTrackSelector { } @Override - public boolean isCompatibleForAdaptationWith(TrackInfo otherTrack) { + public boolean isCompatibleForAdaptationWith(TextTrackInfo otherTrack) { return false; } diff --git a/library/core/src/test/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelectorTest.java b/library/core/src/test/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelectorTest.java index 6f01cd7e1e..8eefe2d16a 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelectorTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelectorTest.java @@ -20,6 +20,10 @@ import static com.google.android.exoplayer2.C.FORMAT_HANDLED; import static com.google.android.exoplayer2.C.FORMAT_UNSUPPORTED_SUBTYPE; import static com.google.android.exoplayer2.C.FORMAT_UNSUPPORTED_TYPE; import static com.google.android.exoplayer2.RendererCapabilities.ADAPTIVE_NOT_SEAMLESS; +import static com.google.android.exoplayer2.RendererCapabilities.DECODER_SUPPORT_FALLBACK; +import static com.google.android.exoplayer2.RendererCapabilities.DECODER_SUPPORT_PRIMARY; +import static com.google.android.exoplayer2.RendererCapabilities.HARDWARE_ACCELERATION_NOT_SUPPORTED; +import static com.google.android.exoplayer2.RendererCapabilities.HARDWARE_ACCELERATION_SUPPORTED; import static com.google.android.exoplayer2.RendererCapabilities.TUNNELING_NOT_SUPPORTED; import static com.google.android.exoplayer2.RendererConfiguration.DEFAULT; import static com.google.common.truth.Truth.assertThat; @@ -37,6 +41,7 @@ import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.ExoPlaybackException; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.RendererCapabilities; +import com.google.android.exoplayer2.RendererCapabilities.Capabilities; import com.google.android.exoplayer2.RendererConfiguration; import com.google.android.exoplayer2.Timeline; import com.google.android.exoplayer2.TracksInfo; @@ -1623,6 +1628,122 @@ public final class DefaultTrackSelectorTest { assertNoSelection(result.selections[0]); } + @Test + public void selectTracksWithMultipleAudioTracksWithMixedDecoderSupportLevels() throws Exception { + Format.Builder formatBuilder = AUDIO_FORMAT.buildUpon(); + Format format0 = formatBuilder.setId("0").setAverageBitrate(200).build(); + Format format1 = formatBuilder.setId("1").setAverageBitrate(400).build(); + Format format2 = formatBuilder.setId("2").setAverageBitrate(600).build(); + Format format3 = formatBuilder.setId("3").setAverageBitrate(800).build(); + TrackGroupArray trackGroups = singleTrackGroup(format0, format1, format2, format3); + @Capabilities int unsupported = RendererCapabilities.create(FORMAT_UNSUPPORTED_TYPE); + @Capabilities + int primaryHardware = + RendererCapabilities.create( + FORMAT_HANDLED, + ADAPTIVE_NOT_SEAMLESS, + TUNNELING_NOT_SUPPORTED, + HARDWARE_ACCELERATION_SUPPORTED, + DECODER_SUPPORT_PRIMARY); + @Capabilities + int primarySoftware = + RendererCapabilities.create( + FORMAT_HANDLED, + ADAPTIVE_NOT_SEAMLESS, + TUNNELING_NOT_SUPPORTED, + HARDWARE_ACCELERATION_NOT_SUPPORTED, + DECODER_SUPPORT_PRIMARY); + @Capabilities + int fallbackHardware = + RendererCapabilities.create( + FORMAT_HANDLED, + ADAPTIVE_NOT_SEAMLESS, + TUNNELING_NOT_SUPPORTED, + HARDWARE_ACCELERATION_SUPPORTED, + DECODER_SUPPORT_FALLBACK); + @Capabilities + int fallbackSoftware = + RendererCapabilities.create( + FORMAT_HANDLED, + ADAPTIVE_NOT_SEAMLESS, + TUNNELING_NOT_SUPPORTED, + HARDWARE_ACCELERATION_NOT_SUPPORTED, + DECODER_SUPPORT_FALLBACK); + + // Select all tracks supported by primary, hardware decoder by default. + ImmutableMap rendererCapabilitiesMap = + ImmutableMap.of( + "0", + primaryHardware, + "1", + primaryHardware, + "2", + primarySoftware, + "3", + fallbackHardware); + RendererCapabilities rendererCapabilities = + new FakeMappedRendererCapabilities(C.TRACK_TYPE_AUDIO, rendererCapabilitiesMap); + TrackSelectorResult result = + trackSelector.selectTracks( + new RendererCapabilities[] {rendererCapabilities}, trackGroups, periodId, TIMELINE); + assertAdaptiveSelection(result.selections[0], trackGroups.get(0), 1, 0); + + // Select all tracks supported by primary, software decoder by default if no primary, hardware + // decoder is available. + rendererCapabilitiesMap = + ImmutableMap.of( + "0", + fallbackHardware, + "1", + fallbackHardware, + "2", + primarySoftware, + "3", + fallbackSoftware); + rendererCapabilities = + new FakeMappedRendererCapabilities(C.TRACK_TYPE_AUDIO, rendererCapabilitiesMap); + result = + trackSelector.selectTracks( + new RendererCapabilities[] {rendererCapabilities}, trackGroups, periodId, TIMELINE); + assertFixedSelection(result.selections[0], trackGroups.get(0), 2); + + // Select all tracks supported by fallback, hardware decoder if no primary decoder is + // available. + rendererCapabilitiesMap = + ImmutableMap.of( + "0", fallbackHardware, "1", unsupported, "2", fallbackSoftware, "3", fallbackHardware); + rendererCapabilities = + new FakeMappedRendererCapabilities(C.TRACK_TYPE_AUDIO, rendererCapabilitiesMap); + result = + trackSelector.selectTracks( + new RendererCapabilities[] {rendererCapabilities}, trackGroups, periodId, TIMELINE); + assertAdaptiveSelection(result.selections[0], trackGroups.get(0), 3, 0); + + // Select all tracks supported by fallback, software decoder if no other decoder is available. + rendererCapabilitiesMap = + ImmutableMap.of( + "0", fallbackSoftware, "1", fallbackSoftware, "2", unsupported, "3", fallbackSoftware); + rendererCapabilities = + new FakeMappedRendererCapabilities(C.TRACK_TYPE_AUDIO, rendererCapabilitiesMap); + result = + trackSelector.selectTracks( + new RendererCapabilities[] {rendererCapabilities}, trackGroups, periodId, TIMELINE); + assertAdaptiveSelection(result.selections[0], trackGroups.get(0), 3, 1, 0); + + // Select all tracks if mixed decoder support is allowed. + rendererCapabilitiesMap = + ImmutableMap.of( + "0", primaryHardware, "1", unsupported, "2", primarySoftware, "3", fallbackHardware); + rendererCapabilities = + new FakeMappedRendererCapabilities(C.TRACK_TYPE_AUDIO, rendererCapabilitiesMap); + trackSelector.setParameters( + defaultParameters.buildUpon().setAllowAudioMixedDecoderSupportAdaptiveness(true)); + result = + trackSelector.selectTracks( + new RendererCapabilities[] {rendererCapabilities}, trackGroups, periodId, TIMELINE); + assertAdaptiveSelection(result.selections[0], trackGroups.get(0), 3, 2, 0); + } + @Test public void selectTracksWithMultipleAudioTracksOverrideReturnsAdaptiveTrackSelection() throws Exception { @@ -1773,6 +1894,122 @@ public final class DefaultTrackSelectorTest { assertAdaptiveSelection(result.selections[0], trackGroups.get(0), 0, 1); } + @Test + public void selectTracksWithMultipleVideoTracksWithMixedDecoderSupportLevels() throws Exception { + Format.Builder formatBuilder = VIDEO_FORMAT.buildUpon(); + Format format0 = formatBuilder.setId("0").setAverageBitrate(200).build(); + Format format1 = formatBuilder.setId("1").setAverageBitrate(400).build(); + Format format2 = formatBuilder.setId("2").setAverageBitrate(600).build(); + Format format3 = formatBuilder.setId("3").setAverageBitrate(800).build(); + TrackGroupArray trackGroups = singleTrackGroup(format0, format1, format2, format3); + @Capabilities int unsupported = RendererCapabilities.create(FORMAT_UNSUPPORTED_TYPE); + @Capabilities + int primaryHardware = + RendererCapabilities.create( + FORMAT_HANDLED, + ADAPTIVE_NOT_SEAMLESS, + TUNNELING_NOT_SUPPORTED, + HARDWARE_ACCELERATION_SUPPORTED, + DECODER_SUPPORT_PRIMARY); + @Capabilities + int primarySoftware = + RendererCapabilities.create( + FORMAT_HANDLED, + ADAPTIVE_NOT_SEAMLESS, + TUNNELING_NOT_SUPPORTED, + HARDWARE_ACCELERATION_NOT_SUPPORTED, + DECODER_SUPPORT_PRIMARY); + @Capabilities + int fallbackHardware = + RendererCapabilities.create( + FORMAT_HANDLED, + ADAPTIVE_NOT_SEAMLESS, + TUNNELING_NOT_SUPPORTED, + HARDWARE_ACCELERATION_SUPPORTED, + DECODER_SUPPORT_FALLBACK); + @Capabilities + int fallbackSoftware = + RendererCapabilities.create( + FORMAT_HANDLED, + ADAPTIVE_NOT_SEAMLESS, + TUNNELING_NOT_SUPPORTED, + HARDWARE_ACCELERATION_NOT_SUPPORTED, + DECODER_SUPPORT_FALLBACK); + + // Select all tracks supported by primary, hardware decoder by default. + ImmutableMap rendererCapabilitiesMap = + ImmutableMap.of( + "0", + primaryHardware, + "1", + primaryHardware, + "2", + primarySoftware, + "3", + fallbackHardware); + RendererCapabilities rendererCapabilities = + new FakeMappedRendererCapabilities(C.TRACK_TYPE_VIDEO, rendererCapabilitiesMap); + TrackSelectorResult result = + trackSelector.selectTracks( + new RendererCapabilities[] {rendererCapabilities}, trackGroups, periodId, TIMELINE); + assertAdaptiveSelection(result.selections[0], trackGroups.get(0), 1, 0); + + // Select all tracks supported by primary, software decoder by default if no primary, hardware + // decoder is available. + rendererCapabilitiesMap = + ImmutableMap.of( + "0", + fallbackHardware, + "1", + fallbackHardware, + "2", + primarySoftware, + "3", + fallbackSoftware); + rendererCapabilities = + new FakeMappedRendererCapabilities(C.TRACK_TYPE_VIDEO, rendererCapabilitiesMap); + result = + trackSelector.selectTracks( + new RendererCapabilities[] {rendererCapabilities}, trackGroups, periodId, TIMELINE); + assertFixedSelection(result.selections[0], trackGroups.get(0), 2); + + // Select all tracks supported by fallback, hardware decoder if no primary decoder is + // available. + rendererCapabilitiesMap = + ImmutableMap.of( + "0", fallbackHardware, "1", unsupported, "2", fallbackSoftware, "3", fallbackHardware); + rendererCapabilities = + new FakeMappedRendererCapabilities(C.TRACK_TYPE_VIDEO, rendererCapabilitiesMap); + result = + trackSelector.selectTracks( + new RendererCapabilities[] {rendererCapabilities}, trackGroups, periodId, TIMELINE); + assertAdaptiveSelection(result.selections[0], trackGroups.get(0), 3, 0); + + // Select all tracks supported by fallback, software decoder if no other decoder is available. + rendererCapabilitiesMap = + ImmutableMap.of( + "0", fallbackSoftware, "1", fallbackSoftware, "2", unsupported, "3", fallbackSoftware); + rendererCapabilities = + new FakeMappedRendererCapabilities(C.TRACK_TYPE_VIDEO, rendererCapabilitiesMap); + result = + trackSelector.selectTracks( + new RendererCapabilities[] {rendererCapabilities}, trackGroups, periodId, TIMELINE); + assertAdaptiveSelection(result.selections[0], trackGroups.get(0), 3, 1, 0); + + // Select all tracks if mixed decoder support is allowed. + rendererCapabilitiesMap = + ImmutableMap.of( + "0", primaryHardware, "1", unsupported, "2", primarySoftware, "3", fallbackHardware); + rendererCapabilities = + new FakeMappedRendererCapabilities(C.TRACK_TYPE_VIDEO, rendererCapabilitiesMap); + trackSelector.setParameters( + defaultParameters.buildUpon().setAllowVideoMixedDecoderSupportAdaptiveness(true)); + result = + trackSelector.selectTracks( + new RendererCapabilities[] {rendererCapabilities}, trackGroups, periodId, TIMELINE); + assertAdaptiveSelection(result.selections[0], trackGroups.get(0), 3, 2, 0); + } + @Test public void selectTracksWithMultipleVideoTracksOverrideReturnsAdaptiveTrackSelection() throws Exception { @@ -2119,6 +2356,7 @@ public final class DefaultTrackSelectorTest { .setExceedVideoConstraintsIfNecessary(false) .setAllowVideoMixedMimeTypeAdaptiveness(true) .setAllowVideoNonSeamlessAdaptiveness(false) + .setAllowVideoMixedDecoderSupportAdaptiveness(true) .setViewportSize( /* viewportWidth= */ 8, /* viewportHeight= */ 9, @@ -2133,6 +2371,7 @@ public final class DefaultTrackSelectorTest { .setAllowAudioMixedMimeTypeAdaptiveness(true) .setAllowAudioMixedSampleRateAdaptiveness(false) .setAllowAudioMixedChannelCountAdaptiveness(true) + .setAllowAudioMixedDecoderSupportAdaptiveness(false) .setPreferredAudioMimeTypes(MimeTypes.AUDIO_AC3, MimeTypes.AUDIO_E_AC3) // Text .setPreferredTextLanguages("de", "en") From 6b3187ccf12a4d8850a58c87c0208ae955111c0b Mon Sep 17 00:00:00 2001 From: Marcel Dopita Date: Sat, 15 Jan 2022 20:37:40 +0100 Subject: [PATCH 12/12] Support MKV embedded WebVTT captions --- .../extractor/mkv/MatroskaExtractor.java | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mkv/MatroskaExtractor.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mkv/MatroskaExtractor.java index e8876ca52c..2cfefe2cef 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mkv/MatroskaExtractor.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/mkv/MatroskaExtractor.java @@ -138,6 +138,7 @@ public class MatroskaExtractor implements Extractor { private static final String CODEC_ID_PCM_FLOAT = "A_PCM/FLOAT/IEEE"; private static final String CODEC_ID_SUBRIP = "S_TEXT/UTF8"; private static final String CODEC_ID_ASS = "S_TEXT/ASS"; + private static final String CODEC_ID_VTT = "S_TEXT/WEBVTT"; private static final String CODEC_ID_VOBSUB = "S_VOBSUB"; private static final String CODEC_ID_PGS = "S_HDMV/PGS"; private static final String CODEC_ID_DVBSUB = "S_DVBSUB"; @@ -323,6 +324,32 @@ public class MatroskaExtractor implements Extractor { /** The format of an SSA timecode. */ private static final String SSA_TIMECODE_FORMAT = "%01d:%02d:%02d:%02d"; + /** + * A template for the prefix that must be added to each VTT sample. + * + *

      The display time of each subtitle is passed as {@code timeUs} to {@link + * TrackOutput#sampleMetadata}. The start and end timecodes in this template are relative to + * {@code timeUs}. Hence the start timecode is always zero. The 12 byte end timecode starting at + * {@link #VTT_PREFIX_END_TIMECODE_OFFSET} is set to a placeholder value, and must be replaced + * with the duration of the subtitle. + * + *

      Equivalent to the UTF-8 string: "WEBVTT\n\n00:00:00.000 --> 00:00:00.000\n". + */ + private static final byte[] VTT_PREFIX = + new byte[]{ + 87, 69, 66, 86, 84, 84, 10, 10, 48, 48, 58, 48, 48, 58, 48, 48, 46, 48, 48, 48, 32, 45, + 45, 62, 32, 48, 48, 58, 48, 48, 58, 48, 48, 46, 48, 48, 48, 10 + }; + /** The byte offset of the end timecode in {@link #VTT_PREFIX}. */ + private static final int VTT_PREFIX_END_TIMECODE_OFFSET = 25; + /** + * The value by which to divide a time in microseconds to convert it to the unit of the last value + * in a VTT timecode (milliseconds). + */ + private static final long VTT_TIMECODE_LAST_VALUE_SCALING_FACTOR = 1000; + /** The format of a VTT timecode. */ + private static final String VTT_TIMECODE_FORMAT = "%02d:%02d:%02d.%03d"; + /** The length in bytes of a WAVEFORMATEX structure. */ private static final int WAVE_FORMAT_SIZE = 18; /** Format tag indicating a WAVEFORMATEXTENSIBLE structure. */ @@ -1342,7 +1369,8 @@ public class MatroskaExtractor implements Extractor { track.trueHdSampleRechunker.sampleMetadata( track.output, timeUs, flags, size, offset, track.cryptoData); } else { - if (CODEC_ID_SUBRIP.equals(track.codecId) || CODEC_ID_ASS.equals(track.codecId)) { + if (CODEC_ID_SUBRIP.equals(track.codecId) || CODEC_ID_ASS.equals(track.codecId) + || CODEC_ID_VTT.equals(track.codecId)) { if (blockSampleCount > 1) { Log.w(TAG, "Skipping subtitle sample in laced block."); } else if (blockDurationUs == C.TIME_UNSET) { @@ -1415,6 +1443,9 @@ public class MatroskaExtractor implements Extractor { } else if (CODEC_ID_ASS.equals(track.codecId)) { writeSubtitleSampleData(input, SSA_PREFIX, size); return finishWriteSampleData(); + } else if (CODEC_ID_VTT.equals(track.codecId)) { + writeSubtitleSampleData(input, VTT_PREFIX, size); + return finishWriteSampleData(); } TrackOutput output = track.output; @@ -1641,7 +1672,8 @@ public class MatroskaExtractor implements Extractor { *

      See documentation on {@link #SSA_DIALOGUE_FORMAT} and {@link #SUBRIP_PREFIX} for why we use * the duration as the end timecode. * - * @param codecId The subtitle codec; must be {@link #CODEC_ID_SUBRIP} or {@link #CODEC_ID_ASS}. + * @param codecId The subtitle codec; must be {@link #CODEC_ID_SUBRIP}, {@link #CODEC_ID_ASS} or + * {@link #CODEC_ID_VTT}. * @param durationUs The duration of the sample, in microseconds. * @param subtitleData The subtitle sample in which to overwrite the end timecode (output * parameter). @@ -1662,6 +1694,12 @@ public class MatroskaExtractor implements Extractor { durationUs, SSA_TIMECODE_FORMAT, SSA_TIMECODE_LAST_VALUE_SCALING_FACTOR); endTimecodeOffset = SSA_PREFIX_END_TIMECODE_OFFSET; break; + case CODEC_ID_VTT: + endTimecode = + formatSubtitleTimecode( + durationUs, VTT_TIMECODE_FORMAT, VTT_TIMECODE_LAST_VALUE_SCALING_FACTOR); + endTimecodeOffset = VTT_PREFIX_END_TIMECODE_OFFSET; + break; default: throw new IllegalArgumentException(); } @@ -1830,6 +1868,7 @@ public class MatroskaExtractor implements Extractor { case CODEC_ID_PCM_FLOAT: case CODEC_ID_SUBRIP: case CODEC_ID_ASS: + case CODEC_ID_VTT: case CODEC_ID_VOBSUB: case CODEC_ID_PGS: case CODEC_ID_DVBSUB: @@ -2157,6 +2196,9 @@ public class MatroskaExtractor implements Extractor { mimeType = MimeTypes.TEXT_SSA; initializationData = ImmutableList.of(SSA_DIALOGUE_FORMAT, getCodecPrivate(codecId)); break; + case CODEC_ID_VTT: + mimeType = MimeTypes.TEXT_VTT; + break; case CODEC_ID_VOBSUB: mimeType = MimeTypes.APPLICATION_VOBSUB; initializationData = ImmutableList.of(getCodecPrivate(codecId)); @@ -2245,6 +2287,7 @@ public class MatroskaExtractor implements Extractor { .setColorInfo(colorInfo); } else if (MimeTypes.APPLICATION_SUBRIP.equals(mimeType) || MimeTypes.TEXT_SSA.equals(mimeType) + || MimeTypes.TEXT_VTT.equals(mimeType) || MimeTypes.APPLICATION_VOBSUB.equals(mimeType) || MimeTypes.APPLICATION_PGS.equals(mimeType) || MimeTypes.APPLICATION_DVBSUBS.equals(mimeType)) {