loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add video language to DefaultTrackSelector

Browse Source 
PiperOrigin-RevId: 688155680
This commit is contained in:
ibaker 2024-10-21 08:59:35 -07:00 committed by Copybara-Service
parent e926b0df1e
commit b04b37074b
5 changed files with 218 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
RELEASENOTES.md
View File

@ -4,6 +4,13 @@
* Common Library:
* ExoPlayer:
* Consider language when selecting a video track. By default select a
'main' video track that matches the language of the selected audio
track, if available. Explicit video language preferences can be
expressed with
`TrackSelectionParameters.Builder.setPreferredVideoLanguage(s)`.
* Add `selectedAudioLanguage` parameter to
`DefaultTrackSelector.selectVideoTrack()` method.
* Transformer:
* Extractors:
* Fix media duration parsing in `mdhd` box of MP4 files to handle `-1`

48
libraries/common/src/main/java/androidx/media3/common/TrackSelectionParameters.java
View File

@ -90,6 +90,7 @@ public class TrackSelectionParameters {
private int viewportHeight;
private boolean viewportOrientationMayChange;
private ImmutableList<String> preferredVideoMimeTypes;
private ImmutableList<String> preferredVideoLanguages;
private @C.RoleFlags int preferredVideoRoleFlags;
// Audio
private ImmutableList<String> preferredAudioLanguages;
@ -127,6 +128,7 @@ public class TrackSelectionParameters {
viewportHeight = Integer.MAX_VALUE;
viewportOrientationMayChange = true;
preferredVideoMimeTypes = ImmutableList.of();
preferredVideoLanguages = ImmutableList.of();
preferredVideoRoleFlags = 0;
// Audio
preferredAudioLanguages = ImmutableList.of();
@ -194,6 +196,9 @@ public class TrackSelectionParameters {
preferredVideoMimeTypes =
ImmutableList.copyOf(
firstNonNull(bundle.getStringArray(FIELD_PREFERRED_VIDEO_MIMETYPES), new String[0]));
preferredVideoLanguages =
ImmutableList.copyOf(
firstNonNull(bundle.getStringArray(FIELD_PREFERRED_VIDEO_LANGUAGES), new String[0]));
preferredVideoRoleFlags =
bundle.getInt(
FIELD_PREFERRED_VIDEO_ROLE_FLAGS, DEFAULT_WITHOUT_CONTEXT.preferredVideoRoleFlags);
@ -284,6 +289,7 @@ public class TrackSelectionParameters {
/** Overrides the value of the builder with the value of {@link TrackSelectionParameters}. */
@EnsuresNonNull({
"preferredVideoMimeTypes",
"preferredVideoLanguages",
"preferredAudioLanguages",
"preferredAudioMimeTypes",
"audioOffloadPreferences",
@ -305,6 +311,7 @@ public class TrackSelectionParameters {
viewportHeight = parameters.viewportHeight;
viewportOrientationMayChange = parameters.viewportOrientationMayChange;
preferredVideoMimeTypes = parameters.preferredVideoMimeTypes;
preferredVideoLanguages = parameters.preferredVideoLanguages;
preferredVideoRoleFlags = parameters.preferredVideoRoleFlags;
// Audio
preferredAudioLanguages = parameters.preferredAudioLanguages;
@ -504,6 +511,36 @@ public class TrackSelectionParameters {
return this;
}
/**
* Sets the preferred language for video tracks.
*
* @param preferredVideoLanguage Preferred video language as an IETF BCP 47 conformant tag, or
* {@code null} to express no language preference for video track selection.
* @return This builder.
*/
@UnstableApi
@CanIgnoreReturnValue
public Builder setPreferredVideoLanguage(@Nullable String preferredVideoLanguage) {
return preferredVideoLanguage == null
? setPreferredVideoLanguages()
: setPreferredVideoLanguages(preferredVideoLanguage);
}
/**
* Sets the preferred languages for video tracks.
*
* @param preferredVideoLanguages Preferred video languages as IETF BCP 47 conformant tags in
* order of preference, or an empty array to express no language preference for video track
* selection.
* @return This builder.
*/
@UnstableApi
@CanIgnoreReturnValue
public Builder setPreferredVideoLanguages(String... preferredVideoLanguages) {
this.preferredVideoLanguages = normalizeLanguageCodes(preferredVideoLanguages);
return this;
}
/**
* Sets the preferred {@link C.RoleFlags} for video tracks.
*
@ -1148,6 +1185,11 @@ public class TrackSelectionParameters {
*/
public final ImmutableList<String> preferredVideoMimeTypes;
/**
* The preferred languages for video tracks as IETF BCP 47 conformant tags in order of preference.
*/
@UnstableApi public final ImmutableList<String> preferredVideoLanguages;
/**
* The preferred {@link C.RoleFlags} for video tracks. {@code 0} selects the default track if
* there is one, or the first track if there's no default. The default value is {@code 0}.
@ -1267,6 +1309,7 @@ public class TrackSelectionParameters {
this.viewportHeight = builder.viewportHeight;
this.viewportOrientationMayChange = builder.viewportOrientationMayChange;
this.preferredVideoMimeTypes = builder.preferredVideoMimeTypes;
this.preferredVideoLanguages = builder.preferredVideoLanguages;
this.preferredVideoRoleFlags = builder.preferredVideoRoleFlags;
// Audio
this.preferredAudioLanguages = builder.preferredAudioLanguages;
@ -1317,6 +1360,7 @@ public class TrackSelectionParameters {
&& viewportWidth == other.viewportWidth
&& viewportHeight == other.viewportHeight
&& preferredVideoMimeTypes.equals(other.preferredVideoMimeTypes)
&& preferredVideoLanguages.equals(other.preferredVideoLanguages)
&& preferredVideoRoleFlags == other.preferredVideoRoleFlags
// Audio
&& preferredAudioLanguages.equals(other.preferredAudioLanguages)
@ -1355,6 +1399,7 @@ public class TrackSelectionParameters {
result = 31 * result + viewportWidth;
result = 31 * result + viewportHeight;
result = 31 * result + preferredVideoMimeTypes.hashCode();
result = 31 * result + preferredVideoLanguages.hashCode();
result = 31 * result + preferredVideoRoleFlags;
// Audio
result = 31 * result + preferredAudioLanguages.hashCode();
@ -1410,6 +1455,7 @@ public class TrackSelectionParameters {
private static final String FIELD_AUDIO_OFFLOAD_PREFERENCES = Util.intToStringMaxRadix(30);
private static final String FIELD_IS_PREFER_IMAGE_OVER_VIDEO_ENABLED =
Util.intToStringMaxRadix(31);
private static final String FIELD_PREFERRED_VIDEO_LANGUAGES = Util.intToStringMaxRadix(32);
/**
* Defines a minimum field ID value for subclasses to use when implementing {@link #toBundle()}
@ -1439,6 +1485,8 @@ public class TrackSelectionParameters {
bundle.putBoolean(FIELD_VIEWPORT_ORIENTATION_MAY_CHANGE, viewportOrientationMayChange);
bundle.putStringArray(
FIELD_PREFERRED_VIDEO_MIMETYPES, preferredVideoMimeTypes.toArray(new String[0]));
bundle.putStringArray(
FIELD_PREFERRED_VIDEO_LANGUAGES, preferredVideoLanguages.toArray(new String[0]));
bundle.putInt(FIELD_PREFERRED_VIDEO_ROLE_FLAGS, preferredVideoRoleFlags);
// Audio
bundle.putStringArray(

107
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java
View File

@ -343,6 +343,20 @@ public class DefaultTrackSelector extends MappingTrackSelector
return this;
}
@CanIgnoreReturnValue
@Override
public ParametersBuilder setPreferredVideoLanguage(@Nullable String preferredVideoLanguage) {
super.setPreferredVideoLanguage(preferredVideoLanguage);
return this;
}
@CanIgnoreReturnValue
@Override
public ParametersBuilder setPreferredVideoLanguages(String... preferredVideoLanguages) {
super.setPreferredVideoLanguages(preferredVideoLanguages);
return this;
}
@SuppressWarnings("deprecation") // Intentionally returning deprecated type
@CanIgnoreReturnValue
@Override
@ -1153,6 +1167,20 @@ public class DefaultTrackSelector extends MappingTrackSelector
return this;
}
@CanIgnoreReturnValue
@Override
public Builder setPreferredVideoLanguage(@Nullable String preferredVideoLanguage) {
super.setPreferredVideoLanguage(preferredVideoLanguage);
return this;
}
@CanIgnoreReturnValue
@Override
public Builder setPreferredVideoLanguages(String... preferredVideoLanguages) {
super.setPreferredVideoLanguages(preferredVideoLanguages);
return this;
}
@CanIgnoreReturnValue
@Override
public Builder setPreferredVideoRoleFlags(@RoleFlags int preferredVideoRoleFlags) {
@ -2663,13 +2691,30 @@ public class DefaultTrackSelector extends MappingTrackSelector
ExoTrackSelection.@NullableType Definition[] definitions =
new ExoTrackSelection.Definition[rendererCount];
@Nullable
Pair<ExoTrackSelection.Definition, Integer> selectedAudio =
selectAudioTrack(
mappedTrackInfo,
rendererFormatSupports,
rendererMixedMimeTypeAdaptationSupports,
params);
if (selectedAudio != null) {
definitions[selectedAudio.second] = selectedAudio.first;
}
@Nullable
String selectedAudioLanguage =
selectedAudio == null
? null
: selectedAudio.first.group.getFormat(selectedAudio.first.tracks[0]).language;
@Nullable
Pair<ExoTrackSelection.Definition, Integer> selectedVideo =
selectVideoTrack(
mappedTrackInfo,
rendererFormatSupports,
rendererMixedMimeTypeAdaptationSupports,
params);
params,
selectedAudioLanguage);
@Nullable
Pair<ExoTrackSelection.Definition, Integer> selectedImage =
@ -2683,22 +2728,6 @@ public class DefaultTrackSelector extends MappingTrackSelector
definitions[selectedVideo.second] = selectedVideo.first;
}
@Nullable
Pair<ExoTrackSelection.Definition, Integer> selectedAudio =
selectAudioTrack(
mappedTrackInfo,
rendererFormatSupports,
rendererMixedMimeTypeAdaptationSupports,
params);
if (selectedAudio != null) {
definitions[selectedAudio.second] = selectedAudio.first;
}
@Nullable
String selectedAudioLanguage =
selectedAudio == null
? null
: selectedAudio.first.group.getFormat(selectedAudio.first.tracks[0]).language;
@Nullable
Pair<ExoTrackSelection.Definition, Integer> selectedText =
selectTextTrack(mappedTrackInfo, rendererFormatSupports, params, selectedAudioLanguage);
@ -2732,6 +2761,8 @@ public class DefaultTrackSelector extends MappingTrackSelector
* @param mixedMimeTypeSupports The {@link AdaptiveSupport} for mixed MIME type adaptation for the
* renderer.
* @param params The selector's current constraint parameters.
* @param selectedAudioLanguage The language of the selected audio track. May be null if the
* selected audio track declares no language or no audio track was selected.
* @return A pair of the selected {@link ExoTrackSelection.Definition} and the corresponding
* renderer index, or null if no selection was made.
* @throws ExoPlaybackException If an error occurs while selecting the tracks.
@ -2741,7 +2772,8 @@ public class DefaultTrackSelector extends MappingTrackSelector
MappedTrackInfo mappedTrackInfo,
@Capabilities int[][][] rendererFormatSupports,
@AdaptiveSupport int[] mixedMimeTypeSupports,
Parameters params)
Parameters params,
@Nullable String selectedAudioLanguage)
throws ExoPlaybackException {
if (params.audioOffloadPreferences.audioOffloadMode == AUDIO_OFFLOAD_MODE_REQUIRED) {
return null;
@ -2752,7 +2784,12 @@ public class DefaultTrackSelector extends MappingTrackSelector
rendererFormatSupports,
(int rendererIndex, TrackGroup group, @Capabilities int[] support) ->
VideoTrackInfo.createForTrackGroup(
rendererIndex, group, params, support, mixedMimeTypeSupports[rendererIndex]),
rendererIndex,
group,
params,
support,
selectedAudioLanguage,
mixedMimeTypeSupports[rendererIndex]),
VideoTrackInfo::compareSelections);
}
@ -3486,6 +3523,7 @@ public class DefaultTrackSelector extends MappingTrackSelector
TrackGroup trackGroup,
Parameters params,
@Capabilities int[] formatSupport,
@Nullable String selectedAudioLanguage,
@AdaptiveSupport int mixedMimeTypeAdaptationSupport) {
int maxPixelsToRetainForViewport =
getMaxVideoPixelsToRetainForViewport(
@ -3506,6 +3544,7 @@ public class DefaultTrackSelector extends MappingTrackSelector
/* trackIndex= */ i,
params,
formatSupport[i],
selectedAudioLanguage,
mixedMimeTypeAdaptationSupport,
isSuitableForViewport));
}
@ -3525,8 +3564,11 @@ public class DefaultTrackSelector extends MappingTrackSelector
private final int bitrate;
private final int pixelCount;
private final int preferredMimeTypeMatchIndex;
private final int preferredLanguageIndex;
private final int preferredLanguageScore;
private final int preferredRoleFlagsScore;
private final boolean hasMainOrNoRoleFlag;
private final int selectedAudioLanguageScore;
private final boolean allowMixedMimeTypes;
private final @SelectionEligibility int selectionEligibility;
private final boolean usesPrimaryDecoder;
@ -3539,6 +3581,7 @@ public class DefaultTrackSelector extends MappingTrackSelector
int trackIndex,
Parameters parameters,
@Capabilities int formatSupport,
@Nullable String selectedAudioLanguage,
@AdaptiveSupport int mixedMimeTypeAdaptationSupport,
boolean isSuitableForViewport) {
super(rendererIndex, trackGroup, trackIndex);
@ -3574,9 +3617,29 @@ public class DefaultTrackSelector extends MappingTrackSelector
format.frameRate != Format.NO_VALUE && format.frameRate >= MIN_REASONABLE_FRAME_RATE;
bitrate = format.bitrate;
pixelCount = format.getPixelCount();
int bestLanguageIndex = Integer.MAX_VALUE;
int bestLanguageScore = 0;
for (int i = 0; i < parameters.preferredVideoLanguages.size(); i++) {
int score =
getFormatLanguageScore(
format,
parameters.preferredVideoLanguages.get(i),
/* allowUndeterminedFormatLanguage= */ false);
if (score > 0) {
bestLanguageIndex = i;
bestLanguageScore = score;
break;
}
}
preferredLanguageIndex = bestLanguageIndex;
preferredLanguageScore = bestLanguageScore;
preferredRoleFlagsScore =
getRoleFlagMatchScore(format.roleFlags, parameters.preferredVideoRoleFlags);
hasMainOrNoRoleFlag = format.roleFlags == 0 || (format.roleFlags & C.ROLE_FLAG_MAIN) != 0;
boolean selectedAudioLanguageUndetermined =
normalizeUndeterminedLanguageToNull(selectedAudioLanguage) == null;
selectedAudioLanguageScore =
getFormatLanguageScore(format, selectedAudioLanguage, selectedAudioLanguageUndetermined);
int bestMimeTypeMatchIndex = Integer.MAX_VALUE;
for (int i = 0; i < parameters.preferredVideoMimeTypes.size(); i++) {
if (format.sampleMimeType != null
@ -3639,9 +3702,15 @@ public class DefaultTrackSelector extends MappingTrackSelector
.compareFalseFirst(
info1.isWithinRendererCapabilities, info2.isWithinRendererCapabilities)
// 1. Compare match with specific content preferences set by the parameters.
.compare(
info1.preferredLanguageIndex,
info2.preferredLanguageIndex,
Ordering.natural().reverse())
.compare(info1.preferredLanguageScore, info2.preferredLanguageScore)
.compare(info1.preferredRoleFlagsScore, info2.preferredRoleFlagsScore)
// 2. Compare match with implicit content preferences set by the media.
.compareFalseFirst(info1.hasMainOrNoRoleFlag, info2.hasMainOrNoRoleFlag)
.compare(info1.selectedAudioLanguageScore, info2.selectedAudioLanguageScore)
// 3. Compare match with 'reasonable' frame rate threshold.
.compareFalseFirst(info1.hasReasonableFrameRate, info2.hasReasonableFrameRate)
// 4. Compare match with technical preferences set by the parameters.

68
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java
View File

@ -743,6 +743,74 @@ public final class DefaultTrackSelectorTest {
assertFixedSelection(result.selections[0], trackGroups, enNonDefaultFormat);
}
/**
* Tests that track selector will select a video track with a language that matches the preferred
* language given by {@link Parameters}.
*/
@Test
public void selectTracksSelectPreferredAudioVideoLanguage() throws Exception {
Format.Builder formatBuilder = VIDEO_FORMAT.buildUpon();
Format frVideoFormat = formatBuilder.setLanguage("fra").build();
Format enVideoFormat = formatBuilder.setLanguage("eng").build();
TrackGroupArray trackGroups = wrapFormats(frVideoFormat, enVideoFormat);
trackSelector.setParameters(defaultParameters.buildUpon().setPreferredVideoLanguage("eng"));
TrackSelectorResult result =
trackSelector.selectTracks(
new RendererCapabilities[] {ALL_VIDEO_FORMAT_EXCEEDED_RENDERER_CAPABILITIES},
wrapFormats(frVideoFormat, enVideoFormat),
periodId,
TIMELINE);
assertFixedSelection(result.selections[0], trackGroups, enVideoFormat);
}
/**
* Tests that the default track selector will select:
*
* <ul>
* <li>A main video track matching the selected audio language when a main video track in
* another language is present.
* <li>A main video track that doesn't match the selected audio language when a main video track
* in the selected audio language is not present (but alternate video tracks in this
* language are present).
* </ul>
*/
@Test
public void defaultVideoTracksInteractWithSelectedAudioLanguageAsExpected()
throws ExoPlaybackException {
Format.Builder mainVideoBuilder = VIDEO_FORMAT.buildUpon().setRoleFlags(C.ROLE_FLAG_MAIN);
Format mainEnglish = mainVideoBuilder.setLanguage("eng").build();
Format mainGerman = mainVideoBuilder.setLanguage("deu").build();
Format mainNoLanguage = mainVideoBuilder.setLanguage(C.LANGUAGE_UNDETERMINED).build();
Format alternateGerman =
VIDEO_FORMAT.buildUpon().setRoleFlags(C.ROLE_FLAG_ALTERNATE).setLanguage("deu").build();
Format noLanguageAudio = AUDIO_FORMAT.buildUpon().setLanguage(null).build();
Format germanAudio = AUDIO_FORMAT.buildUpon().setLanguage("deu").build();
RendererCapabilities[] rendererCapabilities =
new RendererCapabilities[] {VIDEO_CAPABILITIES, AUDIO_CAPABILITIES};
// Neither the audio nor the forced text track define a language. We select them both under the
// assumption that they have matching language.
TrackGroupArray trackGroups = wrapFormats(noLanguageAudio, mainNoLanguage);
TrackSelectorResult result =
trackSelector.selectTracks(rendererCapabilities, trackGroups, periodId, TIMELINE);
assertFixedSelection(result.selections[0], trackGroups, mainNoLanguage);
// The audio declares german. The main german track should be selected (in favour of the main
// english track).
trackGroups = wrapFormats(germanAudio, mainGerman, mainEnglish);
result = trackSelector.selectTracks(rendererCapabilities, trackGroups, periodId, TIMELINE);
assertFixedSelection(result.selections[0], trackGroups, mainGerman);
// The audio declares german. The main english track should be selected because there's no
// main german track.
trackGroups = wrapFormats(germanAudio, alternateGerman, mainEnglish);
result = trackSelector.selectTracks(rendererCapabilities, trackGroups, periodId, TIMELINE);
assertFixedSelection(result.selections[0], trackGroups, mainEnglish);
}
/**
* Tests that track selector will prefer tracks that are within renderer's capabilities over track
* that exceed renderer's capabilities.

9
libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java
View File

@ -1155,13 +1155,18 @@ public final class CompositionPlayer extends SimpleBasePlayer
MappedTrackInfo mappedTrackInfo,
@RendererCapabilities.Capabilities int[][][] rendererFormatSupports,
@RendererCapabilities.AdaptiveSupport int[] mixedMimeTypeSupports,
Parameters params)
Parameters params,
@Nullable String selectedAudioLanguage)
throws ExoPlaybackException {
if (disableVideoPlayback) {
return null;
}
return super.selectVideoTrack(
mappedTrackInfo, rendererFormatSupports, mixedMimeTypeSupports, params);
mappedTrackInfo,
rendererFormatSupports,
mixedMimeTypeSupports,
params,
selectedAudioLanguage);
}
@Nullable