loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix decoder fallback logic for Dolby Atmos and Dolby Vision.

Browse Source 
The media codec renderers have fallback logic in getDecoderInfos
to assume that E-AC3 decoders can handle the 2D version of E-AC3-JOC and
that H264/H265 decoders can handle some base layer of Dolby Vision
content. Both fallbacks are useful if there is no decoder for the
enhanced Dolby formats.

Both fallbacks are not applied during track selection at the moment
because the separate MediaCodecInfo.isCodecSupported method verifies
that the mime type corresponding to format.codecs is the same as the
decoder mime type (which isn't true for the fallback case).

To fix the fallback logic, we can just completely remove this additional
check because it's not needed in the context of this method that is only
called after we already established that the decoder can handle the
format.sampleMimeType.

In addition, we need to map the Dolby Vision profiles to the equivalent
H264/H265 profile to make the codec profile comparison sensible again.

PiperOrigin-RevId: 420959104
This commit is contained in:
tonihei 2022-01-11 08:37:46 +00:00 committed by bachinger
parent 66aa1faf7e
commit b208d6d26e
6 changed files with 238 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java
View File

@ -59,9 +59,8 @@ import androidx.media3.exoplayer.mediacodec.MediaCodecRenderer;
import androidx.media3.exoplayer.mediacodec.MediaCodecSelector;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil.DecoderQueryException;
import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
/**
@ -385,27 +384,29 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
throws DecoderQueryException {
@Nullable String mimeType = format.sampleMimeType;
if (mimeType == null) {
return Collections.emptyList();
return ImmutableList.of();
}
if (audioSink.supportsFormat(format)) {
// The format is supported directly, so a codec is only needed for decryption.
@Nullable MediaCodecInfo codecInfo = MediaCodecUtil.getDecryptOnlyDecoderInfo();
if (codecInfo != null) {
return Collections.singletonList(codecInfo);
return ImmutableList.of(codecInfo);
}
}
List<MediaCodecInfo> decoderInfos =
mediaCodecSelector.getDecoderInfos(
mimeType, requiresSecureDecoder, /* requiresTunnelingDecoder= */ false);
if (MimeTypes.AUDIO_E_AC3_JOC.equals(mimeType)) {
// E-AC3 decoders can decode JOC streams, but in 2-D rather than 3-D.
List<MediaCodecInfo> decoderInfosWithEac3 = new ArrayList<>(decoderInfos);
decoderInfosWithEac3.addAll(
mediaCodecSelector.getDecoderInfos(
MimeTypes.AUDIO_E_AC3, requiresSecureDecoder, /* requiresTunnelingDecoder= */ false));
decoderInfos = decoderInfosWithEac3;
@Nullable String alternativeMimeType = MediaCodecUtil.getAlternativeCodecMimeType(format);
if (alternativeMimeType == null) {
return ImmutableList.copyOf(decoderInfos);
}
return Collections.unmodifiableList(decoderInfos);
List<MediaCodecInfo> alternativeDecoderInfos =
mediaCodecSelector.getDecoderInfos(
alternativeMimeType, requiresSecureDecoder, /* requiresTunnelingDecoder= */ false);
return ImmutableList.<MediaCodecInfo>builder()
.addAll(decoderInfos)
.addAll(alternativeDecoderInfos)
.build();
}
@Override

41
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfo.java
View File

@ -243,7 +243,11 @@ public final class MediaCodecInfo {
* @throws MediaCodecUtil.DecoderQueryException Thrown if an error occurs while querying decoders.
*/
public boolean isFormatSupported(Format format) throws MediaCodecUtil.DecoderQueryException {
if (!isCodecSupported(format)) {
if (!isSampleMimeTypeSupported(format)) {
return false;
}
if (!isCodecProfileAndLevelSupported(format)) {
return false;
}
@ -270,25 +274,15 @@ public final class MediaCodecInfo {
}
}
/**
* Whether the decoder supports the codec of the given {@code format}. If there is insufficient
* information to decide, returns true.
*
* @param format The input media format.
* @return True if the codec of the given {@code format} is supported by the decoder.
*/
public boolean isCodecSupported(Format format) {
if (format.codecs == null || mimeType == null) {
return true;
private boolean isSampleMimeTypeSupported(Format format) {
return mimeType.equals(format.sampleMimeType)
|| mimeType.equals(MediaCodecUtil.getAlternativeCodecMimeType(format));
}
String codecMimeType = MimeTypes.getMediaMimeType(format.codecs);
if (codecMimeType == null) {
private boolean isCodecProfileAndLevelSupported(Format format) {
if (format.codecs == null) {
return true;
}
if (!mimeType.equals(codecMimeType)) {
logNoSupport("codec.mime " + format.codecs + ", " + codecMimeType);
return false;
}
Pair<Integer, Integer> codecProfileAndLevel = MediaCodecUtil.getCodecProfileAndLevel(format);
if (codecProfileAndLevel == null) {
// If we don't know any better, we assume that the profile and level are supported.
@ -296,6 +290,19 @@ public final class MediaCodecInfo {
}
int profile = codecProfileAndLevel.first;
int level = codecProfileAndLevel.second;
if (MimeTypes.VIDEO_DOLBY_VISION.equals(format.sampleMimeType)) {
// If this codec is H264 or H265, we only support the Dolby Vision base layer and need to map
// the Dolby Vision profile to the corresponding base layer profile. Also assume all levels of
// this base layer profile are supported.
if (MimeTypes.VIDEO_H264.equals(mimeType)) {
profile = CodecProfileLevel.AVCProfileHigh;
level = 0;
} else if (MimeTypes.VIDEO_H265.equals(mimeType)) {
profile = CodecProfileLevel.HEVCProfileMain10;
level = 0;
}
}
if (!isVideo && profile != CodecProfileLevel.AACObjectXHE) {
// Some devices/builds underreport audio capabilities, so assume support except for xHE-AAC
// which may not be widely supported. See https://github.com/google/ExoPlayer/issues/5145.

42
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecUtil.java
View File

@ -36,6 +36,7 @@ import androidx.media3.common.util.Log;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@ -183,9 +184,9 @@ public final class MediaCodecUtil {
}
}
applyWorkarounds(mimeType, decoderInfos);
List<MediaCodecInfo> unmodifiableDecoderInfos = Collections.unmodifiableList(decoderInfos);
decoderInfosCache.put(key, unmodifiableDecoderInfos);
return unmodifiableDecoderInfos;
ImmutableList<MediaCodecInfo> immutableDecoderInfos = ImmutableList.copyOf(decoderInfos);
decoderInfosCache.put(key, immutableDecoderInfos);
return immutableDecoderInfos;
}
/**
@ -268,6 +269,41 @@ public final class MediaCodecUtil {
}
}
/**
* Returns an alternative codec MIME type (besides the default {@link Format#sampleMimeType}) that
* can be used to decode samples of the provided {@link Format}.
*
* @param format The media format.
* @return An alternative MIME type of a codec that be used decode samples of the provided {@code
* Format} (besides the default {@link Format#sampleMimeType}), or null if no such alternative
* exists.
*/
@Nullable
public static String getAlternativeCodecMimeType(Format format) {
if (MimeTypes.AUDIO_E_AC3_JOC.equals(format.sampleMimeType)) {
// E-AC3 decoders can decode JOC streams, but in 2-D rather than 3-D.
return MimeTypes.AUDIO_E_AC3;
}
if (MimeTypes.VIDEO_DOLBY_VISION.equals(format.sampleMimeType)) {
// H.264/AVC or H.265/HEVC decoders can decode the base layer of some DV profiles. This can't
// be done for profile CodecProfileLevel.DolbyVisionProfileDvheStn and profile
// CodecProfileLevel.DolbyVisionProfileDvheDtb because the first one is not backward
// compatible and the second one is deprecated and is not always backward compatible.
@Nullable
Pair<Integer, Integer> codecProfileAndLevel = MediaCodecUtil.getCodecProfileAndLevel(format);
if (codecProfileAndLevel != null) {
int profile = codecProfileAndLevel.first;
if (profile == CodecProfileLevel.DolbyVisionProfileDvheDtr
|| profile == CodecProfileLevel.DolbyVisionProfileDvheSt) {
return MimeTypes.VIDEO_H265;
} else if (profile == CodecProfileLevel.DolbyVisionProfileDvavSe) {
return MimeTypes.VIDEO_H264;
}
}
}
return null;
}
// Internal methods.
/**

38
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View File

@ -69,7 +69,6 @@ import androidx.media3.exoplayer.mediacodec.MediaCodecUtil.DecoderQueryException
import androidx.media3.exoplayer.video.VideoRendererEventListener.EventDispatcher;
import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.List;
/**
@ -465,42 +464,23 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
throws DecoderQueryException {
@Nullable String mimeType = format.sampleMimeType;
if (mimeType == null) {
return Collections.emptyList();
return ImmutableList.of();
}
List<MediaCodecInfo> decoderInfos =
mediaCodecSelector.getDecoderInfos(
mimeType, requiresSecureDecoder, requiresTunnelingDecoder);
if (MimeTypes.VIDEO_DOLBY_VISION.equals(mimeType)) {
// Fall back to H.264/AVC or H.265/HEVC for the relevant DV profiles. This can't be done for
// profile CodecProfileLevel.DolbyVisionProfileDvheStn and profile
// CodecProfileLevel.DolbyVisionProfileDvheDtb because the first one is not backward
// compatible and the second one is deprecated and is not always backward compatible.
@Nullable
Pair<Integer, Integer> codecProfileAndLevel = MediaCodecUtil.getCodecProfileAndLevel(format);
if (codecProfileAndLevel != null) {
List<MediaCodecInfo> fallbackDecoderInfos;
int profile = codecProfileAndLevel.first;
if (profile == CodecProfileLevel.DolbyVisionProfileDvheDtr
|| profile == CodecProfileLevel.DolbyVisionProfileDvheSt) {
fallbackDecoderInfos =
mediaCodecSelector.getDecoderInfos(
MimeTypes.VIDEO_H265, requiresSecureDecoder, requiresTunnelingDecoder);
} else if (profile == CodecProfileLevel.DolbyVisionProfileDvavSe) {
fallbackDecoderInfos =
mediaCodecSelector.getDecoderInfos(
MimeTypes.VIDEO_H264, requiresSecureDecoder, requiresTunnelingDecoder);
} else {
fallbackDecoderInfos = ImmutableList.of();
@Nullable String alternativeMimeType = MediaCodecUtil.getAlternativeCodecMimeType(format);
if (alternativeMimeType == null) {
return ImmutableList.copyOf(decoderInfos);
}
decoderInfos =
ImmutableList.<MediaCodecInfo>builder()
List<MediaCodecInfo> alternativeDecoderInfos =
mediaCodecSelector.getDecoderInfos(
alternativeMimeType, requiresSecureDecoder, requiresTunnelingDecoder);
return ImmutableList.<MediaCodecInfo>builder()
.addAll(decoderInfos)
.addAll(fallbackDecoderInfos)
.addAll(alternativeDecoderInfos)
.build();
}
}
return Collections.unmodifiableList(decoderInfos);
}
@Override
protected void onEnabled(boolean joining, boolean mayRenderStartOfStream)

48
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/MediaCodecAudioRendererTest.java
View File

@ -18,6 +18,7 @@ package androidx.media3.exoplayer.audio;
import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM;
import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.format;
import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.oneByteSample;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
@ -37,6 +38,8 @@ import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.PlaybackException;
import androidx.media3.exoplayer.ExoPlaybackException;
import androidx.media3.exoplayer.RendererCapabilities;
import androidx.media3.exoplayer.RendererCapabilities.Capabilities;
import androidx.media3.exoplayer.RendererConfiguration;
import androidx.media3.exoplayer.analytics.PlayerId;
import androidx.media3.exoplayer.drm.DrmSessionEventListener;
@ -84,8 +87,14 @@ public class MediaCodecAudioRendererTest {
// audioSink isEnded can always be true because the MediaCodecAudioRenderer isEnded =
// super.isEnded && audioSink.isEnded.
when(audioSink.isEnded()).thenReturn(true);
when(audioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true);
when(audioSink.supportsFormat(any()))
.thenAnswer(
invocation -> {
Format format = invocation.getArgument(/* index= */ 0, Format.class);
return MimeTypes.AUDIO_RAW.equals(format.sampleMimeType)
&& format.pcmEncoding == C.ENCODING_PCM_16BIT;
});
mediaCodecSelector =
(mimeType, requiresSecureDecoder, requiresTunnelingDecoder) ->
@ -315,6 +324,43 @@ public class MediaCodecAudioRendererTest {
verify(audioRendererEventListener).onAudioSinkError(error);
}
@Test
public void supportsFormat_withEac3JocMediaAndEac3Decoder_returnsTrue() throws Exception {
Format mediaFormat =
new Format.Builder()
.setSampleMimeType(MimeTypes.AUDIO_E_AC3_JOC)
.setCodecs(MimeTypes.CODEC_E_AC3_JOC)
.build();
MediaCodecSelector mediaCodecSelector =
(mimeType, requiresSecureDecoder, requiresTunnelingDecoder) ->
!mimeType.equals(MimeTypes.AUDIO_E_AC3)
? ImmutableList.of()
: ImmutableList.of(
MediaCodecInfo.newInstance(
/* name= */ "eac3-codec",
/* mimeType= */ mimeType,
/* codecMimeType= */ mimeType,
/* capabilities= */ null,
/* hardwareAccelerated= */ false,
/* softwareOnly= */ true,
/* vendor= */ false,
/* forceDisableAdaptive= */ false,
/* forceSecure= */ false));
MediaCodecAudioRenderer renderer =
new MediaCodecAudioRenderer(
ApplicationProvider.getApplicationContext(),
mediaCodecSelector,
/* enableDecoderFallback= */ false,
/* eventHandler= */ new Handler(Looper.getMainLooper()),
audioRendererEventListener,
audioSink);
renderer.init(/* index= */ 0, PlayerId.UNSET);
@Capabilities int capabilities = renderer.supportsFormat(mediaFormat);
assertThat(RendererCapabilities.getFormatSupport(capabilities)).isEqualTo(C.FORMAT_HANDLED);
}
private static Format getAudioSinkFormat(Format inputFormat) {
return new Format.Builder()
.setSampleMimeType(MimeTypes.AUDIO_RAW)

103
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java
View File

@ -27,6 +27,8 @@ import static org.mockito.Mockito.verify;
import static org.robolectric.Shadows.shadowOf;
import android.graphics.SurfaceTexture;
import android.media.MediaCodecInfo.CodecCapabilities;
import android.media.MediaCodecInfo.CodecProfileLevel;
import android.media.MediaFormat;
import android.os.Handler;
import android.os.Looper;
@ -39,7 +41,9 @@ import androidx.media3.common.MimeTypes;
import androidx.media3.common.VideoSize;
import androidx.media3.exoplayer.Renderer;
import androidx.media3.exoplayer.RendererCapabilities;
import androidx.media3.exoplayer.RendererCapabilities.Capabilities;
import androidx.media3.exoplayer.RendererConfiguration;
import androidx.media3.exoplayer.analytics.PlayerId;
import androidx.media3.exoplayer.drm.DrmSessionEventListener;
import androidx.media3.exoplayer.drm.DrmSessionManager;
import androidx.media3.exoplayer.mediacodec.MediaCodecInfo;
@ -507,4 +511,103 @@ public class MediaCodecVideoRendererTest {
verify(eventListener, times(2))
.onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong());
}
@Test
public void supportsFormat_withDolbyVisionMedia_returnsTrueWhenFallbackToH265orH264Allowed()
throws Exception {
// Create Dolby media formats that could fall back to H265 or H264.
Format formatDvheDtrFallbackToH265 =
new Format.Builder()
.setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION)
.setCodecs("dvhe.04.01")
.build();
Format formatDvheStFallbackToH265 =
new Format.Builder()
.setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION)
.setCodecs("dvhe.08.01")
.build();
Format formatDvavSeFallbackToH264 =
new Format.Builder()
.setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION)
.setCodecs("dvav.09.01")
.build();
Format formatNoFallbackPossible =
new Format.Builder()
.setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION)
.setCodecs("dvav.01.01")
.build();
// Only provide H264 and H265 decoders with codec profiles needed for fallback.
MediaCodecSelector mediaCodecSelector =
(mimeType, requiresSecureDecoder, requiresTunnelingDecoder) -> {
switch (mimeType) {
case MimeTypes.VIDEO_H264:
CodecCapabilities capabilitiesH264 = new CodecCapabilities();
capabilitiesH264.profileLevels =
new CodecProfileLevel[] {new CodecProfileLevel(), new CodecProfileLevel()};
capabilitiesH264.profileLevels[0].profile = CodecProfileLevel.AVCProfileBaseline;
capabilitiesH264.profileLevels[0].level = CodecProfileLevel.AVCLevel42;
capabilitiesH264.profileLevels[1].profile = CodecProfileLevel.AVCProfileHigh;
capabilitiesH264.profileLevels[1].level = CodecProfileLevel.AVCLevel42;
return ImmutableList.of(
MediaCodecInfo.newInstance(
/* name= */ "h264-codec",
/* mimeType= */ mimeType,
/* codecMimeType= */ mimeType,
/* capabilities= */ capabilitiesH264,
/* hardwareAccelerated= */ false,
/* softwareOnly= */ true,
/* vendor= */ false,
/* forceDisableAdaptive= */ false,
/* forceSecure= */ false));
case MimeTypes.VIDEO_H265:
CodecCapabilities capabilitiesH265 = new CodecCapabilities();
capabilitiesH265.profileLevels =
new CodecProfileLevel[] {new CodecProfileLevel(), new CodecProfileLevel()};
capabilitiesH265.profileLevels[0].profile = CodecProfileLevel.HEVCProfileMain;
capabilitiesH265.profileLevels[0].level = CodecProfileLevel.HEVCMainTierLevel41;
capabilitiesH265.profileLevels[1].profile = CodecProfileLevel.HEVCProfileMain10;
capabilitiesH265.profileLevels[1].level = CodecProfileLevel.HEVCHighTierLevel51;
return ImmutableList.of(
MediaCodecInfo.newInstance(
/* name= */ "h265-codec",
/* mimeType= */ mimeType,
/* codecMimeType= */ mimeType,
/* capabilities= */ capabilitiesH265,
/* hardwareAccelerated= */ false,
/* softwareOnly= */ true,
/* vendor= */ false,
/* forceDisableAdaptive= */ false,
/* forceSecure= */ false));
default:
return ImmutableList.of();
}
};
MediaCodecVideoRenderer renderer =
new MediaCodecVideoRenderer(
ApplicationProvider.getApplicationContext(),
mediaCodecSelector,
/* allowedJoiningTimeMs= */ 0,
/* eventHandler= */ new Handler(testMainLooper),
/* eventListener= */ eventListener,
/* maxDroppedFramesToNotify= */ 1);
renderer.init(/* index= */ 0, PlayerId.UNSET);
@Capabilities
int capabilitiesDvheDtrFallbackToH265 = renderer.supportsFormat(formatDvheDtrFallbackToH265);
@Capabilities
int capabilitiesDvheStFallbackToH265 = renderer.supportsFormat(formatDvheStFallbackToH265);
@Capabilities
int capabilitiesDvavSeFallbackToH264 = renderer.supportsFormat(formatDvavSeFallbackToH264);
@Capabilities
int capabilitiesNoFallbackPossible = renderer.supportsFormat(formatNoFallbackPossible);
assertThat(RendererCapabilities.getFormatSupport(capabilitiesDvheDtrFallbackToH265))
.isEqualTo(C.FORMAT_HANDLED);
assertThat(RendererCapabilities.getFormatSupport(capabilitiesDvheStFallbackToH265))
.isEqualTo(C.FORMAT_HANDLED);
assertThat(RendererCapabilities.getFormatSupport(capabilitiesDvavSeFallbackToH264))
.isEqualTo(C.FORMAT_HANDLED);
assertThat(RendererCapabilities.getFormatSupport(capabilitiesNoFallbackPossible))
.isEqualTo(C.FORMAT_UNSUPPORTED_SUBTYPE);
}
}