From a366590a0425ff138520752e2fc94c70429c20ab Mon Sep 17 00:00:00 2001 From: michaelkatz Date: Mon, 10 Oct 2022 10:35:17 +0000 Subject: [PATCH] Adjust track selection with Dolby Vision if display does not support If the sample type is Dolby Vision and the display does not support Dolby Vision, then the capabilities DecoderSupport flag is set to DECODER_SUPPORT_FALLBACK_MIMETYPE. This denotes that the renderer will use a decoder for a fallback mimetype if possible. This alters track selection as tracks with DecoderSupport DECODER_SUPPORT_PRIMARY are preferred. UnitTests included -DefaultTrackSelector test that checks track selection reordering with DECODER_SUPPORT_FALLBACK_MIMETYPE -MediaCodecVideoRenderer test that checks setting of DecoderSupport flag based on Display's Dolby Vision support Issue: google/ExoPlayer#8944 PiperOrigin-RevId: 480040876 --- .../exoplayer2/RendererCapabilities.java | 16 +-- .../trackselection/DefaultTrackSelector.java | 2 + .../video/MediaCodecVideoRenderer.java | 31 ++++-- .../DefaultTrackSelectorTest.java | 63 +++++++++++ .../video/MediaCodecVideoRendererTest.java | 100 ++++++++++++++++++ 5 files changed, 196 insertions(+), 16 deletions(-) diff --git a/library/core/src/main/java/com/google/android/exoplayer2/RendererCapabilities.java b/library/core/src/main/java/com/google/android/exoplayer2/RendererCapabilities.java index 57578e44d5..04469dedc3 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/RendererCapabilities.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/RendererCapabilities.java @@ -127,21 +127,23 @@ public interface RendererCapabilities { int HARDWARE_ACCELERATION_NOT_SUPPORTED = 0; /** - * Level of decoder support. One of {@link #DECODER_SUPPORT_PRIMARY} and {@link - * #DECODER_SUPPORT_FALLBACK}. + * Level of decoder support. One of {@link #DECODER_SUPPORT_FALLBACK_MIMETYPE}, {@link + * #DECODER_SUPPORT_FALLBACK}, and {@link #DECODER_SUPPORT_PRIMARY}. * *

For video renderers, the level of support is indicated for non-tunneled output. */ @Documented @Retention(RetentionPolicy.SOURCE) @Target(TYPE_USE) - @IntDef({ - DECODER_SUPPORT_PRIMARY, - DECODER_SUPPORT_FALLBACK, - }) + @IntDef({DECODER_SUPPORT_FALLBACK_MIMETYPE, DECODER_SUPPORT_PRIMARY, DECODER_SUPPORT_FALLBACK}) @interface DecoderSupport {} /** A mask to apply to {@link Capabilities} to obtain {@link DecoderSupport} only. */ - int MODE_SUPPORT_MASK = 0b1 << 7; + int MODE_SUPPORT_MASK = 0b11 << 7; + /** + * The renderer will use a decoder for fallback mimetype if possible as format's MIME type is + * unsupported + */ + int DECODER_SUPPORT_FALLBACK_MIMETYPE = 0b10 << 7; /** The renderer is able to use the primary decoder for the format's MIME type. */ int DECODER_SUPPORT_PRIMARY = 0b1 << 7; /** The renderer will use a fallback decoder. */ diff --git a/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java b/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java index 2ae767e384..76c7ba8e55 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelector.java @@ -3100,6 +3100,8 @@ public class DefaultTrackSelector extends MappingTrackSelector { return 0; } switch (mimeType) { + case MimeTypes.VIDEO_DOLBY_VISION: + return 5; case MimeTypes.VIDEO_AV1: return 4; case MimeTypes.VIDEO_H265: diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index c04a544512..615355caf4 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -40,6 +40,7 @@ import android.util.Pair; import android.view.Display; import android.view.Surface; import androidx.annotation.CallSuper; +import androidx.annotation.DoNotInline; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import com.google.android.exoplayer2.C; @@ -409,6 +410,12 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { @DecoderSupport int decoderSupport = isPreferredDecoder ? DECODER_SUPPORT_PRIMARY : DECODER_SUPPORT_FALLBACK; + if (Util.SDK_INT >= 26 + && MimeTypes.VIDEO_DOLBY_VISION.equals(format.sampleMimeType) + && !Api26.doesDisplaySupportDolbyVision(context)) { + decoderSupport = DECODER_SUPPORT_FALLBACK_MIMETYPE; + } + @TunnelingSupport int tunnelingSupport = TUNNELING_NOT_SUPPORTED; if (isFormatSupported) { List tunnelingDecoderInfos = @@ -485,8 +492,20 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { alternativeMimeType, requiresSecureDecoder, requiresTunnelingDecoder); if (Util.SDK_INT >= 26 && MimeTypes.VIDEO_DOLBY_VISION.equals(format.sampleMimeType) - && !alternativeDecoderInfos.isEmpty()) { - // If sample type is Dolby Vision, check if Display supports Dolby Vision + && !alternativeDecoderInfos.isEmpty() + && !Api26.doesDisplaySupportDolbyVision(context)) { + return ImmutableList.copyOf(alternativeDecoderInfos); + } + return ImmutableList.builder() + .addAll(decoderInfos) + .addAll(alternativeDecoderInfos) + .build(); + } + + @RequiresApi(26) + private static final class Api26 { + @DoNotInline + public static boolean doesDisplaySupportDolbyVision(Context context) { boolean supportsDolbyVision = false; DisplayManager displayManager = (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); @@ -501,14 +520,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } } } - if (!supportsDolbyVision) { - return ImmutableList.copyOf(alternativeDecoderInfos); - } + return supportsDolbyVision; } - return ImmutableList.builder() - .addAll(decoderInfos) - .addAll(alternativeDecoderInfos) - .build(); } @Override diff --git a/library/core/src/test/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelectorTest.java b/library/core/src/test/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelectorTest.java index a50e1e1def..c57f611752 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelectorTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/trackselection/DefaultTrackSelectorTest.java @@ -21,6 +21,7 @@ import static com.google.android.exoplayer2.C.FORMAT_UNSUPPORTED_SUBTYPE; import static com.google.android.exoplayer2.C.FORMAT_UNSUPPORTED_TYPE; import static com.google.android.exoplayer2.RendererCapabilities.ADAPTIVE_NOT_SEAMLESS; import static com.google.android.exoplayer2.RendererCapabilities.DECODER_SUPPORT_FALLBACK; +import static com.google.android.exoplayer2.RendererCapabilities.DECODER_SUPPORT_FALLBACK_MIMETYPE; import static com.google.android.exoplayer2.RendererCapabilities.DECODER_SUPPORT_PRIMARY; import static com.google.android.exoplayer2.RendererCapabilities.HARDWARE_ACCELERATION_NOT_SUPPORTED; import static com.google.android.exoplayer2.RendererCapabilities.HARDWARE_ACCELERATION_SUPPORTED; @@ -2242,6 +2243,68 @@ public final class DefaultTrackSelectorTest { assertAdaptiveSelection(result.selections[0], adaptiveGroup, /* expectedTracks...= */ 1, 0); } + /** + * Tests that track selector will select video track with support of its primary decoder over a + * track that will use a decoder for it's format fallback sampleMimetype. + */ + @Test + public void selectTracks_withDecoderSupportFallbackMimetype_selectsTrackWithPrimaryDecoder() + throws Exception { + Format formatDV = + new Format.Builder().setId("0").setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION).build(); + Format formatHevc = + new Format.Builder().setId("1").setSampleMimeType(MimeTypes.VIDEO_H265).build(); + TrackGroupArray trackGroups = + new TrackGroupArray(new TrackGroup(formatDV), new TrackGroup(formatHevc)); + @Capabilities + int capabilitiesDecoderSupportPrimary = + RendererCapabilities.create( + FORMAT_HANDLED, + ADAPTIVE_NOT_SEAMLESS, + TUNNELING_NOT_SUPPORTED, + HARDWARE_ACCELERATION_SUPPORTED, + DECODER_SUPPORT_PRIMARY); + int capabilitiesDecoderSupportFallbackType = + RendererCapabilities.create( + FORMAT_HANDLED, + ADAPTIVE_NOT_SEAMLESS, + TUNNELING_NOT_SUPPORTED, + HARDWARE_ACCELERATION_SUPPORTED, + DECODER_SUPPORT_FALLBACK_MIMETYPE); + + // Select track supported by primary decoder by default. + ImmutableMap rendererCapabilitiesMapDifferingDecoderSupport = + ImmutableMap.of( + "0", capabilitiesDecoderSupportFallbackType, "1", capabilitiesDecoderSupportPrimary); + RendererCapabilities rendererCapabilitiesDifferingDecoderSupport = + new FakeMappedRendererCapabilities( + C.TRACK_TYPE_VIDEO, rendererCapabilitiesMapDifferingDecoderSupport); + TrackSelectorResult result = + trackSelector.selectTracks( + new RendererCapabilities[] {rendererCapabilitiesDifferingDecoderSupport}, + trackGroups, + periodId, + TIMELINE); + + assertFixedSelection(result.selections[0], trackGroups, formatHevc); + + // Select Dolby Vision track over HEVC when renderer supports both equally + ImmutableMap rendererCapabilitiesMapAllPrimaryDecoderSupport = + ImmutableMap.of( + "0", capabilitiesDecoderSupportPrimary, "1", capabilitiesDecoderSupportPrimary); + RendererCapabilities rendererCapabilitiesAllPrimaryDecoderSupport = + new FakeMappedRendererCapabilities( + C.TRACK_TYPE_VIDEO, rendererCapabilitiesMapAllPrimaryDecoderSupport); + result = + trackSelector.selectTracks( + new RendererCapabilities[] {rendererCapabilitiesAllPrimaryDecoderSupport}, + trackGroups, + periodId, + TIMELINE); + + assertFixedSelection(result.selections[0], trackGroups, formatDV); + } + /** * Tests that track selector will select the video track with the highest number of matching role * flags given by {@link Parameters}. diff --git a/library/core/src/test/java/com/google/android/exoplayer2/video/MediaCodecVideoRendererTest.java b/library/core/src/test/java/com/google/android/exoplayer2/video/MediaCodecVideoRendererTest.java index 0e487748de..fc2fa8fc2e 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/video/MediaCodecVideoRendererTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/video/MediaCodecVideoRendererTest.java @@ -15,6 +15,7 @@ */ package com.google.android.exoplayer2.video; +import static android.view.Display.DEFAULT_DISPLAY; import static com.google.android.exoplayer2.testutil.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM; import static com.google.android.exoplayer2.testutil.FakeSampleStream.FakeSampleStreamItem.format; import static com.google.android.exoplayer2.testutil.FakeSampleStream.FakeSampleStreamItem.oneByteSample; @@ -26,13 +27,16 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.robolectric.Shadows.shadowOf; +import android.content.Context; import android.graphics.SurfaceTexture; +import android.hardware.display.DisplayManager; import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaCodecInfo.CodecProfileLevel; import android.media.MediaFormat; import android.os.Handler; import android.os.Looper; import android.os.SystemClock; +import android.view.Display; import android.view.Surface; import androidx.annotation.Nullable; import androidx.test.core.app.ApplicationProvider; @@ -63,6 +67,8 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; +import org.robolectric.Shadows; +import org.robolectric.shadows.ShadowDisplay; import org.robolectric.shadows.ShadowLooper; /** Unit test for {@link MediaCodecVideoRenderer}. */ @@ -610,6 +616,100 @@ public class MediaCodecVideoRendererTest { .isEqualTo(C.FORMAT_UNSUPPORTED_SUBTYPE); } + @Test + public void supportsFormat_withDolbyVision_setsDecoderSupportFlagsByDisplayDolbyVisionSupport() + throws Exception { + Format formatDvheDtr = + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setCodecs("dvhe.04.01") + .build(); + // Provide supporting Dolby Vision and fallback HEVC decoders + MediaCodecSelector mediaCodecSelector = + (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) -> { + switch (mimeType) { + case MimeTypes.VIDEO_DOLBY_VISION: + { + CodecCapabilities capabilitiesDolby = new CodecCapabilities(); + capabilitiesDolby.profileLevels = new CodecProfileLevel[] {new CodecProfileLevel()}; + capabilitiesDolby.profileLevels[0].profile = + CodecProfileLevel.DolbyVisionProfileDvheDtr; + capabilitiesDolby.profileLevels[0].level = CodecProfileLevel.DolbyVisionLevelFhd30; + return ImmutableList.of( + MediaCodecInfo.newInstance( + /* name= */ "dvhe-codec", + /* mimeType= */ mimeType, + /* codecMimeType= */ mimeType, + /* capabilities= */ capabilitiesDolby, + /* hardwareAccelerated= */ true, + /* softwareOnly= */ false, + /* vendor= */ false, + /* forceDisableAdaptive= */ false, + /* forceSecure= */ false)); + } + case MimeTypes.VIDEO_H265: + { + CodecCapabilities capabilitiesH265 = new CodecCapabilities(); + capabilitiesH265.profileLevels = + new CodecProfileLevel[] {new CodecProfileLevel(), new CodecProfileLevel()}; + capabilitiesH265.profileLevels[0].profile = CodecProfileLevel.HEVCProfileMain; + capabilitiesH265.profileLevels[0].level = CodecProfileLevel.HEVCMainTierLevel41; + capabilitiesH265.profileLevels[1].profile = CodecProfileLevel.HEVCProfileMain10; + capabilitiesH265.profileLevels[1].level = CodecProfileLevel.HEVCHighTierLevel51; + return ImmutableList.of( + MediaCodecInfo.newInstance( + /* name= */ "h265-codec", + /* mimeType= */ mimeType, + /* codecMimeType= */ mimeType, + /* capabilities= */ capabilitiesH265, + /* hardwareAccelerated= */ true, + /* softwareOnly= */ false, + /* vendor= */ false, + /* forceDisableAdaptive= */ false, + /* forceSecure= */ false)); + } + default: + return ImmutableList.of(); + } + }; + MediaCodecVideoRenderer renderer = + new MediaCodecVideoRenderer( + ApplicationProvider.getApplicationContext(), + mediaCodecSelector, + /* allowedJoiningTimeMs= */ 0, + /* eventHandler= */ new Handler(testMainLooper), + /* eventListener= */ eventListener, + /* maxDroppedFramesToNotify= */ 1); + renderer.init(/* index= */ 0, PlayerId.UNSET); + + @Capabilities int capabilitiesDvheDtr = renderer.supportsFormat(formatDvheDtr); + + assertThat(RendererCapabilities.getDecoderSupport(capabilitiesDvheDtr)) + .isEqualTo(RendererCapabilities.DECODER_SUPPORT_FALLBACK_MIMETYPE); + + // Set Display to have Dolby Vision support + Context context = ApplicationProvider.getApplicationContext(); + DisplayManager displayManager = + (DisplayManager) context.getSystemService(Context.DISPLAY_SERVICE); + Display display = (displayManager != null) ? displayManager.getDisplay(DEFAULT_DISPLAY) : null; + ShadowDisplay shadowDisplay = Shadows.shadowOf(display); + int[] hdrCapabilities = + new int[] { + Display.HdrCapabilities.HDR_TYPE_HDR10, Display.HdrCapabilities.HDR_TYPE_DOLBY_VISION + }; + shadowDisplay.setDisplayHdrCapabilities( + display.getDisplayId(), + /* maxLuminance= */ 100f, + /* maxAverageLuminance= */ 100f, + /* minLuminance= */ 100f, + hdrCapabilities); + + capabilitiesDvheDtr = renderer.supportsFormat(formatDvheDtr); + + assertThat(RendererCapabilities.getDecoderSupport(capabilitiesDvheDtr)) + .isEqualTo(RendererCapabilities.DECODER_SUPPORT_PRIMARY); + } + @Test public void getCodecMaxInputSize_videoH263() { MediaCodecInfo codecInfo = createMediaCodecInfo(MimeTypes.VIDEO_H263);