From b83b16eba71b4f5c05a99c719e1cbc49c418e20e Mon Sep 17 00:00:00 2001 From: christosts Date: Mon, 15 Aug 2022 11:47:42 +0000 Subject: [PATCH] Increase max sample size for HEVC. Increase the estimated max sample size for HEVC by 2x, and set a minimum size of 2MB. The 2MB will be applied for resolutions up to 1080p, after which the new calculation takes effect. This is in par with the platform's HEVC software decoder. PiperOrigin-RevId: 467641494 --- .../video/MediaCodecVideoRenderer.java | 49 ++- .../video/MediaCodecVideoRendererTest.java | 358 ++++++++++++++++++ 2 files changed, 386 insertions(+), 21 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 9cc8a6e937..968ba993e9 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -114,6 +114,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { /** Magic frame render timestamp that indicates the EOS in tunneling mode. */ private static final long TUNNELING_EOS_PRESENTATION_TIME_US = Long.MAX_VALUE; + /** The minimum input buffer size for HEVC. */ + private static final int HEVC_MAX_INPUT_SIZE_THRESHOLD = 2 * 1024 * 1024; + private static boolean evaluatedDeviceNeedsSetOutputSurfaceWorkaround; private static boolean deviceNeedsSetOutputSurfaceWorkaround; @@ -790,14 +793,20 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } // Attempt to infer a maximum input size from the format. - int maxPixels; - int minCompressionRatio; switch (sampleMimeType) { case MimeTypes.VIDEO_H263: case MimeTypes.VIDEO_MP4V: - maxPixels = width * height; - minCompressionRatio = 2; - break; + case MimeTypes.VIDEO_AV1: + // Assume a min compression of 2 similar to the platform's C2SoftAomDec.cpp. + case MimeTypes.VIDEO_VP8: + // Assume a min compression of 2 similar to the platform's SoftVPX.cpp. + return getMaxSampleSize(/* pixelCount= */ width * height, /* minCompressionRatio= */ 2); + case MimeTypes.VIDEO_H265: + // Assume a min compression of 2 similar to the platform's C2SoftHevcDec.cpp, but restrict + // the minimum size. + return max( + HEVC_MAX_INPUT_SIZE_THRESHOLD, + getMaxSampleSize(/* pixelCount= */ width * height, /* minCompressionRatio= */ 2)); case MimeTypes.VIDEO_H264: if ("BRAVIA 4K 2015".equals(Util.MODEL) // Sony Bravia 4K || ("Amazon".equals(Util.MANUFACTURER) @@ -808,27 +817,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return Format.NO_VALUE; } // Round up width/height to an integer number of macroblocks. - maxPixels = Util.ceilDivide(width, 16) * Util.ceilDivide(height, 16) * 16 * 16; - minCompressionRatio = 2; - break; - case MimeTypes.VIDEO_AV1: - // AV1 does not specify a ratio so use the values from the platform's C2SoftAomDec.cpp. - case MimeTypes.VIDEO_VP8: - // VPX does not specify a ratio so use the values from the platform's SoftVPX.cpp. - maxPixels = width * height; - minCompressionRatio = 2; - break; - case MimeTypes.VIDEO_H265: + int maxPixels = Util.ceilDivide(width, 16) * Util.ceilDivide(height, 16) * 16 * 16; + return getMaxSampleSize(maxPixels, /* minCompressionRatio= */ 2); case MimeTypes.VIDEO_VP9: - maxPixels = width * height; - minCompressionRatio = 4; - break; + return getMaxSampleSize(/* pixelCount= */ width * height, /* minCompressionRatio= */ 4); default: // Leave the default max input size. return Format.NO_VALUE; } - // Estimate the maximum input size assuming three channel 4:2:0 subsampled input frames. - return (maxPixels * 3) / (2 * minCompressionRatio); } @Override @@ -1736,6 +1732,17 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } } + /** + * Returns the maximum sample size assuming three channel 4:2:0 subsampled input frames with the + * specified {@code minCompressionRatio} + * + * @param pixelCount The number of pixels + * @param minCompressionRatio The minimum compression ratio + */ + private static int getMaxSampleSize(int pixelCount, int minCompressionRatio) { + return (pixelCount * 3) / (2 * minCompressionRatio); + } + private static boolean evaluateDeviceNeedsSetOutputSurfaceWorkaround() { if (Util.SDK_INT <= 28) { // Workaround for MiTV and MiBox devices which have been observed broken up to API 28. diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index d12739b2b4..32733b496a 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -610,4 +610,362 @@ public class MediaCodecVideoRendererTest { assertThat(RendererCapabilities.getFormatSupport(capabilitiesNoFallbackPossible)) .isEqualTo(C.FORMAT_UNSUPPORTED_SUBTYPE); } + + @Test + public void getCodecMaxInputSize_videoH263() { + MediaCodecInfo codecInfo = createMediaCodecInfo(MimeTypes.VIDEO_H263); + + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, createFormat(MimeTypes.VIDEO_H263, /* width= */ 640, /* height= */ 480))) + .isEqualTo(230400); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, + createFormat(MimeTypes.VIDEO_H263, /* width= */ 1280, /* height= */ 720))) + .isEqualTo(691200); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, createFormat(MimeTypes.VIDEO_H263, 1920, 1080))) + .isEqualTo(1555200); + } + + @Test + public void getCodecMaxInputSize_videoH264() { + MediaCodecInfo codecInfo = createMediaCodecInfo(MimeTypes.VIDEO_H264); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, createFormat(MimeTypes.VIDEO_H264, /* width= */ 640, /* height= */ 480))) + .isEqualTo(230400); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, + createFormat(MimeTypes.VIDEO_H264, /* width= */ 1280, /* height= */ 720))) + .isEqualTo(691200); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, + createFormat(MimeTypes.VIDEO_H264, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(1566720); + } + + @Test + public void getCodecMaxInputSize_videoHevc() { + MediaCodecInfo codecInfo = createMediaCodecInfo(MimeTypes.VIDEO_H265); + + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, createFormat(MimeTypes.VIDEO_H265, /* width= */ 640, /* height= */ 480))) + .isEqualTo(2097152); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, + createFormat(MimeTypes.VIDEO_H265, /* width= */ 1280, /* height= */ 720))) + .isEqualTo(2097152); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, + createFormat(MimeTypes.VIDEO_H265, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(2097152); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, + createFormat(MimeTypes.VIDEO_H265, /* width= */ 3840, /* height= */ 2160))) + .isEqualTo(6220800); + } + + @Test + public void getCodecMaxInputSize_videoMp4v() { + MediaCodecInfo codecInfo = createMediaCodecInfo(MimeTypes.VIDEO_MP4V); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, createFormat(MimeTypes.VIDEO_MP4V, /* width= */ 640, /* height= */ 480))) + .isEqualTo(230400); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, + createFormat(MimeTypes.VIDEO_MP4V, /* width= */ 1280, /* height= */ 720))) + .isEqualTo(691200); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, + createFormat(MimeTypes.VIDEO_MP4V, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(1555200); + } + + @Test + public void getCodecMaxInputSize_videoAv1() { + MediaCodecInfo codecInfo = createMediaCodecInfo(MimeTypes.VIDEO_AV1); + + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, createFormat(MimeTypes.VIDEO_MP4V, /* width= */ 640, /* height= */ 480))) + .isEqualTo(230400); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, + createFormat(MimeTypes.VIDEO_MP4V, /* width= */ 1280, /* height= */ 720))) + .isEqualTo(691200); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, + createFormat(MimeTypes.VIDEO_MP4V, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(1555200); + } + + @Test + public void getCodecMaxInputSize_videoVp8() { + MediaCodecInfo vp8CodecInfo = createMediaCodecInfo(MimeTypes.VIDEO_VP8); + + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + vp8CodecInfo, + createFormat(MimeTypes.VIDEO_VP8, /* width= */ 640, /* height= */ 480))) + .isEqualTo(230400); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + vp8CodecInfo, + createFormat(MimeTypes.VIDEO_VP8, /* width= */ 1280, /* height= */ 720))) + .isEqualTo(691200); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + vp8CodecInfo, + createFormat(MimeTypes.VIDEO_VP8, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(1555200); + } + + @Test + public void getCodecMaxInputSize_dolbyVision_fallBack() { + MediaCodecInfo dvCodecInfo = createMediaCodecInfo(MimeTypes.VIDEO_DOLBY_VISION); + int h264MaxSampleSize = + MediaCodecVideoRenderer.getCodecMaxInputSize( + createMediaCodecInfo(MimeTypes.VIDEO_H264), + createFormat(MimeTypes.VIDEO_H264, /* width= */ 1920, /* height= */ 1080)); + int hevcMaxSampleSize = + MediaCodecVideoRenderer.getCodecMaxInputSize( + createMediaCodecInfo(MimeTypes.VIDEO_H265), + createFormat(MimeTypes.VIDEO_H265, /* width= */ 1920, /* height= */ 1080)); + + // DV format without codec string fallbacks to HEVC. + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + dvCodecInfo, + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setWidth(1920) + .setHeight(1080) + .build())) + .isEqualTo(hevcMaxSampleSize); + // DV profiles "00", "01" and "09" fallback to H264. + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + dvCodecInfo, + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setCodecs("dvhe.00.01") + .setWidth(1920) + .setHeight(1080) + .build())) + .isEqualTo(h264MaxSampleSize); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + dvCodecInfo, + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setCodecs("dvhe.01.01") + .setWidth(1920) + .setHeight(1080) + .build())) + .isEqualTo(h264MaxSampleSize); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + dvCodecInfo, + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setCodecs("dvhe.09.01") + .setWidth(1920) + .setHeight(1080) + .build())) + .isEqualTo(h264MaxSampleSize); + // DV profiles "02", "03", "04", "05", "06, "07" and "08" fallback to HEVC. + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + dvCodecInfo, + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setCodecs("dvhe.02.01") + .setWidth(1920) + .setHeight(1080) + .build())) + .isEqualTo(hevcMaxSampleSize); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + dvCodecInfo, + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setCodecs("dvhe.03.01") + .setWidth(1920) + .setHeight(1080) + .build())) + .isEqualTo(hevcMaxSampleSize); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + dvCodecInfo, + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setCodecs("dvhe.04.01") + .setWidth(1920) + .setHeight(1080) + .build())) + .isEqualTo(hevcMaxSampleSize); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + dvCodecInfo, + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setCodecs("dvhe.05.01") + .setWidth(1920) + .setHeight(1080) + .build())) + .isEqualTo(hevcMaxSampleSize); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + dvCodecInfo, + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setCodecs("dvhe.06.01") + .setWidth(1920) + .setHeight(1080) + .build())) + .isEqualTo(hevcMaxSampleSize); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + dvCodecInfo, + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setCodecs("dvhe.07.01") + .setWidth(1920) + .setHeight(1080) + .build())) + .isEqualTo(hevcMaxSampleSize); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + dvCodecInfo, + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_DOLBY_VISION) + .setCodecs("dvhe.08.01") + .setWidth(1920) + .setHeight(1080) + .build())) + .isEqualTo(hevcMaxSampleSize); + } + + @Test + public void getCodecMaxInputSize_videoVp9() { + MediaCodecInfo codecInfo = createMediaCodecInfo(MimeTypes.VIDEO_VP9); + + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, createFormat(MimeTypes.VIDEO_VP9, /* width= */ 640, /* height= */ 480))) + .isEqualTo(115200); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, createFormat(MimeTypes.VIDEO_VP9, /* width= */ 1280, /* height= */ 720))) + .isEqualTo(345600); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + codecInfo, + createFormat(MimeTypes.VIDEO_VP9, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(777600); + } + + @Test + public void getCodecMaxInputSize_withUnsupportedFormat_returnsNoValue() { + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + createMediaCodecInfo(MimeTypes.VIDEO_MP43), + createFormat(MimeTypes.VIDEO_MP43, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(Format.NO_VALUE); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + createMediaCodecInfo(MimeTypes.VIDEO_MP42), + createFormat(MimeTypes.VIDEO_MP42, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(Format.NO_VALUE); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + createMediaCodecInfo(MimeTypes.VIDEO_MJPEG), + createFormat(MimeTypes.VIDEO_MJPEG, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(Format.NO_VALUE); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + createMediaCodecInfo(MimeTypes.VIDEO_AVI), + createFormat(MimeTypes.VIDEO_AVI, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(Format.NO_VALUE); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + createMediaCodecInfo(MimeTypes.VIDEO_OGG), + createFormat(MimeTypes.VIDEO_OGG, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(Format.NO_VALUE); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + createMediaCodecInfo(MimeTypes.VIDEO_FLV), + createFormat(MimeTypes.VIDEO_FLV, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(Format.NO_VALUE); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + createMediaCodecInfo(MimeTypes.VIDEO_VC1), + createFormat(MimeTypes.VIDEO_VC1, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(Format.NO_VALUE); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + createMediaCodecInfo(MimeTypes.VIDEO_MPEG2), + createFormat(MimeTypes.VIDEO_MPEG2, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(Format.NO_VALUE); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + createMediaCodecInfo(MimeTypes.VIDEO_PS), + createFormat(MimeTypes.VIDEO_PS, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(Format.NO_VALUE); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + createMediaCodecInfo(MimeTypes.VIDEO_MPEG), + createFormat(MimeTypes.VIDEO_MPEG, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(Format.NO_VALUE); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + createMediaCodecInfo(MimeTypes.VIDEO_MP2T), + createFormat(MimeTypes.VIDEO_MP2T, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(Format.NO_VALUE); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + createMediaCodecInfo(MimeTypes.VIDEO_WEBM), + createFormat(MimeTypes.VIDEO_WEBM, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(Format.NO_VALUE); + assertThat( + MediaCodecVideoRenderer.getCodecMaxInputSize( + createMediaCodecInfo(MimeTypes.VIDEO_DIVX), + createFormat(MimeTypes.VIDEO_DIVX, /* width= */ 1920, /* height= */ 1080))) + .isEqualTo(Format.NO_VALUE); + } + + private static MediaCodecInfo createMediaCodecInfo(String mimeType) { + return MediaCodecInfo.newInstance( + /* name= */ mimeType, + /* mimeType= */ mimeType, + /* codecMimeType= */ mimeType, + /* capabilities= */ new CodecCapabilities(), + /* hardwareAccelerated= */ true, + /* softwareOnly= */ false, + /* vendor= */ true, + /* forceDisableAdaptive= */ false, + /* forceSecure= */ false); + } + + private static Format createFormat(String mimeType, int width, int height) { + return new Format.Builder() + .setSampleMimeType(mimeType) + .setWidth(width) + .setHeight(height) + .build(); + } }