From 7e873b5121500fc7b52dec0bc7b4c2b88a1b6d8f Mon Sep 17 00:00:00 2001 From: claincly Date: Wed, 2 Feb 2022 17:14:56 +0000 Subject: [PATCH] Improve MIME type fallback logic. - The MIME type should ideally default to HEVC if there is an encoder for it. - Next, check if AVC is supported. - If there is no encoder for AVC, then we should pick an encoder in the list of existing encoders instead of abandoning the transformation. PiperOrigin-RevId: 425900638 --- .../transformer/DefaultEncoderFactory.java | 69 +++++++++++-------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java index a8f7ee6133..9fe038957b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java @@ -18,6 +18,7 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.common.util.Util.SDK_INT; import static androidx.media3.transformer.CodecFactoryUtil.createCodec; @@ -41,8 +42,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @UnstableApi public final class DefaultEncoderFactory implements Codec.EncoderFactory { // TODO(b/214973843): Add option to disable fallback. - // TODO(b/210591626): Fall back adaptively to H265 if possible. - private static final String DEFAULT_FALLBACK_MIME_TYPE = MimeTypes.VIDEO_H264; private static final int DEFAULT_COLOR_FORMAT = MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; private static final int DEFAULT_FRAME_RATE = 60; @@ -147,22 +146,15 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { @Nullable private static Pair findEncoderWithClosestFormatSupport( Format requestedFormat, EncoderSelector encoderSelector, List allowedMimeTypes) { - String mimeType = requestedFormat.sampleMimeType; - - // TODO(b/210591626) Improve MIME type selection. - List encodersForMimeType = encoderSelector.selectEncoderInfos(mimeType); - if (!allowedMimeTypes.contains(mimeType) || encodersForMimeType.isEmpty()) { - mimeType = - allowedMimeTypes.contains(DEFAULT_FALLBACK_MIME_TYPE) - ? DEFAULT_FALLBACK_MIME_TYPE - : allowedMimeTypes.get(0); - encodersForMimeType = encoderSelector.selectEncoderInfos(mimeType); - if (encodersForMimeType.isEmpty()) { - return null; - } + @Nullable + String mimeType = + findFallbackMimeType(encoderSelector, requestedFormat.sampleMimeType, allowedMimeTypes); + if (mimeType == null) { + return null; } - String finalMimeType = mimeType; + List encodersForMimeType = encoderSelector.selectEncoderInfos(mimeType); + checkState(!encodersForMimeType.isEmpty()); ImmutableList filteredEncoders = filterEncoders( encodersForMimeType, @@ -170,7 +162,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { @Nullable Pair closestSupportedResolution = EncoderUtil.getClosestSupportedResolution( - encoderInfo, finalMimeType, requestedFormat.width, requestedFormat.height); + encoderInfo, mimeType, requestedFormat.width, requestedFormat.height); if (closestSupportedResolution == null) { // Drops encoder. return Integer.MAX_VALUE; @@ -186,10 +178,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { Pair finalResolution = checkNotNull( EncoderUtil.getClosestSupportedResolution( - filteredEncoders.get(0), - finalMimeType, - requestedFormat.width, - requestedFormat.height)); + filteredEncoders.get(0), mimeType, requestedFormat.width, requestedFormat.height)); int requestedBitrate = requestedFormat.averageBitrate == Format.NO_VALUE @@ -205,8 +194,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { filteredEncoders, /* cost= */ (encoderInfo) -> { int achievableBitrate = - EncoderUtil.getClosestSupportedBitrate( - encoderInfo, finalMimeType, requestedBitrate); + EncoderUtil.getClosestSupportedBitrate(encoderInfo, mimeType, requestedBitrate); return abs(achievableBitrate - requestedBitrate); }); if (filteredEncoders.isEmpty()) { @@ -218,10 +206,10 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { Pair profileLevel = MediaCodecUtil.getCodecProfileAndLevel(requestedFormat); @Nullable String codecs = null; if (profileLevel != null - && requestedFormat.sampleMimeType.equals(finalMimeType) + && requestedFormat.sampleMimeType.equals(mimeType) && EncoderUtil.isProfileLevelSupported( pickedEncoder, - finalMimeType, + mimeType, /* profile= */ profileLevel.first, /* level= */ profileLevel.second)) { codecs = requestedFormat.codecs; @@ -230,7 +218,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { Format encoderSupportedFormat = requestedFormat .buildUpon() - .setSampleMimeType(finalMimeType) + .setSampleMimeType(mimeType) .setCodecs(codecs) .setWidth(finalResolution.first) .setHeight(finalResolution.second) @@ -239,8 +227,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { ? requestedFormat.frameRate : DEFAULT_FRAME_RATE) .setAverageBitrate( - EncoderUtil.getClosestSupportedBitrate( - pickedEncoder, finalMimeType, requestedBitrate)) + EncoderUtil.getClosestSupportedBitrate(pickedEncoder, mimeType, requestedBitrate)) .build(); return Pair.create(pickedEncoder, encoderSupportedFormat); } @@ -279,6 +266,32 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { return ImmutableList.copyOf(filteredEncoders); } + @Nullable + private static String findFallbackMimeType( + EncoderSelector encoderSelector, String requestedMimeType, List allowedMimeTypes) { + if (mimeTypeIsSupported(encoderSelector, requestedMimeType, allowedMimeTypes)) { + return requestedMimeType; + } else if (mimeTypeIsSupported(encoderSelector, MimeTypes.VIDEO_H265, allowedMimeTypes)) { + return MimeTypes.VIDEO_H265; + } else if (mimeTypeIsSupported(encoderSelector, MimeTypes.VIDEO_H264, allowedMimeTypes)) { + return MimeTypes.VIDEO_H264; + } else { + for (int i = 0; i < allowedMimeTypes.size(); i++) { + String allowedMimeType = allowedMimeTypes.get(i); + if (mimeTypeIsSupported(encoderSelector, allowedMimeType, allowedMimeTypes)) { + return allowedMimeType; + } + } + } + return null; + } + + private static boolean mimeTypeIsSupported( + EncoderSelector encoderSelector, String mimeType, List allowedMimeTypes) { + return !encoderSelector.selectEncoderInfos(mimeType).isEmpty() + && allowedMimeTypes.contains(mimeType); + } + /** Computes the video bit rate using the Kush Gauge. */ private static int getSuggestedBitrate(int width, int height, float frameRate) { // TODO(b/210591626) Implement bitrate estimation.