From d767be4ca67873a9bfc0e3b228a9f97b9b803fc8 Mon Sep 17 00:00:00 2001 From: claincly Date: Tue, 1 Feb 2022 14:40:20 +0000 Subject: [PATCH] Add format fallback ranking. Introduce an interface EncoderSelector for developers to filter out unwanted encoders. PiperOrigin-RevId: 425611421 --- .../android/exoplayer2/transformer/Codec.java | 4 +- .../transformer/DefaultCodecFactory.java | 238 +++++++++++++----- .../transformer/EncoderSelector.java | 42 ++++ .../exoplayer2/transformer/EncoderUtil.java | 53 ++++ 4 files changed, 273 insertions(+), 64 deletions(-) create mode 100644 library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderSelector.java diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java index 834b54a404..e348e8674d 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java @@ -47,7 +47,7 @@ public final class Codec { public interface DecoderFactory { /** A default {@code DecoderFactory} implementation. */ - DecoderFactory DEFAULT = new DefaultCodecFactory(); + DecoderFactory DEFAULT = new DefaultCodecFactory(/* videoEncoderSelector= */ null); /** * Returns a {@link Codec} for audio decoding. @@ -76,7 +76,7 @@ public final class Codec { public interface EncoderFactory { /** A default {@code EncoderFactory} implementation. */ - EncoderFactory DEFAULT = new DefaultCodecFactory(); + EncoderFactory DEFAULT = new DefaultCodecFactory(EncoderSelector.DEFAULT); /** * Returns a {@link Codec} for audio encoding. diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodecFactory.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodecFactory.java index b4a7cf68b5..6910b7809f 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodecFactory.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodecFactory.java @@ -18,7 +18,9 @@ package com.google.android.exoplayer2.transformer; import static com.google.android.exoplayer2.util.Assertions.checkArgument; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; +import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; import static com.google.android.exoplayer2.util.Util.SDK_INT; +import static java.lang.Math.abs; import android.annotation.SuppressLint; import android.media.MediaCodec; @@ -33,7 +35,9 @@ import com.google.android.exoplayer2.mediacodec.MediaCodecUtil; import com.google.android.exoplayer2.util.MediaFormatUtil; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.TraceUtil; +import com.google.common.collect.ImmutableList; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -48,6 +52,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private static final int DEFAULT_FRAME_RATE = 60; private static final int DEFAULT_I_FRAME_INTERVAL_SECS = 1; + @Nullable private final EncoderSelector videoEncoderSelector; + + /** Creates a new instance. */ + public DefaultCodecFactory(@Nullable EncoderSelector videoEncoderSelector) { + this.videoEncoderSelector = videoEncoderSelector; + } + @Override public Codec createForAudioDecoding(Format format) throws TransformationException { MediaFormat mediaFormat = @@ -60,6 +71,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return createCodec( format, mediaFormat, + /* mediaCodecName= */ null, /* isVideo= */ false, /* isDecoder= */ true, /* outputSurface= */ null); @@ -83,12 +95,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } return createCodec( - format, mediaFormat, /* isVideo= */ true, /* isDecoder= */ true, outputSurface); + format, + mediaFormat, + /* mediaCodecName= */ null, + /* isVideo= */ true, + /* isDecoder= */ true, + outputSurface); } @Override public Codec createForAudioEncoding(Format format, List allowedMimeTypes) throws TransformationException { + // TODO(b/210591626) Add encoder selection for audio. checkArgument(!allowedMimeTypes.isEmpty()); if (!allowedMimeTypes.contains(format.sampleMimeType)) { // TODO(b/210591626): Pick fallback MIME type using same strategy as for encoder @@ -103,6 +121,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return createCodec( format, mediaFormat, + /* mediaCodecName= */ null, /* isVideo= */ false, /* isDecoder= */ false, /* outputSurface= */ null); @@ -118,11 +137,23 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; checkArgument(format.height <= format.width); checkArgument(format.rotationDegrees == 0); checkNotNull(format.sampleMimeType); - checkArgument(!allowedMimeTypes.isEmpty()); + checkStateNotNull(videoEncoderSelector); - format = getVideoEncoderSupportedFormat(format, allowedMimeTypes); + @Nullable + EncoderAndSupportedFormat encoderAndSupportedFormat = + findEncoderWithClosestFormatSupport(format, videoEncoderSelector, allowedMimeTypes); + if (encoderAndSupportedFormat == null) { + throw createTransformationException( + new IllegalArgumentException( + "No encoder available that supports the requested output format."), + format, + /* isVideo= */ true, + /* isDecoder= */ false, + /* mediaCodecName= */ null); + } + format = encoderAndSupportedFormat.supportedFormat; MediaFormat mediaFormat = MediaFormat.createVideoFormat( checkNotNull(format.sampleMimeType), format.width, format.height); @@ -144,6 +175,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return createCodec( format, mediaFormat, + encoderAndSupportedFormat.encoderInfo.getName(), /* isVideo= */ true, /* isDecoder= */ false, /* outputSurface= */ null); @@ -153,6 +185,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private static Codec createCodec( Format format, MediaFormat mediaFormat, + @Nullable String mediaCodecName, boolean isVideo, boolean isDecoder, @Nullable Surface outputSurface) @@ -161,9 +194,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @Nullable Surface inputSurface = null; try { mediaCodec = - isDecoder - ? MediaCodec.createDecoderByType(format.sampleMimeType) - : MediaCodec.createEncoderByType(format.sampleMimeType); + mediaCodecName != null + ? MediaCodec.createByCodecName(mediaCodecName) + : isDecoder + ? MediaCodec.createDecoderByType(format.sampleMimeType) + : MediaCodec.createEncoderByType(format.sampleMimeType); configureCodec(mediaCodec, mediaFormat, isDecoder, outputSurface); if (isVideo && !isDecoder) { inputSurface = mediaCodec.createInputSurface(); @@ -173,7 +208,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; if (inputSurface != null) { inputSurface.release(); } - @Nullable String mediaCodecName = null; if (mediaCodec != null) { mediaCodecName = mediaCodec.getName(); mediaCodec.release(); @@ -203,80 +237,144 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; TraceUtil.endSection(); } + /** + * Finds the {@link EncoderAndSupportedFormat} whose {@link EncoderAndSupportedFormat#encoderInfo + * encoder} supports the {@code requestedFormat} most closely; {@code null} if none is found. + */ @RequiresNonNull("#1.sampleMimeType") - private static Format getVideoEncoderSupportedFormat( - Format requestedFormat, List allowedMimeTypes) throws TransformationException { + @Nullable + private static EncoderAndSupportedFormat findEncoderWithClosestFormatSupport( + Format requestedFormat, EncoderSelector encoderSelector, List allowedMimeTypes) { String mimeType = requestedFormat.sampleMimeType; - Format.Builder formatBuilder = requestedFormat.buildUpon(); - // TODO(b/210591626) Implement encoder filtering. - if (!allowedMimeTypes.contains(mimeType) - || EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) { + // TODO(b/210591626) Improve MIME type selection. + List encodersForMimeType = encoderSelector.selectEncoderInfos(mimeType); + if (!allowedMimeTypes.contains(mimeType) || encodersForMimeType.isEmpty()) { mimeType = allowedMimeTypes.contains(DEFAULT_FALLBACK_MIME_TYPE) ? DEFAULT_FALLBACK_MIME_TYPE : allowedMimeTypes.get(0); - if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) { - throw createTransformationException( - new IllegalArgumentException( - "No encoder is found for requested MIME type " + requestedFormat.sampleMimeType), - requestedFormat, - /* isVideo= */ true, - /* isDecoder= */ false, - /* mediaCodecName= */ null); + encodersForMimeType = encoderSelector.selectEncoderInfos(mimeType); + if (encodersForMimeType.isEmpty()) { + return null; } } - formatBuilder.setSampleMimeType(mimeType); - MediaCodecInfo encoderInfo = EncoderUtil.getSupportedEncoders(mimeType).get(0); - - int width = requestedFormat.width; - int height = requestedFormat.height; - @Nullable - Pair encoderSupportedResolution = - EncoderUtil.getClosestSupportedResolution(encoderInfo, mimeType, width, height); - if (encoderSupportedResolution == null) { - throw createTransformationException( - new IllegalArgumentException( - "Cannot find fallback resolution for resolution " + width + " x " + height), - requestedFormat, - /* isVideo= */ true, - /* isDecoder= */ false, - /* mediaCodecName= */ null); + String finalMimeType = mimeType; + ImmutableList filteredEncoders = + filterEncoders( + encodersForMimeType, + /* cost= */ (encoderInfo) -> { + @Nullable + Pair closestSupportedResolution = + EncoderUtil.getClosestSupportedResolution( + encoderInfo, finalMimeType, requestedFormat.width, requestedFormat.height); + if (closestSupportedResolution == null) { + // Drops encoder. + return Integer.MAX_VALUE; + } + return abs( + requestedFormat.width * requestedFormat.height + - closestSupportedResolution.first * closestSupportedResolution.second); + }); + if (filteredEncoders.isEmpty()) { + return null; } - width = encoderSupportedResolution.first; - height = encoderSupportedResolution.second; - formatBuilder.setWidth(width).setHeight(height); + // The supported resolution is the same for all remaining encoders. + Pair finalResolution = + checkNotNull( + EncoderUtil.getClosestSupportedResolution( + filteredEncoders.get(0), + finalMimeType, + requestedFormat.width, + requestedFormat.height)); - // The frameRate does not affect the resulting frame rate. It affects the encoder's rate control - // algorithm. Setting it too high may lead to video quality degradation. - float frameRate = - requestedFormat.frameRate != Format.NO_VALUE - ? requestedFormat.frameRate - : DEFAULT_FRAME_RATE; - int bitrate = - EncoderUtil.getClosestSupportedBitrate( - encoderInfo, - mimeType, - /* bitrate= */ requestedFormat.averageBitrate != Format.NO_VALUE - ? requestedFormat.averageBitrate - : getSuggestedBitrate(width, height, frameRate)); - formatBuilder.setFrameRate(frameRate).setAverageBitrate(bitrate); + int requestedBitrate = + requestedFormat.averageBitrate == Format.NO_VALUE + ? getSuggestedBitrate( + /* width= */ finalResolution.first, + /* height= */ finalResolution.second, + requestedFormat.frameRate == Format.NO_VALUE + ? DEFAULT_FRAME_RATE + : requestedFormat.frameRate) + : requestedFormat.averageBitrate; + filteredEncoders = + filterEncoders( + filteredEncoders, + /* cost= */ (encoderInfo) -> { + int achievableBitrate = + EncoderUtil.getClosestSupportedBitrate( + encoderInfo, finalMimeType, requestedBitrate); + return abs(achievableBitrate - requestedBitrate); + }); + if (filteredEncoders.isEmpty()) { + return null; + } + MediaCodecInfo pickedEncoder = filteredEncoders.get(0); @Nullable Pair profileLevel = MediaCodecUtil.getCodecProfileAndLevel(requestedFormat); - if (profileLevel == null - // Transcoding to another MIME type. - || !requestedFormat.sampleMimeType.equals(mimeType) - || !EncoderUtil.isProfileLevelSupported( - encoderInfo, - mimeType, + @Nullable String codecs = null; + if (profileLevel != null + && requestedFormat.sampleMimeType.equals(finalMimeType) + && EncoderUtil.isProfileLevelSupported( + pickedEncoder, + finalMimeType, /* profile= */ profileLevel.first, /* level= */ profileLevel.second)) { - formatBuilder.setCodecs(null); + codecs = requestedFormat.codecs; } - return formatBuilder.build(); + Format encoderSupportedFormat = + requestedFormat + .buildUpon() + .setSampleMimeType(finalMimeType) + .setCodecs(codecs) + .setWidth(finalResolution.first) + .setHeight(finalResolution.second) + .setFrameRate( + requestedFormat.frameRate != Format.NO_VALUE + ? requestedFormat.frameRate + : DEFAULT_FRAME_RATE) + .setAverageBitrate( + EncoderUtil.getClosestSupportedBitrate( + pickedEncoder, finalMimeType, requestedBitrate)) + .build(); + return new EncoderAndSupportedFormat(pickedEncoder, encoderSupportedFormat); + } + + private interface EncoderFallbackCost { + /** + * Returns a cost that represents the gap between the requested encoding parameter(s) and the + * {@link MediaCodecInfo encoder}'s support for them. + * + *

The method must return {@link Integer#MAX_VALUE} when the {@link MediaCodecInfo encoder} + * does not support the encoding parameters. + */ + int getParameterSupportGap(MediaCodecInfo encoderInfo); + } + + private static ImmutableList filterEncoders( + List encoders, EncoderFallbackCost cost) { + List filteredEncoders = new ArrayList<>(encoders.size()); + + int minGap = Integer.MAX_VALUE; + for (int i = 0; i < encoders.size(); i++) { + MediaCodecInfo encoderInfo = encoders.get(i); + int gap = cost.getParameterSupportGap(encoderInfo); + if (gap == Integer.MAX_VALUE) { + continue; + } + + if (gap < minGap) { + minGap = gap; + filteredEncoders.clear(); + filteredEncoders.add(encoderInfo); + } else if (gap == minGap) { + filteredEncoders.add(encoderInfo); + } + } + return ImmutableList.copyOf(filteredEncoders); } /** Computes the video bit rate using the Kush Gauge. */ @@ -315,4 +413,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } return TransformationException.createForUnexpected(cause); } + + /** + * A class wrapping a selected {@link MediaCodecInfo encoder} and its supported {@link Format}. + */ + private static class EncoderAndSupportedFormat { + /** The {@link MediaCodecInfo} that describes the encoder. */ + public final MediaCodecInfo encoderInfo; + /** The {@link Format} that this encoder supports. */ + public final Format supportedFormat; + + /** Creates a new instance. */ + public EncoderAndSupportedFormat(MediaCodecInfo encoderInfo, Format supportedFormat) { + this.encoderInfo = encoderInfo; + this.supportedFormat = supportedFormat; + } + } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderSelector.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderSelector.java new file mode 100644 index 0000000000..aa27383db6 --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderSelector.java @@ -0,0 +1,42 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.exoplayer2.transformer; + +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import com.google.android.exoplayer2.util.MimeTypes; +import java.util.List; + +/** Selector of {@link MediaCodec} encoder instances. */ +public interface EncoderSelector { + + /** + * Default implementation of {@link EncoderSelector}, which returns the preferred encoders for the + * given {@link MimeTypes MIME type}. + */ + EncoderSelector DEFAULT = EncoderUtil::getSupportedEncoders; + + /** + * Returns a list of encoders that can encode media in the specified {@code mimeType}, in priority + * order. + * + * @param mimeType The {@link MimeTypes MIME type} for which an encoder is required. + * @return An unmodifiable list of {@link MediaCodecInfo encoders} that supports the {@code + * mimeType}. The list may be empty. + */ + List selectEncoderInfos(String mimeType); +} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java index 65e96efbc8..b1e60f435e 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/EncoderUtil.java @@ -22,7 +22,11 @@ import android.media.MediaCodec; import android.media.MediaCodecInfo; import android.media.MediaCodecList; import android.util.Pair; +import androidx.annotation.DoNotInline; import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import com.google.android.exoplayer2.util.MimeTypes; +import com.google.android.exoplayer2.util.Util; import com.google.common.base.Ascii; import com.google.common.collect.ImmutableList; import java.util.ArrayList; @@ -128,6 +132,42 @@ public final class EncoderUtil { .clamp(bitrate); } + /** Checks if a {@link MediaCodecInfo codec} is hardware-accelerated. */ + public static boolean isHardwareAccelerated(MediaCodecInfo encoderInfo, String mimeType) { + // TODO(b/214964116): Merge into MediaCodecUtil. + if (Util.SDK_INT >= 29) { + return Api29.isHardwareAccelerated(encoderInfo); + } + // codecInfo.isHardwareAccelerated() == !codecInfo.isSoftwareOnly() is not necessarily true. + // However, we assume this to be true as an approximation. + return !isSoftwareOnly(encoderInfo, mimeType); + } + + private static boolean isSoftwareOnly(MediaCodecInfo encoderInfo, String mimeType) { + if (Util.SDK_INT >= 29) { + return Api29.isSoftwareOnly(encoderInfo); + } + + if (MimeTypes.isAudio(mimeType)) { + // Assume audio decoders are software only. + return true; + } + String codecName = Ascii.toLowerCase(encoderInfo.getName()); + if (codecName.startsWith("arc.")) { + // App Runtime for Chrome (ARC) codecs + return false; + } + + // Estimate whether a codec is software-only, to emulate isSoftwareOnly on API < 29. + return codecName.startsWith("omx.google.") + || codecName.startsWith("omx.ffmpeg.") + || (codecName.startsWith("omx.sec.") && codecName.contains(".sw.")) + || codecName.equals("omx.qcom.video.decoder.hevcswvdec") + || codecName.startsWith("c2.android.") + || codecName.startsWith("c2.google.") + || (!codecName.startsWith("omx.") && !codecName.startsWith("c2.")); + } + /** * Align to the closest resolution that respects the encoder's supported alignment. * @@ -152,5 +192,18 @@ public final class EncoderUtil { } } + @RequiresApi(29) + private static final class Api29 { + @DoNotInline + public static boolean isHardwareAccelerated(MediaCodecInfo encoderInfo) { + return encoderInfo.isHardwareAccelerated(); + } + + @DoNotInline + public static boolean isSoftwareOnly(MediaCodecInfo encoderInfo) { + return encoderInfo.isSoftwareOnly(); + } + } + private EncoderUtil() {} }