Add format fallback ranking.

Introduce an interface EncoderSelector for developers to filter out unwanted
encoders.

PiperOrigin-RevId: 425611421

libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java

@@ -49,7 +49,7 @@ public final class Codec {
   public interface DecoderFactory {
 
     /** A default {@code DecoderFactory} implementation. */
-    DecoderFactory DEFAULT = new DefaultCodecFactory();
+    DecoderFactory DEFAULT = new DefaultCodecFactory(/* videoEncoderSelector= */ null);
 
     /**
      * Returns a {@link Codec} for audio decoding.
@@ -78,7 +78,7 @@ public final class Codec {
   public interface EncoderFactory {
 
     /** A default {@code EncoderFactory} implementation. */
-    EncoderFactory DEFAULT = new DefaultCodecFactory();
+    EncoderFactory DEFAULT = new DefaultCodecFactory(EncoderSelector.DEFAULT);
 
     /**
      * Returns a {@link Codec} for audio encoding.

libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodecFactory.java

@@ -18,7 +18,9 @@ package androidx.media3.transformer;
 
 import static androidx.media3.common.util.Assertions.checkArgument;
 import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.common.util.Assertions.checkStateNotNull;
 import static androidx.media3.common.util.Util.SDK_INT;
+import static java.lang.Math.abs;
 
 import android.annotation.SuppressLint;
 import android.media.MediaCodec;
@@ -33,7 +35,9 @@ import androidx.media3.common.MimeTypes;
 import androidx.media3.common.util.MediaFormatUtil;
 import androidx.media3.common.util.TraceUtil;
 import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
+import com.google.common.collect.ImmutableList;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 
@@ -48,6 +52,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   private static final int DEFAULT_FRAME_RATE = 60;
   private static final int DEFAULT_I_FRAME_INTERVAL_SECS = 1;
 
+  @Nullable private final EncoderSelector videoEncoderSelector;
+
+  /** Creates a new instance. */
+  public DefaultCodecFactory(@Nullable EncoderSelector videoEncoderSelector) {
+    this.videoEncoderSelector = videoEncoderSelector;
+  }
+
   @Override
   public Codec createForAudioDecoding(Format format) throws TransformationException {
     MediaFormat mediaFormat =
@@ -60,6 +71,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     return createCodec(
         format,
         mediaFormat,
+        /* mediaCodecName= */ null,
         /* isVideo= */ false,
         /* isDecoder= */ true,
         /* outputSurface= */ null);
@@ -83,12 +95,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     }
 
     return createCodec(
-        format, mediaFormat, /* isVideo= */ true, /* isDecoder= */ true, outputSurface);
+        format,
+        mediaFormat,
+        /* mediaCodecName= */ null,
+        /* isVideo= */ true,
+        /* isDecoder= */ true,
+        outputSurface);
   }
 
   @Override
   public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
       throws TransformationException {
+    // TODO(b/210591626) Add encoder selection for audio.
     checkArgument(!allowedMimeTypes.isEmpty());
     if (!allowedMimeTypes.contains(format.sampleMimeType)) {
       // TODO(b/210591626): Pick fallback MIME type using same strategy as for encoder
@@ -103,6 +121,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     return createCodec(
         format,
         mediaFormat,
+        /* mediaCodecName= */ null,
         /* isVideo= */ false,
         /* isDecoder= */ false,
         /* outputSurface= */ null);
@@ -118,11 +137,23 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     checkArgument(format.height <= format.width);
     checkArgument(format.rotationDegrees == 0);
     checkNotNull(format.sampleMimeType);
-
     checkArgument(!allowedMimeTypes.isEmpty());
+    checkStateNotNull(videoEncoderSelector);
 
-    format = getVideoEncoderSupportedFormat(format, allowedMimeTypes);
+    @Nullable
+    EncoderAndSupportedFormat encoderAndSupportedFormat =
+        findEncoderWithClosestFormatSupport(format, videoEncoderSelector, allowedMimeTypes);
+    if (encoderAndSupportedFormat == null) {
+      throw createTransformationException(
+          new IllegalArgumentException(
+              "No encoder available that supports the requested output format."),
+          format,
+          /* isVideo= */ true,
+          /* isDecoder= */ false,
+          /* mediaCodecName= */ null);
+    }
 
+    format = encoderAndSupportedFormat.supportedFormat;
     MediaFormat mediaFormat =
         MediaFormat.createVideoFormat(
             checkNotNull(format.sampleMimeType), format.width, format.height);
@@ -144,6 +175,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     return createCodec(
         format,
         mediaFormat,
+        encoderAndSupportedFormat.encoderInfo.getName(),
         /* isVideo= */ true,
         /* isDecoder= */ false,
         /* outputSurface= */ null);
@@ -153,6 +185,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   private static Codec createCodec(
       Format format,
       MediaFormat mediaFormat,
+      @Nullable String mediaCodecName,
       boolean isVideo,
       boolean isDecoder,
       @Nullable Surface outputSurface)
@@ -161,7 +194,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     @Nullable Surface inputSurface = null;
     try {
       mediaCodec =
-          isDecoder
+          mediaCodecName != null
+              ? MediaCodec.createByCodecName(mediaCodecName)
+              : isDecoder
                   ? MediaCodec.createDecoderByType(format.sampleMimeType)
                   : MediaCodec.createEncoderByType(format.sampleMimeType);
       configureCodec(mediaCodec, mediaFormat, isDecoder, outputSurface);
@@ -173,7 +208,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
       if (inputSurface != null) {
         inputSurface.release();
       }
-      @Nullable String mediaCodecName = null;
       if (mediaCodec != null) {
         mediaCodecName = mediaCodec.getName();
         mediaCodec.release();
@@ -203,80 +237,144 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     TraceUtil.endSection();
   }
 
+  /**
+   * Finds the {@link EncoderAndSupportedFormat} whose {@link EncoderAndSupportedFormat#encoderInfo
+   * encoder} supports the {@code requestedFormat} most closely; {@code null} if none is found.
+   */
   @RequiresNonNull("#1.sampleMimeType")
-  private static Format getVideoEncoderSupportedFormat(
+  @Nullable
-      Format requestedFormat, List<String> allowedMimeTypes) throws TransformationException {
+  private static EncoderAndSupportedFormat findEncoderWithClosestFormatSupport(
+      Format requestedFormat, EncoderSelector encoderSelector, List<String> allowedMimeTypes) {
     String mimeType = requestedFormat.sampleMimeType;
-    Format.Builder formatBuilder = requestedFormat.buildUpon();
 
-    // TODO(b/210591626) Implement encoder filtering.
+    // TODO(b/210591626) Improve MIME type selection.
-    if (!allowedMimeTypes.contains(mimeType)
+    List<MediaCodecInfo> encodersForMimeType = encoderSelector.selectEncoderInfos(mimeType);
-        || EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) {
+    if (!allowedMimeTypes.contains(mimeType) || encodersForMimeType.isEmpty()) {
       mimeType =
           allowedMimeTypes.contains(DEFAULT_FALLBACK_MIME_TYPE)
               ? DEFAULT_FALLBACK_MIME_TYPE
               : allowedMimeTypes.get(0);
-      if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) {
+      encodersForMimeType = encoderSelector.selectEncoderInfos(mimeType);
-        throw createTransformationException(
+      if (encodersForMimeType.isEmpty()) {
-            new IllegalArgumentException(
+        return null;
-                "No encoder is found for requested MIME type " + requestedFormat.sampleMimeType),
-            requestedFormat,
-            /* isVideo= */ true,
-            /* isDecoder= */ false,
-            /* mediaCodecName= */ null);
       }
     }
 
-    formatBuilder.setSampleMimeType(mimeType);
+    String finalMimeType = mimeType;
-    MediaCodecInfo encoderInfo = EncoderUtil.getSupportedEncoders(mimeType).get(0);
+    ImmutableList<MediaCodecInfo> filteredEncoders =
-
+        filterEncoders(
-    int width = requestedFormat.width;
+            encodersForMimeType,
-    int height = requestedFormat.height;
+            /* cost= */ (encoderInfo) -> {
               @Nullable
-    Pair<Integer, Integer> encoderSupportedResolution =
+              Pair<Integer, Integer> closestSupportedResolution =
-        EncoderUtil.getClosestSupportedResolution(encoderInfo, mimeType, width, height);
+                  EncoderUtil.getClosestSupportedResolution(
-    if (encoderSupportedResolution == null) {
+                      encoderInfo, finalMimeType, requestedFormat.width, requestedFormat.height);
-      throw createTransformationException(
+              if (closestSupportedResolution == null) {
-          new IllegalArgumentException(
+                // Drops encoder.
-              "Cannot find fallback resolution for resolution " + width + " x " + height),
+                return Integer.MAX_VALUE;
-          requestedFormat,
-          /* isVideo= */ true,
-          /* isDecoder= */ false,
-          /* mediaCodecName= */ null);
               }
-    width = encoderSupportedResolution.first;
+              return abs(
-    height = encoderSupportedResolution.second;
+                  requestedFormat.width * requestedFormat.height
-    formatBuilder.setWidth(width).setHeight(height);
+                      - closestSupportedResolution.first * closestSupportedResolution.second);
+            });
+    if (filteredEncoders.isEmpty()) {
+      return null;
+    }
+    // The supported resolution is the same for all remaining encoders.
+    Pair<Integer, Integer> finalResolution =
+        checkNotNull(
+            EncoderUtil.getClosestSupportedResolution(
+                filteredEncoders.get(0),
+                finalMimeType,
+                requestedFormat.width,
+                requestedFormat.height));
 
-    // The frameRate does not affect the resulting frame rate. It affects the encoder's rate control
+    int requestedBitrate =
-    // algorithm. Setting it too high may lead to video quality degradation.
+        requestedFormat.averageBitrate == Format.NO_VALUE
-    float frameRate =
+            ? getSuggestedBitrate(
-        requestedFormat.frameRate != Format.NO_VALUE
+                /* width= */ finalResolution.first,
-            ? requestedFormat.frameRate
+                /* height= */ finalResolution.second,
-            : DEFAULT_FRAME_RATE;
+                requestedFormat.frameRate == Format.NO_VALUE
-    int bitrate =
+                    ? DEFAULT_FRAME_RATE
+                    : requestedFormat.frameRate)
+            : requestedFormat.averageBitrate;
+    filteredEncoders =
+        filterEncoders(
+            filteredEncoders,
+            /* cost= */ (encoderInfo) -> {
+              int achievableBitrate =
                   EncoderUtil.getClosestSupportedBitrate(
-            encoderInfo,
+                      encoderInfo, finalMimeType, requestedBitrate);
-            mimeType,
+              return abs(achievableBitrate - requestedBitrate);
-            /* bitrate= */ requestedFormat.averageBitrate != Format.NO_VALUE
+            });
-                ? requestedFormat.averageBitrate
+    if (filteredEncoders.isEmpty()) {
-                : getSuggestedBitrate(width, height, frameRate));
+      return null;
-    formatBuilder.setFrameRate(frameRate).setAverageBitrate(bitrate);
+    }
 
+    MediaCodecInfo pickedEncoder = filteredEncoders.get(0);
     @Nullable
     Pair<Integer, Integer> profileLevel = MediaCodecUtil.getCodecProfileAndLevel(requestedFormat);
-    if (profileLevel == null
+    @Nullable String codecs = null;
-        // Transcoding to another MIME type.
+    if (profileLevel != null
-        || !requestedFormat.sampleMimeType.equals(mimeType)
+        && requestedFormat.sampleMimeType.equals(finalMimeType)
-        || !EncoderUtil.isProfileLevelSupported(
+        && EncoderUtil.isProfileLevelSupported(
-            encoderInfo,
+            pickedEncoder,
-            mimeType,
+            finalMimeType,
             /* profile= */ profileLevel.first,
             /* level= */ profileLevel.second)) {
-      formatBuilder.setCodecs(null);
+      codecs = requestedFormat.codecs;
     }
 
-    return formatBuilder.build();
+    Format encoderSupportedFormat =
+        requestedFormat
+            .buildUpon()
+            .setSampleMimeType(finalMimeType)
+            .setCodecs(codecs)
+            .setWidth(finalResolution.first)
+            .setHeight(finalResolution.second)
+            .setFrameRate(
+                requestedFormat.frameRate != Format.NO_VALUE
+                    ? requestedFormat.frameRate
+                    : DEFAULT_FRAME_RATE)
+            .setAverageBitrate(
+                EncoderUtil.getClosestSupportedBitrate(
+                    pickedEncoder, finalMimeType, requestedBitrate))
+            .build();
+    return new EncoderAndSupportedFormat(pickedEncoder, encoderSupportedFormat);
+  }
+
+  private interface EncoderFallbackCost {
+    /**
+     * Returns a cost that represents the gap between the requested encoding parameter(s) and the
+     * {@link MediaCodecInfo encoder}'s support for them.
+     *
+     * <p>The method must return {@link Integer#MAX_VALUE} when the {@link MediaCodecInfo encoder}
+     * does not support the encoding parameters.
+     */
+    int getParameterSupportGap(MediaCodecInfo encoderInfo);
+  }
+
+  private static ImmutableList<MediaCodecInfo> filterEncoders(
+      List<MediaCodecInfo> encoders, EncoderFallbackCost cost) {
+    List<MediaCodecInfo> filteredEncoders = new ArrayList<>(encoders.size());
+
+    int minGap = Integer.MAX_VALUE;
+    for (int i = 0; i < encoders.size(); i++) {
+      MediaCodecInfo encoderInfo = encoders.get(i);
+      int gap = cost.getParameterSupportGap(encoderInfo);
+      if (gap == Integer.MAX_VALUE) {
+        continue;
+      }
+
+      if (gap < minGap) {
+        minGap = gap;
+        filteredEncoders.clear();
+        filteredEncoders.add(encoderInfo);
+      } else if (gap == minGap) {
+        filteredEncoders.add(encoderInfo);
+      }
+    }
+    return ImmutableList.copyOf(filteredEncoders);
   }
 
   /** Computes the video bit rate using the Kush Gauge. */
@@ -315,4 +413,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     }
     return TransformationException.createForUnexpected(cause);
   }
+
+  /**
+   * A class wrapping a selected {@link MediaCodecInfo encoder} and its supported {@link Format}.
+   */
+  private static class EncoderAndSupportedFormat {
+    /** The {@link MediaCodecInfo} that describes the encoder. */
+    public final MediaCodecInfo encoderInfo;
+    /** The {@link Format} that this encoder supports. */
+    public final Format supportedFormat;
+
+    /** Creates a new instance. */
+    public EncoderAndSupportedFormat(MediaCodecInfo encoderInfo, Format supportedFormat) {
+      this.encoderInfo = encoderInfo;
+      this.supportedFormat = supportedFormat;
+    }
+  }
 }

libraries/transformer/src/main/java/androidx/media3/transformer/EncoderSelector.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.transformer;
+
+import android.media.MediaCodec;
+import android.media.MediaCodecInfo;
+import androidx.media3.common.MimeTypes;
+import androidx.media3.common.util.UnstableApi;
+import java.util.List;
+
+/** Selector of {@link MediaCodec} encoder instances. */
+@UnstableApi
+public interface EncoderSelector {
+
+  /**
+   * Default implementation of {@link EncoderSelector}, which returns the preferred encoders for the
+   * given {@link MimeTypes MIME type}.
+   */
+  EncoderSelector DEFAULT = EncoderUtil::getSupportedEncoders;
+
+  /**
+   * Returns a list of encoders that can encode media in the specified {@code mimeType}, in priority
+   * order.
+   *
+   * @param mimeType The {@link MimeTypes MIME type} for which an encoder is required.
+   * @return An unmodifiable list of {@link MediaCodecInfo encoders} that supports the {@code
+   *     mimeType}. The list may be empty.
+   */
+  List<MediaCodecInfo> selectEncoderInfos(String mimeType);
+}

libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java

@@ -22,8 +22,12 @@ import android.media.MediaCodec;
 import android.media.MediaCodecInfo;
 import android.media.MediaCodecList;
 import android.util.Pair;
+import androidx.annotation.DoNotInline;
 import androidx.annotation.Nullable;
+import androidx.annotation.RequiresApi;
+import androidx.media3.common.MimeTypes;
 import androidx.media3.common.util.UnstableApi;
+import androidx.media3.common.util.Util;
 import com.google.common.base.Ascii;
 import com.google.common.collect.ImmutableList;
 import java.util.ArrayList;
@@ -130,6 +134,42 @@ public final class EncoderUtil {
         .clamp(bitrate);
   }
 
+  /** Checks if a {@link MediaCodecInfo codec} is hardware-accelerated. */
+  public static boolean isHardwareAccelerated(MediaCodecInfo encoderInfo, String mimeType) {
+    // TODO(b/214964116): Merge into MediaCodecUtil.
+    if (Util.SDK_INT >= 29) {
+      return Api29.isHardwareAccelerated(encoderInfo);
+    }
+    // codecInfo.isHardwareAccelerated() == !codecInfo.isSoftwareOnly() is not necessarily true.
+    // However, we assume this to be true as an approximation.
+    return !isSoftwareOnly(encoderInfo, mimeType);
+  }
+
+  private static boolean isSoftwareOnly(MediaCodecInfo encoderInfo, String mimeType) {
+    if (Util.SDK_INT >= 29) {
+      return Api29.isSoftwareOnly(encoderInfo);
+    }
+
+    if (MimeTypes.isAudio(mimeType)) {
+      // Assume audio decoders are software only.
+      return true;
+    }
+    String codecName = Ascii.toLowerCase(encoderInfo.getName());
+    if (codecName.startsWith("arc.")) {
+      // App Runtime for Chrome (ARC) codecs
+      return false;
+    }
+
+    // Estimate whether a codec is software-only, to emulate isSoftwareOnly on API < 29.
+    return codecName.startsWith("omx.google.")
+        || codecName.startsWith("omx.ffmpeg.")
+        || (codecName.startsWith("omx.sec.") && codecName.contains(".sw."))
+        || codecName.equals("omx.qcom.video.decoder.hevcswvdec")
+        || codecName.startsWith("c2.android.")
+        || codecName.startsWith("c2.google.")
+        || (!codecName.startsWith("omx.") && !codecName.startsWith("c2."));
+  }
+
   /**
    * Align to the closest resolution that respects the encoder's supported alignment.
    *
@@ -154,5 +194,18 @@ public final class EncoderUtil {
     }
   }
 
+  @RequiresApi(29)
+  private static final class Api29 {
+    @DoNotInline
+    public static boolean isHardwareAccelerated(MediaCodecInfo encoderInfo) {
+      return encoderInfo.isHardwareAccelerated();
+    }
+
+    @DoNotInline
+    public static boolean isSoftwareOnly(MediaCodecInfo encoderInfo) {
+      return encoderInfo.isSoftwareOnly();
+    }
+  }
+
   private EncoderUtil() {}
 }