Make minor fixes to HDR handling

- Update profile selection logic to pick an HDR-compatible profile when doing HDR editing on H.264/AVC videos. - Handle doing the capabilities check for all MIME types that support HDR (not just H.265/HEVC). - Fix a bug where we would pass an HDR input color format to the encoder when using tone-mapping. - Tweak how `EncoderWrapper` works so decisions at made at construction time. Manually tested cases: - Transformation of an SDR video. - Transformation of an HDR video to AVC (which triggers fallback/tone-mapping on a device that doesn't support HDR editing for AVC). - Transformation of an HDR video with HDR editing. PiperOrigin-RevId: 461572973 (cherry picked from commit 604ab7fcdaa759025536feb673a3abb93196a829)
2025-05-14 11:09:53 +08:00 · 2022-07-18 10:09:35 +00:00 · 2022-07-18 10:09:35 +00:00 · 04fa2fda2a
commit 04fa2fda2a
parent c4e64c3d0d
4 changed files with 136 additions and 73 deletions
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java
@ -275,7 +275,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
    }

    if (mimeType.equals(MimeTypes.VIDEO_H264)) {
-      adjustMediaFormatForH264EncoderSettings(mediaFormat, encoderInfo);
+      adjustMediaFormatForH264EncoderSettings(format.colorInfo, encoderInfo, mediaFormat);
    }

    MediaFormatUtil.maybeSetColorInfo(mediaFormat, encoderSupportedFormat.colorInfo);
@ -525,12 +525,21 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
   * <p>The adjustment is applied in-place to {@code mediaFormat}.
   */
  private static void adjustMediaFormatForH264EncoderSettings(
-      MediaFormat mediaFormat, MediaCodecInfo encoderInfo) {
+      @Nullable ColorInfo colorInfo, MediaCodecInfo encoderInfo, MediaFormat mediaFormat) {
    // TODO(b/210593256): Remove overriding profile/level (before API 29) after switching to in-app
    // muxing.
    String mimeType = MimeTypes.VIDEO_H264;
    if (Util.SDK_INT >= 29) {
      int expectedEncodingProfile = MediaCodecInfo.CodecProfileLevel.AVCProfileHigh;
+      if (colorInfo != null) {
+        int colorTransfer = colorInfo.colorTransfer;
+        ImmutableList<Integer> codecProfiles =
+            EncoderUtil.getCodecProfilesForHdrFormat(mimeType, colorTransfer);
+        if (!codecProfiles.isEmpty()) {
+          // Default to the most compatible profile, which is first in the list.
+          expectedEncodingProfile = codecProfiles.get(0);
+        }
+      }
      int supportedEncodingLevel =
          EncoderUtil.findHighestSupportedEncodingLevel(
              encoderInfo, mimeType, expectedEncodingProfile);
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java
@ -31,6 +31,9 @@ import android.util.Size;
 import androidx.annotation.DoNotInline;
 import androidx.annotation.Nullable;
 import androidx.annotation.RequiresApi;
+import androidx.media3.common.C;
+import androidx.media3.common.C.ColorTransfer;
+import androidx.media3.common.ColorInfo;
 import androidx.media3.common.Format;
 import androidx.media3.common.MimeTypes;
 import androidx.media3.common.util.MediaFormatUtil;
@ -67,6 +70,83 @@ public final class EncoderUtil {
    return checkNotNull(MIME_TYPE_TO_ENCODERS.get()).keySet();
  }

+  /**
+   * Returns the names of encoders that support HDR editing for the given format, or an empty list
+   * if the format is unknown or not supported for HDR encoding.
+   */
+  public static ImmutableList<String> getSupportedEncoderNamesForHdrEditing(
+      String mimeType, @Nullable ColorInfo colorInfo) {
+    if (Util.SDK_INT < 31 || colorInfo == null) {
+      return ImmutableList.of();
+    }
+
+    @ColorTransfer int colorTransfer = colorInfo.colorTransfer;
+    ImmutableList<Integer> profiles = getCodecProfilesForHdrFormat(mimeType, colorTransfer);
+    ImmutableList.Builder<String> resultBuilder = ImmutableList.builder();
+    ImmutableList<MediaCodecInfo> mediaCodecInfos =
+        EncoderSelector.DEFAULT.selectEncoderInfos(mimeType);
+    for (int i = 0; i < mediaCodecInfos.size(); i++) {
+      MediaCodecInfo mediaCodecInfo = mediaCodecInfos.get(i);
+      if (mediaCodecInfo.isAlias()
+          || !EncoderUtil.isFeatureSupported(
+              mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing)) {
+        continue;
+      }
+      for (MediaCodecInfo.CodecProfileLevel codecProfileLevel :
+          mediaCodecInfo.getCapabilitiesForType(mimeType).profileLevels) {
+        if (profiles.contains(codecProfileLevel.profile)) {
+          resultBuilder.add(mediaCodecInfo.getName());
+        }
+      }
+    }
+    return resultBuilder.build();
+  }
+
+  /**
+   * Returns the {@linkplain MediaCodecInfo.CodecProfileLevel#profile profile} constants that can be
+   * used to encode the given HDR format, if supported by the device (this method does not check
+   * device capabilities). If multiple profiles are returned, they are ordered by expected level of
+   * compatibility, with the most widely compatible profile first.
+   */
+  @SuppressWarnings("InlinedApi") // Safe use of inlined constants from newer API versions.
+  public static ImmutableList<Integer> getCodecProfilesForHdrFormat(
+      String mimeType, @ColorTransfer int colorTransfer) {
+    // TODO(b/239174610): Add a way to determine profiles for DV and HDR10+.
+    switch (mimeType) {
+      case MimeTypes.VIDEO_VP9:
+        if (colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084) {
+          // Profiles support both HLG and PQ.
+          return ImmutableList.of(
+              MediaCodecInfo.CodecProfileLevel.VP9Profile2HDR,
+              MediaCodecInfo.CodecProfileLevel.VP9Profile3HDR);
+        }
+        break;
+      case MimeTypes.VIDEO_H264:
+        if (colorTransfer == C.COLOR_TRANSFER_HLG) {
+          return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.AVCProfileHigh10);
+        }
+        break;
+      case MimeTypes.VIDEO_H265:
+        if (colorTransfer == C.COLOR_TRANSFER_HLG) {
+          return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10);
+        } else if (colorTransfer == C.COLOR_TRANSFER_ST2084) {
+          return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10HDR10);
+        }
+        break;
+      case MimeTypes.VIDEO_AV1:
+        if (colorTransfer == C.COLOR_TRANSFER_HLG) {
+          return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.AV1ProfileMain10);
+        } else if (colorTransfer == C.COLOR_TRANSFER_ST2084) {
+          return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.AV1ProfileMain10HDR10);
+        }
+        break;
+      default:
+        break;
+    }
+    // There are no profiles defined for the HDR format, or it's invalid.
+    return ImmutableList.of();
+  }
+
  /** Returns whether the {@linkplain MediaCodecInfo encoder} supports the given resolution. */
  public static boolean isSizeSupported(
      MediaCodecInfo encoderInfo, String mimeType, int width, int height) {
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
@ -17,25 +17,20 @@
 package androidx.media3.transformer;

 import static androidx.media3.common.util.Assertions.checkNotNull;
-import static androidx.media3.common.util.Assertions.checkState;

 import android.content.Context;
 import android.media.MediaCodec;
-import android.media.MediaCodecInfo;
 import android.view.Surface;
 import androidx.annotation.Nullable;
 import androidx.annotation.VisibleForTesting;
 import androidx.media3.common.C;
 import androidx.media3.common.ColorInfo;
 import androidx.media3.common.Format;
-import androidx.media3.common.MimeTypes;
-import androidx.media3.common.util.Log;
 import androidx.media3.common.util.Util;
 import androidx.media3.decoder.DecoderInputBuffer;
 import com.google.common.collect.ImmutableList;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
-import java.util.HashSet;
 import java.util.List;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 import org.checkerframework.dataflow.qual.Pure;
@ -105,17 +100,6 @@ import org.checkerframework.dataflow.qual.Pure;
            transformationRequest,
            fallbackListener);

-    boolean enableRequestSdrToneMapping = transformationRequest.enableRequestSdrToneMapping;
-    // TODO(b/237674316): While HLG10 is correctly reported, HDR10 currently will be incorrectly
-    //  processed as SDR, because the inputFormat.colorInfo reports the wrong value.
-    boolean useHdr =
-        transformationRequest.enableHdrEditing && ColorInfo.isHdr(inputFormat.colorInfo);
-    if (useHdr && !encoderWrapper.supportsHdr()) {
-      useHdr = false;
-      enableRequestSdrToneMapping = true;
-      encoderWrapper.signalFallbackToSdr();
-    }
-
    try {
      frameProcessor =
          GlEffectsFrameProcessor.create(
@ -153,7 +137,7 @@ import org.checkerframework.dataflow.qual.Pure;
              // HDR is only used if the MediaCodec encoder supports FEATURE_HdrEditing. This
              // implies that the OpenGL EXT_YUV_target extension is supported and hence the
              // GlEffectsFrameProcessor also supports HDR.
-              useHdr);
+              /* useHdr= */ encoderWrapper.isHdrEditingEnabled());
    } catch (FrameProcessingException e) {
      throw TransformationException.createForFrameProcessingException(
          e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
@ -161,9 +145,11 @@ import org.checkerframework.dataflow.qual.Pure;
    frameProcessor.setInputFrameInfo(
        new FrameInfo(decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio));

+    boolean isToneMappingRequired =
+        ColorInfo.isHdr(inputFormat.colorInfo) && !encoderWrapper.isHdrEditingEnabled();
    decoder =
        decoderFactory.createForVideoDecoding(
-            inputFormat, frameProcessor.getInputSurface(), enableRequestSdrToneMapping);
+            inputFormat, frameProcessor.getInputSurface(), isToneMappingRequired);
    // TODO(b/236316454): Check in the decoder output format whether tone-mapping was actually
    //  applied and throw an exception if not.
    maxPendingFrameCount = decoder.getMaxPendingFrameCount();
@ -331,14 +317,14 @@ import org.checkerframework.dataflow.qual.Pure;
    private final List<String> allowedOutputMimeTypes;
    private final TransformationRequest transformationRequest;
    private final FallbackListener fallbackListener;
-    private final HashSet<String> hdrMediaCodecNames;
+    private final String requestedOutputMimeType;
+    private final ImmutableList<String> supportedEncoderNamesForHdrEditing;

    private @MonotonicNonNull SurfaceInfo encoderSurfaceInfo;

    private volatile @MonotonicNonNull Codec encoder;
    private volatile int outputRotationDegrees;
    private volatile boolean releaseEncoder;
-    private boolean fallbackToSdr;

    public EncoderWrapper(
        Codec.EncoderFactory encoderFactory,
@ -346,14 +332,26 @@ import org.checkerframework.dataflow.qual.Pure;
        List<String> allowedOutputMimeTypes,
        TransformationRequest transformationRequest,
        FallbackListener fallbackListener) {
-
      this.encoderFactory = encoderFactory;
      this.inputFormat = inputFormat;
      this.allowedOutputMimeTypes = allowedOutputMimeTypes;
      this.transformationRequest = transformationRequest;
      this.fallbackListener = fallbackListener;

-      hdrMediaCodecNames = new HashSet<>();
+      requestedOutputMimeType =
+          transformationRequest.videoMimeType != null
+              ? transformationRequest.videoMimeType
+              : checkNotNull(inputFormat.sampleMimeType);
+      supportedEncoderNamesForHdrEditing =
+          EncoderUtil.getSupportedEncoderNamesForHdrEditing(
+              requestedOutputMimeType, inputFormat.colorInfo);
+    }
+
+    /** Returns whether the wrapped encoder is expecting HDR input for the HDR editing use case. */
+    public boolean isHdrEditingEnabled() {
+      return transformationRequest.enableHdrEditing
+          && !transformationRequest.enableRequestSdrToneMapping
+          && !supportedEncoderNamesForHdrEditing.isEmpty();
    }

    @Nullable
@ -378,37 +376,39 @@ import org.checkerframework.dataflow.qual.Pure;
        outputRotationDegrees = 90;
      }

+      boolean isInputToneMapped = ColorInfo.isHdr(inputFormat.colorInfo) && !isHdrEditingEnabled();
      Format requestedEncoderFormat =
          new Format.Builder()
              .setWidth(requestedWidth)
              .setHeight(requestedHeight)
              .setRotationDegrees(0)
              .setFrameRate(inputFormat.frameRate)
-              .setSampleMimeType(
-                  transformationRequest.videoMimeType != null
-                      ? transformationRequest.videoMimeType
-                      : inputFormat.sampleMimeType)
-              .setColorInfo(fallbackToSdr ? null : inputFormat.colorInfo)
+              .setSampleMimeType(requestedOutputMimeType)
+              .setColorInfo(isInputToneMapped ? null : inputFormat.colorInfo)
              .build();

      encoder =
          encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes);
-      if (!hdrMediaCodecNames.isEmpty() && !hdrMediaCodecNames.contains(encoder.getName())) {
-        Log.d(
-            TAG,
-            "Selected encoder "
-                + encoder.getName()
-                + " does not report sufficient HDR capabilities");
-      }

      Format encoderSupportedFormat = encoder.getConfigurationFormat();
+      if (isHdrEditingEnabled()) {
+        if (!requestedOutputMimeType.equals(encoderSupportedFormat.sampleMimeType)) {
+          throw createEncodingException(
+              new IllegalStateException("MIME type fallback unsupported with HDR editing"),
+              encoderSupportedFormat);
+        } else if (!supportedEncoderNamesForHdrEditing.contains(encoder.getName())) {
+          throw createEncodingException(
+              new IllegalStateException("Selected encoder doesn't support HDR editing"),
+              encoderSupportedFormat);
+        }
+      }
      fallbackListener.onTransformationRequestFinalized(
          createFallbackTransformationRequest(
              transformationRequest,
              /* hasOutputFormatRotation= */ flipOrientation,
              requestedEncoderFormat,
              encoderSupportedFormat,
-              fallbackToSdr));
+              isInputToneMapped));

      encoderSurfaceInfo =
          new SurfaceInfo(
@ -468,41 +468,14 @@ import org.checkerframework.dataflow.qual.Pure;
      releaseEncoder = true;
    }

-    /**
-     * Checks whether at least one MediaCodec encoder on the device has sufficient capabilities to
-     * encode HDR (only checks support for HLG at this time).
-     */
-    public boolean supportsHdr() {
-      if (Util.SDK_INT < 31) {
-        return false;
-      }
-
-      // The only output MIME type that Transformer currently supports that can be used with HDR
-      // is H265/HEVC. So we assume that the EncoderFactory will pick this if HDR is requested.
-      String mimeType = MimeTypes.VIDEO_H265;
-
-      List<MediaCodecInfo> mediaCodecInfos = EncoderSelector.DEFAULT.selectEncoderInfos(mimeType);
-      for (int i = 0; i < mediaCodecInfos.size(); i++) {
-        MediaCodecInfo mediaCodecInfo = mediaCodecInfos.get(i);
-        if (EncoderUtil.isFeatureSupported(
-            mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing)) {
-          for (MediaCodecInfo.CodecProfileLevel capabilities :
-              mediaCodecInfo.getCapabilitiesForType(MimeTypes.VIDEO_H265).profileLevels) {
-            // TODO(b/227624622): What profile to check depends on the HDR format. Once other
-            //  formats besides HLG are supported, check the corresponding profiles here.
-            if (capabilities.profile == MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10) {
-              return hdrMediaCodecNames.add(mediaCodecInfo.getCanonicalName());
-            }
-          }
-        }
-      }
-      return !hdrMediaCodecNames.isEmpty();
-    }
-
-    public void signalFallbackToSdr() {
-      checkState(encoder == null, "Fallback to SDR is only allowed before encoder initialization");
-      fallbackToSdr = true;
-      hdrMediaCodecNames.clear();
+    private TransformationException createEncodingException(Exception cause, Format format) {
+      return TransformationException.createForCodec(
+          cause,
+          /* isVideo= */ true,
+          /* isDecoder= */ false,
+          format,
+          checkNotNull(encoder).getName(),
+          TransformationException.ERROR_CODE_ENCODING_FAILED);
    }
  }
 }
--- a/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java
+++ b/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java
@ -24,6 +24,7 @@ import android.os.Looper;
 import androidx.media3.common.C;
 import androidx.media3.common.Format;
 import androidx.media3.common.MediaItem;
+import androidx.media3.common.MimeTypes;
 import androidx.media3.common.util.Clock;
 import androidx.media3.common.util.ListenerSet;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
@ -47,7 +48,7 @@ public final class VideoEncoderWrapperTest {
  private final VideoTranscodingSamplePipeline.EncoderWrapper encoderWrapper =
      new VideoTranscodingSamplePipeline.EncoderWrapper(
          fakeEncoderFactory,
-          /* inputFormat= */ new Format.Builder().build(),
+          /* inputFormat= */ new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_H265).build(),
          /* allowedOutputMimeTypes= */ ImmutableList.of(),
          emptyTransformationRequest,
          fallbackListener);