diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java index b41e4e99ea..a1df6d6934 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java @@ -275,7 +275,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { } if (mimeType.equals(MimeTypes.VIDEO_H264)) { - adjustMediaFormatForH264EncoderSettings(mediaFormat, encoderInfo); + adjustMediaFormatForH264EncoderSettings(format.colorInfo, encoderInfo, mediaFormat); } MediaFormatUtil.maybeSetColorInfo(mediaFormat, encoderSupportedFormat.colorInfo); @@ -525,12 +525,21 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { *

The adjustment is applied in-place to {@code mediaFormat}. */ private static void adjustMediaFormatForH264EncoderSettings( - MediaFormat mediaFormat, MediaCodecInfo encoderInfo) { + @Nullable ColorInfo colorInfo, MediaCodecInfo encoderInfo, MediaFormat mediaFormat) { // TODO(b/210593256): Remove overriding profile/level (before API 29) after switching to in-app // muxing. String mimeType = MimeTypes.VIDEO_H264; if (Util.SDK_INT >= 29) { int expectedEncodingProfile = MediaCodecInfo.CodecProfileLevel.AVCProfileHigh; + if (colorInfo != null) { + int colorTransfer = colorInfo.colorTransfer; + ImmutableList codecProfiles = + EncoderUtil.getCodecProfilesForHdrFormat(mimeType, colorTransfer); + if (!codecProfiles.isEmpty()) { + // Default to the most compatible profile, which is first in the list. + expectedEncodingProfile = codecProfiles.get(0); + } + } int supportedEncodingLevel = EncoderUtil.findHighestSupportedEncodingLevel( encoderInfo, mimeType, expectedEncodingProfile); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java index 1123d39893..aceb651236 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java @@ -31,6 +31,9 @@ import android.util.Size; import androidx.annotation.DoNotInline; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; +import androidx.media3.common.C; +import androidx.media3.common.C.ColorTransfer; +import androidx.media3.common.ColorInfo; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.MediaFormatUtil; @@ -67,6 +70,83 @@ public final class EncoderUtil { return checkNotNull(MIME_TYPE_TO_ENCODERS.get()).keySet(); } + /** + * Returns the names of encoders that support HDR editing for the given format, or an empty list + * if the format is unknown or not supported for HDR encoding. + */ + public static ImmutableList getSupportedEncoderNamesForHdrEditing( + String mimeType, @Nullable ColorInfo colorInfo) { + if (Util.SDK_INT < 31 || colorInfo == null) { + return ImmutableList.of(); + } + + @ColorTransfer int colorTransfer = colorInfo.colorTransfer; + ImmutableList profiles = getCodecProfilesForHdrFormat(mimeType, colorTransfer); + ImmutableList.Builder resultBuilder = ImmutableList.builder(); + ImmutableList mediaCodecInfos = + EncoderSelector.DEFAULT.selectEncoderInfos(mimeType); + for (int i = 0; i < mediaCodecInfos.size(); i++) { + MediaCodecInfo mediaCodecInfo = mediaCodecInfos.get(i); + if (mediaCodecInfo.isAlias() + || !EncoderUtil.isFeatureSupported( + mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing)) { + continue; + } + for (MediaCodecInfo.CodecProfileLevel codecProfileLevel : + mediaCodecInfo.getCapabilitiesForType(mimeType).profileLevels) { + if (profiles.contains(codecProfileLevel.profile)) { + resultBuilder.add(mediaCodecInfo.getName()); + } + } + } + return resultBuilder.build(); + } + + /** + * Returns the {@linkplain MediaCodecInfo.CodecProfileLevel#profile profile} constants that can be + * used to encode the given HDR format, if supported by the device (this method does not check + * device capabilities). If multiple profiles are returned, they are ordered by expected level of + * compatibility, with the most widely compatible profile first. + */ + @SuppressWarnings("InlinedApi") // Safe use of inlined constants from newer API versions. + public static ImmutableList getCodecProfilesForHdrFormat( + String mimeType, @ColorTransfer int colorTransfer) { + // TODO(b/239174610): Add a way to determine profiles for DV and HDR10+. + switch (mimeType) { + case MimeTypes.VIDEO_VP9: + if (colorTransfer == C.COLOR_TRANSFER_HLG || colorTransfer == C.COLOR_TRANSFER_ST2084) { + // Profiles support both HLG and PQ. + return ImmutableList.of( + MediaCodecInfo.CodecProfileLevel.VP9Profile2HDR, + MediaCodecInfo.CodecProfileLevel.VP9Profile3HDR); + } + break; + case MimeTypes.VIDEO_H264: + if (colorTransfer == C.COLOR_TRANSFER_HLG) { + return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.AVCProfileHigh10); + } + break; + case MimeTypes.VIDEO_H265: + if (colorTransfer == C.COLOR_TRANSFER_HLG) { + return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10); + } else if (colorTransfer == C.COLOR_TRANSFER_ST2084) { + return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10HDR10); + } + break; + case MimeTypes.VIDEO_AV1: + if (colorTransfer == C.COLOR_TRANSFER_HLG) { + return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.AV1ProfileMain10); + } else if (colorTransfer == C.COLOR_TRANSFER_ST2084) { + return ImmutableList.of(MediaCodecInfo.CodecProfileLevel.AV1ProfileMain10HDR10); + } + break; + default: + break; + } + // There are no profiles defined for the HDR format, or it's invalid. + return ImmutableList.of(); + } + /** Returns whether the {@linkplain MediaCodecInfo encoder} supports the given resolution. */ public static boolean isSizeSupported( MediaCodecInfo encoderInfo, String mimeType, int width, int height) { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 0c97b1268f..3eb54168c9 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -17,25 +17,20 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkState; import android.content.Context; import android.media.MediaCodec; -import android.media.MediaCodecInfo; import android.view.Surface; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; import androidx.media3.common.ColorInfo; import androidx.media3.common.Format; -import androidx.media3.common.MimeTypes; -import androidx.media3.common.util.Log; import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.dataflow.qual.Pure; @@ -105,17 +100,6 @@ import org.checkerframework.dataflow.qual.Pure; transformationRequest, fallbackListener); - boolean enableRequestSdrToneMapping = transformationRequest.enableRequestSdrToneMapping; - // TODO(b/237674316): While HLG10 is correctly reported, HDR10 currently will be incorrectly - // processed as SDR, because the inputFormat.colorInfo reports the wrong value. - boolean useHdr = - transformationRequest.enableHdrEditing && ColorInfo.isHdr(inputFormat.colorInfo); - if (useHdr && !encoderWrapper.supportsHdr()) { - useHdr = false; - enableRequestSdrToneMapping = true; - encoderWrapper.signalFallbackToSdr(); - } - try { frameProcessor = GlEffectsFrameProcessor.create( @@ -153,7 +137,7 @@ import org.checkerframework.dataflow.qual.Pure; // HDR is only used if the MediaCodec encoder supports FEATURE_HdrEditing. This // implies that the OpenGL EXT_YUV_target extension is supported and hence the // GlEffectsFrameProcessor also supports HDR. - useHdr); + /* useHdr= */ encoderWrapper.isHdrEditingEnabled()); } catch (FrameProcessingException e) { throw TransformationException.createForFrameProcessingException( e, TransformationException.ERROR_CODE_GL_INIT_FAILED); @@ -161,9 +145,11 @@ import org.checkerframework.dataflow.qual.Pure; frameProcessor.setInputFrameInfo( new FrameInfo(decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio)); + boolean isToneMappingRequired = + ColorInfo.isHdr(inputFormat.colorInfo) && !encoderWrapper.isHdrEditingEnabled(); decoder = decoderFactory.createForVideoDecoding( - inputFormat, frameProcessor.getInputSurface(), enableRequestSdrToneMapping); + inputFormat, frameProcessor.getInputSurface(), isToneMappingRequired); // TODO(b/236316454): Check in the decoder output format whether tone-mapping was actually // applied and throw an exception if not. maxPendingFrameCount = decoder.getMaxPendingFrameCount(); @@ -331,14 +317,14 @@ import org.checkerframework.dataflow.qual.Pure; private final List allowedOutputMimeTypes; private final TransformationRequest transformationRequest; private final FallbackListener fallbackListener; - private final HashSet hdrMediaCodecNames; + private final String requestedOutputMimeType; + private final ImmutableList supportedEncoderNamesForHdrEditing; private @MonotonicNonNull SurfaceInfo encoderSurfaceInfo; private volatile @MonotonicNonNull Codec encoder; private volatile int outputRotationDegrees; private volatile boolean releaseEncoder; - private boolean fallbackToSdr; public EncoderWrapper( Codec.EncoderFactory encoderFactory, @@ -346,14 +332,26 @@ import org.checkerframework.dataflow.qual.Pure; List allowedOutputMimeTypes, TransformationRequest transformationRequest, FallbackListener fallbackListener) { - this.encoderFactory = encoderFactory; this.inputFormat = inputFormat; this.allowedOutputMimeTypes = allowedOutputMimeTypes; this.transformationRequest = transformationRequest; this.fallbackListener = fallbackListener; - hdrMediaCodecNames = new HashSet<>(); + requestedOutputMimeType = + transformationRequest.videoMimeType != null + ? transformationRequest.videoMimeType + : checkNotNull(inputFormat.sampleMimeType); + supportedEncoderNamesForHdrEditing = + EncoderUtil.getSupportedEncoderNamesForHdrEditing( + requestedOutputMimeType, inputFormat.colorInfo); + } + + /** Returns whether the wrapped encoder is expecting HDR input for the HDR editing use case. */ + public boolean isHdrEditingEnabled() { + return transformationRequest.enableHdrEditing + && !transformationRequest.enableRequestSdrToneMapping + && !supportedEncoderNamesForHdrEditing.isEmpty(); } @Nullable @@ -378,37 +376,39 @@ import org.checkerframework.dataflow.qual.Pure; outputRotationDegrees = 90; } + boolean isInputToneMapped = ColorInfo.isHdr(inputFormat.colorInfo) && !isHdrEditingEnabled(); Format requestedEncoderFormat = new Format.Builder() .setWidth(requestedWidth) .setHeight(requestedHeight) .setRotationDegrees(0) .setFrameRate(inputFormat.frameRate) - .setSampleMimeType( - transformationRequest.videoMimeType != null - ? transformationRequest.videoMimeType - : inputFormat.sampleMimeType) - .setColorInfo(fallbackToSdr ? null : inputFormat.colorInfo) + .setSampleMimeType(requestedOutputMimeType) + .setColorInfo(isInputToneMapped ? null : inputFormat.colorInfo) .build(); encoder = encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes); - if (!hdrMediaCodecNames.isEmpty() && !hdrMediaCodecNames.contains(encoder.getName())) { - Log.d( - TAG, - "Selected encoder " - + encoder.getName() - + " does not report sufficient HDR capabilities"); - } Format encoderSupportedFormat = encoder.getConfigurationFormat(); + if (isHdrEditingEnabled()) { + if (!requestedOutputMimeType.equals(encoderSupportedFormat.sampleMimeType)) { + throw createEncodingException( + new IllegalStateException("MIME type fallback unsupported with HDR editing"), + encoderSupportedFormat); + } else if (!supportedEncoderNamesForHdrEditing.contains(encoder.getName())) { + throw createEncodingException( + new IllegalStateException("Selected encoder doesn't support HDR editing"), + encoderSupportedFormat); + } + } fallbackListener.onTransformationRequestFinalized( createFallbackTransformationRequest( transformationRequest, /* hasOutputFormatRotation= */ flipOrientation, requestedEncoderFormat, encoderSupportedFormat, - fallbackToSdr)); + isInputToneMapped)); encoderSurfaceInfo = new SurfaceInfo( @@ -468,41 +468,14 @@ import org.checkerframework.dataflow.qual.Pure; releaseEncoder = true; } - /** - * Checks whether at least one MediaCodec encoder on the device has sufficient capabilities to - * encode HDR (only checks support for HLG at this time). - */ - public boolean supportsHdr() { - if (Util.SDK_INT < 31) { - return false; - } - - // The only output MIME type that Transformer currently supports that can be used with HDR - // is H265/HEVC. So we assume that the EncoderFactory will pick this if HDR is requested. - String mimeType = MimeTypes.VIDEO_H265; - - List mediaCodecInfos = EncoderSelector.DEFAULT.selectEncoderInfos(mimeType); - for (int i = 0; i < mediaCodecInfos.size(); i++) { - MediaCodecInfo mediaCodecInfo = mediaCodecInfos.get(i); - if (EncoderUtil.isFeatureSupported( - mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing)) { - for (MediaCodecInfo.CodecProfileLevel capabilities : - mediaCodecInfo.getCapabilitiesForType(MimeTypes.VIDEO_H265).profileLevels) { - // TODO(b/227624622): What profile to check depends on the HDR format. Once other - // formats besides HLG are supported, check the corresponding profiles here. - if (capabilities.profile == MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10) { - return hdrMediaCodecNames.add(mediaCodecInfo.getCanonicalName()); - } - } - } - } - return !hdrMediaCodecNames.isEmpty(); - } - - public void signalFallbackToSdr() { - checkState(encoder == null, "Fallback to SDR is only allowed before encoder initialization"); - fallbackToSdr = true; - hdrMediaCodecNames.clear(); + private TransformationException createEncodingException(Exception cause, Format format) { + return TransformationException.createForCodec( + cause, + /* isVideo= */ true, + /* isDecoder= */ false, + format, + checkNotNull(encoder).getName(), + TransformationException.ERROR_CODE_ENCODING_FAILED); } } } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java index 1b8b8b4502..4f546b88af 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java @@ -24,6 +24,7 @@ import android.os.Looper; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MediaItem; +import androidx.media3.common.MimeTypes; import androidx.media3.common.util.Clock; import androidx.media3.common.util.ListenerSet; import androidx.test.ext.junit.runners.AndroidJUnit4; @@ -47,7 +48,7 @@ public final class VideoEncoderWrapperTest { private final VideoTranscodingSamplePipeline.EncoderWrapper encoderWrapper = new VideoTranscodingSamplePipeline.EncoderWrapper( fakeEncoderFactory, - /* inputFormat= */ new Format.Builder().build(), + /* inputFormat= */ new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_H265).build(), /* allowedOutputMimeTypes= */ ImmutableList.of(), emptyTransformationRequest, fallbackListener);