From ba84de02f60a2f95e9526b59f1dc36f5394db2c3 Mon Sep 17 00:00:00 2001 From: claincly Date: Wed, 31 May 2023 12:27:53 +0000 Subject: [PATCH] Fix codec's MIME type is not used In some cases the codec selected for decoding has a different MIME type than the media. In thoses cases Transformer continued to use the media's MIME type and that caused codec configuration failures. Removed `EncoderUtil.findCodecForFormat()` as we stopped using the method it uses for finding a codec. Plus, the method is only used in the test. See also `MediaCodecUtil.getALternativeCodecMimeType()`. PiperOrigin-RevId: 536683663 (cherry picked from commit 208eefc0fdf7b5320f5468aa9da864fefba3cfbe) --- .../media3/transformer/AndroidTestUtil.java | 23 +++- .../mh/ToneMapHdrToSdrUsingOpenGlTest.java | 109 ++++++++---------- .../transformer/DefaultDecoderFactory.java | 32 ++++- .../media3/transformer/EncoderUtil.java | 33 ------ 4 files changed, 91 insertions(+), 106 deletions(-) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index 7d24c881f1..b80a209fb4 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -16,6 +16,7 @@ package androidx.media3.transformer; import static androidx.media3.common.MimeTypes.VIDEO_AV1; +import static androidx.media3.common.MimeTypes.VIDEO_DOLBY_VISION; import static androidx.media3.common.MimeTypes.VIDEO_H264; import static androidx.media3.common.MimeTypes.VIDEO_H265; import static androidx.media3.common.util.Assertions.checkNotNull; @@ -167,7 +168,23 @@ public final class AndroidTestUtil { .setCodecs("hvc1.2.4.L153") .build(); + // This file needs alternative MIME type, meaning the decoder needs to be configured with + // video/hevc instead of video/dolby-vision. public static final String MP4_ASSET_DOLBY_VISION_HDR = "asset:///media/mp4/dolbyVision-hdr.MOV"; + public static final Format MP4_ASSET_DOLBY_VISION_HDR_FORMAT = + new Format.Builder() + .setSampleMimeType(VIDEO_DOLBY_VISION) + .setWidth(1280) + .setHeight(720) + .setFrameRate(30.00f) + .setCodecs("hev1.08.02") + .setColorInfo( + new ColorInfo.Builder() + .setColorTransfer(C.COLOR_TRANSFER_HLG) + .setColorRange(C.COLOR_RANGE_LIMITED) + .setColorSpace(C.COLOR_SPACE_BT2020) + .build()) + .build(); public static final String MP4_ASSET_4K60_PORTRAIT_URI_STRING = "asset:///media/mp4/portrait_4k60.mp4"; @@ -707,7 +724,7 @@ public final class AndroidTestUtil { */ public static boolean skipAndLogIfFormatsUnsupported( Context context, String testId, Format inputFormat, @Nullable Format outputFormat) - throws IOException, JSONException { + throws IOException, JSONException, MediaCodecUtil.DecoderQueryException { // TODO(b/278657595): Make this capability check match the default codec factory selection code. boolean canDecode = canDecode(inputFormat); @@ -809,7 +826,7 @@ public final class AndroidTestUtil { } } - private static boolean canDecode(Format format) { + private static boolean canDecode(Format format) throws MediaCodecUtil.DecoderQueryException { // Check decoding capability in the same way as the default decoder factory. MediaFormat mediaFormat = MediaFormatUtil.createMediaFormatFromFormat(format); @Nullable @@ -818,7 +835,7 @@ public final class AndroidTestUtil { MediaFormatUtil.maybeSetInteger( mediaFormat, MediaFormat.KEY_PROFILE, codecProfileAndLevel.first); } - return EncoderUtil.findCodecForFormat(mediaFormat, /* isDecoder= */ true) != null; + return DefaultDecoderFactory.getDecoderInfo(format) != null; } private static boolean canEncode(Format format) { diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlTest.java index cdd5d8ff91..246a17fb0a 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlTest.java @@ -19,17 +19,20 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECO import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_720P_4_SECOND_HDR10; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_720P_4_SECOND_HDR10_FORMAT; +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_DOLBY_VISION_HDR; +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_DOLBY_VISION_HDR_FORMAT; import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped; import static androidx.media3.transformer.mh.FileUtil.maybeAssertFileHasColorTransfer; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import android.content.Context; -import android.net.Uri; import androidx.media3.common.C; +import androidx.media3.common.Format; import androidx.media3.common.MediaItem; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Log; import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import androidx.media3.transformer.AndroidTestUtil; import androidx.media3.transformer.ExportException; import androidx.media3.transformer.ExportTestResult; @@ -38,6 +41,8 @@ import androidx.media3.transformer.Transformer; import androidx.media3.transformer.TransformerAndroidTestRunner; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import java.io.IOException; +import org.json.JSONException; import org.junit.Test; import org.junit.runner.RunWith; @@ -49,85 +54,42 @@ import org.junit.runner.RunWith; public class ToneMapHdrToSdrUsingOpenGlTest { public static final String TAG = "ToneMapHdrToSdrUsingOpenGlTest"; + private final Context context = ApplicationProvider.getApplicationContext(); + @Test public void export_toneMap_hlg10File_toneMapsOrThrows() throws Exception { String testId = "export_glToneMap_hlg10File_toneMapsOrThrows"; - - if (Util.SDK_INT < 29) { - recordTestSkipped( - ApplicationProvider.getApplicationContext(), - testId, - /* reason= */ "OpenGL-based HDR to SDR tone mapping is only supported on API 29+."); + if (!deviceSupportsOpenGlToneMapping( + testId, /* inputFormat= */ MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT)) { return; } - if (!GlUtil.isYuvTargetExtensionSupported()) { - recordTestSkipped( - getApplicationContext(), testId, /* reason= */ "Device lacks YUV extension support."); - return; - } - - if (AndroidTestUtil.skipAndLogIfFormatsUnsupported( - getApplicationContext(), - testId, - /* inputFormat= */ MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT, - /* outputFormat= */ null)) { - return; - } - - Context context = ApplicationProvider.getApplicationContext(); - - Transformer transformer = - new Transformer.Builder(context) - .setTransformationRequest( - new TransformationRequest.Builder() - .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL) - .build()) - .build(); - MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET_1080P_5_SECOND_HLG10)); - try { - ExportTestResult exportTestResult = - new TransformerAndroidTestRunner.Builder(context, transformer) - .build() - .run(testId, mediaItem); - Log.i(TAG, "Tone mapped."); - maybeAssertFileHasColorTransfer(exportTestResult.filePath, C.COLOR_TRANSFER_SDR); - } catch (ExportException exception) { - Log.e(TAG, "Error during export.", exception); - if (exception.errorCode != ExportException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED) { - throw exception; - } - } + runTransformerWithOpenGlToneMapping(testId, MP4_ASSET_1080P_5_SECOND_HLG10); } @Test public void export_toneMap_hdr10File_toneMapsOrThrows() throws Exception { String testId = "export_glToneMap_hdr10File_toneMapsOrThrows"; - - if (Util.SDK_INT < 29) { - recordTestSkipped( - ApplicationProvider.getApplicationContext(), - testId, - /* reason= */ "OpenGL-based HDR to SDR tone mapping is only supported on API 29+."); + if (!deviceSupportsOpenGlToneMapping( + testId, /* inputFormat= */ MP4_ASSET_720P_4_SECOND_HDR10_FORMAT)) { return; } - if (!GlUtil.isYuvTargetExtensionSupported()) { - recordTestSkipped( - getApplicationContext(), testId, /* reason= */ "Device lacks YUV extension support."); + runTransformerWithOpenGlToneMapping(testId, MP4_ASSET_720P_4_SECOND_HDR10); + } + + @Test + public void export_toneMap_dolbyVisionFile_toneMapsOrThrows() throws Exception { + String testId = "export_toneMap_dolbyVisionFile_toneMapsOrThrows"; + if (!deviceSupportsOpenGlToneMapping( + testId, /* inputFormat= */ MP4_ASSET_DOLBY_VISION_HDR_FORMAT)) { return; } - if (AndroidTestUtil.skipAndLogIfFormatsUnsupported( - getApplicationContext(), - testId, - /* inputFormat= */ MP4_ASSET_720P_4_SECOND_HDR10_FORMAT, - /* outputFormat= */ null)) { - return; - } - - Context context = ApplicationProvider.getApplicationContext(); + runTransformerWithOpenGlToneMapping(testId, MP4_ASSET_DOLBY_VISION_HDR); + } + private void runTransformerWithOpenGlToneMapping(String testId, String fileUri) throws Exception { Transformer transformer = new Transformer.Builder(context) .setTransformationRequest( @@ -135,12 +97,11 @@ public class ToneMapHdrToSdrUsingOpenGlTest { .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL) .build()) .build(); - MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET_720P_4_SECOND_HDR10)); try { ExportTestResult exportTestResult = new TransformerAndroidTestRunner.Builder(context, transformer) .build() - .run(testId, mediaItem); + .run(testId, MediaItem.fromUri(fileUri)); Log.i(TAG, "Tone mapped."); maybeAssertFileHasColorTransfer(exportTestResult.filePath, C.COLOR_TRANSFER_SDR); } catch (ExportException exception) { @@ -150,4 +111,24 @@ public class ToneMapHdrToSdrUsingOpenGlTest { } } } + + private static boolean deviceSupportsOpenGlToneMapping(String testId, Format inputFormat) + throws JSONException, IOException, MediaCodecUtil.DecoderQueryException { + if (Util.SDK_INT < 29) { + recordTestSkipped( + ApplicationProvider.getApplicationContext(), + testId, + /* reason= */ "OpenGL-based HDR to SDR tone mapping is only supported on API 29+."); + return false; + } + + if (!GlUtil.isYuvTargetExtensionSupported()) { + recordTestSkipped( + getApplicationContext(), testId, /* reason= */ "Device lacks YUV extension support."); + return false; + } + + return !AndroidTestUtil.skipAndLogIfFormatsUnsupported( + getApplicationContext(), testId, /* inputFormat= */ inputFormat, /* outputFormat= */ null); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java index 6c143d154a..856de2e5a8 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java @@ -27,6 +27,7 @@ import android.os.Build; import android.util.Pair; import android.view.Surface; import androidx.annotation.Nullable; +import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; import androidx.media3.common.ColorInfo; import androidx.media3.common.Format; @@ -64,7 +65,15 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; String mediaCodecName; try { - mediaCodecName = getMediaCodecNameForDecoding(format); + @Nullable MediaCodecInfo decoderInfo = getDecoderInfo(format); + if (decoderInfo == null) { + throw createExportException(format, /* reason= */ "No decoders for format"); + } + mediaCodecName = decoderInfo.name; + String codecMimeType = decoderInfo.codecMimeType; + // Does not alter format.sampleMimeType to keep the original MimeType. + // The MIME type of the selected decoder may differ from Format.sampleMimeType. + mediaFormat.setString(MediaFormat.KEY_MIME, codecMimeType); } catch (MediaCodecUtil.DecoderQueryException e) { Log.e(TAG, "Error querying decoders", e); throw createExportException(format, /* reason= */ "Querying codecs failed."); @@ -113,7 +122,16 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; String mediaCodecName; try { - mediaCodecName = getMediaCodecNameForDecoding(format); + @Nullable MediaCodecInfo decoderInfo = getDecoderInfo(format); + if (decoderInfo == null) { + throw createExportException(format, /* reason= */ "No decoders for format"); + } + mediaCodecName = decoderInfo.name; + String codecMimeType = decoderInfo.codecMimeType; + // Does not alter format.sampleMimeType to keep the original MimeType. + // The MIME type of the selected decoder may differ from Format.sampleMimeType, for example, + // video/hevc is used instead of video/dolby-vision for some specific DolbyVision videos. + mediaFormat.setString(MediaFormat.KEY_MIME, codecMimeType); } catch (MediaCodecUtil.DecoderQueryException e) { Log.e(TAG, "Error querying decoders", e); throw createExportException(format, /* reason= */ "Querying codecs failed"); @@ -158,8 +176,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; format); } - private static String getMediaCodecNameForDecoding(Format format) - throws MediaCodecUtil.DecoderQueryException, ExportException { + @VisibleForTesting + @Nullable + /* package */ static MediaCodecInfo getDecoderInfo(Format format) + throws MediaCodecUtil.DecoderQueryException { checkNotNull(format.sampleMimeType); List decoderInfos = MediaCodecUtil.getDecoderInfosSortedByFormatSupport( @@ -170,8 +190,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; /* requiresTunnelingDecoder= */ false), format); if (decoderInfos.isEmpty()) { - throw createExportException(format, /* reason= */ "No decoders for format"); + return null; } - return decoderInfos.get(0).name; + return decoderInfos.get(0); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java index d3071a63db..acf8205c83 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java @@ -24,7 +24,6 @@ import android.media.CamcorderProfile; import android.media.MediaCodec; import android.media.MediaCodecInfo; import android.media.MediaCodecList; -import android.media.MediaFormat; import android.util.Pair; import android.util.Range; import android.util.Size; @@ -38,7 +37,6 @@ import androidx.media3.common.C.ColorTransfer; import androidx.media3.common.ColorInfo; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; -import androidx.media3.common.util.MediaFormatUtil; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import com.google.common.base.Ascii; @@ -303,37 +301,6 @@ public final class EncoderUtil { return maxSupportedLevel; } - /** - * Finds a {@link MediaCodec} that supports the {@link MediaFormat}, or {@code null} if none is - * found. - */ - @Nullable - public static String findCodecForFormat(MediaFormat format, boolean isDecoder) { - MediaCodecList mediaCodecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS); - // Format must not include KEY_FRAME_RATE on API21. - // https://developer.android.com/reference/android/media/MediaCodecList#findDecoderForFormat(android.media.MediaFormat) - float frameRate = Format.NO_VALUE; - if (Util.SDK_INT == 21 && format.containsKey(MediaFormat.KEY_FRAME_RATE)) { - try { - frameRate = format.getFloat(MediaFormat.KEY_FRAME_RATE); - } catch (ClassCastException e) { - frameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE); - } - // Clears the frame rate field. - format.setString(MediaFormat.KEY_FRAME_RATE, null); - } - - String mediaCodecName = - isDecoder - ? mediaCodecList.findDecoderForFormat(format) - : mediaCodecList.findEncoderForFormat(format); - - if (Util.SDK_INT == 21) { - MediaFormatUtil.maybeSetInteger(format, MediaFormat.KEY_FRAME_RATE, round(frameRate)); - } - return mediaCodecName; - } - /** Returns the range of supported bitrates for the given {@linkplain MimeTypes MIME type}. */ public static Range getSupportedBitrateRange( MediaCodecInfo encoderInfo, String mimeType) {