Fix codec's MIME type is not used

In some cases the codec selected for decoding has a different MIME type than
the media. In thoses cases Transformer continued to use the media's MIME type
and that caused codec configuration failures.

Removed `EncoderUtil.findCodecForFormat()` as we stopped using the method it
uses for finding a codec. Plus, the method is only used in the test.

See also `MediaCodecUtil.getALternativeCodecMimeType()`.

PiperOrigin-RevId: 536683663
(cherry picked from commit 208eefc0fdf7b5320f5468aa9da864fefba3cfbe)
This commit is contained in:
claincly 2023-05-31 12:27:53 +00:00 committed by Tofunmi Adigun-Hameed
parent db1b5f148d
commit ba84de02f6
4 changed files with 91 additions and 106 deletions

View File

@ -16,6 +16,7 @@
package androidx.media3.transformer; package androidx.media3.transformer;
import static androidx.media3.common.MimeTypes.VIDEO_AV1; import static androidx.media3.common.MimeTypes.VIDEO_AV1;
import static androidx.media3.common.MimeTypes.VIDEO_DOLBY_VISION;
import static androidx.media3.common.MimeTypes.VIDEO_H264; import static androidx.media3.common.MimeTypes.VIDEO_H264;
import static androidx.media3.common.MimeTypes.VIDEO_H265; import static androidx.media3.common.MimeTypes.VIDEO_H265;
import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkNotNull;
@ -167,7 +168,23 @@ public final class AndroidTestUtil {
.setCodecs("hvc1.2.4.L153") .setCodecs("hvc1.2.4.L153")
.build(); .build();
// This file needs alternative MIME type, meaning the decoder needs to be configured with
// video/hevc instead of video/dolby-vision.
public static final String MP4_ASSET_DOLBY_VISION_HDR = "asset:///media/mp4/dolbyVision-hdr.MOV"; public static final String MP4_ASSET_DOLBY_VISION_HDR = "asset:///media/mp4/dolbyVision-hdr.MOV";
public static final Format MP4_ASSET_DOLBY_VISION_HDR_FORMAT =
new Format.Builder()
.setSampleMimeType(VIDEO_DOLBY_VISION)
.setWidth(1280)
.setHeight(720)
.setFrameRate(30.00f)
.setCodecs("hev1.08.02")
.setColorInfo(
new ColorInfo.Builder()
.setColorTransfer(C.COLOR_TRANSFER_HLG)
.setColorRange(C.COLOR_RANGE_LIMITED)
.setColorSpace(C.COLOR_SPACE_BT2020)
.build())
.build();
public static final String MP4_ASSET_4K60_PORTRAIT_URI_STRING = public static final String MP4_ASSET_4K60_PORTRAIT_URI_STRING =
"asset:///media/mp4/portrait_4k60.mp4"; "asset:///media/mp4/portrait_4k60.mp4";
@ -707,7 +724,7 @@ public final class AndroidTestUtil {
*/ */
public static boolean skipAndLogIfFormatsUnsupported( public static boolean skipAndLogIfFormatsUnsupported(
Context context, String testId, Format inputFormat, @Nullable Format outputFormat) Context context, String testId, Format inputFormat, @Nullable Format outputFormat)
throws IOException, JSONException { throws IOException, JSONException, MediaCodecUtil.DecoderQueryException {
// TODO(b/278657595): Make this capability check match the default codec factory selection code. // TODO(b/278657595): Make this capability check match the default codec factory selection code.
boolean canDecode = canDecode(inputFormat); boolean canDecode = canDecode(inputFormat);
@ -809,7 +826,7 @@ public final class AndroidTestUtil {
} }
} }
private static boolean canDecode(Format format) { private static boolean canDecode(Format format) throws MediaCodecUtil.DecoderQueryException {
// Check decoding capability in the same way as the default decoder factory. // Check decoding capability in the same way as the default decoder factory.
MediaFormat mediaFormat = MediaFormatUtil.createMediaFormatFromFormat(format); MediaFormat mediaFormat = MediaFormatUtil.createMediaFormatFromFormat(format);
@Nullable @Nullable
@ -818,7 +835,7 @@ public final class AndroidTestUtil {
MediaFormatUtil.maybeSetInteger( MediaFormatUtil.maybeSetInteger(
mediaFormat, MediaFormat.KEY_PROFILE, codecProfileAndLevel.first); mediaFormat, MediaFormat.KEY_PROFILE, codecProfileAndLevel.first);
} }
return EncoderUtil.findCodecForFormat(mediaFormat, /* isDecoder= */ true) != null; return DefaultDecoderFactory.getDecoderInfo(format) != null;
} }
private static boolean canEncode(Format format) { private static boolean canEncode(Format format) {

View File

@ -19,17 +19,20 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECO
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_720P_4_SECOND_HDR10; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_720P_4_SECOND_HDR10;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_720P_4_SECOND_HDR10_FORMAT; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_720P_4_SECOND_HDR10_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_DOLBY_VISION_HDR;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_DOLBY_VISION_HDR_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped; import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped;
import static androidx.media3.transformer.mh.FileUtil.maybeAssertFileHasColorTransfer; import static androidx.media3.transformer.mh.FileUtil.maybeAssertFileHasColorTransfer;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import android.content.Context; import android.content.Context;
import android.net.Uri;
import androidx.media3.common.C; import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MediaItem; import androidx.media3.common.MediaItem;
import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Log; import androidx.media3.common.util.Log;
import androidx.media3.common.util.Util; import androidx.media3.common.util.Util;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
import androidx.media3.transformer.AndroidTestUtil; import androidx.media3.transformer.AndroidTestUtil;
import androidx.media3.transformer.ExportException; import androidx.media3.transformer.ExportException;
import androidx.media3.transformer.ExportTestResult; import androidx.media3.transformer.ExportTestResult;
@ -38,6 +41,8 @@ import androidx.media3.transformer.Transformer;
import androidx.media3.transformer.TransformerAndroidTestRunner; import androidx.media3.transformer.TransformerAndroidTestRunner;
import androidx.test.core.app.ApplicationProvider; import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import java.io.IOException;
import org.json.JSONException;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
@ -49,85 +54,42 @@ import org.junit.runner.RunWith;
public class ToneMapHdrToSdrUsingOpenGlTest { public class ToneMapHdrToSdrUsingOpenGlTest {
public static final String TAG = "ToneMapHdrToSdrUsingOpenGlTest"; public static final String TAG = "ToneMapHdrToSdrUsingOpenGlTest";
private final Context context = ApplicationProvider.getApplicationContext();
@Test @Test
public void export_toneMap_hlg10File_toneMapsOrThrows() throws Exception { public void export_toneMap_hlg10File_toneMapsOrThrows() throws Exception {
String testId = "export_glToneMap_hlg10File_toneMapsOrThrows"; String testId = "export_glToneMap_hlg10File_toneMapsOrThrows";
if (!deviceSupportsOpenGlToneMapping(
if (Util.SDK_INT < 29) { testId, /* inputFormat= */ MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT)) {
recordTestSkipped(
ApplicationProvider.getApplicationContext(),
testId,
/* reason= */ "OpenGL-based HDR to SDR tone mapping is only supported on API 29+.");
return; return;
} }
if (!GlUtil.isYuvTargetExtensionSupported()) { runTransformerWithOpenGlToneMapping(testId, MP4_ASSET_1080P_5_SECOND_HLG10);
recordTestSkipped(
getApplicationContext(), testId, /* reason= */ "Device lacks YUV extension support.");
return;
}
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
getApplicationContext(),
testId,
/* inputFormat= */ MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT,
/* outputFormat= */ null)) {
return;
}
Context context = ApplicationProvider.getApplicationContext();
Transformer transformer =
new Transformer.Builder(context)
.setTransformationRequest(
new TransformationRequest.Builder()
.setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL)
.build())
.build();
MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET_1080P_5_SECOND_HLG10));
try {
ExportTestResult exportTestResult =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, mediaItem);
Log.i(TAG, "Tone mapped.");
maybeAssertFileHasColorTransfer(exportTestResult.filePath, C.COLOR_TRANSFER_SDR);
} catch (ExportException exception) {
Log.e(TAG, "Error during export.", exception);
if (exception.errorCode != ExportException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED) {
throw exception;
}
}
} }
@Test @Test
public void export_toneMap_hdr10File_toneMapsOrThrows() throws Exception { public void export_toneMap_hdr10File_toneMapsOrThrows() throws Exception {
String testId = "export_glToneMap_hdr10File_toneMapsOrThrows"; String testId = "export_glToneMap_hdr10File_toneMapsOrThrows";
if (!deviceSupportsOpenGlToneMapping(
if (Util.SDK_INT < 29) { testId, /* inputFormat= */ MP4_ASSET_720P_4_SECOND_HDR10_FORMAT)) {
recordTestSkipped(
ApplicationProvider.getApplicationContext(),
testId,
/* reason= */ "OpenGL-based HDR to SDR tone mapping is only supported on API 29+.");
return; return;
} }
if (!GlUtil.isYuvTargetExtensionSupported()) { runTransformerWithOpenGlToneMapping(testId, MP4_ASSET_720P_4_SECOND_HDR10);
recordTestSkipped( }
getApplicationContext(), testId, /* reason= */ "Device lacks YUV extension support.");
@Test
public void export_toneMap_dolbyVisionFile_toneMapsOrThrows() throws Exception {
String testId = "export_toneMap_dolbyVisionFile_toneMapsOrThrows";
if (!deviceSupportsOpenGlToneMapping(
testId, /* inputFormat= */ MP4_ASSET_DOLBY_VISION_HDR_FORMAT)) {
return; return;
} }
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported( runTransformerWithOpenGlToneMapping(testId, MP4_ASSET_DOLBY_VISION_HDR);
getApplicationContext(),
testId,
/* inputFormat= */ MP4_ASSET_720P_4_SECOND_HDR10_FORMAT,
/* outputFormat= */ null)) {
return;
} }
Context context = ApplicationProvider.getApplicationContext(); private void runTransformerWithOpenGlToneMapping(String testId, String fileUri) throws Exception {
Transformer transformer = Transformer transformer =
new Transformer.Builder(context) new Transformer.Builder(context)
.setTransformationRequest( .setTransformationRequest(
@ -135,12 +97,11 @@ public class ToneMapHdrToSdrUsingOpenGlTest {
.setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL) .setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL)
.build()) .build())
.build(); .build();
MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET_720P_4_SECOND_HDR10));
try { try {
ExportTestResult exportTestResult = ExportTestResult exportTestResult =
new TransformerAndroidTestRunner.Builder(context, transformer) new TransformerAndroidTestRunner.Builder(context, transformer)
.build() .build()
.run(testId, mediaItem); .run(testId, MediaItem.fromUri(fileUri));
Log.i(TAG, "Tone mapped."); Log.i(TAG, "Tone mapped.");
maybeAssertFileHasColorTransfer(exportTestResult.filePath, C.COLOR_TRANSFER_SDR); maybeAssertFileHasColorTransfer(exportTestResult.filePath, C.COLOR_TRANSFER_SDR);
} catch (ExportException exception) { } catch (ExportException exception) {
@ -150,4 +111,24 @@ public class ToneMapHdrToSdrUsingOpenGlTest {
} }
} }
} }
private static boolean deviceSupportsOpenGlToneMapping(String testId, Format inputFormat)
throws JSONException, IOException, MediaCodecUtil.DecoderQueryException {
if (Util.SDK_INT < 29) {
recordTestSkipped(
ApplicationProvider.getApplicationContext(),
testId,
/* reason= */ "OpenGL-based HDR to SDR tone mapping is only supported on API 29+.");
return false;
}
if (!GlUtil.isYuvTargetExtensionSupported()) {
recordTestSkipped(
getApplicationContext(), testId, /* reason= */ "Device lacks YUV extension support.");
return false;
}
return !AndroidTestUtil.skipAndLogIfFormatsUnsupported(
getApplicationContext(), testId, /* inputFormat= */ inputFormat, /* outputFormat= */ null);
}
} }

View File

@ -27,6 +27,7 @@ import android.os.Build;
import android.util.Pair; import android.util.Pair;
import android.view.Surface; import android.view.Surface;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C; import androidx.media3.common.C;
import androidx.media3.common.ColorInfo; import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format; import androidx.media3.common.Format;
@ -64,7 +65,15 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
String mediaCodecName; String mediaCodecName;
try { try {
mediaCodecName = getMediaCodecNameForDecoding(format); @Nullable MediaCodecInfo decoderInfo = getDecoderInfo(format);
if (decoderInfo == null) {
throw createExportException(format, /* reason= */ "No decoders for format");
}
mediaCodecName = decoderInfo.name;
String codecMimeType = decoderInfo.codecMimeType;
// Does not alter format.sampleMimeType to keep the original MimeType.
// The MIME type of the selected decoder may differ from Format.sampleMimeType.
mediaFormat.setString(MediaFormat.KEY_MIME, codecMimeType);
} catch (MediaCodecUtil.DecoderQueryException e) { } catch (MediaCodecUtil.DecoderQueryException e) {
Log.e(TAG, "Error querying decoders", e); Log.e(TAG, "Error querying decoders", e);
throw createExportException(format, /* reason= */ "Querying codecs failed."); throw createExportException(format, /* reason= */ "Querying codecs failed.");
@ -113,7 +122,16 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
String mediaCodecName; String mediaCodecName;
try { try {
mediaCodecName = getMediaCodecNameForDecoding(format); @Nullable MediaCodecInfo decoderInfo = getDecoderInfo(format);
if (decoderInfo == null) {
throw createExportException(format, /* reason= */ "No decoders for format");
}
mediaCodecName = decoderInfo.name;
String codecMimeType = decoderInfo.codecMimeType;
// Does not alter format.sampleMimeType to keep the original MimeType.
// The MIME type of the selected decoder may differ from Format.sampleMimeType, for example,
// video/hevc is used instead of video/dolby-vision for some specific DolbyVision videos.
mediaFormat.setString(MediaFormat.KEY_MIME, codecMimeType);
} catch (MediaCodecUtil.DecoderQueryException e) { } catch (MediaCodecUtil.DecoderQueryException e) {
Log.e(TAG, "Error querying decoders", e); Log.e(TAG, "Error querying decoders", e);
throw createExportException(format, /* reason= */ "Querying codecs failed"); throw createExportException(format, /* reason= */ "Querying codecs failed");
@ -158,8 +176,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
format); format);
} }
private static String getMediaCodecNameForDecoding(Format format) @VisibleForTesting
throws MediaCodecUtil.DecoderQueryException, ExportException { @Nullable
/* package */ static MediaCodecInfo getDecoderInfo(Format format)
throws MediaCodecUtil.DecoderQueryException {
checkNotNull(format.sampleMimeType); checkNotNull(format.sampleMimeType);
List<MediaCodecInfo> decoderInfos = List<MediaCodecInfo> decoderInfos =
MediaCodecUtil.getDecoderInfosSortedByFormatSupport( MediaCodecUtil.getDecoderInfosSortedByFormatSupport(
@ -170,8 +190,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/* requiresTunnelingDecoder= */ false), /* requiresTunnelingDecoder= */ false),
format); format);
if (decoderInfos.isEmpty()) { if (decoderInfos.isEmpty()) {
throw createExportException(format, /* reason= */ "No decoders for format"); return null;
} }
return decoderInfos.get(0).name; return decoderInfos.get(0);
} }
} }

View File

@ -24,7 +24,6 @@ import android.media.CamcorderProfile;
import android.media.MediaCodec; import android.media.MediaCodec;
import android.media.MediaCodecInfo; import android.media.MediaCodecInfo;
import android.media.MediaCodecList; import android.media.MediaCodecList;
import android.media.MediaFormat;
import android.util.Pair; import android.util.Pair;
import android.util.Range; import android.util.Range;
import android.util.Size; import android.util.Size;
@ -38,7 +37,6 @@ import androidx.media3.common.C.ColorTransfer;
import androidx.media3.common.ColorInfo; import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format; import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes; import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.MediaFormatUtil;
import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util; import androidx.media3.common.util.Util;
import com.google.common.base.Ascii; import com.google.common.base.Ascii;
@ -303,37 +301,6 @@ public final class EncoderUtil {
return maxSupportedLevel; return maxSupportedLevel;
} }
/**
* Finds a {@link MediaCodec} that supports the {@link MediaFormat}, or {@code null} if none is
* found.
*/
@Nullable
public static String findCodecForFormat(MediaFormat format, boolean isDecoder) {
MediaCodecList mediaCodecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
// Format must not include KEY_FRAME_RATE on API21.
// https://developer.android.com/reference/android/media/MediaCodecList#findDecoderForFormat(android.media.MediaFormat)
float frameRate = Format.NO_VALUE;
if (Util.SDK_INT == 21 && format.containsKey(MediaFormat.KEY_FRAME_RATE)) {
try {
frameRate = format.getFloat(MediaFormat.KEY_FRAME_RATE);
} catch (ClassCastException e) {
frameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE);
}
// Clears the frame rate field.
format.setString(MediaFormat.KEY_FRAME_RATE, null);
}
String mediaCodecName =
isDecoder
? mediaCodecList.findDecoderForFormat(format)
: mediaCodecList.findEncoderForFormat(format);
if (Util.SDK_INT == 21) {
MediaFormatUtil.maybeSetInteger(format, MediaFormat.KEY_FRAME_RATE, round(frameRate));
}
return mediaCodecName;
}
/** Returns the range of supported bitrates for the given {@linkplain MimeTypes MIME type}. */ /** Returns the range of supported bitrates for the given {@linkplain MimeTypes MIME type}. */
public static Range<Integer> getSupportedBitrateRange( public static Range<Integer> getSupportedBitrateRange(
MediaCodecInfo encoderInfo, String mimeType) { MediaCodecInfo encoderInfo, String mimeType) {