loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix codec's MIME type is not used

Browse Source 
In some cases the codec selected for decoding has a different MIME type than
the media. In thoses cases Transformer continued to use the media's MIME type
and that caused codec configuration failures.

Removed `EncoderUtil.findCodecForFormat()` as we stopped using the method it
uses for finding a codec. Plus, the method is only used in the test.

See also `MediaCodecUtil.getALternativeCodecMimeType()`.

PiperOrigin-RevId: 536683663
This commit is contained in:
claincly 2023-05-31 12:27:53 +00:00 committed by Tofunmi Adigun-Hameed
parent e2821f10f5
commit 208eefc0fd
4 changed files with 91 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View File

@ -16,6 +16,7 @@
package androidx.media3.transformer;
import static androidx.media3.common.MimeTypes.VIDEO_AV1;
import static androidx.media3.common.MimeTypes.VIDEO_DOLBY_VISION;
import static androidx.media3.common.MimeTypes.VIDEO_H264;
import static androidx.media3.common.MimeTypes.VIDEO_H265;
import static androidx.media3.common.util.Assertions.checkNotNull;
@ -167,7 +168,23 @@ public final class AndroidTestUtil {
.setCodecs("hvc1.2.4.L153")
.build();
// This file needs alternative MIME type, meaning the decoder needs to be configured with
// video/hevc instead of video/dolby-vision.
public static final String MP4_ASSET_DOLBY_VISION_HDR = "asset:///media/mp4/dolbyVision-hdr.MOV";
public static final Format MP4_ASSET_DOLBY_VISION_HDR_FORMAT =
new Format.Builder()
.setSampleMimeType(VIDEO_DOLBY_VISION)
.setWidth(1280)
.setHeight(720)
.setFrameRate(30.00f)
.setCodecs("hev1.08.02")
.setColorInfo(
new ColorInfo.Builder()
.setColorTransfer(C.COLOR_TRANSFER_HLG)
.setColorRange(C.COLOR_RANGE_LIMITED)
.setColorSpace(C.COLOR_SPACE_BT2020)
.build())
.build();
public static final String MP4_ASSET_4K60_PORTRAIT_URI_STRING =
"asset:///media/mp4/portrait_4k60.mp4";
@ -707,7 +724,7 @@ public final class AndroidTestUtil {
*/
public static boolean skipAndLogIfFormatsUnsupported(
Context context, String testId, Format inputFormat, @Nullable Format outputFormat)
throws IOException, JSONException {
throws IOException, JSONException, MediaCodecUtil.DecoderQueryException {
// TODO(b/278657595): Make this capability check match the default codec factory selection code.
boolean canDecode = canDecode(inputFormat);
@ -809,7 +826,7 @@ public final class AndroidTestUtil {
}
}
private static boolean canDecode(Format format) {
private static boolean canDecode(Format format) throws MediaCodecUtil.DecoderQueryException {
// Check decoding capability in the same way as the default decoder factory.
MediaFormat mediaFormat = MediaFormatUtil.createMediaFormatFromFormat(format);
@Nullable
@ -818,7 +835,7 @@ public final class AndroidTestUtil {
MediaFormatUtil.maybeSetInteger(
mediaFormat, MediaFormat.KEY_PROFILE, codecProfileAndLevel.first);
}
return EncoderUtil.findCodecForFormat(mediaFormat, /* isDecoder= */ true) != null;
return DefaultDecoderFactory.getDecoderInfo(format) != null;
}
private static boolean canEncode(Format format) {

109
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlTest.java
View File

@ -19,17 +19,20 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECO
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_720P_4_SECOND_HDR10;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_720P_4_SECOND_HDR10_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_DOLBY_VISION_HDR;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_DOLBY_VISION_HDR_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped;
import static androidx.media3.transformer.mh.FileUtil.maybeAssertFileHasColorTransfer;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import android.content.Context;
import android.net.Uri;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MediaItem;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.Util;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
import androidx.media3.transformer.AndroidTestUtil;
import androidx.media3.transformer.ExportException;
import androidx.media3.transformer.ExportTestResult;
@ -38,6 +41,8 @@ import androidx.media3.transformer.Transformer;
import androidx.media3.transformer.TransformerAndroidTestRunner;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import java.io.IOException;
import org.json.JSONException;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -49,85 +54,42 @@ import org.junit.runner.RunWith;
public class ToneMapHdrToSdrUsingOpenGlTest {
public static final String TAG = "ToneMapHdrToSdrUsingOpenGlTest";
private final Context context = ApplicationProvider.getApplicationContext();
@Test
public void export_toneMap_hlg10File_toneMapsOrThrows() throws Exception {
String testId = "export_glToneMap_hlg10File_toneMapsOrThrows";
if (Util.SDK_INT < 29) {
recordTestSkipped(
ApplicationProvider.getApplicationContext(),
testId,
/* reason= */ "OpenGL-based HDR to SDR tone mapping is only supported on API 29+.");
if (!deviceSupportsOpenGlToneMapping(
testId, /* inputFormat= */ MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT)) {
return;
}
if (!GlUtil.isYuvTargetExtensionSupported()) {
recordTestSkipped(
getApplicationContext(), testId, /* reason= */ "Device lacks YUV extension support.");
return;
}
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
getApplicationContext(),
testId,
/* inputFormat= */ MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT,
/* outputFormat= */ null)) {
return;
}
Context context = ApplicationProvider.getApplicationContext();
Transformer transformer =
new Transformer.Builder(context)
.setTransformationRequest(
new TransformationRequest.Builder()
.setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL)
.build())
.build();
MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET_1080P_5_SECOND_HLG10));
try {
ExportTestResult exportTestResult =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, mediaItem);
Log.i(TAG, "Tone mapped.");
maybeAssertFileHasColorTransfer(exportTestResult.filePath, C.COLOR_TRANSFER_SDR);
} catch (ExportException exception) {
Log.e(TAG, "Error during export.", exception);
if (exception.errorCode != ExportException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED) {
throw exception;
}
}
runTransformerWithOpenGlToneMapping(testId, MP4_ASSET_1080P_5_SECOND_HLG10);
}
@Test
public void export_toneMap_hdr10File_toneMapsOrThrows() throws Exception {
String testId = "export_glToneMap_hdr10File_toneMapsOrThrows";
if (Util.SDK_INT < 29) {
recordTestSkipped(
ApplicationProvider.getApplicationContext(),
testId,
/* reason= */ "OpenGL-based HDR to SDR tone mapping is only supported on API 29+.");
if (!deviceSupportsOpenGlToneMapping(
testId, /* inputFormat= */ MP4_ASSET_720P_4_SECOND_HDR10_FORMAT)) {
return;
}
if (!GlUtil.isYuvTargetExtensionSupported()) {
recordTestSkipped(
getApplicationContext(), testId, /* reason= */ "Device lacks YUV extension support.");
runTransformerWithOpenGlToneMapping(testId, MP4_ASSET_720P_4_SECOND_HDR10);
}
@Test
public void export_toneMap_dolbyVisionFile_toneMapsOrThrows() throws Exception {
String testId = "export_toneMap_dolbyVisionFile_toneMapsOrThrows";
if (!deviceSupportsOpenGlToneMapping(
testId, /* inputFormat= */ MP4_ASSET_DOLBY_VISION_HDR_FORMAT)) {
return;
}
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
getApplicationContext(),
testId,
/* inputFormat= */ MP4_ASSET_720P_4_SECOND_HDR10_FORMAT,
/* outputFormat= */ null)) {
return;
}
Context context = ApplicationProvider.getApplicationContext();
runTransformerWithOpenGlToneMapping(testId, MP4_ASSET_DOLBY_VISION_HDR);
}
private void runTransformerWithOpenGlToneMapping(String testId, String fileUri) throws Exception {
Transformer transformer =
new Transformer.Builder(context)
.setTransformationRequest(
@ -135,12 +97,11 @@ public class ToneMapHdrToSdrUsingOpenGlTest {
.setHdrMode(TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL)
.build())
.build();
MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET_720P_4_SECOND_HDR10));
try {
ExportTestResult exportTestResult =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, mediaItem);
.run(testId, MediaItem.fromUri(fileUri));
Log.i(TAG, "Tone mapped.");
maybeAssertFileHasColorTransfer(exportTestResult.filePath, C.COLOR_TRANSFER_SDR);
} catch (ExportException exception) {
@ -150,4 +111,24 @@ public class ToneMapHdrToSdrUsingOpenGlTest {
}
}
}
private static boolean deviceSupportsOpenGlToneMapping(String testId, Format inputFormat)
throws JSONException, IOException, MediaCodecUtil.DecoderQueryException {
if (Util.SDK_INT < 29) {
recordTestSkipped(
ApplicationProvider.getApplicationContext(),
testId,
/* reason= */ "OpenGL-based HDR to SDR tone mapping is only supported on API 29+.");
return false;
}
if (!GlUtil.isYuvTargetExtensionSupported()) {
recordTestSkipped(
getApplicationContext(), testId, /* reason= */ "Device lacks YUV extension support.");
return false;
}
return !AndroidTestUtil.skipAndLogIfFormatsUnsupported(
getApplicationContext(), testId, /* inputFormat= */ inputFormat, /* outputFormat= */ null);
}
}

32
libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java
View File

@ -27,6 +27,7 @@ import android.os.Build;
import android.util.Pair;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
@ -64,7 +65,15 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
String mediaCodecName;
try {
mediaCodecName = getMediaCodecNameForDecoding(format);
@Nullable MediaCodecInfo decoderInfo = getDecoderInfo(format);
if (decoderInfo == null) {
throw createExportException(format, /* reason= */ "No decoders for format");
}
mediaCodecName = decoderInfo.name;
String codecMimeType = decoderInfo.codecMimeType;
// Does not alter format.sampleMimeType to keep the original MimeType.
// The MIME type of the selected decoder may differ from Format.sampleMimeType.
mediaFormat.setString(MediaFormat.KEY_MIME, codecMimeType);
} catch (MediaCodecUtil.DecoderQueryException e) {
Log.e(TAG, "Error querying decoders", e);
throw createExportException(format, /* reason= */ "Querying codecs failed.");
@ -113,7 +122,16 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
String mediaCodecName;
try {
mediaCodecName = getMediaCodecNameForDecoding(format);
@Nullable MediaCodecInfo decoderInfo = getDecoderInfo(format);
if (decoderInfo == null) {
throw createExportException(format, /* reason= */ "No decoders for format");
}
mediaCodecName = decoderInfo.name;
String codecMimeType = decoderInfo.codecMimeType;
// Does not alter format.sampleMimeType to keep the original MimeType.
// The MIME type of the selected decoder may differ from Format.sampleMimeType, for example,
// video/hevc is used instead of video/dolby-vision for some specific DolbyVision videos.
mediaFormat.setString(MediaFormat.KEY_MIME, codecMimeType);
} catch (MediaCodecUtil.DecoderQueryException e) {
Log.e(TAG, "Error querying decoders", e);
throw createExportException(format, /* reason= */ "Querying codecs failed");
@ -158,8 +176,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
format);
}
private static String getMediaCodecNameForDecoding(Format format)
throws MediaCodecUtil.DecoderQueryException, ExportException {
@VisibleForTesting
@Nullable
/* package */ static MediaCodecInfo getDecoderInfo(Format format)
throws MediaCodecUtil.DecoderQueryException {
checkNotNull(format.sampleMimeType);
List<MediaCodecInfo> decoderInfos =
MediaCodecUtil.getDecoderInfosSortedByFormatSupport(
@ -170,8 +190,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/* requiresTunnelingDecoder= */ false),
format);
if (decoderInfos.isEmpty()) {
throw createExportException(format, /* reason= */ "No decoders for format");
return null;
}
return decoderInfos.get(0).name;
return decoderInfos.get(0);
}
}

33
libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java
View File

@ -24,7 +24,6 @@ import android.media.CamcorderProfile;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaCodecList;
import android.media.MediaFormat;
import android.util.Pair;
import android.util.Range;
import android.util.Size;
@ -38,7 +37,6 @@ import androidx.media3.common.C.ColorTransfer;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.MediaFormatUtil;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import com.google.common.base.Ascii;
@ -303,37 +301,6 @@ public final class EncoderUtil {
return maxSupportedLevel;
}
/**
* Finds a {@link MediaCodec} that supports the {@link MediaFormat}, or {@code null} if none is
* found.
*/
@Nullable
public static String findCodecForFormat(MediaFormat format, boolean isDecoder) {
MediaCodecList mediaCodecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
// Format must not include KEY_FRAME_RATE on API21.
// https://developer.android.com/reference/android/media/MediaCodecList#findDecoderForFormat(android.media.MediaFormat)
float frameRate = Format.NO_VALUE;
if (Util.SDK_INT == 21 && format.containsKey(MediaFormat.KEY_FRAME_RATE)) {
try {
frameRate = format.getFloat(MediaFormat.KEY_FRAME_RATE);
} catch (ClassCastException e) {
frameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE);
}
// Clears the frame rate field.
format.setString(MediaFormat.KEY_FRAME_RATE, null);
}
String mediaCodecName =
isDecoder
? mediaCodecList.findDecoderForFormat(format)
: mediaCodecList.findEncoderForFormat(format);
if (Util.SDK_INT == 21) {
MediaFormatUtil.maybeSetInteger(format, MediaFormat.KEY_FRAME_RATE, round(frameRate));
}
return mediaCodecName;
}
/** Returns the range of supported bitrates for the given {@linkplain MimeTypes MIME type}. */
public static Range<Integer> getSupportedBitrateRange(
MediaCodecInfo encoderInfo, String mimeType) {