loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Transformer: Add codec support for Dolby Vision HDR video

Browse Source 
Allow use of H265/H264 codecs for Dolby Vision video.

Also, reflow ExoPlayer code to use this new utility class

PiperOrigin-RevId: 530619388
This commit is contained in:
huangdarwin 2023-05-09 15:42:56 +00:00 committed by Tofunmi Adigun-Hameed
parent 129a6e0cc6
commit 2db2de5993
8 changed files with 166 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java
View File

@ -398,20 +398,8 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
return ImmutableList.of(codecInfo);
}
}
List<MediaCodecInfo> decoderInfos =
mediaCodecSelector.getDecoderInfos(
mimeType, requiresSecureDecoder, /* requiresTunnelingDecoder= */ false);
@Nullable String alternativeMimeType = MediaCodecUtil.getAlternativeCodecMimeType(format);
if (alternativeMimeType == null) {
return ImmutableList.copyOf(decoderInfos);
}
List<MediaCodecInfo> alternativeDecoderInfos =
mediaCodecSelector.getDecoderInfos(
alternativeMimeType, requiresSecureDecoder, /* requiresTunnelingDecoder= */ false);
return ImmutableList.<MediaCodecInfo>builder()
.addAll(decoderInfos)
.addAll(alternativeDecoderInfos)
.build();
return MediaCodecUtil.getDecoderInfosSoftMatch(
mediaCodecSelector, format, requiresSecureDecoder, /* requiresTunnelingDecoder= */ false);
}
@Override

75
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecUtil.java
View File

@ -44,6 +44,7 @@ import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/** A utility class for querying the available codecs. */
@SuppressLint("InlinedApi")
@ -189,6 +190,77 @@ public final class MediaCodecUtil {
return immutableDecoderInfos;
}
/**
* Returns a list of decoders that can decode media in the specified format, in the priority order
* specified by the {@link MediaCodecSelector}.
*
* <p>Since the {@link MediaCodecSelector} only has access to {@link Format#sampleMimeType}, the
* list is not ordered to account for whether each decoder supports the details of the format
* (e.g., taking into account the format's profile, level, resolution and so on). {@link
* #getDecoderInfosSortedByFormatSupport} can be used to further sort the list into an order where
* decoders that fully support the format come first.
*
* <p>This list is more complete than {@link #getDecoderInfos}, as it also considers alternative
* MIME types that are a close match using {@link #getAlternativeCodecMimeType}.
*
* @param mediaCodecSelector The decoder selector.
* @param format The {@link Format} for which a decoder is required.
* @param requiresSecureDecoder Whether a secure decoder is required.
* @param requiresTunnelingDecoder Whether a tunneling decoder is required.
* @return A list of {@link MediaCodecInfo}s corresponding to decoders. May be empty.
* @throws DecoderQueryException Thrown if there was an error querying decoders.
*/
@RequiresNonNull("#2.sampleMimeType")
public static List<MediaCodecInfo> getDecoderInfosSoftMatch(
MediaCodecSelector mediaCodecSelector,
Format format,
boolean requiresSecureDecoder,
boolean requiresTunnelingDecoder)
throws DecoderQueryException {
List<MediaCodecInfo> decoderInfos =
mediaCodecSelector.getDecoderInfos(
format.sampleMimeType, requiresSecureDecoder, requiresTunnelingDecoder);
List<MediaCodecInfo> alternativeDecoderInfos =
getAlternativeDecoderInfos(
mediaCodecSelector, format, requiresSecureDecoder, requiresTunnelingDecoder);
return ImmutableList.<MediaCodecInfo>builder()
.addAll(decoderInfos)
.addAll(alternativeDecoderInfos)
.build();
}
/**
* Returns a list of decoders for {@linkplain #getAlternativeCodecMimeType alternative MIME types}
* that can decode samples of the provided {@link Format}, in the priority order specified by the
* {@link MediaCodecSelector}.
*
* <p>Since the {@link MediaCodecSelector} only has access to {@link Format#sampleMimeType}, the
* list is not ordered to account for whether each decoder supports the details of the format
* (e.g., taking into account the format's profile, level, resolution and so on). {@link
* #getDecoderInfosSortedByFormatSupport} can be used to further sort the list into an order where
* decoders that fully support the format come first.
*
* @param mediaCodecSelector The decoder selector.
* @param format The {@link Format} for which an alternative decoder is required.
* @param requiresSecureDecoder Whether a secure decoder is required.
* @param requiresTunnelingDecoder Whether a tunneling decoder is required.
* @return A list of {@link MediaCodecInfo}s corresponding to alternative decoders. May be empty.
* @throws DecoderQueryException Thrown if there was an error querying decoders.
*/
public static List<MediaCodecInfo> getAlternativeDecoderInfos(
MediaCodecSelector mediaCodecSelector,
Format format,
boolean requiresSecureDecoder,
boolean requiresTunnelingDecoder)
throws DecoderQueryException {
@Nullable String alternativeMimeType = getAlternativeCodecMimeType(format);
if (alternativeMimeType == null) {
return ImmutableList.of();
}
return mediaCodecSelector.getDecoderInfos(
alternativeMimeType, requiresSecureDecoder, requiresTunnelingDecoder);
}
/**
* Returns a copy of the provided decoder list sorted such that decoders with functional format
* support are listed first. The returned list is modifiable for convenience.
@ -282,8 +354,7 @@ public final class MediaCodecUtil {
// be done for profile CodecProfileLevel.DolbyVisionProfileDvheStn and profile
// CodecProfileLevel.DolbyVisionProfileDvheDtb because the first one is not backward
// compatible and the second one is deprecated and is not always backward compatible.
@Nullable
Pair<Integer, Integer> codecProfileAndLevel = MediaCodecUtil.getCodecProfileAndLevel(format);
@Nullable Pair<Integer, Integer> codecProfileAndLevel = getCodecProfileAndLevel(format);
if (codecProfileAndLevel != null) {
int profile = codecProfileAndLevel.first;
if (profile == CodecProfileLevel.DolbyVisionProfileDvheDtr

27
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View File

@ -499,30 +499,21 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
boolean requiresSecureDecoder,
boolean requiresTunnelingDecoder)
throws DecoderQueryException {
@Nullable String mimeType = format.sampleMimeType;
if (mimeType == null) {
if (format.sampleMimeType == null) {
return ImmutableList.of();
}
List<MediaCodecInfo> decoderInfos =
mediaCodecSelector.getDecoderInfos(
mimeType, requiresSecureDecoder, requiresTunnelingDecoder);
@Nullable String alternativeMimeType = MediaCodecUtil.getAlternativeCodecMimeType(format);
if (alternativeMimeType == null) {
return ImmutableList.copyOf(decoderInfos);
}
List<MediaCodecInfo> alternativeDecoderInfos =
mediaCodecSelector.getDecoderInfos(
alternativeMimeType, requiresSecureDecoder, requiresTunnelingDecoder);
if (Util.SDK_INT >= 26
&& MimeTypes.VIDEO_DOLBY_VISION.equals(format.sampleMimeType)
&& !alternativeDecoderInfos.isEmpty()
&& !Api26.doesDisplaySupportDolbyVision(context)) {
return ImmutableList.copyOf(alternativeDecoderInfos);
List<MediaCodecInfo> alternativeDecoderInfos =
MediaCodecUtil.getAlternativeDecoderInfos(
mediaCodecSelector, format, requiresSecureDecoder, requiresTunnelingDecoder);
if (!alternativeDecoderInfos.isEmpty()) {
return alternativeDecoderInfos;
}
}
return ImmutableList.<MediaCodecInfo>builder()
.addAll(decoderInfos)
.addAll(alternativeDecoderInfos)
.build();
return MediaCodecUtil.getDecoderInfosSoftMatch(
mediaCodecSelector, format, requiresSecureDecoder, requiresTunnelingDecoder);
}
@RequiresApi(26)

BIN
libraries/test_data/src/test/assets/media/mp4/dolbyVision-hdr.MOV Normal file
View File

Binary file not shown.

2
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View File

@ -148,6 +148,8 @@ public final class AndroidTestUtil {
.setCodecs("hvc1.2.4.L153")
.build();
public static final String MP4_ASSET_DOLBY_VISION_HDR = "asset:///media/mp4/dolbyVision-hdr.MOV";
public static final String MP4_ASSET_4K60_PORTRAIT_URI_STRING =
"asset:///media/mp4/portrait_4k60.mp4";
public static final Format MP4_ASSET_4K60_PORTRAIT_FORMAT =

28
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java
View File

@ -19,6 +19,7 @@ import static androidx.media3.common.MimeTypes.VIDEO_H265;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_720P_4_SECOND_HDR10;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_DOLBY_VISION_HDR;
import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped;
import static androidx.media3.transformer.mh.FileUtil.maybeAssertFileHasColorTransfer;
import static com.google.common.truth.Truth.assertThat;
@ -166,6 +167,33 @@ public class HdrEditingTest {
maybeAssertFileHasColorTransfer(exportTestResult.filePath, C.COLOR_TRANSFER_HLG);
}
@Test
public void exportAndTranscode_dolbyVisionFile_whenHdrEditingIsSupported_exports()
throws Exception {
String testId = "exportAndTranscode_dolbyVisionFile_whenHdrEditingIsSupported_exports";
Context context = ApplicationProvider.getApplicationContext();
// This dolby vision file has a ColorInfo identical to HLG10.
if (!deviceSupportsHdrEditing(VIDEO_H265, HLG10_DEFAULT_COLOR_INFO)) {
recordTestSkipped(context, testId, /* reason= */ "Device lacks HLG10 editing support.");
return;
}
Transformer transformer = new Transformer.Builder(context).build();
MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET_DOLBY_VISION_HDR));
ImmutableList<Effect> videoEffects =
ImmutableList.of(
new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build());
Effects effects = new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects);
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem).setEffects(effects).build();
ExportTestResult exportTestResult =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
maybeAssertFileHasColorTransfer(exportTestResult.filePath, C.COLOR_TRANSFER_HLG);
}
@Test
public void exportAndTranscode_hdr10File_whenHdrEditingUnsupported_toneMapsOrThrows()
throws Exception {

9
libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java
View File

@ -34,6 +34,7 @@ import androidx.media3.common.util.Log;
import androidx.media3.common.util.MediaFormatUtil;
import androidx.media3.common.util.Util;
import androidx.media3.exoplayer.mediacodec.MediaCodecInfo;
import androidx.media3.exoplayer.mediacodec.MediaCodecSelector;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
import java.util.List;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@ -145,10 +146,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private static String getMediaCodecNameForDecoding(Format format)
throws MediaCodecUtil.DecoderQueryException, ExportException {
checkNotNull(format.sampleMimeType);
List<MediaCodecInfo> decoderInfos =
MediaCodecUtil.getDecoderInfosSortedByFormatSupport(
MediaCodecUtil.getDecoderInfos(
checkNotNull(format.sampleMimeType), /* secure= */ false, /* tunneling= */ false),
MediaCodecUtil.getDecoderInfosSoftMatch(
MediaCodecSelector.DEFAULT,
format,
/* requiresSecureDecoder= */ false,
/* requiresTunnelingDecoder= */ false),
format);
if (decoderInfos.isEmpty()) {
throw createExportException(format, /* reason= */ "No decoders for format");

72
libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java
View File

@ -31,6 +31,8 @@ import static androidx.media3.transformer.TransformationRequest.HDR_MODE_TONE_MA
import android.content.Context;
import android.graphics.Bitmap;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.util.Pair;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
@ -51,6 +53,7 @@ import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer;
import androidx.media3.effect.DebugTraceUtil;
import androidx.media3.effect.Presentation;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.MoreExecutors;
import java.nio.ByteBuffer;
@ -350,8 +353,16 @@ import org.checkerframework.dataflow.qual.Pure;
this.muxerSupportedMimeTypes = muxerSupportedMimeTypes;
this.transformationRequest = transformationRequest;
this.fallbackListener = fallbackListener;
String inputSampleMimeType = checkNotNull(inputFormat.sampleMimeType);
Pair<String, Integer> outputMimeTypeAndHdrModeAfterFallback =
getRequestedOutputMimeTypeAndHdrModeAfterFallback(inputFormat, transformationRequest);
requestedOutputMimeType = outputMimeTypeAndHdrModeAfterFallback.first;
hdrModeAfterFallback = outputMimeTypeAndHdrModeAfterFallback.second;
}
private static Pair<String, Integer> getRequestedOutputMimeTypeAndHdrModeAfterFallback(
Format inputFormat, TransformationRequest transformationRequest) {
String inputSampleMimeType = checkNotNull(inputFormat.sampleMimeType);
String requestedOutputMimeType;
if (transformationRequest.videoMimeType != null) {
requestedOutputMimeType = transformationRequest.videoMimeType;
} else if (MimeTypes.isImage(inputSampleMimeType)) {
@ -362,33 +373,25 @@ import org.checkerframework.dataflow.qual.Pure;
// HdrMode fallback is only supported from HDR_MODE_KEEP_HDR to
// HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC.
boolean fallbackToMediaCodec =
isTransferHdr(inputFormat.colorInfo)
&& transformationRequest.hdrMode == HDR_MODE_KEEP_HDR
&& getSupportedEncodersForHdrEditing(requestedOutputMimeType, inputFormat.colorInfo)
.isEmpty();
hdrModeAfterFallback =
fallbackToMediaCodec
? HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC
: transformationRequest.hdrMode;
}
@TransformationRequest.HdrMode int hdrMode = transformationRequest.hdrMode;
if (hdrMode == HDR_MODE_KEEP_HDR && isTransferHdr(inputFormat.colorInfo)) {
ImmutableList<MediaCodecInfo> hdrEncoders =
getSupportedEncodersForHdrEditing(requestedOutputMimeType, inputFormat.colorInfo);
if (hdrEncoders.isEmpty()) {
@Nullable
String alternativeMimeType = MediaCodecUtil.getAlternativeCodecMimeType(inputFormat);
if (alternativeMimeType != null) {
requestedOutputMimeType = alternativeMimeType;
hdrEncoders =
getSupportedEncodersForHdrEditing(alternativeMimeType, inputFormat.colorInfo);
}
}
if (hdrEncoders.isEmpty()) {
hdrMode = HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC;
}
}
/** Returns the {@link ColorInfo} expected from the input surface. */
public ColorInfo getSupportedInputColor() {
boolean isHdrEditingEnabled =
transformationRequest.hdrMode == HDR_MODE_KEEP_HDR
&& !getSupportedEncodersForHdrEditing(requestedOutputMimeType, inputFormat.colorInfo)
.isEmpty();
boolean isInputToneMapped = !isHdrEditingEnabled && isTransferHdr(inputFormat.colorInfo);
if (isInputToneMapped) {
// When tone-mapping HDR to SDR is enabled, assume we get BT.709 to avoid having the encoder
// populate default color info, which depends on the resolution.
return ColorInfo.SDR_BT709_LIMITED;
}
if (SRGB_BT709_FULL.equals(inputFormat.colorInfo)) {
return ColorInfo.SDR_BT709_LIMITED;
}
return checkNotNull(inputFormat.colorInfo);
return Pair.create(requestedOutputMimeType, hdrMode);
}
public @TransformationRequest.HdrMode int getHdrModeAfterFallback() {
@ -458,6 +461,21 @@ import org.checkerframework.dataflow.qual.Pure;
return encoderSurfaceInfo;
}
/** Returns the {@link ColorInfo} expected from the input surface. */
private ColorInfo getSupportedInputColor() {
boolean isInputToneMapped =
isTransferHdr(inputFormat.colorInfo) && hdrModeAfterFallback != HDR_MODE_KEEP_HDR;
if (isInputToneMapped) {
// When tone-mapping HDR to SDR is enabled, assume we get BT.709 to avoid having the encoder
// populate default color info, which depends on the resolution.
return ColorInfo.SDR_BT709_LIMITED;
}
if (SRGB_BT709_FULL.equals(inputFormat.colorInfo)) {
return ColorInfo.SDR_BT709_LIMITED;
}
return checkNotNull(inputFormat.colorInfo);
}
/**
* Creates a {@link TransformationRequest}, based on an original {@code TransformationRequest}
* and parameters specifying alterations to it that indicate device support.