loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fallback to SDR if encoder doesn't support HDR (HLG only).

Browse Source 
If the input is HDR (HLG), check encoder capabilities for HDR support
and request tone-mapping to SDR during decoder configuration otherwise.
Capabilities are only checked for API 31 and above, as HDR editing is
not supported before.

As the encoder capabilities check needs to happen before selecting the
encoder to use (as this may depend on the resolution output by the
effects chain), the EncoderWrapper checks all candidate encoders
for the MIME type for HDR capabilities and only requests fallback to
SDR if none of them support it.

When the actual encoder is selected, the wrapper checks that it matches
one of the encoders is checked capabilities for.

PiperOrigin-RevId: 458511599
This commit is contained in:
hschlueter 2022-07-01 18:34:02 +00:00 committed by Marc Baechinger
parent a0870a42be
commit 9c8dcb402b
3 changed files with 102 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
libraries/common/src/main/java/androidx/media3/common/ColorInfo.java
View File

@ -31,6 +31,13 @@ import org.checkerframework.dataflow.qual.Pure;
/** Stores color info. */
@UnstableApi
public final class ColorInfo implements Bundleable {
/** Standard Dynamic Range (SDR). */
public static final ColorInfo SDR =
new ColorInfo(
C.COLOR_SPACE_BT709,
C.COLOR_RANGE_LIMITED,
C.COLOR_TRANSFER_SDR,
/* hdrStaticInfo= */ null);
/**
* Returns the {@link C.ColorSpace} corresponding to the given ISO color primary code, as per

9
libraries/transformer/src/main/java/androidx/media3/transformer/FallbackListener.java
View File

@ -91,6 +91,15 @@ import androidx.media3.common.util.Util;
if (transformationRequest.outputHeight != originalTransformationRequest.outputHeight) {
fallbackRequestBuilder.setResolution(transformationRequest.outputHeight);
}
if (transformationRequest.enableHdrEditing != originalTransformationRequest.enableHdrEditing) {
fallbackRequestBuilder.experimental_setEnableHdrEditing(
transformationRequest.enableHdrEditing);
}
if (transformationRequest.enableRequestSdrToneMapping
!= originalTransformationRequest.enableRequestSdrToneMapping) {
fallbackRequestBuilder.setEnableRequestSdrToneMapping(
transformationRequest.enableRequestSdrToneMapping);
}
fallbackTransformationRequest = fallbackRequestBuilder.build();
if (trackCount == 0 && !originalTransformationRequest.equals(fallbackTransformationRequest)) {

93
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -17,20 +17,25 @@
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import android.content.Context;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer;
import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.dataflow.qual.Pure;
@ -39,6 +44,8 @@ import org.checkerframework.dataflow.qual.Pure;
* Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them.
*/
/* package */ final class VideoTranscodingSamplePipeline implements SamplePipeline {
private static final String TAG = "VideoTranscodingSP";
private final int maxPendingFrameCount;
private final DecoderInputBuffer decoderInputBuffer;
@ -98,9 +105,17 @@ import org.checkerframework.dataflow.qual.Pure;
transformationRequest,
fallbackListener);
boolean enableRequestSdrToneMapping = transformationRequest.enableRequestSdrToneMapping;
// TODO(b/237674316): While HLG10 is correctly reported, HDR10 currently will be incorrectly
// processed as SDR, because the inputFormat.colorInfo reports the wrong value.
boolean useHdr = transformationRequest.enableHdrEditing && isHdr(inputFormat.colorInfo);
if (useHdr && !encoderWrapper.supportsHdr()) {
// TODO(b/236316454): Also check whether GlEffectsFrameProcessor supports HDR, i.e., whether
// EXT_YUV_target is supported.
useHdr = false;
enableRequestSdrToneMapping = true;
encoderWrapper.signalFallbackToSdr();
}
try {
frameProcessor =
@ -146,9 +161,9 @@ import org.checkerframework.dataflow.qual.Pure;
decoder =
decoderFactory.createForVideoDecoding(
inputFormat,
frameProcessor.getInputSurface(),
transformationRequest.enableRequestSdrToneMapping);
inputFormat, frameProcessor.getInputSurface(), enableRequestSdrToneMapping);
// TODO(b/236316454): Check in the decoder output format whether tone-mapping was actually
// applied and throw an exception if not.
maxPendingFrameCount = decoder.getMaxPendingFrameCount();
}
@ -232,23 +247,33 @@ import org.checkerframework.dataflow.qual.Pure;
* processing, with {@link Format#rotationDegrees} of 90 added to the output format.
* @param requestedFormat The requested format.
* @param supportedFormat A format supported by the device.
* @param fallbackToSdr Whether HDR editing was requested via the TransformationRequest or
* inferred from the input and tone-mapping to SDR was used instead due to lack of encoder
* capabilities.
*/
@Pure
private static TransformationRequest createFallbackTransformationRequest(
TransformationRequest transformationRequest,
boolean hasOutputFormatRotation,
Format requestedFormat,
Format supportedFormat) {
Format supportedFormat,
boolean fallbackToSdr) {
// TODO(b/210591626): Also update bitrate etc. once encoder configuration and fallback are
// implemented.
if (Util.areEqual(requestedFormat.sampleMimeType, supportedFormat.sampleMimeType)
if (!fallbackToSdr
&& Util.areEqual(requestedFormat.sampleMimeType, supportedFormat.sampleMimeType)
&& (hasOutputFormatRotation
? requestedFormat.width == supportedFormat.width
: requestedFormat.height == supportedFormat.height)) {
return transformationRequest;
}
return transformationRequest
.buildUpon()
TransformationRequest.Builder transformationRequestBuilder = transformationRequest.buildUpon();
if (fallbackToSdr) {
transformationRequestBuilder
.setEnableRequestSdrToneMapping(true)
.experimental_setEnableHdrEditing(false);
}
return transformationRequestBuilder
.setVideoMimeType(supportedFormat.sampleMimeType)
.setResolution(hasOutputFormatRotation ? requestedFormat.width : requestedFormat.height)
.build();
@ -309,12 +334,14 @@ import org.checkerframework.dataflow.qual.Pure;
private final List<String> allowedOutputMimeTypes;
private final TransformationRequest transformationRequest;
private final FallbackListener fallbackListener;
private final HashSet<String> hdrMediaCodecNames;
private @MonotonicNonNull SurfaceInfo encoderSurfaceInfo;
private volatile @MonotonicNonNull Codec encoder;
private volatile int outputRotationDegrees;
private volatile boolean releaseEncoder;
private boolean fallbackToSdr;
public EncoderWrapper(
Codec.EncoderFactory encoderFactory,
@ -328,6 +355,8 @@ import org.checkerframework.dataflow.qual.Pure;
this.allowedOutputMimeTypes = allowedOutputMimeTypes;
this.transformationRequest = transformationRequest;
this.fallbackListener = fallbackListener;
hdrMediaCodecNames = new HashSet<>();
}
@Nullable
@ -362,18 +391,27 @@ import org.checkerframework.dataflow.qual.Pure;
transformationRequest.videoMimeType != null
? transformationRequest.videoMimeType
: inputFormat.sampleMimeType)
.setColorInfo(inputFormat.colorInfo)
.setColorInfo(fallbackToSdr ? ColorInfo.SDR : inputFormat.colorInfo)
.build();
encoder =
encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes);
if (!hdrMediaCodecNames.isEmpty() && !hdrMediaCodecNames.contains(encoder.getName())) {
Log.d(
TAG,
"Selected encoder "
+ encoder.getName()
+ " does not report sufficient HDR capabilities");
}
Format encoderSupportedFormat = encoder.getConfigurationFormat();
fallbackListener.onTransformationRequestFinalized(
createFallbackTransformationRequest(
transformationRequest,
/* hasOutputFormatRotation= */ flipOrientation,
requestedEncoderFormat,
encoderSupportedFormat));
encoderSupportedFormat,
fallbackToSdr));
encoderSurfaceInfo =
new SurfaceInfo(
@ -432,5 +470,42 @@ import org.checkerframework.dataflow.qual.Pure;
}
releaseEncoder = true;
}
/**
* Checks whether at least one MediaCodec encoder on the device has sufficient capabilities to
* encode HDR (only checks support for HLG at this time).
*/
public boolean supportsHdr() {
if (Util.SDK_INT < 31) {
return false;
}
// The only output MIME type that Transformer currently supports that can be used with HDR
// is H265/HEVC. So we assume that the EncoderFactory will pick this if HDR is requested.
String mimeType = MimeTypes.VIDEO_H265;
List<MediaCodecInfo> mediaCodecInfos = EncoderSelector.DEFAULT.selectEncoderInfos(mimeType);
for (int i = 0; i < mediaCodecInfos.size(); i++) {
MediaCodecInfo mediaCodecInfo = mediaCodecInfos.get(i);
if (EncoderUtil.isFeatureSupported(
mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing)) {
for (MediaCodecInfo.CodecProfileLevel capabilities :
mediaCodecInfo.getCapabilitiesForType(MimeTypes.VIDEO_H265).profileLevels) {
// TODO(b/227624622): What profile to check depends on the HDR format. Once other
// formats besides HLG are supported, check the corresponding profiles here.
if (capabilities.profile == MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10) {
return hdrMediaCodecNames.add(mediaCodecInfo.getCanonicalName());
}
}
}
}
return !hdrMediaCodecNames.isEmpty();
}
public void signalFallbackToSdr() {
checkState(encoder == null, "Fallback to SDR is only allowed before encoder initialization");
fallbackToSdr = true;
hdrMediaCodecNames.clear();
}
}
}