From d2a9419ad377fcbb35ddfd49dc94be58b041fbea Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Mon, 28 Mar 2022 16:29:56 +0100 Subject: [PATCH] Add support for requesting color transfer to SDR From Android T onwards `MediaCodec` supports requesting tone-mapping down to SDR. Add an option to request this behavior and document that it isn't supported before T. Also add an option in the demo app to try it out. Tested manually on a prerelease build. PiperOrigin-RevId: 437765325 --- .../transformer/ConfigurationActivity.java | 11 +++++++ .../demo/transformer/TransformerActivity.java | 2 ++ .../res/layout/configuration_activity.xml | 10 +++++++ .../src/main/res/values/strings.xml | 3 +- .../androidx/media3/transformer/Codec.java | 4 ++- .../transformer/DefaultDecoderFactory.java | 7 ++++- .../transformer/TransformationRequest.java | 30 ++++++++++++++++++- .../transformer/TransformerVideoRenderer.java | 3 ++ .../VideoTranscodingSamplePipeline.java | 5 +++- 9 files changed, 70 insertions(+), 5 deletions(-) diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java index a24ab9e46e..bc0219f6f5 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java @@ -54,6 +54,7 @@ public final class ConfigurationActivity extends AppCompatActivity { public static final String SCALE_Y = "scale_y"; public static final String ROTATE_DEGREES = "rotate_degrees"; public static final String ENABLE_FALLBACK = "enable_fallback"; + public static final String ENABLE_REQUEST_SDR_TONE_MAPPING = "enable_request_sdr_tone_mapping"; public static final String ENABLE_HDR_EDITING = "enable_hdr_editing"; private static final String[] INPUT_URIS = { "https://html5demos.com/assets/dizzy.mp4", @@ -84,6 +85,7 @@ public final class ConfigurationActivity extends AppCompatActivity { private @MonotonicNonNull Spinner scaleSpinner; private @MonotonicNonNull Spinner rotateSpinner; private @MonotonicNonNull CheckBox enableFallbackCheckBox; + private @MonotonicNonNull CheckBox enableRequestSdrToneMappingCheckBox; private @MonotonicNonNull CheckBox enableHdrEditingCheckBox; private int inputUriPosition; @@ -150,6 +152,7 @@ public final class ConfigurationActivity extends AppCompatActivity { rotateAdapter.addAll(SAME_AS_INPUT_OPTION, "0", "10", "45", "60", "90", "180"); enableFallbackCheckBox = findViewById(R.id.enable_fallback_checkbox); + enableRequestSdrToneMappingCheckBox = findViewById(R.id.request_sdr_tone_mapping_checkbox); enableHdrEditingCheckBox = findViewById(R.id.hdr_editing_checkbox); } @@ -179,6 +182,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "scaleSpinner", "rotateSpinner", "enableFallbackCheckBox", + "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox" }) private void startTransformation(View view) { @@ -211,6 +215,8 @@ public final class ConfigurationActivity extends AppCompatActivity { bundle.putFloat(ROTATE_DEGREES, Float.parseFloat(selectedRotate)); } bundle.putBoolean(ENABLE_FALLBACK, enableFallbackCheckBox.isChecked()); + bundle.putBoolean( + ENABLE_REQUEST_SDR_TONE_MAPPING, enableRequestSdrToneMappingCheckBox.isChecked()); bundle.putBoolean(ENABLE_HDR_EDITING, enableHdrEditingCheckBox.isChecked()); transformerIntent.putExtras(bundle); @@ -243,6 +249,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "resolutionHeightSpinner", "scaleSpinner", "rotateSpinner", + "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox" }) private void onRemoveAudio(View view) { @@ -261,6 +268,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "resolutionHeightSpinner", "scaleSpinner", "rotateSpinner", + "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox" }) private void onRemoveVideo(View view) { @@ -278,6 +286,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "resolutionHeightSpinner", "scaleSpinner", "rotateSpinner", + "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox" }) private void enableTrackSpecificOptions(boolean isAudioEnabled, boolean isVideoEnabled) { @@ -286,6 +295,7 @@ public final class ConfigurationActivity extends AppCompatActivity { resolutionHeightSpinner.setEnabled(isVideoEnabled); scaleSpinner.setEnabled(isVideoEnabled); rotateSpinner.setEnabled(isVideoEnabled); + enableRequestSdrToneMappingCheckBox.setEnabled(isVideoEnabled); enableHdrEditingCheckBox.setEnabled(isVideoEnabled); findViewById(R.id.audio_mime_text_view).setEnabled(isAudioEnabled); @@ -293,6 +303,7 @@ public final class ConfigurationActivity extends AppCompatActivity { findViewById(R.id.resolution_height_text_view).setEnabled(isVideoEnabled); findViewById(R.id.scale).setEnabled(isVideoEnabled); findViewById(R.id.rotate).setEnabled(isVideoEnabled); + findViewById(R.id.request_sdr_tone_mapping).setEnabled(isVideoEnabled); findViewById(R.id.hdr_editing).setEnabled(isVideoEnabled); } } diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index ad83ce75f0..3a957cba3e 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -225,6 +225,8 @@ public final class TransformerActivity extends AppCompatActivity { bundle.getFloat(ConfigurationActivity.ROTATE_DEGREES, /* defaultValue= */ 0); requestBuilder.setRotationDegrees(rotateDegrees); + requestBuilder.setEnableRequestSdrToneMapping( + bundle.getBoolean(ConfigurationActivity.ENABLE_REQUEST_SDR_TONE_MAPPING)); requestBuilder.experimental_setEnableHdrEditing( bundle.getBoolean(ConfigurationActivity.ENABLE_HDR_EDITING)); transformerBuilder diff --git a/demos/transformer/src/main/res/layout/configuration_activity.xml b/demos/transformer/src/main/res/layout/configuration_activity.xml index 1ff3cafc6b..c973bf4137 100644 --- a/demos/transformer/src/main/res/layout/configuration_activity.xml +++ b/demos/transformer/src/main/res/layout/configuration_activity.xml @@ -169,6 +169,16 @@ android:layout_gravity="right" android:checked="true"/> + + + + diff --git a/demos/transformer/src/main/res/values/strings.xml b/demos/transformer/src/main/res/values/strings.xml index 8e8f97ecf9..8ab9fe25f2 100644 --- a/demos/transformer/src/main/res/values/strings.xml +++ b/demos/transformer/src/main/res/values/strings.xml @@ -27,8 +27,9 @@ Scale video Rotate video (degrees) Enable fallback - Transform + Request SDR tone-mapping [Experimental] HDR editing + Transform Debug preview: No debug preview available. Transformation started diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java index 104a48ccbd..bf04a863cf 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java @@ -58,10 +58,12 @@ public interface Codec { * @param format The {@link Format} (of the input data) used to determine the underlying decoder * and its configuration values. * @param outputSurface The {@link Surface} to which the decoder output is rendered. + * @param enableRequestSdrToneMapping Whether to request tone-mapping to SDR. * @return A {@link Codec} for video decoding. * @throws TransformationException If no suitable {@link Codec} can be created. */ - Codec createForVideoDecoding(Format format, Surface outputSurface) + Codec createForVideoDecoding( + Format format, Surface outputSurface, boolean enableRequestSdrToneMapping) throws TransformationException; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java index a044351682..7b525416a3 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java @@ -49,7 +49,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } @Override - public Codec createForVideoDecoding(Format format, Surface outputSurface) + public Codec createForVideoDecoding( + Format format, Surface outputSurface, boolean enableRequestSdrToneMapping) throws TransformationException { MediaFormat mediaFormat = MediaFormat.createVideoFormat( @@ -63,6 +64,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; // cycle. This key ensures no frame dropping when the decoder's output surface is full. mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0); } + if (SDK_INT >= 31 && enableRequestSdrToneMapping) { + mediaFormat.setInteger( + MediaFormat.KEY_COLOR_TRANSFER_REQUEST, MediaFormat.COLOR_TRANSFER_SDR_VIDEO); + } @Nullable String mediaCodecName = EncoderUtil.findCodecForFormat(mediaFormat, /* isDecoder= */ true); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java index 7658dc61d8..a0ca504c08 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java @@ -40,6 +40,7 @@ public final class TransformationRequest { private int outputHeight; @Nullable private String audioMimeType; @Nullable private String videoMimeType; + private boolean enableRequestSdrToneMapping; private boolean enableHdrEditing; /** @@ -62,6 +63,7 @@ public final class TransformationRequest { this.outputHeight = transformationRequest.outputHeight; this.audioMimeType = transformationRequest.audioMimeType; this.videoMimeType = transformationRequest.videoMimeType; + this.enableRequestSdrToneMapping = transformationRequest.enableRequestSdrToneMapping; this.enableHdrEditing = transformationRequest.enableHdrEditing; } @@ -194,13 +196,30 @@ public final class TransformationRequest { return this; } + /** + * Sets whether to request tone-mapping to standard dynamic range (SDR). If enabled and + * supported, high dynamic range (HDR) input will be tone-mapped into an SDR opto-electrical + * transfer function before processing. + * + *

The setting has no effect if the input is already in SDR, or if tone-mapping is not + * supported. Currently tone-mapping is only guaranteed to be supported from Android T onwards. + * + * @param enableRequestSdrToneMapping Whether to request tone-mapping down to SDR. + * @return This builder. + */ + public Builder setEnableRequestSdrToneMapping(boolean enableRequestSdrToneMapping) { + this.enableRequestSdrToneMapping = enableRequestSdrToneMapping; + return this; + } + /** * Sets whether to attempt to process any input video stream as a high dynamic range (HDR) * signal. * *

This method is experimental, and will be renamed or removed in a future release. The HDR * editing feature is under development and is intended for developing/testing HDR processing - * and encoding support. + * and encoding support. HDR editing can't be enabled at the same time as {@link + * #setEnableRequestSdrToneMapping(boolean) SDR tone-mapping}. * * @param enableHdrEditing Whether to attempt to process any input video stream as a high * dynamic range (HDR) signal. @@ -221,6 +240,7 @@ public final class TransformationRequest { outputHeight, audioMimeType, videoMimeType, + enableRequestSdrToneMapping, enableHdrEditing); } } @@ -271,6 +291,9 @@ public final class TransformationRequest { * @see Builder#setVideoMimeType(String) */ @Nullable public final String videoMimeType; + /** Whether to request tone-mapping to standard dynamic range (SDR). */ + public final boolean enableRequestSdrToneMapping; + /** * Whether to attempt to process any input video stream as a high dynamic range (HDR) signal. * @@ -286,7 +309,9 @@ public final class TransformationRequest { int outputHeight, @Nullable String audioMimeType, @Nullable String videoMimeType, + boolean enableRequestSdrToneMapping, boolean enableHdrEditing) { + checkArgument(!enableHdrEditing || !enableRequestSdrToneMapping); this.flattenForSlowMotion = flattenForSlowMotion; this.scaleX = scaleX; this.scaleY = scaleY; @@ -294,6 +319,7 @@ public final class TransformationRequest { this.outputHeight = outputHeight; this.audioMimeType = audioMimeType; this.videoMimeType = videoMimeType; + this.enableRequestSdrToneMapping = enableRequestSdrToneMapping; this.enableHdrEditing = enableHdrEditing; } @@ -313,6 +339,7 @@ public final class TransformationRequest { && outputHeight == that.outputHeight && Util.areEqual(audioMimeType, that.audioMimeType) && Util.areEqual(videoMimeType, that.videoMimeType) + && enableRequestSdrToneMapping == that.enableRequestSdrToneMapping && enableHdrEditing == that.enableHdrEditing; } @@ -325,6 +352,7 @@ public final class TransformationRequest { result = 31 * result + outputHeight; result = 31 * result + (audioMimeType != null ? audioMimeType.hashCode() : 0); result = 31 * result + (videoMimeType != null ? videoMimeType.hashCode() : 0); + result = 31 * result + (enableRequestSdrToneMapping ? 1 : 0); result = 31 * result + (enableHdrEditing ? 1 : 0); return result; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java index b3042a922b..d8789fc382 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java @@ -107,6 +107,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; if (encoderFactory.videoNeedsEncoding()) { return false; } + if (transformationRequest.enableRequestSdrToneMapping) { + return false; + } if (transformationRequest.enableHdrEditing) { return false; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 5b4da873c7..068495d5e2 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -124,7 +124,10 @@ import org.checkerframework.dataflow.qual.Pure; encoderSupportedFormat.width, encoderSupportedFormat.height)); decoder = - decoderFactory.createForVideoDecoding(inputFormat, frameProcessorChain.getInputSurface()); + decoderFactory.createForVideoDecoding( + inputFormat, + frameProcessorChain.getInputSurface(), + transformationRequest.enableRequestSdrToneMapping); maxPendingFrameCount = getMaxPendingFrameCount(); }