From fde6a321566b286e11107831244179dbfc7f80d7 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Wed, 21 Feb 2024 12:12:18 -0800 Subject: [PATCH] Transformer: Input decoder output color to VideoSampleExporter. Previously, the track format was used in VideoSampleExporter. Now, we use a simulated decoder output format. As the last expected change for this bug, also adds release notes PiperOrigin-RevId: 609080629 --- RELEASENOTES.md | 1 + .../ExoAssetLoaderVideoRenderer.java | 2 +- .../transformer/TextureAssetLoader.java | 3 +- .../transformer/TransformerInternal.java | 13 +++++++- .../media3/transformer/TransformerUtil.java | 12 ++++++++ .../transformer/VideoSampleExporter.java | 30 +++---------------- 6 files changed, 32 insertions(+), 29 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 6491948706..4f3ed97933 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -11,6 +11,7 @@ * Add workaround for exception thrown due to `MediaMuxer` not supporting negative presentation timestamps before API 30. * Relax trim optimization H.264 level checks. + * Add support for changing between SDR and HDR input media in a sequence. * Track Selection: * Extractors: * Audio: diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderVideoRenderer.java index a64921cfe9..15d9edf6b5 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderVideoRenderer.java @@ -83,7 +83,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; protected Format overrideOutputFormat(Format format) { // Gets the expected output color from the decoder, based on the input track format, if // tone-mapping is applied. - ColorInfo validColor = VideoSampleExporter.getValidColor(format.colorInfo); + ColorInfo validColor = TransformerUtil.getValidColor(format.colorInfo); boolean isDecoderToneMappingRequested = hdrMode == Composition.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC; ColorInfo outputColor = getDecoderOutputColor(validColor, isDecoderToneMappingRequested); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TextureAssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TextureAssetLoader.java index bedcb4293d..2b03678b04 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TextureAssetLoader.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TextureAssetLoader.java @@ -22,6 +22,7 @@ import static androidx.media3.transformer.SampleConsumer.INPUT_RESULT_END_OF_STR import static androidx.media3.transformer.SampleConsumer.INPUT_RESULT_TRY_AGAIN_LATER; import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE; import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED; +import static androidx.media3.transformer.TransformerUtil.getValidColor; import static java.lang.Math.round; import androidx.annotation.Nullable; @@ -83,7 +84,7 @@ public final class TextureAssetLoader implements AssetLoader { this.format = format .buildUpon() - .setColorInfo(VideoSampleExporter.getValidColor(format.colorInfo)) + .setColorInfo(getValidColor(format.colorInfo)) .setSampleMimeType(MimeTypes.VIDEO_RAW) .build(); this.frameProcessedListener = frameProcessedListener; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index 63dde87dfc..fbe1ad3722 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -23,11 +23,14 @@ import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Util.contains; import static androidx.media3.transformer.AssetLoader.SUPPORTED_OUTPUT_TYPE_DECODED; import static androidx.media3.transformer.AssetLoader.SUPPORTED_OUTPUT_TYPE_ENCODED; +import static androidx.media3.transformer.Composition.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC; +import static androidx.media3.transformer.ExoAssetLoaderVideoRenderer.getDecoderOutputColor; import static androidx.media3.transformer.ExportException.ERROR_CODE_FAILED_RUNTIME_CHECK; import static androidx.media3.transformer.ExportException.ERROR_CODE_MUXING_FAILED; import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE; import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED; import static androidx.media3.transformer.TransformerUtil.getProcessedTrackType; +import static androidx.media3.transformer.TransformerUtil.getValidColor; import static androidx.media3.transformer.TransformerUtil.maybeSetMuxerWrapperAdditionalRotationDegrees; import static androidx.media3.transformer.TransformerUtil.shouldTranscodeAudio; import static androidx.media3.transformer.TransformerUtil.shouldTranscodeVideo; @@ -45,6 +48,7 @@ import androidx.annotation.IntDef; import androidx.annotation.IntRange; import androidx.annotation.Nullable; import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Format; import androidx.media3.common.MediaItem; @@ -642,11 +646,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; fallbackListener)); } else { // TODO(b/267301878): Pass firstAssetLoaderOutputFormat once surface creation not in VSP. + boolean isMediaCodecToneMappingRequested = + transformationRequest.hdrMode == HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC; + ColorInfo decoderOutputColor = + getDecoderOutputColor( + getValidColor(firstAssetLoaderInputFormat.colorInfo), + isMediaCodecToneMappingRequested); + assetLoaderInputTracker.registerSampleExporter( C.TRACK_TYPE_VIDEO, new VideoSampleExporter( context, - firstAssetLoaderInputFormat, + firstAssetLoaderInputFormat.buildUpon().setColorInfo(decoderOutputColor).build(), transformationRequest, composition.videoCompositorSettings, composition.effects.videoEffects, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java index e06853145d..00eb06e449 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java @@ -22,6 +22,7 @@ import static java.lang.Math.round; import android.media.MediaCodec; import androidx.annotation.Nullable; import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; import androidx.media3.common.Effect; import androidx.media3.common.Format; import androidx.media3.common.Metadata; @@ -208,4 +209,15 @@ import com.google.common.collect.ImmutableList; muxerWrapper.setAdditionalRotationDegrees(360 - round(rotationDegrees)); } } + + /** + * Adjust for invalid {@link ColorInfo} values, by defaulting to {@link + * ColorInfo#SDR_BT709_LIMITED}. + */ + public static ColorInfo getValidColor(@Nullable ColorInfo colorInfo) { + if (colorInfo == null || !colorInfo.isDataSpaceValid()) { + return ColorInfo.SDR_BT709_LIMITED; + } + return colorInfo; + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index 6f177de0a4..9568d177bd 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -22,7 +22,6 @@ import static androidx.media3.common.ColorInfo.isTransferHdr; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.transformer.Composition.HDR_MODE_KEEP_HDR; -import static androidx.media3.transformer.Composition.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC; import static androidx.media3.transformer.Composition.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL; import static androidx.media3.transformer.EncoderUtil.getSupportedEncodersForHdrEditing; @@ -74,18 +73,6 @@ import org.checkerframework.dataflow.qual.Pure; private boolean hasMuxedTimestampZero; - // TODO: b/307952514 - Move this method to a color utility. - /** - * Adjust for invalid {@link ColorInfo} values, by defaulting to {@link - * ColorInfo#SDR_BT709_LIMITED}. - */ - public static ColorInfo getValidColor(@Nullable ColorInfo colorInfo) { - if (colorInfo == null || !colorInfo.isDataSpaceValid()) { - return ColorInfo.SDR_BT709_LIMITED; - } - return colorInfo; - } - public VideoSampleExporter( Context context, Format firstInputFormat, @@ -107,29 +94,20 @@ import org.checkerframework.dataflow.qual.Pure; this.initialTimestampOffsetUs = initialTimestampOffsetUs; finalFramePresentationTimeUs = C.TIME_UNSET; - ColorInfo decoderInputColor = getValidColor(firstInputFormat.colorInfo); encoderWrapper = new EncoderWrapper( encoderFactory, - firstInputFormat.buildUpon().setColorInfo(decoderInputColor).build(), + firstInputFormat, muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_VIDEO), transformationRequest, fallbackListener); encoderOutputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); - @Composition.HdrMode int hdrModeAfterFallback = encoderWrapper.getHdrModeAfterFallback(); - boolean isMediaCodecToneMappingRequested = - hdrModeAfterFallback == HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC; - // TODO: b/278259383 - After solving the bug, we can use the decoder output format, and no - // longer need to import this color conversion method. - ColorInfo videoGraphInputColor = - ExoAssetLoaderVideoRenderer.getDecoderOutputColor( - decoderInputColor, isMediaCodecToneMappingRequested); - + ColorInfo videoGraphInputColor = checkNotNull(firstInputFormat.colorInfo); boolean isGlToneMapping = - hdrModeAfterFallback == HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL - && ColorInfo.isTransferHdr(decoderInputColor); + encoderWrapper.getHdrModeAfterFallback() == HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL + && ColorInfo.isTransferHdr(videoGraphInputColor); ColorInfo videoGraphOutputColor; if (videoGraphInputColor.colorTransfer == C.COLOR_TRANSFER_SRGB) { // The sRGB color transfer is only used for images, so when an image gets transcoded into a