From b1c1c2a6dd5289f9b6cadef7aa570f7e400d4ea8 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Fri, 26 Jan 2024 08:32:37 -0800 Subject: [PATCH] Transformer: Plumb hdrMode into AssetLoaders instead of forceInterpret. This allows us to also plumb whether we're doing MediaCodec tone-mapping, which will be used in a follow-up CL in the ExoAssetLoaderVideoRenderer. PiperOrigin-RevId: 601774435 --- .../transformer/ForceEndOfStreamTest.java | 2 +- .../DefaultAssetLoaderFactory.java | 36 ++++++++++-------- .../ExoAssetLoaderVideoRenderer.java | 11 +++--- .../transformer/ExoPlayerAssetLoader.java | 38 +++++++++---------- .../media3/transformer/Transformer.java | 7 +--- .../transformer/ExoPlayerAssetLoaderTest.java | 2 +- .../transformer/MediaItemExportTest.java | 2 +- 7 files changed, 48 insertions(+), 50 deletions(-) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/ForceEndOfStreamTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/ForceEndOfStreamTest.java index eb4418a822..3bb28d4765 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/ForceEndOfStreamTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/ForceEndOfStreamTest.java @@ -118,7 +118,7 @@ public class ForceEndOfStreamTest { new DefaultAssetLoaderFactory( context, new FrameDroppingDecoderFactory(context, MP4_ASSET_FRAME_COUNT, framesToSkip), - /* forceInterpretHdrAsSdr= */ false, + Composition.HDR_MODE_KEEP_HDR, Clock.DEFAULT)) .build(); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java index bfb4b5afd9..dc420d774a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java @@ -46,7 +46,7 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory { private final Context context; private final Codec.DecoderFactory decoderFactory; - private final boolean forceInterpretHdrAsSdr; + private final @Composition.HdrMode int hdrMode; private final Clock clock; private final MediaSource.@MonotonicNonNull Factory mediaSourceFactory; private final BitmapLoader bitmapLoader; @@ -64,19 +64,22 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory { * @param context The {@link Context}. * @param decoderFactory The {@link Codec.DecoderFactory} to use to decode the samples (if * necessary). - * @param forceInterpretHdrAsSdr Whether to apply {@link - * Composition#HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR}. + * @param hdrMode The {@link Composition.HdrMode} to apply. Only {@link + * Composition#HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC} and {@link + * Composition#HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR} are applied in this + * component. * @param clock The {@link Clock} to use. It should always be {@link Clock#DEFAULT}, except for * testing. */ public DefaultAssetLoaderFactory( Context context, Codec.DecoderFactory decoderFactory, - boolean forceInterpretHdrAsSdr, + @Composition.HdrMode int hdrMode, Clock clock) { this.context = context.getApplicationContext(); this.decoderFactory = decoderFactory; - this.forceInterpretHdrAsSdr = forceInterpretHdrAsSdr; + // TODO: b/307952514 - MediaCodec tone-mapping information isn't actually used yet. Use it. + this.hdrMode = hdrMode; this.clock = clock; this.mediaSourceFactory = null; @Nullable BitmapFactory.Options options = null; @@ -98,15 +101,16 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory { * The frame loaded is determined by the {@link BitmapLoader} implementation. * * @param context The {@link Context}. - * @param forceInterpretHdrAsSdr Whether to apply {@link - * Composition#HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR}. + * @param hdrMode The {@link Composition.HdrMode} to apply. Only {@link + * Composition#HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC} and {@link + * Composition#HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR} are applied. * @param bitmapLoader The {@link BitmapLoader} to use to load and decode images. */ public DefaultAssetLoaderFactory( - Context context, boolean forceInterpretHdrAsSdr, BitmapLoader bitmapLoader) { + Context context, @Composition.HdrMode int hdrMode, BitmapLoader bitmapLoader) { this.context = context.getApplicationContext(); this.decoderFactory = new DefaultDecoderFactory(context); - this.forceInterpretHdrAsSdr = forceInterpretHdrAsSdr; + this.hdrMode = hdrMode; this.clock = Clock.DEFAULT; this.mediaSourceFactory = null; this.bitmapLoader = bitmapLoader; @@ -118,8 +122,9 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory { * @param context The {@link Context}. * @param decoderFactory The {@link Codec.DecoderFactory} to use to decode the samples (if * necessary). - * @param forceInterpretHdrAsSdr Whether to apply {@link - * Composition#HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR}. + * @param hdrMode The {@link Composition.HdrMode} to apply. Only {@link + * Composition#HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC} and {@link + * Composition#HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR} are applied. * @param clock The {@link Clock} to use. It should always be {@link Clock#DEFAULT}, except for * testing. * @param mediaSourceFactory The {@link MediaSource.Factory} to use to retrieve the samples to @@ -129,13 +134,13 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory { public DefaultAssetLoaderFactory( Context context, Codec.DecoderFactory decoderFactory, - boolean forceInterpretHdrAsSdr, + @Composition.HdrMode int hdrMode, Clock clock, MediaSource.Factory mediaSourceFactory, BitmapLoader bitmapLoader) { this.context = context.getApplicationContext(); this.decoderFactory = decoderFactory; - this.forceInterpretHdrAsSdr = forceInterpretHdrAsSdr; + this.hdrMode = hdrMode; this.clock = clock; this.mediaSourceFactory = mediaSourceFactory; this.bitmapLoader = bitmapLoader; @@ -155,9 +160,8 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory { exoPlayerAssetLoaderFactory = mediaSourceFactory != null ? new ExoPlayerAssetLoader.Factory( - context, decoderFactory, forceInterpretHdrAsSdr, clock, mediaSourceFactory) - : new ExoPlayerAssetLoader.Factory( - context, decoderFactory, forceInterpretHdrAsSdr, clock); + context, decoderFactory, hdrMode, clock, mediaSourceFactory) + : new ExoPlayerAssetLoader.Factory(context, decoderFactory, hdrMode, clock); } return exoPlayerAssetLoaderFactory.createAssetLoader(editedMediaItem, looper, listener); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderVideoRenderer.java index f8e3689b88..b522bd4f2d 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderVideoRenderer.java @@ -37,7 +37,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private final boolean flattenForSlowMotion; private final Codec.DecoderFactory decoderFactory; - private final boolean forceInterpretHdrAsSdr; + private final @Composition.HdrMode int hdrMode; private final List decodeOnlyPresentationTimestamps; private @MonotonicNonNull SefSlowMotionFlattener sefVideoSlowMotionFlattener; @@ -46,13 +46,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; public ExoAssetLoaderVideoRenderer( boolean flattenForSlowMotion, Codec.DecoderFactory decoderFactory, - boolean forceInterpretHdrAsSdr, + @Composition.HdrMode int hdrMode, TransformerMediaClock mediaClock, AssetLoader.Listener assetLoaderListener) { super(C.TRACK_TYPE_VIDEO, mediaClock, assetLoaderListener); this.flattenForSlowMotion = flattenForSlowMotion; this.decoderFactory = decoderFactory; - this.forceInterpretHdrAsSdr = forceInterpretHdrAsSdr; + this.hdrMode = hdrMode; decodeOnlyPresentationTimestamps = new ArrayList<>(); } @@ -63,7 +63,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @Override protected Format overrideFormat(Format inputFormat) { - if (forceInterpretHdrAsSdr && ColorInfo.isTransferHdr(inputFormat.colorInfo)) { + if (hdrMode == Composition.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR + && ColorInfo.isTransferHdr(inputFormat.colorInfo)) { return inputFormat.buildUpon().setColorInfo(ColorInfo.SDR_BT709_LIMITED).build(); } return inputFormat; @@ -84,7 +85,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @Override protected void initDecoder(Format inputFormat) throws ExportException { // TODO(b/278259383): Move surface creation out of sampleConsumer. Init decoder before - // sampleConsumer. + // sampleConsumer. checkStateNotNull(sampleConsumer); boolean isDecoderToneMappingRequired = ColorInfo.isTransferHdr(inputFormat.colorInfo) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java index c2b53daa52..45ba9272b6 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java @@ -65,7 +65,7 @@ public final class ExoPlayerAssetLoader implements AssetLoader { private final Context context; private final Codec.DecoderFactory decoderFactory; - private final boolean forceInterpretHdrAsSdr; + private final @Composition.HdrMode int hdrMode; private final Clock clock; @Nullable private final MediaSource.Factory mediaSourceFactory; @@ -75,19 +75,20 @@ public final class ExoPlayerAssetLoader implements AssetLoader { * @param context The {@link Context}. * @param decoderFactory The {@link Codec.DecoderFactory} to use to decode the samples (if * necessary). - * @param forceInterpretHdrAsSdr Whether to apply {@link - * Composition#HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR}. + * @param hdrMode The {@link Composition.HdrMode} to apply. Only {@link + * Composition#HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC} and {@link + * Composition#HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR} are applied. * @param clock The {@link Clock} to use. It should always be {@link Clock#DEFAULT}, except for * testing. */ public Factory( Context context, Codec.DecoderFactory decoderFactory, - boolean forceInterpretHdrAsSdr, + @Composition.HdrMode int hdrMode, Clock clock) { this.context = context; this.decoderFactory = decoderFactory; - this.forceInterpretHdrAsSdr = forceInterpretHdrAsSdr; + this.hdrMode = hdrMode; this.clock = clock; this.mediaSourceFactory = null; } @@ -98,8 +99,9 @@ public final class ExoPlayerAssetLoader implements AssetLoader { * @param context The {@link Context}. * @param decoderFactory The {@link Codec.DecoderFactory} to use to decode the samples (if * necessary). - * @param forceInterpretHdrAsSdr Whether to apply {@link - * Composition#HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR}. + * @param hdrMode The {@link Composition.HdrMode} to apply. Only {@link + * Composition#HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC} and {@link + * Composition#HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR} are applied. * @param clock The {@link Clock} to use. It should always be {@link Clock#DEFAULT}, except for * testing. * @param mediaSourceFactory The {@link MediaSource.Factory} to use to retrieve the samples to @@ -108,12 +110,12 @@ public final class ExoPlayerAssetLoader implements AssetLoader { public Factory( Context context, Codec.DecoderFactory decoderFactory, - boolean forceInterpretHdrAsSdr, + @Composition.HdrMode int hdrMode, Clock clock, MediaSource.Factory mediaSourceFactory) { this.context = context; this.decoderFactory = decoderFactory; - this.forceInterpretHdrAsSdr = forceInterpretHdrAsSdr; + this.hdrMode = hdrMode; this.clock = clock; this.mediaSourceFactory = mediaSourceFactory; } @@ -134,7 +136,7 @@ public final class ExoPlayerAssetLoader implements AssetLoader { editedMediaItem, mediaSourceFactory, decoderFactory, - forceInterpretHdrAsSdr, + hdrMode, looper, listener, clock); @@ -158,7 +160,7 @@ public final class ExoPlayerAssetLoader implements AssetLoader { EditedMediaItem editedMediaItem, MediaSource.Factory mediaSourceFactory, Codec.DecoderFactory decoderFactory, - boolean forceInterpretHdrAsSdr, + @Composition.HdrMode int hdrMode, Looper looper, Listener listener, Clock clock) { @@ -188,7 +190,7 @@ public final class ExoPlayerAssetLoader implements AssetLoader { editedMediaItem.removeVideo, editedMediaItem.flattenForSlowMotion, this.decoderFactory, - forceInterpretHdrAsSdr, + hdrMode, listener)) .setMediaSourceFactory(mediaSourceFactory) .setTrackSelector(trackSelector) @@ -252,7 +254,7 @@ public final class ExoPlayerAssetLoader implements AssetLoader { private final boolean removeVideo; private final boolean flattenForSlowMotion; private final Codec.DecoderFactory decoderFactory; - private final boolean forceInterpretHdrAsSdr; + private final @Composition.HdrMode int hdrMode; private final Listener assetLoaderListener; public RenderersFactoryImpl( @@ -260,13 +262,13 @@ public final class ExoPlayerAssetLoader implements AssetLoader { boolean removeVideo, boolean flattenForSlowMotion, Codec.DecoderFactory decoderFactory, - boolean forceInterpretHdrAsSdr, + @Composition.HdrMode int hdrMode, Listener assetLoaderListener) { this.removeAudio = removeAudio; this.removeVideo = removeVideo; this.flattenForSlowMotion = flattenForSlowMotion; this.decoderFactory = decoderFactory; - this.forceInterpretHdrAsSdr = forceInterpretHdrAsSdr; + this.hdrMode = hdrMode; this.assetLoaderListener = assetLoaderListener; mediaClock = new TransformerMediaClock(); } @@ -289,11 +291,7 @@ public final class ExoPlayerAssetLoader implements AssetLoader { if (!removeVideo) { renderers[index] = new ExoAssetLoaderVideoRenderer( - flattenForSlowMotion, - decoderFactory, - forceInterpretHdrAsSdr, - mediaClock, - assetLoaderListener); + flattenForSlowMotion, decoderFactory, hdrMode, mediaClock, assetLoaderListener); index++; } return renderers; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 53ad7ce317..a87bb40322 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -20,7 +20,6 @@ import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.extractor.AacUtil.AAC_LC_AUDIO_SAMPLE_COUNT; -import static androidx.media3.transformer.Composition.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR; import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED_KEYFRAME_PLACEMENT_OPTIMAL_FOR_TRIM; import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED_OTHER; import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED_TRIM_AND_TRANSCODING_TRANSFORMATION_REQUESTED; @@ -1532,11 +1531,7 @@ public final class Transformer { if (assetLoaderFactory == null) { assetLoaderFactory = new DefaultAssetLoaderFactory( - context, - new DefaultDecoderFactory(context), - /* forceInterpretHdrAsSdr= */ transformationRequest.hdrMode - == HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR, - clock); + context, new DefaultDecoderFactory(context), transformationRequest.hdrMode, clock); } DebugTraceUtil.reset(); transformerInternal = diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java index af7061ff1f..63b37bf0dc 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java @@ -144,7 +144,7 @@ public class ExoPlayerAssetLoaderTest { EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(MediaItem.fromUri("asset:///media/mp4/sample.mp4")).build(); return new ExoPlayerAssetLoader.Factory( - context, decoderFactory, /* forceInterpretHdrAsSdr= */ false, clock) + context, decoderFactory, Composition.HDR_MODE_KEEP_HDR, clock) .createAssetLoader(editedMediaItem, Looper.myLooper(), listener); } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/MediaItemExportTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/MediaItemExportTest.java index 3dda3f8681..c8315f364f 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/MediaItemExportTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/MediaItemExportTest.java @@ -872,7 +872,7 @@ public final class MediaItemExportTest { new ExoPlayerAssetLoader.Factory( context, decoderFactory, - /* forceInterpretHdrAsSdr= */ false, + Composition.HDR_MODE_KEEP_HDR, new FakeClock(/* isAutoAdvancing= */ true), mediaSourceFactory); CapturingMuxer.Factory muxerFactory =