diff --git a/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java b/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java index bd2052f610..6d46ef9a14 100644 --- a/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java +++ b/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java @@ -161,6 +161,7 @@ public final class MimeTypes { // image/ MIME types public static final String IMAGE_JPEG = BASE_TYPE_IMAGE + "/jpeg"; + @UnstableApi public static final String IMAGE_JPEG_R = BASE_TYPE_IMAGE + "/jpeg_r"; @UnstableApi public static final String IMAGE_PNG = BASE_TYPE_IMAGE + "/png"; @UnstableApi public static final String IMAGE_HEIF = BASE_TYPE_IMAGE + "/heif"; @UnstableApi public static final String IMAGE_BMP = BASE_TYPE_IMAGE + "/bmp"; diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/TransformerUltraHdrPixelTest/exportUltraHdrImage_withUltraHdrEnabledOnSupportedDevice_succeeds_0.png b/libraries/test_data/src/test/assets/test-generated-goldens/TransformerUltraHdrPixelTest/exportUltraHdrImage_withUltraHdrEnabledOnSupportedDevice_succeeds_0.png new file mode 100644 index 0000000000..34bff3f6b7 Binary files /dev/null and b/libraries/test_data/src/test/assets/test-generated-goldens/TransformerUltraHdrPixelTest/exportUltraHdrImage_withUltraHdrEnabledOnSupportedDevice_succeeds_0.png differ diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java index da59413133..8ad424624b 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java @@ -226,7 +226,8 @@ public class BitmapPixelTestUtil { /** * Returns a grayscale bitmap from the Luma channel in the {@link ImageFormat#YUV_420_888} image. */ - public static Bitmap createGrayscaleArgb8888BitmapFromYuv420888Image(Image image) { + public static Bitmap createGrayscaleBitmapFromYuv420888Image( + Image image, Bitmap.Config bitmapConfig) { int width = image.getWidth(); int height = image.getHeight(); assertThat(image.getPlanes()).hasLength(3); @@ -247,7 +248,7 @@ public class BitmapPixelTestUtil { /* blue= */ lumaValue); } } - return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888); + return Bitmap.createBitmap(colors, width, height, bitmapConfig); } /** diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index 94cf98e06a..8bc61f3d0c 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -26,6 +26,7 @@ import static org.junit.Assume.assumeFalse; import android.content.Context; import android.graphics.Bitmap; +import android.graphics.Bitmap.Config; import android.media.Image; import android.media.MediaFormat; import android.opengl.EGLContext; @@ -69,6 +70,7 @@ public final class AndroidTestUtil { public static final String PNG_ASSET_URI_STRING = "asset:///media/png/media3test.png"; public static final String JPG_ASSET_URI_STRING = "asset:///media/jpeg/london.jpg"; public static final String JPG_PORTRAIT_ASSET_URI_STRING = "asset:///media/jpeg/tokyo.jpg"; + public static final String ULTRA_HDR_URI_STRING = "asset:///media/jpeg/ultraHDR.jpg"; public static final String MP4_TRIM_OPTIMIZATION_URI_STRING = "asset:///media/mp4/internal_emulator_transformer_output.mp4"; @@ -633,6 +635,12 @@ public final class AndroidTestUtil { public static ImmutableList extractBitmapsFromVideo(Context context, String filePath) throws IOException, InterruptedException { + return extractBitmapsFromVideo(context, filePath, Config.ARGB_8888); + } + + public static ImmutableList extractBitmapsFromVideo( + Context context, String filePath, Bitmap.Config config) + throws IOException, InterruptedException { // b/298599172 - runUntilComparisonFrameOrEnded fails on this device because reading decoder // output as a bitmap doesn't work. assumeFalse(Util.SDK_INT == 21 && Ascii.toLowerCase(Util.MODEL).contains("nexus")); @@ -645,7 +653,7 @@ public final class AndroidTestUtil { if (image == null) { break; } - bitmaps.add(BitmapPixelTestUtil.createGrayscaleArgb8888BitmapFromYuv420888Image(image)); + bitmaps.add(BitmapPixelTestUtil.createGrayscaleBitmapFromYuv420888Image(image, config)); image.close(); } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SequenceEffectTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SequenceEffectTestUtil.java index 756c614e91..531c33a097 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SequenceEffectTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SequenceEffectTestUtil.java @@ -34,6 +34,7 @@ import java.util.List; /** Utility class for checking testing {@link EditedMediaItemSequence} instances. */ public final class SequenceEffectTestUtil { + public static final ImmutableList NO_EFFECT = ImmutableList.of(); private static final String PNG_ASSET_BASE_PATH = "test-generated-goldens/transformer_sequence_effect_test"; public static final long SINGLE_30_FPS_VIDEO_FRAME_THRESHOLD_MS = 50; diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerSequenceEffectTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerSequenceEffectTest.java index c1bf403471..034ea284a6 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerSequenceEffectTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerSequenceEffectTest.java @@ -29,6 +29,7 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.MP4_PORTRAIT_ASSET_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.assumeFormatsSupported; import static androidx.media3.transformer.AndroidTestUtil.extractBitmapsFromVideo; +import static androidx.media3.transformer.SequenceEffectTestUtil.NO_EFFECT; import static androidx.media3.transformer.SequenceEffectTestUtil.SINGLE_30_FPS_VIDEO_FRAME_THRESHOLD_MS; import static androidx.media3.transformer.SequenceEffectTestUtil.assertBitmapsMatchExpectedAndSave; import static androidx.media3.transformer.SequenceEffectTestUtil.clippedVideo; @@ -64,7 +65,6 @@ import org.junit.runner.RunWith; @RunWith(AndroidJUnit4.class) public final class TransformerSequenceEffectTest { - private static final ImmutableList NO_EFFECT = ImmutableList.of(); private static final String OVERLAY_PNG_ASSET_PATH = "media/png/media3test.png"; private static final int EXPORT_WIDTH = 360; private static final int EXPORT_HEIGHT = 240; diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerUltraHdrTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerUltraHdrTest.java new file mode 100644 index 0000000000..42a481167a --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerUltraHdrTest.java @@ -0,0 +1,198 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package androidx.media3.transformer; + +import static androidx.media3.test.utils.TestUtil.retrieveTrackFormat; +import static androidx.media3.transformer.AndroidTestUtil.JPG_ASSET_URI_STRING; +import static androidx.media3.transformer.AndroidTestUtil.ULTRA_HDR_URI_STRING; +import static androidx.media3.transformer.Composition.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL; +import static androidx.media3.transformer.SequenceEffectTestUtil.NO_EFFECT; +import static androidx.media3.transformer.SequenceEffectTestUtil.oneFrameFromImage; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assume.assumeTrue; + +import android.content.Context; +import android.graphics.Bitmap; +import android.net.Uri; +import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.util.BitmapLoader; +import androidx.media3.common.util.Util; +import androidx.media3.datasource.DataSourceBitmapLoader; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import java.util.concurrent.ExecutionException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +/** + * Tests for Ultra HDR support in Transformer that can run on an emulator. + * + *

See {@code TransformerMhUltraHdrPixelTest} for other UltraHdr tests. + */ +@RunWith(AndroidJUnit4.class) +public final class TransformerUltraHdrTest { + + @Rule public final TestName testName = new TestName(); + private final Context context = ApplicationProvider.getApplicationContext(); + + private String testId; + + @Before + public void setUpTestId() { + testId = testName.getMethodName(); + } + + @Test + public void exportUltraHdrImage_withUltraHdrEnabledOnUnsupportedApiLevel_fallbackToExportSdr() + throws Exception { + assumeTrue(Util.SDK_INT < 34); + Composition composition = + createUltraHdrComposition( + /* tonemap= */ false, oneFrameFromImage(ULTRA_HDR_URI_STRING, NO_EFFECT)); + + // Downscale source bitmap to avoid "video encoding format not supported" errors on emulators. + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, createDownscalingTransformer()) + .build() + .run(testId, composition); + + assertThat(result.filePath).isNotNull(); + ColorInfo colorInfo = + retrieveTrackFormat(context, result.filePath, C.TRACK_TYPE_VIDEO).colorInfo; + assertThat(colorInfo.colorSpace).isEqualTo(C.COLOR_SPACE_BT709); + assertThat(colorInfo.colorTransfer).isEqualTo(C.COLOR_TRANSFER_SDR); + } + + @Test + public void exportUltraHdrImage_withUltraHdrAndTonemappingEnabled_exportsSdr() throws Exception { + Composition composition = + createUltraHdrComposition( + /* tonemap= */ true, oneFrameFromImage(ULTRA_HDR_URI_STRING, NO_EFFECT)); + + // Downscale source bitmap to avoid "video encoding format not supported" errors on emulators. + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, createDownscalingTransformer()) + .build() + .run(testId, composition); + + assertThat(result.filePath).isNotNull(); + ColorInfo colorInfo = + retrieveTrackFormat(context, result.filePath, C.TRACK_TYPE_VIDEO).colorInfo; + assertThat(colorInfo.colorSpace).isEqualTo(C.COLOR_SPACE_BT709); + assertThat(colorInfo.colorTransfer).isEqualTo(C.COLOR_TRANSFER_SDR); + } + + @Test + public void exportUltraHdrImage_withUltraHdrDisabled_exportsSdr() throws Exception { + Composition composition = + new Composition.Builder( + new EditedMediaItemSequence(oneFrameFromImage(ULTRA_HDR_URI_STRING, NO_EFFECT))) + .build(); + + // Downscale source bitmap to avoid "video encoding format not supported" errors on emulators. + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, createDownscalingTransformer()) + .build() + .run(testId, composition); + + assertThat(result.filePath).isNotNull(); + ColorInfo colorInfo = + retrieveTrackFormat(context, result.filePath, C.TRACK_TYPE_VIDEO).colorInfo; + assertThat(colorInfo.colorSpace).isEqualTo(C.COLOR_SPACE_BT709); + assertThat(colorInfo.colorTransfer).isEqualTo(C.COLOR_TRANSFER_SDR); + } + + @Test + public void exportNonUltraHdrImage_withUltraHdrEnabled_exportsSdr() throws Exception { + Composition composition = + createUltraHdrComposition( + /* tonemap= */ false, oneFrameFromImage(JPG_ASSET_URI_STRING, NO_EFFECT)); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build() + .run(testId, composition); + + assertThat(result.filePath).isNotNull(); + ColorInfo colorInfo = + retrieveTrackFormat(context, result.filePath, C.TRACK_TYPE_VIDEO).colorInfo; + assertThat(colorInfo.colorSpace).isEqualTo(C.COLOR_SPACE_BT709); + assertThat(colorInfo.colorTransfer).isEqualTo(C.COLOR_TRANSFER_SDR); + } + + private static Composition createUltraHdrComposition( + boolean tonemap, EditedMediaItem editedMediaItem, EditedMediaItem... editedMediaItems) { + Composition.Builder builder = + new Composition.Builder(new EditedMediaItemSequence(editedMediaItem, editedMediaItems)) + .experimentalSetRetainHdrFromUltraHdrImage(true); + if (tonemap) { + builder.setHdrMode(HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL); + } + return builder.build(); + } + + private Transformer createDownscalingTransformer() { + BitmapLoader downscalingBitmapLoader = + new BitmapLoader() { + + static final int DOWNSCALED_WIDTH_HEIGHT = 120; + final BitmapLoader bitmapLoader; + + { + bitmapLoader = new DataSourceBitmapLoader(context); + } + + @Override + public boolean supportsMimeType(String mimeType) { + return bitmapLoader.supportsMimeType(mimeType); + } + + @Override + public ListenableFuture decodeBitmap(byte[] data) { + return bitmapLoader.decodeBitmap(data); + } + + @Override + public ListenableFuture loadBitmap(Uri uri) { + SettableFuture outputFuture = SettableFuture.create(); + try { + Bitmap bitmap = + Bitmap.createScaledBitmap( + bitmapLoader.loadBitmap(uri).get(), + DOWNSCALED_WIDTH_HEIGHT, + DOWNSCALED_WIDTH_HEIGHT, + /* filter= */ true); + outputFuture.set(bitmap); + return outputFuture; + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + } + }; + + return new Transformer.Builder(context) + .setAssetLoaderFactory(new DefaultAssetLoaderFactory(context, downscalingBitmapLoader)) + .build(); + } +} diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformerMhUltraHdrPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformerMhUltraHdrPixelTest.java new file mode 100644 index 0000000000..664b21de06 --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformerMhUltraHdrPixelTest.java @@ -0,0 +1,141 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package androidx.media3.transformer.mh; + +import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_LUMA; +import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceFp16; +import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap; +import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; +import static androidx.media3.test.utils.TestUtil.retrieveTrackFormat; +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT; +import static androidx.media3.transformer.AndroidTestUtil.ULTRA_HDR_URI_STRING; +import static androidx.media3.transformer.AndroidTestUtil.extractBitmapsFromVideo; +import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped; +import static androidx.media3.transformer.SequenceEffectTestUtil.NO_EFFECT; +import static androidx.media3.transformer.SequenceEffectTestUtil.oneFrameFromImage; +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; + +import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.Bitmap.Config; +import androidx.annotation.RequiresApi; +import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.mediacodec.MediaCodecUtil.DecoderQueryException; +import androidx.media3.transformer.AndroidTestUtil; +import androidx.media3.transformer.Composition; +import androidx.media3.transformer.EditedMediaItem; +import androidx.media3.transformer.EditedMediaItemSequence; +import androidx.media3.transformer.ExportTestResult; +import androidx.media3.transformer.Transformer; +import androidx.media3.transformer.TransformerAndroidTestRunner; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import java.io.IOException; +import java.util.List; +import org.json.JSONException; +import org.junit.AssumptionViolatedException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +/** Tests for Ultra HDR support in Transformer that should only run in mobile harness. */ +@RunWith(AndroidJUnit4.class) +public final class TransformerMhUltraHdrPixelTest { + + private static final String PNG_ASSET_BASE_PATH = + "test-generated-goldens/TransformerUltraHdrPixelTest"; + + @Rule public final TestName testName = new TestName(); + private final Context context = ApplicationProvider.getApplicationContext(); + + private String testId; + + @Before + public void setUpTestId() { + testId = testName.getMethodName(); + } + + @Test + public void exportUltraHdrImage_withUltraHdrEnabledOnSupportedDevice_succeeds() throws Exception { + assumeDeviceSupportsUltraHdrEditing(); + Composition composition = + createUltraHdrComposition(oneFrameFromImage(ULTRA_HDR_URI_STRING, NO_EFFECT)); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build() + .run(testId, composition); + + assertThat(result.filePath).isNotNull(); + ColorInfo colorInfo = + retrieveTrackFormat(context, result.filePath, C.TRACK_TYPE_VIDEO).colorInfo; + assertThat(colorInfo.colorSpace).isEqualTo(C.COLOR_SPACE_BT2020); + assertThat(colorInfo.colorTransfer).isEqualTo(C.COLOR_TRANSFER_HLG); + assertFp16BitmapsMatchExpectedAndSave( + extractBitmapsFromVideo(context, result.filePath, Config.RGBA_F16), testId); + } + + @RequiresApi(29) // getBitmapAveragePixelAbsoluteDifferenceFp16() + public static void assertFp16BitmapsMatchExpectedAndSave( + List actualBitmaps, String testId) throws IOException { + for (int i = 0; i < actualBitmaps.size(); i++) { + maybeSaveTestBitmap( + testId, /* bitmapLabel= */ String.valueOf(i), actualBitmaps.get(i), /* path= */ null); + } + + for (int i = 0; i < actualBitmaps.size(); i++) { + String subTestId = testId + "_" + i; + String expectedPath = Util.formatInvariant("%s/%s.png", PNG_ASSET_BASE_PATH, subTestId); + Bitmap expectedBitmap = readBitmap(expectedPath); + + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceFp16(expectedBitmap, actualBitmaps.get(i)); + assertWithMessage("For expected bitmap " + expectedPath) + .that(averagePixelAbsoluteDifference) + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_LUMA); + } + } + + private void assumeDeviceSupportsUltraHdrEditing() + throws JSONException, IOException, DecoderQueryException { + if (Util.SDK_INT < 34) { + recordTestSkipped( + getApplicationContext(), testId, "Ultra HDR is not supported on this API level."); + throw new AssumptionViolatedException("Ultra HDR is not supported on this API level."); + } + AndroidTestUtil.assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT, + /* outputFormat= */ null); + } + + private static Composition createUltraHdrComposition( + EditedMediaItem editedMediaItem, EditedMediaItem... editedMediaItems) { + Composition.Builder builder = + new Composition.Builder(new EditedMediaItemSequence(editedMediaItem, editedMediaItems)) + .experimentalSetRetainHdrFromUltraHdrImage(true); + return builder.build(); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AssetLoader.java index 751a4d7853..1a26c3240e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AssetLoader.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AssetLoader.java @@ -24,6 +24,7 @@ import androidx.annotation.IntRange; import androidx.annotation.Nullable; import androidx.media3.common.Format; import androidx.media3.common.util.UnstableApi; +import androidx.media3.transformer.Composition.HdrMode; import com.google.common.collect.ImmutableMap; import java.lang.annotation.Documented; import java.lang.annotation.Retention; @@ -144,14 +145,12 @@ public interface AssetLoader { */ /* package */ class CompositionSettings { - public final @Composition.HdrMode int hdrMode; + public final @HdrMode int hdrMode; + public final boolean retainHdrFromUltraHdrImage; - public CompositionSettings() { - this.hdrMode = Composition.HDR_MODE_KEEP_HDR; - } - - public CompositionSettings(@Composition.HdrMode int hdrMode) { + public CompositionSettings(@HdrMode int hdrMode, boolean retainHdrFromUltraHdrImage) { this.hdrMode = hdrMode; + this.retainHdrFromUltraHdrImage = retainHdrFromUltraHdrImage; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java index b3067cb35c..601ca952fe 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java @@ -50,6 +50,7 @@ public final class Composition { private boolean transmuxAudio; private boolean transmuxVideo; private @HdrMode int hdrMode; + private boolean retainHdrFromUltraHdrImage; /** * Creates an instance. @@ -88,6 +89,7 @@ public final class Composition { transmuxAudio = composition.transmuxAudio; transmuxVideo = composition.transmuxVideo; hdrMode = composition.hdrMode; + retainHdrFromUltraHdrImage = composition.retainHdrFromUltraHdrImage; } /** @@ -230,6 +232,33 @@ public final class Composition { return this; } + /** + * Sets whether to use produce an HDR output video from Ultra HDR image input. + * + *

If the {@link HdrMode} is {@link #HDR_MODE_KEEP_HDR}, then setting this to {@code true} + * applies the recovery map (i.e. the gainmap) to the base image to produce HDR video frames. + * + *

The output video will have the same color encoding as the first {@link EditedMediaItem} + * the sequence. If the Ultra HDR image is first in the sequence, output video will default to + * BT2020 HLG full range colors. + * + *

Ignored if {@link HdrMode} is not {@link #HDR_MODE_KEEP_HDR}. + * + *

Supported on API 34+, by some device and HDR format combinations. Ignored if unsupported + * by device or API level. + * + *

The default value is {@code false}. + * + * @param retainHdrFromUltraHdrImage Whether to use produce an HDR output video from Ultra HDR + * image input. + * @return This builder. + */ + @CanIgnoreReturnValue + public Builder experimentalSetRetainHdrFromUltraHdrImage(boolean retainHdrFromUltraHdrImage) { + this.retainHdrFromUltraHdrImage = retainHdrFromUltraHdrImage; + return this; + } + /** Builds a {@link Composition} instance. */ public Composition build() { return new Composition( @@ -239,7 +268,8 @@ public final class Composition { forceAudioTrack, transmuxAudio, transmuxVideo, - hdrMode); + hdrMode, + retainHdrFromUltraHdrImage && hdrMode == HDR_MODE_KEEP_HDR); } /** @@ -377,6 +407,14 @@ public final class Composition { */ public final @HdrMode int hdrMode; + /** + * Sets whether to use produce an HDR output video from Ultra HDR image input. + * + *

For more information, see {@link + * Builder#experimentalSetRetainHdrFromUltraHdrImage(boolean)}. + */ + public final boolean retainHdrFromUltraHdrImage; + /** Returns a {@link Composition.Builder} initialized with the values of this instance. */ /* package */ Builder buildUpon() { return new Builder(this); @@ -389,7 +427,8 @@ public final class Composition { boolean forceAudioTrack, boolean transmuxAudio, boolean transmuxVideo, - @HdrMode int hdrMode) { + @HdrMode int hdrMode, + boolean retainHdrFromUltraHdrImage) { checkArgument( !transmuxAudio || !forceAudioTrack, "Audio transmuxing and audio track forcing are not allowed together."); @@ -400,5 +439,6 @@ public final class Composition { this.transmuxVideo = transmuxVideo; this.forceAudioTrack = forceAudioTrack; this.hdrMode = hdrMode; + this.retainHdrFromUltraHdrImage = retainHdrFromUltraHdrImage; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java index 1bc9c9753e..ff46b59182 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java @@ -92,13 +92,9 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory { * The frame loaded is determined by the {@link BitmapLoader} implementation. * * @param context The {@link Context}. - * @param hdrMode The {@link Composition.HdrMode} to apply. Only {@link - * Composition#HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC} and {@link - * Composition#HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR} are applied. * @param bitmapLoader The {@link BitmapLoader} to use to load and decode images. */ - public DefaultAssetLoaderFactory( - Context context, @Composition.HdrMode int hdrMode, BitmapLoader bitmapLoader) { + public DefaultAssetLoaderFactory(Context context, BitmapLoader bitmapLoader) { this.context = context.getApplicationContext(); this.decoderFactory = new DefaultDecoderFactory(context); this.clock = Clock.DEFAULT; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java index 4cd3d6b851..f85f72a1e3 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java @@ -38,6 +38,7 @@ import androidx.media3.common.MimeTypes; import androidx.media3.common.util.BitmapLoader; import androidx.media3.common.util.ConstantRateTimestampIterator; import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.util.Util; import androidx.media3.transformer.SampleConsumer.InputResult; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.FutureCallback; @@ -56,6 +57,8 @@ import java.util.concurrent.ScheduledExecutorService; @UnstableApi public final class ImageAssetLoader implements AssetLoader { + private final boolean retainHdrFromUltraHdrImage; + /** An {@link AssetLoader.Factory} for {@link ImageAssetLoader} instances. */ public static final class Factory implements AssetLoader.Factory { @@ -76,7 +79,8 @@ public final class ImageAssetLoader implements AssetLoader { Looper looper, Listener listener, CompositionSettings compositionSettings) { - return new ImageAssetLoader(editedMediaItem, listener, bitmapLoader); + return new ImageAssetLoader( + editedMediaItem, listener, bitmapLoader, compositionSettings.retainHdrFromUltraHdrImage); } } @@ -95,7 +99,11 @@ public final class ImageAssetLoader implements AssetLoader { private volatile int progress; private ImageAssetLoader( - EditedMediaItem editedMediaItem, Listener listener, BitmapLoader bitmapLoader) { + EditedMediaItem editedMediaItem, + Listener listener, + BitmapLoader bitmapLoader, + boolean retainHdrFromUltraHdrImage) { + this.retainHdrFromUltraHdrImage = retainHdrFromUltraHdrImage; checkState(editedMediaItem.durationUs != C.TIME_UNSET); checkState(editedMediaItem.frameRate != C.RATE_UNSET_INT); this.editedMediaItem = editedMediaItem; @@ -124,16 +132,20 @@ public final class ImageAssetLoader implements AssetLoader { @Override public void onSuccess(Bitmap bitmap) { progress = 50; + Format inputFormat = + new Format.Builder() + .setHeight(bitmap.getHeight()) + .setWidth(bitmap.getWidth()) + .setSampleMimeType(MIME_TYPE_IMAGE_ALL) + .setColorInfo(ColorInfo.SRGB_BT709_FULL) + .build(); + Format outputFormat = + retainHdrFromUltraHdrImage && Util.SDK_INT >= 34 && bitmap.hasGainmap() + ? inputFormat.buildUpon().setSampleMimeType(MimeTypes.IMAGE_JPEG_R).build() + : inputFormat; try { - Format format = - new Format.Builder() - .setHeight(bitmap.getHeight()) - .setWidth(bitmap.getWidth()) - .setSampleMimeType(MIME_TYPE_IMAGE_ALL) - .setColorInfo(ColorInfo.SRGB_BT709_FULL) - .build(); - listener.onTrackAdded(format, SUPPORTED_OUTPUT_TYPE_DECODED); - scheduledExecutorService.submit(() -> queueBitmapInternal(bitmap, format)); + listener.onTrackAdded(inputFormat, SUPPORTED_OUTPUT_TYPE_DECODED); + scheduledExecutorService.submit(() -> queueBitmapInternal(bitmap, outputFormat)); } catch (RuntimeException e) { listener.onError(ExportException.createForAssetLoader(e, ERROR_CODE_UNSPECIFIED)); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index 8d50725b86..7a26c9d2f5 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -238,7 +238,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; sequence, composition.forceAudioTrack, assetLoaderFactory, - new CompositionSettings(transformationRequest.hdrMode), + new CompositionSettings( + transformationRequest.hdrMode, composition.retainHdrFromUltraHdrImage), sequenceAssetLoaderListener, clock, internalLooper)); @@ -689,17 +690,23 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; muxerWrapper, fallbackListener)); } else { - ColorInfo decoderOutputColor; + Format firstFormat; if (MimeTypes.isVideo(assetLoaderOutputFormat.sampleMimeType)) { // TODO(b/267301878): Pass firstAssetLoaderOutputFormat once surface creation not in VSP. boolean isMediaCodecToneMappingRequested = transformationRequest.hdrMode == HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC; - decoderOutputColor = + ColorInfo decoderOutputColor = getDecoderOutputColor( getValidColor(firstAssetLoaderInputFormat.colorInfo), isMediaCodecToneMappingRequested); + firstFormat = + firstAssetLoaderInputFormat.buildUpon().setColorInfo(decoderOutputColor).build(); } else if (MimeTypes.isImage(assetLoaderOutputFormat.sampleMimeType)) { - decoderOutputColor = getValidColor(assetLoaderOutputFormat.colorInfo); + firstFormat = + firstAssetLoaderInputFormat + .buildUpon() + .setColorInfo(getValidColor(assetLoaderOutputFormat.colorInfo)) + .build(); } else { throw ExportException.createForUnexpected( new IllegalArgumentException( @@ -710,7 +717,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; C.TRACK_TYPE_VIDEO, new VideoSampleExporter( context, - firstAssetLoaderInputFormat.buildUpon().setColorInfo(decoderOutputColor).build(), + firstFormat, transformationRequest, composition.videoCompositorSettings, composition.effects.videoEffects, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index 9ee3032562..b15c3ee523 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -16,6 +16,9 @@ package androidx.media3.transformer; +import static androidx.media3.common.C.COLOR_RANGE_FULL; +import static androidx.media3.common.C.COLOR_SPACE_BT2020; +import static androidx.media3.common.C.COLOR_TRANSFER_HLG; import static androidx.media3.common.ColorInfo.SDR_BT709_LIMITED; import static androidx.media3.common.ColorInfo.SRGB_BT709_FULL; import static androidx.media3.common.ColorInfo.isTransferHdr; @@ -52,6 +55,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.MoreExecutors; import java.nio.ByteBuffer; import java.util.List; +import java.util.Objects; import org.checkerframework.checker.initialization.qual.Initialized; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.dataflow.qual.Pure; @@ -94,26 +98,40 @@ import org.checkerframework.dataflow.qual.Pure; this.initialTimestampOffsetUs = initialTimestampOffsetUs; finalFramePresentationTimeUs = C.TIME_UNSET; + ColorInfo videoGraphInputColor = checkNotNull(firstInputFormat.colorInfo); + ColorInfo videoGraphOutputColor; + if (videoGraphInputColor.colorTransfer == C.COLOR_TRANSFER_SRGB) { + // The sRGB color transfer is only used for images. + // When an Ultra HDR image transcoded into a video, we use BT2020 HLG full range colors in the + // resulting HDR video. + // When an SDR image gets transcoded into a video, we use the SMPTE 170M transfer function for + // the resulting video. + videoGraphOutputColor = + Objects.equals(firstInputFormat.sampleMimeType, MimeTypes.IMAGE_JPEG_R) + ? new ColorInfo.Builder() + .setColorSpace(COLOR_SPACE_BT2020) + .setColorTransfer(COLOR_TRANSFER_HLG) + .setColorRange(COLOR_RANGE_FULL) + .build() + : SDR_BT709_LIMITED; + } else { + videoGraphOutputColor = videoGraphInputColor; + } + encoderWrapper = new EncoderWrapper( encoderFactory, - firstInputFormat, + firstInputFormat.buildUpon().setColorInfo(videoGraphOutputColor).build(), muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_VIDEO), transformationRequest, fallbackListener); encoderOutputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); - ColorInfo videoGraphInputColor = checkNotNull(firstInputFormat.colorInfo); boolean isGlToneMapping = encoderWrapper.getHdrModeAfterFallback() == HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL && ColorInfo.isTransferHdr(videoGraphInputColor); - ColorInfo videoGraphOutputColor; - if (videoGraphInputColor.colorTransfer == C.COLOR_TRANSFER_SRGB) { - // The sRGB color transfer is only used for images, so when an image gets transcoded into a - // video, we use the SMPTE 170M transfer function for the resulting video. - videoGraphOutputColor = SDR_BT709_LIMITED; - } else if (isGlToneMapping) { + if (isGlToneMapping) { // For consistency with the Android platform, OpenGL tone mapping outputs colors with // C.COLOR_TRANSFER_GAMMA_2_2 instead of C.COLOR_TRANSFER_SDR, and outputs this as // C.COLOR_TRANSFER_SDR to the encoder. @@ -123,8 +141,6 @@ import org.checkerframework.dataflow.qual.Pure; .setColorRange(C.COLOR_RANGE_LIMITED) .setColorTransfer(C.COLOR_TRANSFER_GAMMA_2_2) .build(); - } else { - videoGraphOutputColor = videoGraphInputColor; } try { diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java index 70aaab9cae..300ec7f0b1 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/ExoPlayerAssetLoaderTest.java @@ -145,7 +145,12 @@ public class ExoPlayerAssetLoaderTest { EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(MediaItem.fromUri("asset:///media/mp4/sample.mp4")).build(); return new ExoPlayerAssetLoader.Factory(context, decoderFactory, clock) - .createAssetLoader(editedMediaItem, Looper.myLooper(), listener, new CompositionSettings()); + .createAssetLoader( + editedMediaItem, + Looper.myLooper(), + listener, + new CompositionSettings( + Composition.HDR_MODE_KEEP_HDR, /* retainHdrFromUltraHdrImage= */ false)); } private static final class FakeSampleConsumer implements SampleConsumer { diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/ImageAssetLoaderTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/ImageAssetLoaderTest.java index 87afb4c2a5..a20f547a16 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/ImageAssetLoaderTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/ImageAssetLoaderTest.java @@ -123,7 +123,12 @@ public class ImageAssetLoaderTest { .build(); return new ImageAssetLoader.Factory( new DataSourceBitmapLoader(ApplicationProvider.getApplicationContext())) - .createAssetLoader(editedMediaItem, Looper.myLooper(), listener, new CompositionSettings()); + .createAssetLoader( + editedMediaItem, + Looper.myLooper(), + listener, + new CompositionSettings( + Composition.HDR_MODE_KEEP_HDR, /* retainHdrFromUltraHdrImage= */ false)); } private static final class FakeSampleConsumer implements SampleConsumer {