diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 836e557b32..353d2a1fa9 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -29,6 +29,7 @@ `VideoFrameProcessor.Listener.onInputStreamRegistered` to use `Format`. * Add support for transmuxing into alternative backwards compatible formats. + * Generate HDR static metadata when using `DefaultEncoderFactory`. * Extractors: * MP3: Don't stop playback early when a `VBRI` frame's table of contents doesn't cover all the MP3 data in a file diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/DecodeOneFrameUtil.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/DecodeOneFrameUtil.java index 2592954f0b..675b6070d1 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/DecodeOneFrameUtil.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/DecodeOneFrameUtil.java @@ -18,6 +18,7 @@ package androidx.media3.test.utils; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.MediaFormatUtil.createMediaFormatFromFormat; +import static androidx.media3.common.util.Util.postOrRun; import static androidx.media3.test.utils.TestUtil.buildAssetUri; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; @@ -70,6 +71,21 @@ public final class DecodeOneFrameUtil { @SuppressWarnings("CatchingUnchecked") public static void decodeOneAssetFileFrame( String assetFilePath, Listener listener, Surface surface) throws Exception { + decodeOneMediaItemFrame(MediaItem.fromUri(buildAssetUri(assetFilePath)), listener, surface); + } + + /** + * Reads and decodes one frame synchronously from the {@code mediaItem} and renders it to the + * {@code surface}. + * + *

This method blocks until the frame has been rendered to the {@code surface}. + * + * @param mediaItem The {@link MediaItem} from which to decode a frame. + * @param listener A {@link Listener} implementation. + * @param surface The {@link Surface} to render the decoded frame to. + */ + public static void decodeOneMediaItemFrame( + MediaItem mediaItem, Listener listener, Surface surface) throws Exception { Context context = getApplicationContext(); AtomicReference<@NullableType Exception> unexpectedExceptionReference = new AtomicReference<>(); AtomicReference<@NullableType PlaybackException> playbackExceptionReference = @@ -77,6 +93,12 @@ public final class DecodeOneFrameUtil { ConditionVariable firstFrameRenderedOrError = new ConditionVariable(); ExoPlayer exoPlayer = new ExoPlayer.Builder(context).build(); + postOrRun( + new Handler(exoPlayer.getApplicationLooper()), + () -> + exoPlayer.setVideoFrameMetadataListener( + (presentationTimeUs, releaseTimeNs, format, mediaFormat) -> + listener.onFrameDecoded(checkNotNull(mediaFormat)))); Handler handler = new Handler(exoPlayer.getApplicationLooper()); AnalyticsListener analyticsListener = new AnalyticsListener() { @@ -93,8 +115,6 @@ public final class DecodeOneFrameUtil { if (exoPlayer.isReleased()) { return; } - listener.onFrameDecoded( - createMediaFormatFromFormat(checkNotNull(exoPlayer.getVideoFormat()))); firstFrameRenderedOrError.open(); } @@ -115,7 +135,7 @@ public final class DecodeOneFrameUtil { try { exoPlayer.setVideoSurface(surface); exoPlayer.addAnalyticsListener(analyticsListener); - exoPlayer.setMediaItem(MediaItem.fromUri(buildAssetUri(assetFilePath))); + exoPlayer.setMediaItem(mediaItem); exoPlayer.setPlayWhenReady(false); exoPlayer.prepare(); // Catch all exceptions to report. Exceptions thrown here and not caught will not diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java index e16156a86b..e415bb1ce2 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java @@ -16,6 +16,7 @@ package androidx.media3.transformer.mh; import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_ORIGINAL; +import static androidx.media3.test.utils.DecodeOneFrameUtil.decodeOneMediaItemFrame; import static androidx.media3.test.utils.TestUtil.retrieveTrackFormat; import static androidx.media3.transformer.AndroidTestUtil.FORCE_TRANSCODE_VIDEO_EFFECTS; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10; @@ -30,9 +31,12 @@ import static androidx.media3.transformer.mh.HdrCapabilitiesUtil.assumeDeviceDoe import static androidx.media3.transformer.mh.HdrCapabilitiesUtil.assumeDeviceSupportsHdrEditing; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; +import static java.util.Collections.max; import android.content.Context; +import android.media.MediaFormat; import android.net.Uri; +import android.view.Surface; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; @@ -40,6 +44,8 @@ import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.Util; import androidx.media3.effect.DefaultVideoFrameProcessor; +import androidx.media3.exoplayer.video.PlaceholderSurface; +import androidx.media3.test.utils.DecodeOneFrameUtil; import androidx.media3.transformer.Composition; import androidx.media3.transformer.EditedMediaItem; import androidx.media3.transformer.EncoderUtil; @@ -50,8 +56,13 @@ import androidx.media3.transformer.Transformer; import androidx.media3.transformer.TransformerAndroidTestRunner; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.After; import org.junit.AssumptionViolatedException; import org.junit.Before; import org.junit.Rule; @@ -69,12 +80,20 @@ public final class HdrEditingTest { @Rule public final TestName testName = new TestName(); private String testId; + @Nullable private Surface placeholderSurface; @Before public void setUpTestId() { testId = testName.getMethodName(); } + @After + public void tearDown() { + if (placeholderSurface != null) { + placeholderSurface.release(); + } + } + @Test public void export_transmuxHdr10File() throws Exception { Context context = ApplicationProvider.getApplicationContext(); @@ -154,12 +173,12 @@ public final class HdrEditingTest { new TransformerAndroidTestRunner.Builder(context, transformer) .build() .run(testId, editedMediaItem); - @C.ColorTransfer - int actualColorTransfer = - retrieveTrackFormat(context, exportTestResult.filePath, C.TRACK_TYPE_VIDEO) - .colorInfo - .colorTransfer; - assertThat(actualColorTransfer).isEqualTo(C.COLOR_TRANSFER_ST2084); + MediaFormat mediaFormat = getVideoMediaFormatFromDecoder(context, exportTestResult.filePath); + ByteBuffer hdrStaticInfo = mediaFormat.getByteBuffer(MediaFormat.KEY_HDR_STATIC_INFO); + + assertThat(max(byteList(hdrStaticInfo))).isAtLeast((byte) 1); + assertThat(mediaFormat.getInteger(MediaFormat.KEY_COLOR_TRANSFER)) + .isEqualTo(MediaFormat.COLOR_TRANSFER_ST2084); } @Test @@ -246,10 +265,14 @@ public final class HdrEditingTest { new TransformerAndroidTestRunner.Builder(context, transformer) .build() .run(testId, editedMediaItem); + MediaFormat mediaFormat = getVideoMediaFormatFromDecoder(context, exportTestResult.filePath); + ByteBuffer hdrStaticInfo = mediaFormat.getByteBuffer(MediaFormat.KEY_HDR_STATIC_INFO); + Format outputFormat = retrieveTrackFormat(context, exportTestResult.filePath, C.TRACK_TYPE_VIDEO); assertThat(outputFormat.colorInfo.colorTransfer).isEqualTo(C.COLOR_TRANSFER_ST2084); assertThat(outputFormat.sampleMimeType).isEqualTo(MimeTypes.VIDEO_H265); + assertThat(max(byteList(hdrStaticInfo))).isAtLeast((byte) 1); } @Test @@ -401,4 +424,39 @@ public final class HdrEditingTest { throw exception; } } + + private static List byteList(ByteBuffer buffer) { + ArrayList outputBytes = new ArrayList<>(); + while (buffer.hasRemaining()) { + outputBytes.add(buffer.get()); + } + return outputBytes; + } + + /** + * Returns the {@link MediaFormat} corresponding to the video track in {@code filePath}. + * + *

HDR metadata is optional in both the container and bitstream. Return the {@link MediaFormat} + * produced by the decoder which should include any metadata from either container or bitstream. + */ + private MediaFormat getVideoMediaFormatFromDecoder(Context context, String filePath) + throws Exception { + AtomicReference decodedFrameFormat = new AtomicReference<>(); + if (placeholderSurface == null) { + placeholderSurface = PlaceholderSurface.newInstance(context, false); + } + decodeOneMediaItemFrame( + MediaItem.fromUri(filePath), + new DecodeOneFrameUtil.Listener() { + @Override + public void onContainerExtracted(MediaFormat mediaFormat) {} + + @Override + public void onFrameDecoded(MediaFormat mediaFormat) { + decodedFrameFormat.set(mediaFormat); + } + }, + placeholderSurface); + return decodedFrameFormat.get(); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java index 623574040e..3b26bff9f2 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java @@ -23,6 +23,7 @@ import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.common.util.MediaFormatUtil.createMediaFormatFromFormat; import static androidx.media3.common.util.Util.SDK_INT; +import static androidx.media3.transformer.EncoderUtil.getCodecProfilesForHdrFormat; import static java.lang.Math.abs; import static java.lang.Math.floor; import static java.lang.Math.max; @@ -318,6 +319,10 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { // the values. mediaFormat.setInteger(MediaFormat.KEY_PROFILE, supportedVideoEncoderSettings.profile); mediaFormat.setInteger(MediaFormat.KEY_LEVEL, supportedVideoEncoderSettings.level); + } else if (SDK_INT >= 24 && ColorInfo.isTransferHdr(format.colorInfo)) { + ImmutableList codecProfilesForHdrFormat = + getCodecProfilesForHdrFormat(mimeType, checkNotNull(format.colorInfo).colorTransfer); + mediaFormat.setInteger(MediaFormat.KEY_PROFILE, codecProfilesForHdrFormat.get(0)); } if (mimeType.equals(MimeTypes.VIDEO_H264)) { @@ -417,6 +422,13 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { filteredEncoderInfos.get(0), requestedFormat, videoEncoderSettings); } + filteredEncoderInfos = + filterEncodersByHdrEditingSupport( + filteredEncoderInfos, mimeType, requestedFormat.colorInfo); + if (filteredEncoderInfos.isEmpty()) { + return null; + } + filteredEncoderInfos = filterEncodersByResolution( filteredEncoderInfos, mimeType, requestedFormat.width, requestedFormat.height); @@ -542,6 +554,23 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { : Integer.MAX_VALUE); // Drops encoder. } + /** + * Returns a list of encoders that support the requested {@link ColorInfo#colorTransfer}, or all + * input encoders if HDR editing is not needed. + */ + private static ImmutableList filterEncodersByHdrEditingSupport( + List encoders, String mimeType, @Nullable ColorInfo colorInfo) { + if (Util.SDK_INT < 33 || !ColorInfo.isTransferHdr(colorInfo)) { + return ImmutableList.copyOf(encoders); + } + return filterEncoders( + encoders, + /* cost= */ (encoderInfo) -> + EncoderUtil.isHdrEditingSupported(encoderInfo, mimeType, checkNotNull(colorInfo)) + ? 0 + : Integer.MAX_VALUE); // Drops encoder. + } + private static final class VideoEncoderQueryResult { public final MediaCodecInfo encoder; public final Format supportedFormat; @@ -614,7 +643,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { if (colorInfo != null) { int colorTransfer = colorInfo.colorTransfer; ImmutableList codecProfiles = - EncoderUtil.getCodecProfilesForHdrFormat(mimeType, colorTransfer); + getCodecProfilesForHdrFormat(mimeType, colorTransfer); if (!codecProfiles.isEmpty()) { // Default to the most compatible profile, which is first in the list. expectedEncodingProfile = codecProfiles.get(0); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java index 92f96ada06..2f8a17576c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java @@ -90,36 +90,51 @@ public final class EncoderUtil { } ImmutableList encoders = getSupportedEncoders(mimeType); - ImmutableList allowedColorProfiles = - getCodecProfilesForHdrFormat(mimeType, colorInfo.colorTransfer); ImmutableList.Builder resultBuilder = new ImmutableList.Builder<>(); for (int i = 0; i < encoders.size(); i++) { MediaCodecInfo mediaCodecInfo = encoders.get(i); if (mediaCodecInfo.isAlias()) { continue; } - boolean hasNeededHdrSupport = - isFeatureSupported( - mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing) - || (colorInfo.colorTransfer == C.COLOR_TRANSFER_HLG - && Util.SDK_INT >= 35 - && isFeatureSupported( - mediaCodecInfo, - mimeType, - MediaCodecInfo.CodecCapabilities.FEATURE_HlgEditing)); - if (!hasNeededHdrSupport) { - continue; - } - for (MediaCodecInfo.CodecProfileLevel codecProfileLevel : - mediaCodecInfo.getCapabilitiesForType(mimeType).profileLevels) { - if (allowedColorProfiles.contains(codecProfileLevel.profile)) { - resultBuilder.add(mediaCodecInfo); - } + if (isHdrEditingSupported(mediaCodecInfo, mimeType, colorInfo)) { + resultBuilder.add(mediaCodecInfo); } } return resultBuilder.build(); } + /** + * Returns whether HDR editing with the given {@linkplain ColorInfo color transfer} is supported + * by the given {@linkplain MediaCodecInfo encoder}. + * + * @param mediaCodecInfo The encoder. + * @param mimeType The MIME type of the video stream. + * @param colorInfo The color info. + */ + @RequiresApi(33) + public static boolean isHdrEditingSupported( + MediaCodecInfo mediaCodecInfo, String mimeType, ColorInfo colorInfo) { + ImmutableList allowedColorProfiles = + getCodecProfilesForHdrFormat(mimeType, colorInfo.colorTransfer); + boolean hasNeededHdrSupport = + isFeatureSupported( + mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing) + || (colorInfo.colorTransfer == C.COLOR_TRANSFER_HLG + && Util.SDK_INT >= 35 + && isFeatureSupported( + mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HlgEditing)); + if (!hasNeededHdrSupport) { + return false; + } + for (MediaCodecInfo.CodecProfileLevel codecProfileLevel : + mediaCodecInfo.getCapabilitiesForType(mimeType).profileLevels) { + if (allowedColorProfiles.contains(codecProfileLevel.profile)) { + return true; + } + } + return false; + } + /** * Returns the {@linkplain MediaCodecInfo.CodecProfileLevel#profile profile} constants that can be * used to encode the given HDR format, if supported by the device (this method does not check