loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Generate static HDR metadata when using DefaultEncoderFactory

Browse Source 
* Select an encoder that supports HDR editing.
* Set KEY_PROFILE to an HDR10 option
* Use DecodeOneFrameUtil test util to return the MediaCodec format,
  which includes HDR_STATIC_INFO

PiperOrigin-RevId: 702752639
This commit is contained in:
dancho 2024-12-04 09:26:31 -08:00 committed by Copybara-Service
parent 3e94bd6125
commit 6193f7c38f
5 changed files with 152 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
RELEASENOTES.md
View File

@ -29,6 +29,7 @@
`VideoFrameProcessor.Listener.onInputStreamRegistered` to use `Format`.
* Add support for transmuxing into alternative backwards compatible
formats.
* Generate HDR static metadata when using `DefaultEncoderFactory`.
* Extractors:
* MP3: Don't stop playback early when a `VBRI` frame's table of contents
doesn't cover all the MP3 data in a file

26
libraries/test_utils/src/main/java/androidx/media3/test/utils/DecodeOneFrameUtil.java
View File

@ -18,6 +18,7 @@ package androidx.media3.test.utils;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.MediaFormatUtil.createMediaFormatFromFormat;
import static androidx.media3.common.util.Util.postOrRun;
import static androidx.media3.test.utils.TestUtil.buildAssetUri;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
@ -70,6 +71,21 @@ public final class DecodeOneFrameUtil {
@SuppressWarnings("CatchingUnchecked")
public static void decodeOneAssetFileFrame(
String assetFilePath, Listener listener, Surface surface) throws Exception {
decodeOneMediaItemFrame(MediaItem.fromUri(buildAssetUri(assetFilePath)), listener, surface);
}
/**
* Reads and decodes one frame synchronously from the {@code mediaItem} and renders it to the
* {@code surface}.
*
* <p>This method blocks until the frame has been rendered to the {@code surface}.
*
* @param mediaItem The {@link MediaItem} from which to decode a frame.
* @param listener A {@link Listener} implementation.
* @param surface The {@link Surface} to render the decoded frame to.
*/
public static void decodeOneMediaItemFrame(
MediaItem mediaItem, Listener listener, Surface surface) throws Exception {
Context context = getApplicationContext();
AtomicReference<@NullableType Exception> unexpectedExceptionReference = new AtomicReference<>();
AtomicReference<@NullableType PlaybackException> playbackExceptionReference =
@ -77,6 +93,12 @@ public final class DecodeOneFrameUtil {
ConditionVariable firstFrameRenderedOrError = new ConditionVariable();
ExoPlayer exoPlayer = new ExoPlayer.Builder(context).build();
postOrRun(
new Handler(exoPlayer.getApplicationLooper()),
() ->
exoPlayer.setVideoFrameMetadataListener(
(presentationTimeUs, releaseTimeNs, format, mediaFormat) ->
listener.onFrameDecoded(checkNotNull(mediaFormat))));
Handler handler = new Handler(exoPlayer.getApplicationLooper());
AnalyticsListener analyticsListener =
new AnalyticsListener() {
@ -93,8 +115,6 @@ public final class DecodeOneFrameUtil {
if (exoPlayer.isReleased()) {
return;
}
listener.onFrameDecoded(
createMediaFormatFromFormat(checkNotNull(exoPlayer.getVideoFormat())));
firstFrameRenderedOrError.open();
}
@ -115,7 +135,7 @@ public final class DecodeOneFrameUtil {
try {
exoPlayer.setVideoSurface(surface);
exoPlayer.addAnalyticsListener(analyticsListener);
exoPlayer.setMediaItem(MediaItem.fromUri(buildAssetUri(assetFilePath)));
exoPlayer.setMediaItem(mediaItem);
exoPlayer.setPlayWhenReady(false);
exoPlayer.prepare();
// Catch all exceptions to report. Exceptions thrown here and not caught will not

70
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java
View File

@ -16,6 +16,7 @@
package androidx.media3.transformer.mh;
import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_ORIGINAL;
import static androidx.media3.test.utils.DecodeOneFrameUtil.decodeOneMediaItemFrame;
import static androidx.media3.test.utils.TestUtil.retrieveTrackFormat;
import static androidx.media3.transformer.AndroidTestUtil.FORCE_TRANSCODE_VIDEO_EFFECTS;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10;
@ -30,9 +31,12 @@ import static androidx.media3.transformer.mh.HdrCapabilitiesUtil.assumeDeviceDoe
import static androidx.media3.transformer.mh.HdrCapabilitiesUtil.assumeDeviceSupportsHdrEditing;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat;
import static java.util.Collections.max;
import android.content.Context;
import android.media.MediaFormat;
import android.net.Uri;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
@ -40,6 +44,8 @@ import androidx.media3.common.MediaItem;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Util;
import androidx.media3.effect.DefaultVideoFrameProcessor;
import androidx.media3.exoplayer.video.PlaceholderSurface;
import androidx.media3.test.utils.DecodeOneFrameUtil;
import androidx.media3.transformer.Composition;
import androidx.media3.transformer.EditedMediaItem;
import androidx.media3.transformer.EncoderUtil;
@ -50,8 +56,13 @@ import androidx.media3.transformer.Transformer;
import androidx.media3.transformer.TransformerAndroidTestRunner;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.After;
import org.junit.AssumptionViolatedException;
import org.junit.Before;
import org.junit.Rule;
@ -69,12 +80,20 @@ public final class HdrEditingTest {
@Rule public final TestName testName = new TestName();
private String testId;
@Nullable private Surface placeholderSurface;
@Before
public void setUpTestId() {
testId = testName.getMethodName();
}
@After
public void tearDown() {
if (placeholderSurface != null) {
placeholderSurface.release();
}
}
@Test
public void export_transmuxHdr10File() throws Exception {
Context context = ApplicationProvider.getApplicationContext();
@ -154,12 +173,12 @@ public final class HdrEditingTest {
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
@C.ColorTransfer
int actualColorTransfer =
retrieveTrackFormat(context, exportTestResult.filePath, C.TRACK_TYPE_VIDEO)
.colorInfo
.colorTransfer;
assertThat(actualColorTransfer).isEqualTo(C.COLOR_TRANSFER_ST2084);
MediaFormat mediaFormat = getVideoMediaFormatFromDecoder(context, exportTestResult.filePath);
ByteBuffer hdrStaticInfo = mediaFormat.getByteBuffer(MediaFormat.KEY_HDR_STATIC_INFO);
assertThat(max(byteList(hdrStaticInfo))).isAtLeast((byte) 1);
assertThat(mediaFormat.getInteger(MediaFormat.KEY_COLOR_TRANSFER))
.isEqualTo(MediaFormat.COLOR_TRANSFER_ST2084);
}
@Test
@ -246,10 +265,14 @@ public final class HdrEditingTest {
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
MediaFormat mediaFormat = getVideoMediaFormatFromDecoder(context, exportTestResult.filePath);
ByteBuffer hdrStaticInfo = mediaFormat.getByteBuffer(MediaFormat.KEY_HDR_STATIC_INFO);
Format outputFormat =
retrieveTrackFormat(context, exportTestResult.filePath, C.TRACK_TYPE_VIDEO);
assertThat(outputFormat.colorInfo.colorTransfer).isEqualTo(C.COLOR_TRANSFER_ST2084);
assertThat(outputFormat.sampleMimeType).isEqualTo(MimeTypes.VIDEO_H265);
assertThat(max(byteList(hdrStaticInfo))).isAtLeast((byte) 1);
}
@Test
@ -401,4 +424,39 @@ public final class HdrEditingTest {
throw exception;
}
}
private static List<Byte> byteList(ByteBuffer buffer) {
ArrayList<Byte> outputBytes = new ArrayList<>();
while (buffer.hasRemaining()) {
outputBytes.add(buffer.get());
}
return outputBytes;
}
/**
* Returns the {@link MediaFormat} corresponding to the video track in {@code filePath}.
*
* <p>HDR metadata is optional in both the container and bitstream. Return the {@link MediaFormat}
* produced by the decoder which should include any metadata from either container or bitstream.
*/
private MediaFormat getVideoMediaFormatFromDecoder(Context context, String filePath)
throws Exception {
AtomicReference<MediaFormat> decodedFrameFormat = new AtomicReference<>();
if (placeholderSurface == null) {
placeholderSurface = PlaceholderSurface.newInstance(context, false);
}
decodeOneMediaItemFrame(
MediaItem.fromUri(filePath),
new DecodeOneFrameUtil.Listener() {
@Override
public void onContainerExtracted(MediaFormat mediaFormat) {}
@Override
public void onFrameDecoded(MediaFormat mediaFormat) {
decodedFrameFormat.set(mediaFormat);
}
},
placeholderSurface);
return decodedFrameFormat.get();
}
}

31
libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java
View File

@ -23,6 +23,7 @@ import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.common.util.MediaFormatUtil.createMediaFormatFromFormat;
import static androidx.media3.common.util.Util.SDK_INT;
import static androidx.media3.transformer.EncoderUtil.getCodecProfilesForHdrFormat;
import static java.lang.Math.abs;
import static java.lang.Math.floor;
import static java.lang.Math.max;
@ -318,6 +319,10 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
// the values.
mediaFormat.setInteger(MediaFormat.KEY_PROFILE, supportedVideoEncoderSettings.profile);
mediaFormat.setInteger(MediaFormat.KEY_LEVEL, supportedVideoEncoderSettings.level);
} else if (SDK_INT >= 24 && ColorInfo.isTransferHdr(format.colorInfo)) {
ImmutableList<Integer> codecProfilesForHdrFormat =
getCodecProfilesForHdrFormat(mimeType, checkNotNull(format.colorInfo).colorTransfer);
mediaFormat.setInteger(MediaFormat.KEY_PROFILE, codecProfilesForHdrFormat.get(0));
}
if (mimeType.equals(MimeTypes.VIDEO_H264)) {
@ -417,6 +422,13 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
filteredEncoderInfos.get(0), requestedFormat, videoEncoderSettings);
}
filteredEncoderInfos =
filterEncodersByHdrEditingSupport(
filteredEncoderInfos, mimeType, requestedFormat.colorInfo);
if (filteredEncoderInfos.isEmpty()) {
return null;
}
filteredEncoderInfos =
filterEncodersByResolution(
filteredEncoderInfos, mimeType, requestedFormat.width, requestedFormat.height);
@ -542,6 +554,23 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
: Integer.MAX_VALUE); // Drops encoder.
}
/**
* Returns a list of encoders that support the requested {@link ColorInfo#colorTransfer}, or all
* input encoders if HDR editing is not needed.
*/
private static ImmutableList<MediaCodecInfo> filterEncodersByHdrEditingSupport(
List<MediaCodecInfo> encoders, String mimeType, @Nullable ColorInfo colorInfo) {
if (Util.SDK_INT < 33 || !ColorInfo.isTransferHdr(colorInfo)) {
return ImmutableList.copyOf(encoders);
}
return filterEncoders(
encoders,
/* cost= */ (encoderInfo) ->
EncoderUtil.isHdrEditingSupported(encoderInfo, mimeType, checkNotNull(colorInfo))
? 0
: Integer.MAX_VALUE); // Drops encoder.
}
private static final class VideoEncoderQueryResult {
public final MediaCodecInfo encoder;
public final Format supportedFormat;
@ -614,7 +643,7 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
if (colorInfo != null) {
int colorTransfer = colorInfo.colorTransfer;
ImmutableList<Integer> codecProfiles =
EncoderUtil.getCodecProfilesForHdrFormat(mimeType, colorTransfer);
getCodecProfilesForHdrFormat(mimeType, colorTransfer);
if (!codecProfiles.isEmpty()) {
// Default to the most compatible profile, which is first in the list.
expectedEncodingProfile = codecProfiles.get(0);

53
libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java
View File

@ -90,36 +90,51 @@ public final class EncoderUtil {
}
ImmutableList<MediaCodecInfo> encoders = getSupportedEncoders(mimeType);
ImmutableList<Integer> allowedColorProfiles =
getCodecProfilesForHdrFormat(mimeType, colorInfo.colorTransfer);
ImmutableList.Builder<MediaCodecInfo> resultBuilder = new ImmutableList.Builder<>();
for (int i = 0; i < encoders.size(); i++) {
MediaCodecInfo mediaCodecInfo = encoders.get(i);
if (mediaCodecInfo.isAlias()) {
continue;
}
boolean hasNeededHdrSupport =
isFeatureSupported(
mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing)
|| (colorInfo.colorTransfer == C.COLOR_TRANSFER_HLG
&& Util.SDK_INT >= 35
&& isFeatureSupported(
mediaCodecInfo,
mimeType,
MediaCodecInfo.CodecCapabilities.FEATURE_HlgEditing));
if (!hasNeededHdrSupport) {
continue;
}
for (MediaCodecInfo.CodecProfileLevel codecProfileLevel :
mediaCodecInfo.getCapabilitiesForType(mimeType).profileLevels) {
if (allowedColorProfiles.contains(codecProfileLevel.profile)) {
resultBuilder.add(mediaCodecInfo);
}
if (isHdrEditingSupported(mediaCodecInfo, mimeType, colorInfo)) {
resultBuilder.add(mediaCodecInfo);
}
}
return resultBuilder.build();
}
/**
* Returns whether HDR editing with the given {@linkplain ColorInfo color transfer} is supported
* by the given {@linkplain MediaCodecInfo encoder}.
*
* @param mediaCodecInfo The encoder.
* @param mimeType The MIME type of the video stream.
* @param colorInfo The color info.
*/
@RequiresApi(33)
public static boolean isHdrEditingSupported(
MediaCodecInfo mediaCodecInfo, String mimeType, ColorInfo colorInfo) {
ImmutableList<Integer> allowedColorProfiles =
getCodecProfilesForHdrFormat(mimeType, colorInfo.colorTransfer);
boolean hasNeededHdrSupport =
isFeatureSupported(
mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HdrEditing)
|| (colorInfo.colorTransfer == C.COLOR_TRANSFER_HLG
&& Util.SDK_INT >= 35
&& isFeatureSupported(
mediaCodecInfo, mimeType, MediaCodecInfo.CodecCapabilities.FEATURE_HlgEditing));
if (!hasNeededHdrSupport) {
return false;
}
for (MediaCodecInfo.CodecProfileLevel codecProfileLevel :
mediaCodecInfo.getCapabilitiesForType(mimeType).profileLevels) {
if (allowedColorProfiles.contains(codecProfileLevel.profile)) {
return true;
}
}
return false;
}
/**
* Returns the {@linkplain MediaCodecInfo.CodecProfileLevel#profile profile} constants that can be
* used to encode the given HDR format, if supported by the device (this method does not check