loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HDR: Throw when unexpected color transfer encountered.

Browse Source 
This may happen when a containers' color transfer incorrectly does not match
the video's color transfer.

An example of a file with such a mismatch is the current Transformer demo HDR10
sample file.

Manually tested by confirming that no errors are emitted for SDR and HLG sample
files, and that errors are emitted for our incorrect HDR10 sample file.

PiperOrigin-RevId: 461583532
This commit is contained in:
huangdarwin 2022-07-18 11:20:31 +00:00 committed by Rohit Singh
parent 9a895cd18f
commit 9f7a159bc4
5 changed files with 108 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
libraries/test_data/src/test/assets/media/mp4/hdr10-video-with-sdr-container.mp4 Normal file
View File

Binary file not shown.

3
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View File

@ -84,6 +84,9 @@ public final class AndroidTestUtil {
.setFrameRate(30.472f)
.build();
public static final String MP4_ASSET_1080P_1_SECOND_HDR10_VIDEO_SDR_CONTAINER =
"asset:///media/mp4/hdr10-video-with-sdr-container.mp4";
public static final String MP4_REMOTE_10_SECONDS_URI_STRING =
"https://storage.googleapis.com/exoplayer-test-media-1/mp4/android-screens-10s.mp4";
public static final Format MP4_REMOTE_10_SECONDS_FORMAT =

63
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetHdrEditingTransformationTest.java Normal file
View File

@ -0,0 +1,63 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer.mh;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_1_SECOND_HDR10_VIDEO_SDR_CONTAINER;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import android.content.Context;
import android.net.Uri;
import androidx.media3.common.MediaItem;
import androidx.media3.common.util.Util;
import androidx.media3.transformer.TransformationException;
import androidx.media3.transformer.TransformationRequest;
import androidx.media3.transformer.Transformer;
import androidx.media3.transformer.TransformerAndroidTestRunner;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Test;
import org.junit.runner.RunWith;
/** {@link Transformer} instrumentation test for applying an HDR frame edit. */
@RunWith(AndroidJUnit4.class)
public class SetHdrEditingTransformationTest {
@Test
public void videoDecoderUnexpectedColorInfo_completesWithError() {
Context context = ApplicationProvider.getApplicationContext();
if (Util.SDK_INT < 24) {
return;
}
Transformer transformer =
new Transformer.Builder(context)
.setTransformationRequest(
new TransformationRequest.Builder().experimental_setEnableHdrEditing(true).build())
.build();
TransformationException exception =
assertThrows(
TransformationException.class,
() ->
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(
/* testId= */ "videoDecoderUnexpectedColorInfo_completesWithError",
MediaItem.fromUri(
Uri.parse(MP4_ASSET_1080P_1_SECOND_HDR10_VIDEO_SDR_CONTAINER))));
assertThat(exception).hasCauseThat().isInstanceOf(IllegalStateException.class);
assertThat(exception.errorCode).isEqualTo(TransformationException.ERROR_CODE_DECODING_FAILED);
}
}

46
libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodec.java
View File

@ -29,6 +29,7 @@ import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.TraceUtil;
@ -53,6 +54,9 @@ public final class DefaultCodec implements Codec {
private final MediaFormat configurationMediaFormat;
private final Format configurationFormat;
/** The expected {@link ColorInfo} output from the codec. */
@Nullable private final ColorInfo configuredOutputColor;
private final MediaCodec mediaCodec;
@Nullable private final Surface inputSurface;
@ -115,6 +119,12 @@ public final class DefaultCodec implements Codec {
e, configurationMediaFormat, isVideo, isDecoder, mediaCodecName);
}
this.mediaCodec = mediaCodec;
boolean toneMapRequested =
SDK_INT >= 31
&& isDecoder
&& (configurationMediaFormat.getInteger(MediaFormat.KEY_COLOR_TRANSFER_REQUEST, 0)
== MediaFormat.COLOR_TRANSFER_SDR_VIDEO);
configuredOutputColor = toneMapRequested ? null : configurationFormat.colorInfo;
this.inputSurface = inputSurface;
decoderNeedsFrameDroppingWorkaround = decoderNeedsFrameDroppingWorkaround(context);
}
@ -308,6 +318,18 @@ public final class DefaultCodec implements Codec {
if (outputBufferIndex < 0) {
if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
outputFormat = getFormat(mediaCodec.getOutputFormat());
if (!isColorTransferEqual(configuredOutputColor, outputFormat.colorInfo)) {
// TODO(b/237674316): These exceptions throw when the container ColorInfo doesn't match
// the video ColorInfo. Instead of throwing when seeing unexpected ColorInfos, consider
// reconfiguring downstream components (ex. FrameProcessor and encoder) when different
// ColorInfo values are output.
throw createTransformationException(
new IllegalStateException(
"Codec output color format does not match configured color format. Configured: "
+ configurationFormat.colorInfo
+ ". Actual: "
+ outputFormat.colorInfo));
}
}
return false;
}
@ -344,12 +366,21 @@ public final class DefaultCodec implements Codec {
isVideo,
isDecoder,
configurationMediaFormat,
mediaCodec.getName(),
getName(),
isDecoder
? TransformationException.ERROR_CODE_DECODING_FAILED
: TransformationException.ERROR_CODE_ENCODING_FAILED);
}
private static boolean isColorTransferEqual(
@Nullable ColorInfo colorInfo1, @Nullable ColorInfo colorInfo2) {
@C.ColorTransfer
int transfer1 = (colorInfo1 != null) ? colorInfo1.colorTransfer : C.COLOR_TRANSFER_SDR;
@C.ColorTransfer
int transfer2 = (colorInfo2 != null) ? colorInfo2.colorTransfer : C.COLOR_TRANSFER_SDR;
return transfer1 == transfer2;
}
private static TransformationException createInitializationTransformationException(
Exception cause,
MediaFormat mediaFormat,
@ -396,13 +427,20 @@ public final class DefaultCodec implements Codec {
}
String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
Format.Builder formatBuilder =
new Format.Builder()
.setSampleMimeType(mediaFormat.getString(MediaFormat.KEY_MIME))
.setInitializationData(csdBuffers.build());
new Format.Builder().setSampleMimeType(mimeType).setInitializationData(csdBuffers.build());
if (MimeTypes.isVideo(mimeType)) {
formatBuilder
.setWidth(mediaFormat.getInteger(MediaFormat.KEY_WIDTH))
.setHeight(mediaFormat.getInteger(MediaFormat.KEY_HEIGHT));
if (SDK_INT >= 24) {
// TODO(b/227624622): Set hdrStaticInfo accordingly using KEY_HDR_STATIC_INFO.
formatBuilder.setColorInfo(
new ColorInfo(
mediaFormat.getInteger(MediaFormat.KEY_COLOR_STANDARD),
mediaFormat.getInteger(MediaFormat.KEY_COLOR_RANGE),
mediaFormat.getInteger(MediaFormat.KEY_COLOR_TRANSFER),
/* hdrStaticInfo= */ null));
}
} else if (MimeTypes.isAudio(mimeType)) {
// TODO(b/178685617): Only set the PCM encoding for audio/raw, once we have a way to
// simulate more realistic codec input/output formats in tests.

2
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -150,8 +150,6 @@ import org.checkerframework.dataflow.qual.Pure;
decoder =
decoderFactory.createForVideoDecoding(
inputFormat, frameProcessor.getInputSurface(), isToneMappingRequired);
// TODO(b/236316454): Check in the decoder output format whether tone-mapping was actually
// applied and throw an exception if not.
maxPendingFrameCount = decoder.getMaxPendingFrameCount();
}