From 407bc4fec9d834ad9ab85c21ad236af6bf6fbc55 Mon Sep 17 00:00:00 2001 From: sheenachhabra Date: Thu, 21 Nov 2024 01:05:12 -0800 Subject: [PATCH] Manage all color value conversions in ColorInfo class This CL also aligns supported color space in `media3 common` and `media3 muxer`. Earlier `muxer` would take even those values which are considered invalid in `media3` in general. Earlier muxer would throw if a given `color standard` is not recognized but with the new change, it will rather put default `unspecified` value. #cherrypick PiperOrigin-RevId: 698683312 --- .../androidx/media3/common/ColorInfo.java | 68 ++++++++++++- .../media3/common/util/MediaFormatUtil.java | 34 +++---- .../java/androidx/media3/muxer/Boxes.java | 66 +++---------- .../androidx/media3/muxer/ColorUtils.java | 97 ------------------- .../transmuxed_with_inappmuxer.dump | 2 + .../transmuxed_with_inappmuxer.dump | 2 + .../transmuxed_with_inappmuxer.dump | 2 + .../transmuxed_with_inappmuxer.dump | 2 + 8 files changed, 97 insertions(+), 176 deletions(-) delete mode 100644 libraries/muxer/src/main/java/androidx/media3/muxer/ColorUtils.java diff --git a/libraries/common/src/main/java/androidx/media3/common/ColorInfo.java b/libraries/common/src/main/java/androidx/media3/common/ColorInfo.java index a003ba1f88..b3ce76567b 100644 --- a/libraries/common/src/main/java/androidx/media3/common/ColorInfo.java +++ b/libraries/common/src/main/java/androidx/media3/common/ColorInfo.java @@ -203,7 +203,7 @@ public final class ColorInfo { /** * Returns the {@link C.ColorSpace} corresponding to the given ISO color primary code, as per - * table A.7.21.1 in Rec. ITU-T T.832 (03/2009), or {@link Format#NO_VALUE} if no mapping can be + * table A.7.21.1 in Rec. ITU-T T.832 (06/2019), or {@link Format#NO_VALUE} if no mapping can be * made. */ @Pure @@ -219,13 +219,52 @@ public final class ColorInfo { case 9: return C.COLOR_SPACE_BT2020; default: + // Remaining color primaries are either reserved or unspecified. return Format.NO_VALUE; } } + /** + * Returns the ISO color primary code corresponding to the given {@link C.ColorSpace}, as per + * table A.7.21.1 in Rec. ITU-T T.832 (06/2019). made. + */ + public static int colorSpaceToIsoColorPrimaries(@C.ColorSpace int colorSpace) { + switch (colorSpace) { + // Default to BT.709 SDR as per the recommendation. + case Format.NO_VALUE: + case C.COLOR_SPACE_BT709: + return 1; + case C.COLOR_SPACE_BT601: + return 5; + case C.COLOR_SPACE_BT2020: + return 9; + } + return 1; + } + + /** + * Returns the ISO matrix coefficients code corresponding to the given {@link C.ColorSpace}, as + * per table A.7.21.3 in Rec. ITU-T T.832 (06/2019). + */ + public static int colorSpaceToIsoMatrixCoefficients(@C.ColorSpace int colorSpace) { + switch (colorSpace) { + // Default to BT.709 SDR as per the recommendation. + case Format.NO_VALUE: + case C.COLOR_SPACE_BT709: + return 1; + case C.COLOR_SPACE_BT601: + return 6; + case C.COLOR_SPACE_BT2020: + return 9; + } + return 1; + } + /** * Returns the {@link C.ColorTransfer} corresponding to the given ISO transfer characteristics - * code, as per table A.7.21.2 in Rec. ITU-T T.832 (03/2009), or {@link Format#NO_VALUE} if no + * code, as per table A.7.21.2 in Rec. ITU-T T.832 (06/2019), or {@link Format#NO_VALUE} if no * mapping can be made. */ @Pure @@ -249,6 +288,31 @@ public final class ColorInfo { } } + /** + * Returns the ISO transfer characteristics code corresponding to the given {@link + * C.ColorTransfer}, as per table A.7.21.2 in Rec. ITU-T T.832 (06/2019). + */ + public static int colorTransferToIsoTransferCharacteristics(@C.ColorTransfer int colorTransfer) { + switch (colorTransfer) { + // Default to BT.709 SDR as per the recommendation. + case C.COLOR_TRANSFER_LINEAR: + return 8; + case C.COLOR_TRANSFER_SRGB: + return 13; + case Format.NO_VALUE: + case C.COLOR_TRANSFER_SDR: + return 1; + case C.COLOR_TRANSFER_ST2084: + return 16; + case C.COLOR_TRANSFER_HLG: + return 18; + case C.COLOR_TRANSFER_GAMMA_2_2: + return 4; + } + return 1; + } + /** * Returns whether the {@code ColorInfo} uses an HDR {@link C.ColorTransfer}. * diff --git a/libraries/common/src/main/java/androidx/media3/common/util/MediaFormatUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/MediaFormatUtil.java index f90f4ccc51..fe80b96069 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/MediaFormatUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/MediaFormatUtil.java @@ -96,8 +96,7 @@ public final class MediaFormatUtil { /* defaultValue= */ Format.NO_VALUE)) .setRotationDegrees( getInteger(mediaFormat, MediaFormat.KEY_ROTATION, /* defaultValue= */ 0)) - // TODO(b/278101856): Disallow invalid values after confirming. - .setColorInfo(getColorInfo(mediaFormat, /* allowInvalidValues= */ true)) + .setColorInfo(getColorInfo(mediaFormat)) .setSampleRate( getInteger( mediaFormat, MediaFormat.KEY_SAMPLE_RATE, /* defaultValue= */ Format.NO_VALUE)) @@ -270,13 +269,6 @@ public final class MediaFormatUtil { */ @Nullable public static ColorInfo getColorInfo(MediaFormat mediaFormat) { - return getColorInfo(mediaFormat, /* allowInvalidValues= */ false); - } - - // Internal methods. - - @Nullable - private static ColorInfo getColorInfo(MediaFormat mediaFormat, boolean allowInvalidValues) { if (SDK_INT < 24) { // MediaFormat KEY_COLOR_TRANSFER and other KEY_COLOR values available from API 24. return null; @@ -294,21 +286,17 @@ public final class MediaFormatUtil { @Nullable byte[] hdrStaticInfo = hdrStaticInfoByteBuffer != null ? getArray(hdrStaticInfoByteBuffer) : null; - - if (!allowInvalidValues) { - // Some devices may produce invalid values from MediaFormat#getInteger. - // See b/239435670 for more information. - if (!isValidColorSpace(colorSpace)) { - colorSpace = Format.NO_VALUE; - } - if (!isValidColorRange(colorRange)) { - colorRange = Format.NO_VALUE; - } - if (!isValidColorTransfer(colorTransfer)) { - colorTransfer = Format.NO_VALUE; - } + // Some devices may produce invalid values from MediaFormat#getInteger. + // See b/239435670 for more information. + if (!isValidColorSpace(colorSpace)) { + colorSpace = Format.NO_VALUE; + } + if (!isValidColorRange(colorRange)) { + colorRange = Format.NO_VALUE; + } + if (!isValidColorTransfer(colorTransfer)) { + colorTransfer = Format.NO_VALUE; } - if (colorSpace != Format.NO_VALUE || colorRange != Format.NO_VALUE || colorTransfer != Format.NO_VALUE diff --git a/libraries/muxer/src/main/java/androidx/media3/muxer/Boxes.java b/libraries/muxer/src/main/java/androidx/media3/muxer/Boxes.java index 40d8055d31..beebe65f85 100644 --- a/libraries/muxer/src/main/java/androidx/media3/muxer/Boxes.java +++ b/libraries/muxer/src/main/java/androidx/media3/muxer/Boxes.java @@ -18,8 +18,6 @@ package androidx.media3.muxer; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; -import static androidx.media3.muxer.ColorUtils.MEDIAFORMAT_STANDARD_TO_PRIMARIES_AND_MATRIX; -import static androidx.media3.muxer.ColorUtils.MEDIAFORMAT_TRANSFER_TO_MP4_TRANSFER; import static androidx.media3.muxer.MuxerUtil.UNSIGNED_INT_MAX_VALUE; import static java.lang.Math.abs; import static java.lang.Math.max; @@ -767,12 +765,7 @@ import org.checkerframework.checker.nullness.qual.PolyNull; contents.put(paspBox()); - // Put in a "colr" box if any of the three color format parameters has a non-default (0) value. - // TODO: b/278101856 - Only null check should be enough once we disallow invalid values. - if (format.colorInfo != null - && (format.colorInfo.colorSpace != 0 - || format.colorInfo.colorTransfer != 0 - || format.colorInfo.colorRange != 0)) { + if (format.colorInfo != null) { contents.put(colrBox(format.colorInfo)); } @@ -1544,24 +1537,18 @@ import org.checkerframework.checker.nullness.qual.PolyNull; // The default values for optional fields as per the : Vp9 webm spec - int colourPrimaries = 1; + int colorPrimaries = 1; int transferCharacteristics = 1; int matrixCoefficients = 1; if (format.colorInfo != null) { - ColorInfo colorInfo = format.colorInfo; - if (colorInfo.colorSpace != Format.NO_VALUE) { - colourPrimaries = - MEDIAFORMAT_STANDARD_TO_PRIMARIES_AND_MATRIX.get(colorInfo.colorSpace).get(0); - matrixCoefficients = - MEDIAFORMAT_STANDARD_TO_PRIMARIES_AND_MATRIX.get(colorInfo.colorSpace).get(1); - } - if (colorInfo.colorTransfer != Format.NO_VALUE) { - transferCharacteristics = MEDIAFORMAT_TRANSFER_TO_MP4_TRANSFER.get(colorInfo.colorTransfer); - } + colorPrimaries = ColorInfo.colorSpaceToIsoColorPrimaries(format.colorInfo.colorSpace); + transferCharacteristics = + ColorInfo.colorTransferToIsoTransferCharacteristics(format.colorInfo.colorTransfer); + matrixCoefficients = ColorInfo.colorSpaceToIsoMatrixCoefficients(format.colorInfo.colorSpace); } - contents.put((byte) colourPrimaries); + contents.put((byte) colorPrimaries); contents.put((byte) transferCharacteristics); contents.put((byte) matrixCoefficients); contents.putShort((short) 0); // codecInitializationDataSize must be 0 for VP9 @@ -1652,40 +1639,11 @@ import org.checkerframework.checker.nullness.qual.PolyNull; contents.put((byte) 'l'); contents.put((byte) 'x'); - short primaries = 0; - short transfer = 0; - short matrix = 0; - byte range = 0; - - if (colorInfo.colorSpace != Format.NO_VALUE) { - int standard = colorInfo.colorSpace; - if (standard < 0 || standard >= MEDIAFORMAT_STANDARD_TO_PRIMARIES_AND_MATRIX.size()) { - throw new IllegalArgumentException("Color standard not implemented: " + standard); - } - - primaries = MEDIAFORMAT_STANDARD_TO_PRIMARIES_AND_MATRIX.get(standard).get(0); - matrix = MEDIAFORMAT_STANDARD_TO_PRIMARIES_AND_MATRIX.get(standard).get(1); - } - - if (colorInfo.colorTransfer != Format.NO_VALUE) { - int transferInFormat = colorInfo.colorTransfer; - if (transferInFormat < 0 || transferInFormat >= MEDIAFORMAT_TRANSFER_TO_MP4_TRANSFER.size()) { - throw new IllegalArgumentException("Color transfer not implemented: " + transferInFormat); - } - - transfer = MEDIAFORMAT_TRANSFER_TO_MP4_TRANSFER.get(transferInFormat); - } - - if (colorInfo.colorRange != Format.NO_VALUE) { - int rangeInFormat = colorInfo.colorRange; - // Handled values are 0 (unknown), 1 (full) and 2 (limited). - if (rangeInFormat < 0 || rangeInFormat > 2) { - throw new IllegalArgumentException("Color range not implemented: " + rangeInFormat); - } - - // Set this to 0x80 only for full range, 0 otherwise. - range = rangeInFormat == C.COLOR_RANGE_FULL ? (byte) 0x80 : 0; - } + short primaries = (short) ColorInfo.colorSpaceToIsoColorPrimaries(colorInfo.colorSpace); + short transfer = + (short) ColorInfo.colorTransferToIsoTransferCharacteristics(colorInfo.colorTransfer); + short matrix = (short) ColorInfo.colorSpaceToIsoMatrixCoefficients(colorInfo.colorSpace); + byte range = colorInfo.colorRange == C.COLOR_RANGE_FULL ? (byte) 0x80 : 0; contents.putShort(primaries); contents.putShort(transfer); diff --git a/libraries/muxer/src/main/java/androidx/media3/muxer/ColorUtils.java b/libraries/muxer/src/main/java/androidx/media3/muxer/ColorUtils.java deleted file mode 100644 index 759b4d21a3..0000000000 --- a/libraries/muxer/src/main/java/androidx/media3/muxer/ColorUtils.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.muxer; - -import android.media.MediaFormat; -import com.google.common.collect.ImmutableList; - -/** Utilities for color information. */ -/* package */ final class ColorUtils { - // The constants are defined as per ISO/IEC 29199-2 (mentioned in MP4 spec ISO/IEC 14496-12: - // 8.5.2.3). - - private static final short TRANSFER_SMPTE170_M = 1; // Main; also 6, 14 and 15 - private static final short TRANSFER_UNSPECIFIED = 2; - private static final short TRANSFER_GAMMA22 = 4; - private static final short TRANSFER_GAMMA28 = 5; - private static final short TRANSFER_SMPTE240_M = 7; - private static final short TRANSFER_LINEAR = 8; - private static final short TRANSFER_OTHER = 9; // Also 10 - private static final short TRANSFER_XV_YCC = 11; - private static final short TRANSFER_BT1361 = 12; - private static final short TRANSFER_SRGB = 13; - private static final short TRANSFER_ST2084 = 16; - private static final short TRANSFER_ST428 = 17; - private static final short TRANSFER_HLG = 18; - - // MediaFormat contains three color-related fields: "standard", "transfer" and "range". The color - // standard maps to "primaries" and "matrix" in the "colr" box, while "transfer" and "range" are - // mapped to a single value each (although for "transfer", it's still not the same enum values). - private static final short PRIMARIES_BT709_5 = 1; - private static final short PRIMARIES_UNSPECIFIED = 2; - private static final short PRIMARIES_BT601_6_625 = 5; - private static final short PRIMARIES_BT601_6_525 = 6; // It's also 7? - private static final short PRIMARIES_GENERIC_FILM = 8; - private static final short PRIMARIES_BT2020 = 9; - private static final short PRIMARIES_BT470_6_M = 4; - - private static final short MATRIX_UNSPECIFIED = 2; - private static final short MATRIX_BT709_5 = 1; - private static final short MATRIX_BT601_6 = 6; - private static final short MATRIX_SMPTE240_M = 7; - private static final short MATRIX_BT2020 = 9; - private static final short MATRIX_BT2020_CONSTANT = 10; - private static final short MATRIX_BT470_6_M = 4; - - /** - * Map from {@link MediaFormat} standards to MP4 primaries and matrix indices. - * - *

The i-th element corresponds to a {@link MediaFormat} value of i. - */ - public static final ImmutableList> - MEDIAFORMAT_STANDARD_TO_PRIMARIES_AND_MATRIX = - ImmutableList.of( - ImmutableList.of(PRIMARIES_UNSPECIFIED, MATRIX_UNSPECIFIED), // Unspecified - ImmutableList.of(PRIMARIES_BT709_5, MATRIX_BT709_5), // BT709 - ImmutableList.of(PRIMARIES_BT601_6_625, MATRIX_BT601_6), // BT601_625 - ImmutableList.of(PRIMARIES_BT601_6_625, MATRIX_BT709_5), // BT601_625_Unadjusted - ImmutableList.of(PRIMARIES_BT601_6_525, MATRIX_BT601_6), // BT601_525 - ImmutableList.of(PRIMARIES_BT601_6_525, MATRIX_SMPTE240_M), // BT601_525_Unadjusted - ImmutableList.of(PRIMARIES_BT2020, MATRIX_BT2020), // BT2020 - ImmutableList.of(PRIMARIES_BT2020, MATRIX_BT2020_CONSTANT), // BT2020Constant - ImmutableList.of(PRIMARIES_BT470_6_M, MATRIX_BT470_6_M), // BT470M - ImmutableList.of(PRIMARIES_GENERIC_FILM, MATRIX_BT2020) // Film - ); - - /** - * Map from {@link MediaFormat} standards to MP4 transfer indices. - * - *

The i-th element corresponds to a {@link MediaFormat} value of i. - */ - public static final ImmutableList MEDIAFORMAT_TRANSFER_TO_MP4_TRANSFER = - ImmutableList.of( - TRANSFER_UNSPECIFIED, // Unspecified - TRANSFER_LINEAR, // Linear - TRANSFER_SRGB, // SRGB - TRANSFER_SMPTE170_M, // SMPTE_170M - TRANSFER_GAMMA22, // Gamma22 - TRANSFER_GAMMA28, // Gamma28 - TRANSFER_ST2084, // ST2084 - TRANSFER_HLG // HLG - ); - - private ColorUtils() {} -} diff --git a/libraries/test_data/src/test/assets/transformerdumps/mp4/bbb_176x144_128kbps_15fps_h263.3gp/transmuxed_with_inappmuxer.dump b/libraries/test_data/src/test/assets/transformerdumps/mp4/bbb_176x144_128kbps_15fps_h263.3gp/transmuxed_with_inappmuxer.dump index 782b157bfc..1a2adc1729 100644 --- a/libraries/test_data/src/test/assets/transformerdumps/mp4/bbb_176x144_128kbps_15fps_h263.3gp/transmuxed_with_inappmuxer.dump +++ b/libraries/test_data/src/test/assets/transformerdumps/mp4/bbb_176x144_128kbps_15fps_h263.3gp/transmuxed_with_inappmuxer.dump @@ -19,7 +19,9 @@ track 0: height = 144 frameRate = 15.00 colorInfo: + colorSpace = 1 colorRange = 2 + colorTransfer = 3 lumaBitdepth = 8 chromaBitdepth = 8 metadata = entries=[Mp4Timestamp: creation time=3000000000, modification time=4000000000, timescale=10000] diff --git a/libraries/test_data/src/test/assets/transformerdumps/mp4/bbb_176x144_192kbps_15fps_mpeg4.mp4/transmuxed_with_inappmuxer.dump b/libraries/test_data/src/test/assets/transformerdumps/mp4/bbb_176x144_192kbps_15fps_mpeg4.mp4/transmuxed_with_inappmuxer.dump index 2aa9fd9f07..4c81f79aa2 100644 --- a/libraries/test_data/src/test/assets/transformerdumps/mp4/bbb_176x144_192kbps_15fps_mpeg4.mp4/transmuxed_with_inappmuxer.dump +++ b/libraries/test_data/src/test/assets/transformerdumps/mp4/bbb_176x144_192kbps_15fps_mpeg4.mp4/transmuxed_with_inappmuxer.dump @@ -21,7 +21,9 @@ track 0: height = 144 frameRate = 15.00 colorInfo: + colorSpace = 1 colorRange = 2 + colorTransfer = 3 lumaBitdepth = 8 chromaBitdepth = 8 metadata = entries=[Mp4Timestamp: creation time=3000000000, modification time=4000000000, timescale=10000] diff --git a/libraries/test_data/src/test/assets/transformerdumps/mp4/sample_av1.mp4/transmuxed_with_inappmuxer.dump b/libraries/test_data/src/test/assets/transformerdumps/mp4/sample_av1.mp4/transmuxed_with_inappmuxer.dump index 1159425fde..c3c0d55723 100644 --- a/libraries/test_data/src/test/assets/transformerdumps/mp4/sample_av1.mp4/transmuxed_with_inappmuxer.dump +++ b/libraries/test_data/src/test/assets/transformerdumps/mp4/sample_av1.mp4/transmuxed_with_inappmuxer.dump @@ -19,7 +19,9 @@ track 0: height = 720 frameRate = 29.97 colorInfo: + colorSpace = 1 colorRange = 2 + colorTransfer = 3 lumaBitdepth = 8 chromaBitdepth = 8 metadata = entries=[Mp4Timestamp: creation time=3000000000, modification time=4000000000, timescale=10000] diff --git a/libraries/test_data/src/test/assets/transformerdumps/ts/sample_h264.ts/transmuxed_with_inappmuxer.dump b/libraries/test_data/src/test/assets/transformerdumps/ts/sample_h264.ts/transmuxed_with_inappmuxer.dump index 8f836ff2b6..a30fefa25a 100644 --- a/libraries/test_data/src/test/assets/transformerdumps/ts/sample_h264.ts/transmuxed_with_inappmuxer.dump +++ b/libraries/test_data/src/test/assets/transformerdumps/ts/sample_h264.ts/transmuxed_with_inappmuxer.dump @@ -21,7 +21,9 @@ track 0: height = 480 frameRate = 30.00 colorInfo: + colorSpace = 1 colorRange = 2 + colorTransfer = 3 lumaBitdepth = 8 chromaBitdepth = 8 metadata = entries=[Mp4Timestamp: creation time=3000000000, modification time=4000000000, timescale=10000]