diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 1ddd49207c..12c02ac609 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -134,6 +134,8 @@ colorspace. * Allow defining indeterminate z-order of EditedMediaItemSequences ([#1055](https://github.com/androidx/media/pull/1055)). + * Maintain a consistent luminance range across different HDR content (uses + the HLG range). * Muxers: * IMA extension: * Promote API that is required for apps to play diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl index 4e2eca5d17..3dfc21bf2d 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_oetf_es3.glsl @@ -31,8 +31,13 @@ uniform int uOutputColorTransfer; uniform mat3 uColorTransform; uniform mat4 uRgbMatrix; -// Output color for an obviously visible error. +// Output colors for an obviously visible error. const vec3 ERROR_COLOR_RED = vec3(1.0, 0.0, 0.0); +const vec3 ERROR_COLOR_BLUE = vec3(0.0, 0.0, 1.0); + +// LINT.IfChange(color_transfer) +const int COLOR_TRANSFER_ST2084 = 6; +const int COLOR_TRANSFER_HLG = 7; // HLG OETF for one channel. highp float hlgOetfSingleChannel(highp float linearChannel) { @@ -75,9 +80,6 @@ highp vec3 pqOetf(highp vec3 linearColor) { // Applies the appropriate OETF to convert linear optical signals to nonlinear // electrical signals. Input and output are both normalized to [0, 1]. highp vec3 applyOetf(highp vec3 linearColor) { - // LINT.IfChange(color_transfer) - const int COLOR_TRANSFER_ST2084 = 6; - const int COLOR_TRANSFER_HLG = 7; if (uOutputColorTransfer == COLOR_TRANSFER_ST2084) { return pqOetf(linearColor); } else if (uOutputColorTransfer == COLOR_TRANSFER_HLG) { @@ -87,9 +89,22 @@ highp vec3 applyOetf(highp vec3 linearColor) { } } +vec3 normalizeHdrLuminance(vec3 inputColor) { + const float PQ_MAX_LUMINANCE = 10000.0; + const float HLG_MAX_LUMINANCE = 1000.0; + if (uOutputColorTransfer == COLOR_TRANSFER_ST2084) { + return inputColor * HLG_MAX_LUMINANCE / PQ_MAX_LUMINANCE; + } else if (uOutputColorTransfer == COLOR_TRANSFER_HLG) { + return inputColor; + } else { + return ERROR_COLOR_BLUE; + } +} + void main() { vec4 inputColor = texture(uTexSampler, vTexSamplingCoord); // transformedColors is an optical color. vec4 transformedColors = uRgbMatrix * vec4(inputColor.rgb, 1); - outColor = vec4(applyOetf(transformedColors.rgb), inputColor.a); + outColor = vec4(applyOetf(normalizeHdrLuminance(transformedColors.rgb)), + inputColor.a); } diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl index 9ea3cbe37c..303ee40243 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_external_yuv_es3.glsl @@ -284,13 +284,25 @@ vec3 yuvToRgb(vec3 yuv) { return clamp(uYuvToRgbColorTransform * (yuv - yuvOffset), 0.0, 1.0); } +vec3 scaleHdrLuminance(vec3 inputColor) { + const float PQ_MAX_LUMINANCE = 10000.0; + const float HLG_MAX_LUMINANCE = 1000.0; + if (uInputColorTransfer == COLOR_TRANSFER_ST2084) { + return inputColor * PQ_MAX_LUMINANCE / HLG_MAX_LUMINANCE; + } else if (uInputColorTransfer == COLOR_TRANSFER_HLG) { + return inputColor; + } else { + return ERROR_COLOR_BLUE; + } +} + void main() { vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz; vec3 opticalColorBt2020 = applyEotf(yuvToRgb(srcYuv)); vec4 opticalColor = (uApplyHdrToSdrToneMapping == 1) ? vec4(applyBt2020ToBt709Ootf(opticalColorBt2020), 1.0) - : vec4(opticalColorBt2020, 1.0); + : vec4(scaleHdrLuminance(opticalColorBt2020), 1.0); vec4 transformedColors = uRgbMatrix * opticalColor; outColor = vec4(applyOetf(transformedColors.rgb), 1.0); } diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_hdr_internal_es3.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_hdr_internal_es3.glsl index 39ef23154f..7ecf19c28e 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_hdr_internal_es3.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_hdr_internal_es3.glsl @@ -271,13 +271,25 @@ highp vec3 applyOetf(highp vec3 linearColor) { } } +vec3 scaleHdrLuminance(vec3 inputColor) { + const float PQ_MAX_LUMINANCE = 10000.0; + const float HLG_MAX_LUMINANCE = 1000.0; + if (uInputColorTransfer == COLOR_TRANSFER_ST2084) { + return inputColor * PQ_MAX_LUMINANCE / HLG_MAX_LUMINANCE; + } else if (uInputColorTransfer == COLOR_TRANSFER_HLG) { + return inputColor; + } else { + return ERROR_COLOR_BLUE; + } +} + void main() { vec3 opticalColorBt2020 = applyEotf(texture(uTexSampler, vTexSamplingCoord).xyz); vec4 opticalColor = (uApplyHdrToSdrToneMapping == 1) ? vec4(applyBt2020ToBt709Ootf(opticalColorBt2020), 1.0) - : vec4(opticalColorBt2020, 1.0); + : vec4(scaleHdrLuminance(opticalColorBt2020), 1.0); vec4 transformedColors = uRgbMatrix * opticalColor; outColor = vec4(applyOetf(transformedColors.rgb), 1.0); } diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/hdr-goldens/original_hdr10_to_hlg.png b/libraries/test_data/src/test/assets/test-generated-goldens/hdr-goldens/original_hdr10_to_hlg.png new file mode 100644 index 0000000000..3782958ff1 Binary files /dev/null and b/libraries/test_data/src/test/assets/test-generated-goldens/hdr-goldens/original_hdr10_to_hlg.png differ diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/hdr-goldens/original_hlg10_to_pq.png b/libraries/test_data/src/test/assets/test-generated-goldens/hdr-goldens/original_hlg10_to_pq.png new file mode 100644 index 0000000000..b31f31cb0b Binary files /dev/null and b/libraries/test_data/src/test/assets/test-generated-goldens/hdr-goldens/original_hlg10_to_pq.png differ diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/hdr-goldens/ultrahdr_to_pq.png b/libraries/test_data/src/test/assets/test-generated-goldens/hdr-goldens/ultrahdr_to_pq.png index 92b1feaabb..f3a82da76b 100644 Binary files a/libraries/test_data/src/test/assets/test-generated-goldens/hdr-goldens/ultrahdr_to_pq.png and b/libraries/test_data/src/test/assets/test-generated-goldens/hdr-goldens/ultrahdr_to_pq.png differ diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java index fdb4663f73..6b72a7c311 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java @@ -90,6 +90,10 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { "test-generated-goldens/hdr-goldens/ultrahdr_to_hlg.png"; private static final String ULTRA_HDR_TO_PQ_PNG_ASSET_PATH = "test-generated-goldens/hdr-goldens/ultrahdr_to_pq.png"; + private static final String HLG_TO_PQ_PNG_ASSET_PATH = + "test-generated-goldens/hdr-goldens/original_hlg10_to_pq.png"; + private static final String PQ_TO_HLG_PNG_ASSET_PATH = + "test-generated-goldens/hdr-goldens/original_hdr10_to_hlg.png"; /** Input SDR video of which we only use the first frame. */ private static final String INPUT_SDR_MP4_ASSET_STRING = "media/mp4/sample.mp4"; @@ -243,6 +247,40 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16); } + @Test + public void noEffects_hlg10InputAndHdr10Output_matchesGoldenFile() throws Exception { + Context context = getApplicationContext(); + Format inputFormat = MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT; + Format outputFormat = + inputFormat + .buildUpon() + .setColorInfo( + new ColorInfo.Builder() + .setColorSpace(C.COLOR_SPACE_BT2020) + .setColorRange(C.COLOR_RANGE_LIMITED) + .setColorTransfer(C.COLOR_TRANSFER_ST2084) + .build()) + .build(); + assumeDeviceSupportsHdrEditing(testId, inputFormat); + assumeFormatsSupported(context, testId, inputFormat, outputFormat); + videoFrameProcessorTestRunner = + getDefaultFrameProcessorTestRunnerBuilder(testId) + .setOutputColorInfo(outputFormat.colorInfo) + .setVideoAssetPath(INPUT_HLG10_MP4_ASSET_STRING) + .build(); + Bitmap expectedBitmap = readBitmap(HLG_TO_PQ_PNG_ASSET_PATH); + + videoFrameProcessorTestRunner.processFirstFrameAndEnd(); + Bitmap actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); + + // TODO(b/207848601): Switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceFp16( + expectedBitmap, actualBitmap); + assertThat(averagePixelAbsoluteDifference) + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16); + } + @Test public void noEffects_hlg10TextureInput_matchesGoldenFile() throws Exception { Context context = getApplicationContext(); @@ -330,6 +368,40 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16); } + @Test + public void noEffects_hdr10InputAndHlg10Output_matchesGoldenFile() throws Exception { + Context context = getApplicationContext(); + Format inputFormat = MP4_ASSET_720P_4_SECOND_HDR10_FORMAT; + Format outputFormat = + inputFormat + .buildUpon() + .setColorInfo( + new ColorInfo.Builder() + .setColorSpace(C.COLOR_SPACE_BT2020) + .setColorRange(C.COLOR_RANGE_LIMITED) + .setColorTransfer(C.COLOR_TRANSFER_HLG) + .build()) + .build(); + assumeDeviceSupportsHdrEditing(testId, inputFormat); + assumeFormatsSupported(context, testId, inputFormat, outputFormat); + videoFrameProcessorTestRunner = + getDefaultFrameProcessorTestRunnerBuilder(testId) + .setOutputColorInfo(outputFormat.colorInfo) + .setVideoAssetPath(INPUT_PQ_MP4_ASSET_STRING) + .build(); + Bitmap expectedBitmap = readBitmap(PQ_TO_HLG_PNG_ASSET_PATH); + + videoFrameProcessorTestRunner.processFirstFrameAndEnd(); + Bitmap actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); + + // TODO(b/207848601): Switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceFp16( + expectedBitmap, actualBitmap); + assertThat(averagePixelAbsoluteDifference) + .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16); + } + @Test public void noEffects_hdr10TextureInput_matchesGoldenFile() throws Exception { Context context = getApplicationContext();