diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl index 66d84116a4..92378ecca0 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl @@ -155,21 +155,8 @@ highp vec3 convertToOutputColors(highp vec3 workingColors) { } } -vec2 getAdjustedTexSamplingCoord(vec2 originalTexSamplingCoord) { - if (uInputColorTransfer == COLOR_TRANSFER_SRGB) { - // Whereas the Android system uses the top-left corner as (0,0) of the - // coordinate system, OpenGL uses the bottom-left corner as (0,0), so the - // texture gets flipped. We flip the texture vertically to ensure the - // orientation of the output is correct. - return vec2(originalTexSamplingCoord.x, 1.0 - originalTexSamplingCoord.y); - } else { - return originalTexSamplingCoord; - } -} - void main() { - vec4 inputColor = - texture2D(uTexSampler, getAdjustedTexSamplingCoord(vTexSamplingCoord)); + vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord); vec3 workingColors = convertToWorkingColors(inputColor.rgb); vec4 transformedColors = uRgbMatrix * vec4(workingColors, 1); gl_FragColor = diff --git a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_ultra_hdr_es3.glsl b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_ultra_hdr_es3.glsl index 4d56aa4d41..a88a294b85 100644 --- a/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_ultra_hdr_es3.glsl +++ b/libraries/effect/src/main/assets/shaders/fragment_shader_transformation_ultra_hdr_es3.glsl @@ -178,20 +178,12 @@ highp vec3 applyOetf(highp vec3 linearColor) { } } -vec2 getVTexSamplingCoord() { - // Whereas the Android system uses the top-left corner as (0,0) of the - // coordinate system, OpenGL uses the bottom-left corner as (0,0), so the - // texture gets flipped. We flip the texture vertically to ensure the - // orientation of the output is correct. - return vec2(vTexSamplingCoord.x, 1.0 - vTexSamplingCoord.y); -} - // Reference: // https://developer.android.com/reference/android/graphics/Gainmap#applying-a-gainmap-manually // Reference Implementation: // https://cs.android.com/android/platform/superproject/main/+/main:frameworks/base/libs/hwui/effects/GainmapRenderer.cpp;l=117-146;drc=fadc20184ccb27fe15bb862e6e03fa6d05d41eac highp vec3 applyGainmapToBase(vec4 S) { - vec4 G = texture(uGainmapTexSampler, getVTexSamplingCoord()); + vec4 G = texture(uGainmapTexSampler, vTexSamplingCoord); float W = clamp((log(HDR_SDR_RATIO) - log(uDisplayRatioSdr)) / (log(uDisplayRatioHdr) - log(uDisplayRatioSdr)), 0.0, 1.0); @@ -225,7 +217,7 @@ highp vec3 bt709ToBt2020(vec3 bt709Color) { } void main() { - vec4 baseElectricalColor = texture(uTexSampler, getVTexSamplingCoord()); + vec4 baseElectricalColor = texture(uTexSampler, vTexSamplingCoord); float alpha = baseElectricalColor.a; vec4 baseOpticalColor = vec4(applyEotf(baseElectricalColor.xyz), alpha); vec3 opticalBt709Color = applyGainmapToBase(baseOpticalColor); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java index 3d299e8318..983bc391af 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java @@ -248,7 +248,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; "uApplyHdrToSdrToneMapping", outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020 ? GL_TRUE : GL_FALSE); } - return createWithSampler(glProgram, inputColorInfo, outputColorInfo, sdrWorkingColorSpace); + ImmutableList matrixTransformations = ImmutableList.of(); + if (inputType == INPUT_TYPE_BITMAP) { + matrixTransformations = + ImmutableList.of( + (MatrixTransformation) + presentationTimeUs -> { + android.graphics.Matrix mirrorY = new android.graphics.Matrix(); + mirrorY.setScale(/* sx= */ 1, /* sy= */ -1); + return mirrorY; + }); + } + return createWithSampler( + glProgram, inputColorInfo, outputColorInfo, sdrWorkingColorSpace, matrixTransformations); } /** @@ -308,7 +320,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } glProgram.setExternalTexturesRequireNearestSampling(sampleWithNearest); - return createWithSampler(glProgram, inputColorInfo, outputColorInfo, sdrWorkingColorSpace); + return createWithSampler( + glProgram, + inputColorInfo, + outputColorInfo, + sdrWorkingColorSpace, + /* matrixTransformations= */ ImmutableList.of()); } /** @@ -379,7 +396,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; GlProgram glProgram, ColorInfo inputColorInfo, ColorInfo outputColorInfo, - @WorkingColorSpace int sdrWorkingColorSpace) { + @WorkingColorSpace int sdrWorkingColorSpace, + ImmutableList matrixTransformations) { boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo); boolean isExpandingColorGamut = (inputColorInfo.colorSpace == C.COLOR_SPACE_BT709 @@ -416,7 +434,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return new DefaultShaderProgram( glProgram, - /* matrixTransformations= */ ImmutableList.of(), + matrixTransformations, /* rgbMatrices= */ ImmutableList.of(), outputColorInfo.colorTransfer, /* useHdr= */ isInputTransferHdr || isExpandingColorGamut); diff --git a/libraries/test_data/src/test/assets/media/png/loremipsum_1920x720.png b/libraries/test_data/src/test/assets/media/png/loremipsum_1920x720.png new file mode 100644 index 0000000000..6ab31ebb84 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/png/loremipsum_1920x720.png differ diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/transformer_sequence_effect_test/export_image_samplesFromTextureCorrectly_0.png b/libraries/test_data/src/test/assets/test-generated-goldens/transformer_sequence_effect_test/export_image_samplesFromTextureCorrectly_0.png new file mode 100644 index 0000000000..92b339b5f8 Binary files /dev/null and b/libraries/test_data/src/test/assets/test-generated-goldens/transformer_sequence_effect_test/export_image_samplesFromTextureCorrectly_0.png differ diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index 80796813f5..c4c52a7692 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -71,6 +71,8 @@ public final class AndroidTestUtil { new ScaleAndRotateTransformation.Builder().setRotationDegrees(45).build())); public static final String PNG_ASSET_URI_STRING = "asset:///media/png/media3test.png"; + public static final String PNG_ASSET_LINES_1080P_URI_STRING = + "asset:///media/png/loremipsum_1920x720.png"; public static final String JPG_ASSET_URI_STRING = "asset:///media/jpeg/london.jpg"; public static final String JPG_PORTRAIT_ASSET_URI_STRING = "asset:///media/jpeg/tokyo.jpg"; public static final String ULTRA_HDR_URI_STRING = "asset:///media/jpeg/ultraHDR.jpg"; diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerSequenceEffectTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerSequenceEffectTest.java index 049c9cc464..ae9041b99c 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerSequenceEffectTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerSequenceEffectTest.java @@ -32,8 +32,10 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_CHECKERBOARD import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_FORMAT; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT; +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_FORMAT; import static androidx.media3.transformer.AndroidTestUtil.MP4_PORTRAIT_ASSET_FORMAT; import static androidx.media3.transformer.AndroidTestUtil.MP4_PORTRAIT_ASSET_URI_STRING; +import static androidx.media3.transformer.AndroidTestUtil.PNG_ASSET_LINES_1080P_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.assumeFormatsSupported; import static androidx.media3.transformer.AndroidTestUtil.extractBitmapsFromVideo; import static androidx.media3.transformer.SequenceEffectTestUtil.NO_EFFECT; @@ -342,6 +344,59 @@ public final class TransformerSequenceEffectTest { assertThat(traceSummary.indexOf(EVENT_SURFACE_TEXTURE_TRANSFORM_FIX)).isNotEqualTo(-1); } + @Test + public void export_image_samplesFromTextureCorrectly() throws Exception { + assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_FORMAT, + /* outputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_FORMAT); + Composition composition = + createComposition( + /* presentation= */ null, + new EditedMediaItem.Builder(MediaItem.fromUri(PNG_ASSET_LINES_1080P_URI_STRING)) + .setFrameRate(30) + .setDurationUs(C.MICROS_PER_SECOND / 4) + .build()); + // Some devices need a very high bitrate to avoid encoding artifacts. + int bitrate = 30_000_000; + if (Ascii.equalsIgnoreCase(Util.MODEL, "mi a2 lite") + || Ascii.equalsIgnoreCase(Util.MODEL, "redmi 8") + || Ascii.equalsIgnoreCase(Util.MODEL, "sm-f711u1") + || Ascii.equalsIgnoreCase(Util.MODEL, "sm-f916u1") + || Ascii.equalsIgnoreCase(Util.MODEL, "sm-f926u1") + || Ascii.equalsIgnoreCase(Util.MODEL, "sm-g981u1") + || Ascii.equalsIgnoreCase(Util.MODEL, "tb-q706")) { + // And some devices need a lower bitrate because VideoDecodingWrapper fails to decode high + // bitrate output, or FrameworkMuxer fails to mux. + bitrate = 10_000_000; + } + Codec.EncoderFactory encoderFactory = + new DefaultEncoderFactory.Builder(context) + .setRequestedVideoEncoderSettings( + new VideoEncoderSettings.Builder().setBitrate(bitrate).build()) + .build(); + Transformer transformer = + new Transformer.Builder(context) + .setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory(encoderFactory)) + .setVideoMimeType("video/avc") + .build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, composition); + + assertThat(checkNotNull(result).filePath).isNotNull(); + // The PSNR threshold was chosen based on: + // Pixel 8 with coordinate rounding error during texture sampling, hits PSNR 23.4. With fix -> + // 29.5 + // Realmi C11 with bug fix hits PSNR 29.94 + // rmx3563 -> 28.8 + assertFirstFrameMatchesExpectedPsnrAndSave( + context, testId, checkNotNull(result.filePath), 28.5f); + } + @Test public void export_withCompositionPresentationAndWithPerMediaItemEffects() throws Exception { // Reference: b/296225823#comment5