From 6dcfe44b89eec8c0483c84f41f08358b4e5d6364 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Thu, 8 Jun 2023 10:52:30 +0000 Subject: [PATCH] Test: Add no-op effect test for GL tone mapping. To ensure no regressions for the potentially confusing pipeline of: * HDR electrical -> SDR linear EOTF+OOTF, and * SDR linear -> SDR electrical OETF PiperOrigin-RevId: 538741079 (cherry picked from commit 0c924fcb4072d7485e0e5b61097c2e7c37eb6b6a) --- .../DefaultVideoFrameProcessorPixelTest.java | 3 - ...oFrameProcessorTextureOutputPixelTest.java | 32 +------ .../ToneMapHdrToSdrUsingOpenGlPixelTest.java | 86 ++++++++++++++++++- .../transformer/mh/UnoptimizedGlEffect.java | 51 +++++++++++ 4 files changed, 136 insertions(+), 36 deletions(-) create mode 100644 libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/UnoptimizedGlEffect.java diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java index bd7b9eb4aa..32f4189be1 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java @@ -577,9 +577,6 @@ public final class DefaultVideoFrameProcessorPixelTest { assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } - // TODO(b/227624622): Add a test for HDR input after BitmapPixelTestUtil can read HDR bitmaps, - // using GlEffectWrapper to ensure usage of intermediate textures. - private VideoFrameProcessorTestRunner.Builder getDefaultFrameProcessorTestRunnerBuilder( String testId) { return new VideoFrameProcessorTestRunner.Builder() diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java index e936606aea..c34e391338 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java @@ -26,6 +26,7 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECO import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_720P_4_SECOND_HDR10_FORMAT; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_FORMAT; import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped; +import static androidx.media3.transformer.mh.UnoptimizedGlEffect.NO_OP_EFFECT; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; @@ -42,10 +43,7 @@ import androidx.media3.common.util.GlUtil; import androidx.media3.effect.BitmapOverlay; import androidx.media3.effect.DefaultGlObjectsProvider; import androidx.media3.effect.DefaultVideoFrameProcessor; -import androidx.media3.effect.GlEffect; -import androidx.media3.effect.GlShaderProgram; import androidx.media3.effect.OverlayEffect; -import androidx.media3.effect.ScaleAndRotateTransformation; import androidx.media3.test.utils.BitmapPixelTestUtil; import androidx.media3.test.utils.VideoFrameProcessorTestRunner; import androidx.media3.transformer.AndroidTestUtil; @@ -86,11 +84,6 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { /** Input HLG video of which we only use the first frame. */ private static final String INPUT_HLG10_MP4_ASSET_STRING = "media/mp4/hlg-1080p.mp4"; - // A passthrough effect allows for testing having an intermediate effect injected, which uses - // different OpenGL shaders from having no effects. - private static final GlEffect NO_OP_EFFECT = - new GlEffectWrapper(new ScaleAndRotateTransformation.Builder().build()); - private @MonotonicNonNull VideoFrameProcessorTestRunner videoFrameProcessorTestRunner; @After @@ -578,27 +571,4 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest { checkNotNull(checkNotNull(format).sampleMimeType), format.colorInfo) .isEmpty(); } - - /** - * Wraps a {@link GlEffect} to prevent the {@link DefaultVideoFrameProcessor} from detecting its - * class and optimizing it. - * - *

This ensures that {@link DefaultVideoFrameProcessor} uses a separate {@link GlShaderProgram} - * for the wrapped {@link GlEffect} rather than merging it with preceding or subsequent {@link - * GlEffect} instances and applying them in one combined {@link GlShaderProgram}. - */ - private static final class GlEffectWrapper implements GlEffect { - - private final GlEffect effect; - - public GlEffectWrapper(GlEffect effect) { - this.effect = effect; - } - - @Override - public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) - throws VideoFrameProcessingException { - return effect.toGlShaderProgram(context, useHdr); - } - } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlPixelTest.java index b88d65b21a..7521ceec9e 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlPixelTest.java @@ -22,6 +22,7 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECO import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_720P_4_SECOND_HDR10_FORMAT; import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped; import static androidx.media3.transformer.AndroidTestUtil.skipAndLogIfFormatsUnsupported; +import static androidx.media3.transformer.mh.UnoptimizedGlEffect.NO_OP_EFFECT; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; @@ -37,6 +38,7 @@ import androidx.media3.effect.DefaultVideoFrameProcessor; import androidx.media3.test.utils.DecodeOneFrameUtil; import androidx.media3.test.utils.VideoFrameProcessorTestRunner; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.junit.After; import org.junit.Test; @@ -58,7 +60,7 @@ public final class ToneMapHdrToSdrUsingOpenGlPixelTest { * across codec/OpenGL versions don't affect whether the test passes for most devices, but * substantial distortions introduced by changes in tested components will cause the test to fail. */ - private static final float MAXIMUM_DEVICE_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 5f; + private static final float MAXIMUM_DEVICE_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 6f; // This file is generated on a Pixel 7, because the emulator isn't able to decode HLG to generate // this file. @@ -133,9 +135,48 @@ public final class ToneMapHdrToSdrUsingOpenGlPixelTest { .isAtMost(MAXIMUM_DEVICE_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } + @Test + public void toneMapWithNoOpEffect_hlgFrame_matchesGoldenFile() throws Exception { + String testId = "toneMapWithNoOpEffect_hlgFrame_matchesGoldenFile"; + if (!deviceSupportsOpenGlToneMapping(testId, MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT)) { + return; + } + videoFrameProcessorTestRunner = + getDefaultFrameProcessorTestRunnerBuilder(testId) + .setVideoAssetPath(INPUT_HLG_MP4_ASSET_STRING) + .setInputColorInfo(checkNotNull(MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT.colorInfo)) + .setOutputColorInfo(TONE_MAP_SDR_COLOR) + .setEffects(ImmutableList.of(NO_OP_EFFECT)) + .build(); + Bitmap expectedBitmap = readBitmap(TONE_MAP_HLG_TO_SDR_PNG_ASSET_PATH); + + Bitmap actualBitmap; + try { + videoFrameProcessorTestRunner.processFirstFrameAndEnd(); + actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); + } catch (UnsupportedOperationException e) { + if (e.getMessage() != null + && e.getMessage().equals(DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING)) { + recordTestSkipped( + getApplicationContext(), + testId, + /* reason= */ DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING); + return; + } else { + throw e; + } + } + + Log.i(TAG, "Successfully tone mapped."); + // TODO(b/207848601): Switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference) + .isAtMost(MAXIMUM_DEVICE_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + @Test public void toneMap_pqFrame_matchesGoldenFile() throws Exception { - // TODO(b/239735341): Move this test to mobileharness testing. String testId = "toneMap_pqFrame_matchesGoldenFile"; if (!deviceSupportsOpenGlToneMapping(testId, MP4_ASSET_720P_4_SECOND_HDR10_FORMAT)) { return; @@ -174,6 +215,47 @@ public final class ToneMapHdrToSdrUsingOpenGlPixelTest { .isAtMost(MAXIMUM_DEVICE_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } + @Test + public void toneMapWithNoOpEffect_pqFrame_matchesGoldenFile() throws Exception { + String testId = "toneMapWithNoOpEffect_pqFrame_matchesGoldenFile"; + if (!deviceSupportsOpenGlToneMapping(testId, MP4_ASSET_720P_4_SECOND_HDR10_FORMAT)) { + return; + } + + videoFrameProcessorTestRunner = + getDefaultFrameProcessorTestRunnerBuilder(testId) + .setVideoAssetPath(INPUT_PQ_MP4_ASSET_STRING) + .setInputColorInfo(checkNotNull(MP4_ASSET_720P_4_SECOND_HDR10_FORMAT.colorInfo)) + .setOutputColorInfo(TONE_MAP_SDR_COLOR) + .setEffects(ImmutableList.of(NO_OP_EFFECT)) + .build(); + Bitmap expectedBitmap = readBitmap(TONE_MAP_PQ_TO_SDR_PNG_ASSET_PATH); + + Bitmap actualBitmap; + try { + videoFrameProcessorTestRunner.processFirstFrameAndEnd(); + actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); + } catch (UnsupportedOperationException e) { + if (e.getMessage() != null + && e.getMessage().equals(DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING)) { + recordTestSkipped( + getApplicationContext(), + testId, + /* reason= */ DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING); + return; + } else { + throw e; + } + } + + Log.i(TAG, "Successfully tone mapped."); + // TODO(b/207848601): Switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference) + .isAtMost(MAXIMUM_DEVICE_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + private static boolean deviceSupportsOpenGlToneMapping(String testId, Format inputFormat) throws Exception { Context context = getApplicationContext(); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/UnoptimizedGlEffect.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/UnoptimizedGlEffect.java new file mode 100644 index 0000000000..3fdd4cfff4 --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/UnoptimizedGlEffect.java @@ -0,0 +1,51 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer.mh; + +import android.content.Context; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.effect.DefaultVideoFrameProcessor; +import androidx.media3.effect.GlEffect; +import androidx.media3.effect.GlShaderProgram; +import androidx.media3.effect.ScaleAndRotateTransformation; + +/** + * Wraps a {@link GlEffect} to prevent the {@link DefaultVideoFrameProcessor} from detecting its + * class and optimizing it. + * + *

This ensures that {@link DefaultVideoFrameProcessor} uses a separate {@link GlShaderProgram} + * for the wrapped {@link GlEffect} rather than merging it with preceding or subsequent {@link + * GlEffect} instances and applying them in one combined {@link GlShaderProgram}. + */ +// TODO(b/263395272): Move this to effects/mh tests. +public final class UnoptimizedGlEffect implements GlEffect { + // A passthrough effect allows for testing having an intermediate effect injected, which uses + // different OpenGL shaders from having no effects. + public static final GlEffect NO_OP_EFFECT = + new UnoptimizedGlEffect(new ScaleAndRotateTransformation.Builder().build()); + + private final GlEffect effect; + + public UnoptimizedGlEffect(GlEffect effect) { + this.effect = effect; + } + + @Override + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) + throws VideoFrameProcessingException { + return effect.toGlShaderProgram(context, useHdr); + } +}