HDR texture asset loading

PiperOrigin-RevId: 532846248
This commit is contained in:
tofunmi 2023-05-17 18:11:48 +00:00 committed by Tofunmi Adigun-Hameed
parent 538524e579
commit 5fe10d7652
5 changed files with 485 additions and 90 deletions

View File

@ -29,7 +29,7 @@
// for outputting to intermediate shaders, or COLOR_TRANSFER_ST2084 /
// COLOR_TRANSFER_HLG to output electrical colors via an OETF (e.g. to an
// encoder).
// The output will be red if an error has occurred.
// The output will be red or blue if an error has occurred.
#extension GL_OES_EGL_image_external : require
#extension GL_EXT_YUV_target : require

View File

@ -0,0 +1,232 @@
#version 300 es
// Copyright 2022 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 3 fragment shader that:
// 1. Samples electrical (HLG or PQ) BT.2020 RGB from an internal texture.
// 2. Applies an EOTF based on uInputColorTransfer, yielding optical linear
// BT.2020 RGB.
// 3. Optionally applies a BT2020 to BT709 OOTF, if OpenGL tone-mapping is
// requested via uApplyHdrToSdrToneMapping.
// 4. Applies a 4x4 RGB color matrix to change the pixel colors.
// 5. Outputs as requested by uOutputColorTransfer. Use COLOR_TRANSFER_LINEAR
// for outputting to intermediate shaders, or COLOR_TRANSFER_ST2084 /
// COLOR_TRANSFER_HLG to output electrical colors via an OETF (e.g. to an
// encoder).
// The output will be red or blue if an error has occurred.
precision mediump float;
uniform sampler2D uTexSampler;
uniform mat4 uRgbMatrix;
// C.java#ColorTransfer value.
// Only COLOR_TRANSFER_ST2084 and COLOR_TRANSFER_HLG are allowed.
uniform int uInputColorTransfer;
uniform int uApplyHdrToSdrToneMapping;
// C.java#ColorTransfer value.
// Only COLOR_TRANSFER_LINEAR, COLOR_TRANSFER_GAMMA_2_2, COLOR_TRANSFER_ST2084,
// and COLOR_TRANSFER_HLG are allowed.
uniform int uOutputColorTransfer;
in vec2 vTexSamplingCoord;
out vec4 outColor;
// LINT.IfChange(color_transfer)
const int COLOR_TRANSFER_LINEAR = 1;
const int COLOR_TRANSFER_GAMMA_2_2 = 10;
const int COLOR_TRANSFER_ST2084 = 6;
const int COLOR_TRANSFER_HLG = 7;
// TODO(b/227624622): Consider using mediump to save precision, if it won't lead
// to noticeable quantization errors.
// BT.2100 / BT.2020 HLG EOTF for one channel.
highp float hlgEotfSingleChannel(highp float hlgChannel) {
// Specification:
// https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG
// Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=265-279;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float a = 0.17883277;
const highp float b = 0.28466892;
const highp float c = 0.55991073;
return hlgChannel <= 0.5 ? hlgChannel * hlgChannel / 3.0 :
(b + exp((hlgChannel - c) / a)) / 12.0;
}
// BT.2100 / BT.2020 HLG EOTF.
highp vec3 hlgEotf(highp vec3 hlgColor) {
return vec3(
hlgEotfSingleChannel(hlgColor.r),
hlgEotfSingleChannel(hlgColor.g),
hlgEotfSingleChannel(hlgColor.b)
);
}
// BT.2100 / BT.2020 PQ EOTF.
highp vec3 pqEotf(highp vec3 pqColor) {
// Specification:
// https://registry.khronos.org/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_PQ
// Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=250-263;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float m1 = (2610.0 / 16384.0);
const highp float m2 = (2523.0 / 4096.0) * 128.0;
const highp float c1 = (3424.0 / 4096.0);
const highp float c2 = (2413.0 / 4096.0) * 32.0;
const highp float c3 = (2392.0 / 4096.0) * 32.0;
highp vec3 temp = pow(clamp(pqColor, 0.0, 1.0), 1.0 / vec3(m2));
temp = max(temp - c1, 0.0) / (c2 - c3 * temp);
return pow(temp, 1.0 / vec3(m1));
}
// Applies the appropriate EOTF to convert nonlinear electrical values to linear
// optical values. Input and output are both normalized to [0, 1].
highp vec3 applyEotf(highp vec3 electricalColor) {
if (uInputColorTransfer == COLOR_TRANSFER_ST2084) {
return pqEotf(electricalColor);
} else if (uInputColorTransfer == COLOR_TRANSFER_HLG) {
return hlgEotf(electricalColor);
} else {
// Output red as an obviously visible error.
return vec3(1.0, 0.0, 0.0);
}
}
// Apply the HLG BT2020 to BT709 OOTF.
highp vec3 applyHlgBt2020ToBt709Ootf(highp vec3 linearRgbBt2020) {
// Reference ("HLG Reference OOTF" section):
// https://www.itu.int/dms_pubrec/itu-r/rec/bt/R-REC-BT.2100-2-201807-I!!PDF-E.pdf
// Matrix values based on computeXYZMatrix(BT2020Primaries, BT2020WhitePoint)
// https://cs.android.com/android/platform/superproject/+/master:frameworks/base/libs/hwui/utils/HostColorSpace.cpp;l=200-232;drc=86bd214059cd6150304888a285941bf74af5b687
const mat3 RGB_TO_XYZ_BT2020 = mat3(
0.63695805f, 0.26270021f, 0.00000000f,
0.14461690f, 0.67799807f, 0.02807269f,
0.16888098f, 0.05930172f, 1.06098506f);
// Matrix values based on computeXYZMatrix(BT709Primaries, BT709WhitePoint)
const mat3 XYZ_TO_RGB_BT709 = mat3(
3.24096994f, -0.96924364f, 0.05563008f,
-1.53738318f, 1.87596750f, -0.20397696f,
-0.49861076f, 0.04155506f, 1.05697151f);
// hlgGamma is 1.2 + 0.42 * log10(nominalPeakLuminance/1000);
// nominalPeakLuminance was selected to use a 500 as a typical value, used
// in https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/tonemap/tonemap.cpp;drc=7a577450e536aa1e99f229a0cb3d3531c82e8a8d;l=62,
// b/199162498#comment35, and
// https://www.microsoft.com/applied-sciences/uploads/projects/investigation-of-hdr-vs-tone-mapped-sdr/investigation-of-hdr-vs-tone-mapped-sdr.pdf.
const float hlgGamma = 1.0735674018211279;
vec3 linearXyzBt2020 = RGB_TO_XYZ_BT2020 * linearRgbBt2020;
vec3 linearXyzBt709 =
linearXyzBt2020 * pow(linearXyzBt2020[1], hlgGamma - 1.0);
vec3 linearRgbBt709 = clamp((XYZ_TO_RGB_BT709 * linearXyzBt709), 0.0, 1.0);
return linearRgbBt709;
}
// Apply the PQ BT2020 to BT709 OOTF.
highp vec3 applyPqBt2020ToBt709Ootf(highp vec3 linearRgbBt2020) {
float pqPeakLuminance = 10000.0;
float sdrPeakLuminance = 500.0;
return linearRgbBt2020 * pqPeakLuminance / sdrPeakLuminance;
}
highp vec3 applyBt2020ToBt709Ootf(highp vec3 linearRgbBt2020) {
if (uInputColorTransfer == COLOR_TRANSFER_ST2084) {
return applyPqBt2020ToBt709Ootf(linearRgbBt2020);
} else if (uInputColorTransfer == COLOR_TRANSFER_HLG) {
return applyHlgBt2020ToBt709Ootf(linearRgbBt2020);
} else {
// Output green as an obviously visible error.
return vec3(0.0, 1.0, 0.0);
}
}
// BT.2100 / BT.2020 HLG OETF for one channel.
highp float hlgOetfSingleChannel(highp float linearChannel) {
// Specification:
// https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG
// Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=529-543;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float a = 0.17883277;
const highp float b = 0.28466892;
const highp float c = 0.55991073;
return linearChannel <= 1.0 / 12.0 ? sqrt(3.0 * linearChannel) :
a * log(12.0 * linearChannel - b) + c;
}
// BT.2100 / BT.2020 HLG OETF.
highp vec3 hlgOetf(highp vec3 linearColor) {
return vec3(
hlgOetfSingleChannel(linearColor.r),
hlgOetfSingleChannel(linearColor.g),
hlgOetfSingleChannel(linearColor.b)
);
}
// BT.2100 / BT.2020, PQ / ST2084 OETF.
highp vec3 pqOetf(highp vec3 linearColor) {
// Specification:
// https://registry.khronos.org/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_PQ
// Reference implementation:
// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=514-527;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
const highp float m1 = (2610.0 / 16384.0);
const highp float m2 = (2523.0 / 4096.0) * 128.0;
const highp float c1 = (3424.0 / 4096.0);
const highp float c2 = (2413.0 / 4096.0) * 32.0;
const highp float c3 = (2392.0 / 4096.0) * 32.0;
highp vec3 temp = pow(linearColor, vec3(m1));
temp = (c1 + c2 * temp) / (1.0 + c3 * temp);
return pow(temp, vec3(m2));
}
// BT.709 gamma 2.2 OETF for one channel.
float gamma22OetfSingleChannel(highp float linearChannel) {
// Reference:
// https://developer.android.com/reference/android/hardware/DataSpace#TRANSFER_GAMMA2_2
return pow(linearChannel, (1.0 / 2.2));
}
// BT.709 gamma 2.2 OETF.
vec3 gamma22Oetf(highp vec3 linearColor) {
return vec3(
gamma22OetfSingleChannel(linearColor.r),
gamma22OetfSingleChannel(linearColor.g),
gamma22OetfSingleChannel(linearColor.b));
}
// Applies the appropriate OETF to convert linear optical signals to nonlinear
// electrical signals. Input and output are both normalized to [0, 1].
highp vec3 applyOetf(highp vec3 linearColor) {
if (uOutputColorTransfer == COLOR_TRANSFER_ST2084) {
return pqOetf(linearColor);
} else if (uOutputColorTransfer == COLOR_TRANSFER_HLG) {
return hlgOetf(linearColor);
} else if (uOutputColorTransfer == COLOR_TRANSFER_GAMMA_2_2) {
return gamma22Oetf(linearColor);
} else if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) {
return linearColor;
} else {
// Output blue as an obviously visible error.
return vec3(0.0, 0.0, 1.0);
}
}
void main() {
vec3 opticalColorBt2020 = applyEotf(
texture(uTexSampler, vTexSamplingCoord).xyz);
vec4 opticalColor = (uApplyHdrToSdrToneMapping == 1)
? vec4(applyBt2020ToBt709Ootf(opticalColorBt2020), 1.0)
: vec4(opticalColorBt2020, 1.0);
vec4 transformedColors = uRgbMatrix * opticalColor;
outColor = vec4(applyOetf(transformedColors.rgb), 1.0);
}

View File

@ -71,6 +71,8 @@ import java.util.List;
"shaders/fragment_shader_transformation_external_yuv_es3.glsl";
private static final String FRAGMENT_SHADER_TRANSFORMATION_SDR_EXTERNAL_PATH =
"shaders/fragment_shader_transformation_sdr_external_es2.glsl";
private static final String FRAGMENT_SHADER_TRANSFORMATION_HDR_INTERNAL_ES3_PATH =
"shaders/fragment_shader_transformation_hdr_internal_es3.glsl";
private static final String FRAGMENT_SHADER_TRANSFORMATION_SDR_INTERNAL_PATH =
"shaders/fragment_shader_transformation_sdr_internal_es2.glsl";
private static final ImmutableList<float[]> NDC_SQUARE =
@ -204,16 +206,18 @@ import java.util.List;
boolean enableColorTransfers,
@InputType int inputType)
throws VideoFrameProcessingException {
checkState(
!ColorInfo.isTransferHdr(inputColorInfo),
"DefaultShaderProgram doesn't support HDR internal sampler input yet.");
checkState(
inputColorInfo.colorTransfer != C.COLOR_TRANSFER_SRGB || inputType == INPUT_TYPE_BITMAP);
GlProgram glProgram =
createGlProgram(
context,
VERTEX_SHADER_TRANSFORMATION_PATH,
FRAGMENT_SHADER_TRANSFORMATION_SDR_INTERNAL_PATH);
boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo);
String vertexShaderFilePath =
isInputTransferHdr
? VERTEX_SHADER_TRANSFORMATION_ES3_PATH
: VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
isInputTransferHdr
? FRAGMENT_SHADER_TRANSFORMATION_HDR_INTERNAL_ES3_PATH
: FRAGMENT_SHADER_TRANSFORMATION_SDR_INTERNAL_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
glProgram.setIntUniform("uInputColorTransfer", inputColorInfo.colorTransfer);
return createWithSampler(
glProgram,
@ -268,8 +272,23 @@ import java.util.List;
isInputTransferHdr
? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_TRANSFORMATION_SDR_EXTERNAL_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
if (isInputTransferHdr) {
// In HDR editing mode the decoder output is sampled in YUV.
if (!GlUtil.isYuvTargetExtensionSupported()) {
throw new VideoFrameProcessingException(
"The EXT_YUV_target extension is required for HDR editing input.");
}
glProgram.setFloatsUniform(
"uYuvToRgbColorTransform",
inputColorInfo.colorRange == C.COLOR_RANGE_FULL
? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX
: BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX);
glProgram.setIntUniform("uInputColorTransfer", inputColorInfo.colorTransfer);
}
return createWithSampler(
createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath),
glProgram,
matrixTransformations,
rgbMatrices,
inputColorInfo,
@ -343,31 +362,16 @@ import java.util.List;
List<RgbMatrix> rgbMatrices,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean enableColorTransfers)
throws VideoFrameProcessingException {
boolean enableColorTransfers) {
boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo);
@C.ColorTransfer int outputColorTransfer = outputColorInfo.colorTransfer;
if (isInputTransferHdr) {
checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020);
checkArgument(enableColorTransfers);
// In HDR editing mode the decoder output is sampled in YUV.
if (!GlUtil.isYuvTargetExtensionSupported()) {
throw new VideoFrameProcessingException(
"The EXT_YUV_target extension is required for HDR editing input.");
}
glProgram.setFloatsUniform(
"uYuvToRgbColorTransform",
inputColorInfo.colorRange == C.COLOR_RANGE_FULL
? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX
: BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX);
checkArgument(ColorInfo.isTransferHdr(inputColorInfo));
glProgram.setIntUniform("uInputColorTransfer", inputColorInfo.colorTransfer);
// TODO(b/239735341): Add a setBooleanUniform method to GlProgram.
glProgram.setIntUniform(
"uApplyHdrToSdrToneMapping",
/* value= */ (outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020) ? 1 : 0);
/* value= */ (outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020) ? GL_TRUE : GL_FALSE);
checkArgument(
outputColorTransfer != Format.NO_VALUE && outputColorTransfer != C.COLOR_TRANSFER_SDR);
glProgram.setIntUniform("uOutputColorTransfer", outputColorTransfer);

View File

@ -620,12 +620,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
inputSwitcher.registerInput(INPUT_TYPE_SURFACE);
if (!ColorInfo.isTransferHdr(inputColorInfo)) {
// HDR bitmap or texture input is not supported.
// HDR bitmap input is not supported.
inputSwitcher.registerInput(INPUT_TYPE_BITMAP);
if (inputColorInfo.colorTransfer != C.COLOR_TRANSFER_SRGB) {
// Image and textureId concatenation not supported.
inputSwitcher.registerInput(INPUT_TYPE_TEXTURE_ID);
}
}
if (inputColorInfo.colorTransfer != C.COLOR_TRANSFER_SRGB) {
// Image and textureId concatenation not supported.
inputSwitcher.registerInput(INPUT_TYPE_TEXTURE_ID);
}
inputSwitcher.setDownstreamShaderProgram(effectsShaderPrograms.get(0));

View File

@ -15,6 +15,7 @@
*/
package androidx.media3.transformer.mh;
import static androidx.media3.common.ColorInfo.SDR_BT709_LIMITED;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
@ -34,6 +35,7 @@ import android.content.Context;
import android.graphics.Bitmap;
import android.view.Surface;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Effect;
import androidx.media3.common.Format;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
@ -55,6 +57,7 @@ import androidx.media3.transformer.AndroidTestUtil;
import androidx.media3.transformer.EncoderUtil;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.util.List;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.junit.After;
@ -89,6 +92,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
/** Input HLG video of which we only use the first frame. */
private static final String INPUT_HLG10_MP4_ASSET_STRING = "media/mp4/hlg-1080p.mp4";
// A passthrough effect allows for testing having an intermediate effect injected, which uses
// different OpenGL shaders from having no effects.
private static final GlEffect NO_OP_EFFECT =
new GlEffectWrapper(new ScaleAndRotateTransformation.Builder().build());
@ -111,7 +116,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
/* outputFormat= */ null)) {
return;
}
videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId).build();
videoFrameProcessorTestRunner = getSurfaceInputFrameProcessorTestRunnerBuilder(testId).build();
Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
videoFrameProcessorTestRunner.processFirstFrameAndEnd();
@ -134,23 +139,14 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
/* outputFormat= */ null)) {
return;
}
TextureBitmapReader producersBitmapReader = new TextureBitmapReader();
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder()
.setTextureOutput(
(outputTexture, presentationTimeUs) ->
inputTextureIntoVideoFrameProcessor(
testId, consumersBitmapReader, outputTexture, presentationTimeUs),
/* textureOutputCapacity= */ 1)
.build();
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
new VideoFrameProcessorTestRunner.Builder()
.setTestId(testId)
.setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactory)
.setVideoAssetPath(INPUT_SDR_MP4_ASSET_STRING)
.setBitmapReader(producersBitmapReader)
.build();
getTexIdProducingFrameProcessorTestRunner(
testId,
consumersBitmapReader,
INPUT_SDR_MP4_ASSET_STRING,
SDR_BT709_LIMITED,
ImmutableList.of());
Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
texIdProducingVideoFrameProcessorTestRunner.processFirstFrameAndEnd();
@ -177,7 +173,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(overlayBitmap);
videoFrameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
getSurfaceInputFrameProcessorTestRunnerBuilder(testId)
.setEffects(new OverlayEffect(ImmutableList.of(bitmapOverlay)))
.build();
Bitmap expectedBitmap = readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH);
@ -203,28 +199,16 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
}
Bitmap overlayBitmap = readBitmap(OVERLAY_PNG_ASSET_PATH);
BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(overlayBitmap);
TextureBitmapReader producersBitmapReader = new TextureBitmapReader();
ImmutableList<Effect> effects =
ImmutableList.of(new OverlayEffect(ImmutableList.of(bitmapOverlay)));
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder()
.setTextureOutput(
(outputTexture, presentationTimeUs) ->
inputTextureIntoVideoFrameProcessor(
testId, consumersBitmapReader, outputTexture, presentationTimeUs),
/* textureOutputCapacity= */ 1)
.build();
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
new VideoFrameProcessorTestRunner.Builder()
.setTestId(testId)
.setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactory)
.setVideoAssetPath(INPUT_SDR_MP4_ASSET_STRING)
.setBitmapReader(producersBitmapReader)
.setEffects(new OverlayEffect(ImmutableList.of(bitmapOverlay)))
.build();
texIdProducingVideoFrameProcessorTestRunner.processFirstFrameAndEnd();
texIdProducingVideoFrameProcessorTestRunner.release();
getTexIdProducingFrameProcessorTestRunner(
testId, consumersBitmapReader, INPUT_SDR_MP4_ASSET_STRING, SDR_BT709_LIMITED, effects);
Bitmap expectedBitmap = readBitmap(BITMAP_OVERLAY_PNG_ASSET_PATH);
texIdProducingVideoFrameProcessorTestRunner.processFirstFrameAndEnd();
texIdProducingVideoFrameProcessorTestRunner.release();
Bitmap actualBitmap = consumersBitmapReader.getBitmap();
// TODO(b/207848601): Switch to using proper tooling for testing against golden data.
@ -249,7 +233,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
}
ColorInfo colorInfo = checkNotNull(format.colorInfo);
videoFrameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
getSurfaceInputFrameProcessorTestRunnerBuilder(testId)
.setInputColorInfo(colorInfo)
.setOutputColorInfo(colorInfo)
.setVideoAssetPath(INPUT_HLG10_MP4_ASSET_STRING)
@ -267,11 +251,9 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
.isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16);
}
// A passthrough effect allows for testing having an intermediate effect injected, which uses
// different OpenGL shaders from having no effects.
@Test
public void noOpEffect_hlg10Input_matchesGoldenFile() throws Exception {
String testId = "noOpEffect_hlg10Input_matchesGoldenFile";
public void noEffects_hlg10TextureInput_matchesGoldenFile() throws Exception {
String testId = "noEffects_hlg10TextureInput_matchesGoldenFile";
Context context = getApplicationContext();
Format format = MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT;
if (!deviceSupportsHdrEditing(format)) {
@ -283,17 +265,19 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
return;
}
ColorInfo colorInfo = checkNotNull(format.colorInfo);
videoFrameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setInputColorInfo(colorInfo)
.setOutputColorInfo(colorInfo)
.setVideoAssetPath(INPUT_HLG10_MP4_ASSET_STRING)
.setEffects(NO_OP_EFFECT)
.build();
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
getTexIdProducingFrameProcessorTestRunner(
testId,
consumersBitmapReader,
INPUT_HLG10_MP4_ASSET_STRING,
colorInfo,
ImmutableList.of());
Bitmap expectedBitmap = readBitmap(ORIGINAL_HLG10_PNG_ASSET_PATH);
videoFrameProcessorTestRunner.processFirstFrameAndEnd();
Bitmap actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap();
texIdProducingVideoFrameProcessorTestRunner.processFirstFrameAndEnd();
texIdProducingVideoFrameProcessorTestRunner.release();
Bitmap actualBitmap = consumersBitmapReader.getBitmap();
// TODO(b/207848601): Switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
@ -309,7 +293,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
Context context = getApplicationContext();
Format format = MP4_ASSET_720P_4_SECOND_HDR10_FORMAT;
if (!deviceSupportsHdrEditing(format)) {
recordTestSkipped(context, testId, "No HLG editing support");
recordTestSkipped(context, testId, "No PQ editing support");
return;
}
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
@ -318,7 +302,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
}
ColorInfo colorInfo = checkNotNull(format.colorInfo);
videoFrameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
getSurfaceInputFrameProcessorTestRunnerBuilder(testId)
.setInputColorInfo(colorInfo)
.setOutputColorInfo(colorInfo)
.setVideoAssetPath(INPUT_PQ_MP4_ASSET_STRING)
@ -336,13 +320,47 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
.isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16);
}
// A passthrough effect allows for testing having an intermediate effect injected, which uses
// different OpenGL shaders from having no effects.
@Test
public void noOpEffect_hdr10Input_matchesGoldenFile() throws Exception {
String testId = "noOpEffect_hdr10Input_matchesGoldenFile";
public void noEffects_hdr10TextureInput_matchesGoldenFile() throws Exception {
String testId = "noEffects_hdr10TextureInput_matchesGoldenFile";
Context context = getApplicationContext();
Format format = MP4_ASSET_720P_4_SECOND_HDR10_FORMAT;
if (!deviceSupportsHdrEditing(format)) {
recordTestSkipped(context, testId, "No PQ editing support");
return;
}
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
context, testId, /* inputFormat= */ format, /* outputFormat= */ null)) {
return;
}
ColorInfo colorInfo = checkNotNull(format.colorInfo);
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
getTexIdProducingFrameProcessorTestRunner(
testId,
consumersBitmapReader,
INPUT_PQ_MP4_ASSET_STRING,
colorInfo,
ImmutableList.of());
Bitmap expectedBitmap = readBitmap(ORIGINAL_HDR10_PNG_ASSET_PATH);
texIdProducingVideoFrameProcessorTestRunner.processFirstFrameAndEnd();
texIdProducingVideoFrameProcessorTestRunner.release();
Bitmap actualBitmap = consumersBitmapReader.getBitmap();
// TODO(b/207848601): Switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceFp16(
expectedBitmap, actualBitmap);
assertThat(averagePixelAbsoluteDifference)
.isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16);
}
@Test
public void noOpEffect_hlg10Input_matchesGoldenFile() throws Exception {
String testId = "noOpEffect_hlg10Input_matchesGoldenFile";
Context context = getApplicationContext();
Format format = MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT;
if (!deviceSupportsHdrEditing(format)) {
recordTestSkipped(context, testId, "No HLG editing support");
return;
@ -353,7 +371,77 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
}
ColorInfo colorInfo = checkNotNull(format.colorInfo);
videoFrameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
getSurfaceInputFrameProcessorTestRunnerBuilder(testId)
.setInputColorInfo(colorInfo)
.setOutputColorInfo(colorInfo)
.setVideoAssetPath(INPUT_HLG10_MP4_ASSET_STRING)
.setEffects(NO_OP_EFFECT)
.build();
Bitmap expectedBitmap = readBitmap(ORIGINAL_HLG10_PNG_ASSET_PATH);
videoFrameProcessorTestRunner.processFirstFrameAndEnd();
Bitmap actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap();
// TODO(b/207848601): Switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceFp16(
expectedBitmap, actualBitmap);
assertThat(averagePixelAbsoluteDifference)
.isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16);
}
@Test
public void noOpEffect_hlg10TextureInput_matchesGoldenFile() throws Exception {
String testId = "noOpEffect_hlg10TextureInput_matchesGoldenFile";
Context context = getApplicationContext();
Format format = MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT;
if (!deviceSupportsHdrEditing(format)) {
recordTestSkipped(context, testId, "No HLG editing support");
return;
}
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
context, testId, /* inputFormat= */ format, /* outputFormat= */ null)) {
return;
}
ColorInfo colorInfo = checkNotNull(format.colorInfo);
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
getTexIdProducingFrameProcessorTestRunner(
testId,
consumersBitmapReader,
INPUT_HLG10_MP4_ASSET_STRING,
colorInfo,
ImmutableList.of(NO_OP_EFFECT));
Bitmap expectedBitmap = readBitmap(ORIGINAL_HLG10_PNG_ASSET_PATH);
texIdProducingVideoFrameProcessorTestRunner.processFirstFrameAndEnd();
texIdProducingVideoFrameProcessorTestRunner.release();
Bitmap actualBitmap = consumersBitmapReader.getBitmap();
// TODO(b/207848601): Switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceFp16(
expectedBitmap, actualBitmap);
assertThat(averagePixelAbsoluteDifference)
.isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16);
}
@Test
public void noOpEffect_hdr10Input_matchesGoldenFile() throws Exception {
String testId = "noOpEffect_hdr10Input_matchesGoldenFile";
Context context = getApplicationContext();
Format format = MP4_ASSET_720P_4_SECOND_HDR10_FORMAT;
if (!deviceSupportsHdrEditing(format)) {
recordTestSkipped(context, testId, "No PQ editing support");
return;
}
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
context, testId, /* inputFormat= */ format, /* outputFormat= */ null)) {
return;
}
ColorInfo colorInfo = checkNotNull(format.colorInfo);
videoFrameProcessorTestRunner =
getSurfaceInputFrameProcessorTestRunnerBuilder(testId)
.setInputColorInfo(colorInfo)
.setOutputColorInfo(colorInfo)
.setVideoAssetPath(INPUT_PQ_MP4_ASSET_STRING)
@ -372,9 +460,78 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
.isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16);
}
@Test
public void noOpEffect_hdr10TextureInput_matchesGoldenFile() throws Exception {
String testId = "noOpEffect_hdr10TextureInput_matchesGoldenFile";
Context context = getApplicationContext();
Format format = MP4_ASSET_720P_4_SECOND_HDR10_FORMAT;
if (!deviceSupportsHdrEditing(format)) {
recordTestSkipped(context, testId, "No PQ editing support");
return;
}
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
context, testId, /* inputFormat= */ format, /* outputFormat= */ null)) {
return;
}
ColorInfo colorInfo = checkNotNull(format.colorInfo);
TextureBitmapReader consumersBitmapReader = new TextureBitmapReader();
VideoFrameProcessorTestRunner texIdProducingVideoFrameProcessorTestRunner =
getTexIdProducingFrameProcessorTestRunner(
testId,
consumersBitmapReader,
INPUT_PQ_MP4_ASSET_STRING,
colorInfo,
ImmutableList.of(NO_OP_EFFECT));
Bitmap expectedBitmap = readBitmap(ORIGINAL_HDR10_PNG_ASSET_PATH);
texIdProducingVideoFrameProcessorTestRunner.processFirstFrameAndEnd();
texIdProducingVideoFrameProcessorTestRunner.release();
Bitmap actualBitmap = consumersBitmapReader.getBitmap();
// TODO(b/207848601): Switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceFp16(
expectedBitmap, actualBitmap);
assertThat(averagePixelAbsoluteDifference)
.isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16);
}
private VideoFrameProcessorTestRunner getTexIdProducingFrameProcessorTestRunner(
String testId,
TextureBitmapReader consumersBitmapReader,
String videoAssetPath,
ColorInfo colorInfo,
List<Effect> effects)
throws VideoFrameProcessingException {
TextureBitmapReader producersBitmapReader = new TextureBitmapReader();
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =
new DefaultVideoFrameProcessor.Factory.Builder()
.setTextureOutput(
(outputTexture, presentationTimeUs) ->
inputTextureIntoVideoFrameProcessor(
testId,
consumersBitmapReader,
colorInfo,
effects,
outputTexture,
presentationTimeUs),
/* textureOutputCapacity= */ 1)
.build();
return new VideoFrameProcessorTestRunner.Builder()
.setTestId(testId)
.setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactory)
.setVideoAssetPath(videoAssetPath)
.setInputColorInfo(colorInfo)
.setOutputColorInfo(colorInfo)
.setBitmapReader(producersBitmapReader)
.build();
}
private void inputTextureIntoVideoFrameProcessor(
String testId,
TextureBitmapReader bitmapReader,
ColorInfo colorInfo,
List<Effect> effects,
GlTextureInfo texture,
long presentationTimeUs)
throws VideoFrameProcessingException {
@ -389,9 +546,11 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
new VideoFrameProcessorTestRunner.Builder()
.setTestId(testId)
.setVideoFrameProcessorFactory(defaultVideoFrameProcessorFactory)
.setVideoAssetPath(INPUT_SDR_MP4_ASSET_STRING)
.setInputColorInfo(colorInfo)
.setOutputColorInfo(colorInfo)
.setBitmapReader(bitmapReader)
.setInputType(VideoFrameProcessor.INPUT_TYPE_TEXTURE_ID)
.setEffects(effects)
.build();
videoFrameProcessorTestRunner.queueInputTexture(texture, presentationTimeUs);
@ -402,7 +561,7 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
}
}
private VideoFrameProcessorTestRunner.Builder getDefaultFrameProcessorTestRunnerBuilder(
private VideoFrameProcessorTestRunner.Builder getSurfaceInputFrameProcessorTestRunnerBuilder(
String testId) {
TextureBitmapReader textureBitmapReader = new TextureBitmapReader();
DefaultVideoFrameProcessor.Factory defaultVideoFrameProcessorFactory =