HDR: Use FP16 color representation for texture processors.

* Introduced `useHdr` for `GlEffect#toGlTextureProcessor`, so
  `TextureProcessor` implementations can decide how to handle HDR.
* Creating FP16 color textures for HDR input.

Tested via manual testing, adding a no-op GlEffectWrapper to the transformation to
force use of intermediate textures, adding a linear ramp to the fragment shader,
and trying to ascertain that there's a real reduction in posterization when
switching from 4-bit to 8-bit unsigned bytes, and again from 8-bit unsigned bytes
to 16-bit floating point.

PiperOrigin-RevId: 461613117
This commit is contained in:
huangdarwin 2022-07-18 14:21:17 +00:00 committed by Rohit Singh
parent fd046bd2f6
commit f67c1a73f4
16 changed files with 149 additions and 45 deletions

View File

@ -15,6 +15,7 @@
*/ */
package com.google.android.exoplayer2.transformerdemo; package com.google.android.exoplayer2.transformerdemo;
import static com.google.android.exoplayer2.util.Assertions.checkArgument;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import android.content.Context; import android.content.Context;
@ -64,9 +65,14 @@ import java.util.Locale;
/** /**
* Creates a new instance. * Creates a new instance.
* *
* @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @throws FrameProcessingException If a problem occurs while reading shader files. * @throws FrameProcessingException If a problem occurs while reading shader files.
*/ */
public BitmapOverlayProcessor(Context context) throws FrameProcessingException { public BitmapOverlayProcessor(Context context, boolean useHdr) throws FrameProcessingException {
super(useHdr);
checkArgument(!useHdr, "BitmapOverlayProcessor does not support HDR colors.");
paint = new Paint(); paint = new Paint();
paint.setTextSize(64); paint.setTextSize(64);
paint.setAntiAlias(true); paint.setAntiAlias(true);
@ -85,7 +91,11 @@ import java.util.Locale;
throw new IllegalStateException(e); throw new IllegalStateException(e);
} }
try { try {
bitmapTexId = GlUtil.createTexture(BITMAP_WIDTH_HEIGHT, BITMAP_WIDTH_HEIGHT); bitmapTexId =
GlUtil.createTexture(
BITMAP_WIDTH_HEIGHT,
BITMAP_WIDTH_HEIGHT,
/* useHighPrecisionColorComponents= */ false);
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, overlayBitmap, /* border= */ 0); GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, overlayBitmap, /* border= */ 0);
glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);

View File

@ -52,6 +52,8 @@ import java.io.IOException;
* <p>The parameters are given in normalized texture coordinates from 0 to 1. * <p>The parameters are given in normalized texture coordinates from 0 to 1.
* *
* @param context The {@link Context}. * @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @param centerX The x-coordinate of the center of the effect. * @param centerX The x-coordinate of the center of the effect.
* @param centerY The y-coordinate of the center of the effect. * @param centerY The y-coordinate of the center of the effect.
* @param minInnerRadius The lower bound of the radius that is unaffected by the effect. * @param minInnerRadius The lower bound of the radius that is unaffected by the effect.
@ -61,12 +63,15 @@ import java.io.IOException;
*/ */
public PeriodicVignetteProcessor( public PeriodicVignetteProcessor(
Context context, Context context,
boolean useHdr,
float centerX, float centerX,
float centerY, float centerY,
float minInnerRadius, float minInnerRadius,
float maxInnerRadius, float maxInnerRadius,
float outerRadius) float outerRadius)
throws FrameProcessingException { throws FrameProcessingException {
super(useHdr);
checkArgument(!useHdr, "PeriodicVignetteProcessor does not support HDR color spaces.");
checkArgument(minInnerRadius <= maxInnerRadius); checkArgument(minInnerRadius <= maxInnerRadius);
checkArgument(maxInnerRadius <= outerRadius); checkArgument(maxInnerRadius <= outerRadius);
this.minInnerRadius = minInnerRadius; this.minInnerRadius = minInnerRadius;

View File

@ -277,13 +277,15 @@ public final class TransformerActivity extends AppCompatActivity {
Class<?> clazz = Class<?> clazz =
Class.forName("com.google.android.exoplayer2.transformerdemo.MediaPipeProcessor"); Class.forName("com.google.android.exoplayer2.transformerdemo.MediaPipeProcessor");
Constructor<?> constructor = Constructor<?> constructor =
clazz.getConstructor(Context.class, String.class, String.class, String.class); clazz.getConstructor(
Context.class, Boolean.class, String.class, String.class, String.class);
effects.add( effects.add(
(Context context) -> { (Context context, boolean useHdr) -> {
try { try {
return (GlTextureProcessor) return (GlTextureProcessor)
constructor.newInstance( constructor.newInstance(
context, context,
useHdr,
/* graphName= */ "edge_detector_mediapipe_graph.binarypb", /* graphName= */ "edge_detector_mediapipe_graph.binarypb",
/* inputStreamName= */ "input_video", /* inputStreamName= */ "input_video",
/* outputStreamName= */ "output_video"); /* outputStreamName= */ "output_video");
@ -298,9 +300,10 @@ public final class TransformerActivity extends AppCompatActivity {
} }
if (selectedEffects[2]) { if (selectedEffects[2]) {
effects.add( effects.add(
(Context context) -> (Context context, boolean useHdr) ->
new PeriodicVignetteProcessor( new PeriodicVignetteProcessor(
context, context,
useHdr,
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X), bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y), bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y),
/* minInnerRadius= */ bundle.getFloat( /* minInnerRadius= */ bundle.getFloat(

View File

@ -15,6 +15,7 @@
*/ */
package com.google.android.exoplayer2.transformerdemo; package com.google.android.exoplayer2.transformerdemo;
import static com.google.android.exoplayer2.util.Assertions.checkArgument;
import static com.google.android.exoplayer2.util.Assertions.checkState; import static com.google.android.exoplayer2.util.Assertions.checkState;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
@ -70,14 +71,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* Creates a new texture processor that wraps a MediaPipe graph. * Creates a new texture processor that wraps a MediaPipe graph.
* *
* @param context The {@link Context}. * @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @param graphName Name of a MediaPipe graph asset to load. * @param graphName Name of a MediaPipe graph asset to load.
* @param inputStreamName Name of the input video stream in the graph. * @param inputStreamName Name of the input video stream in the graph.
* @param outputStreamName Name of the input video stream in the graph. * @param outputStreamName Name of the input video stream in the graph.
*/ */
@SuppressWarnings("AndroidConcurrentHashMap") // Only used on API >= 23. @SuppressWarnings("AndroidConcurrentHashMap") // Only used on API >= 23.
public MediaPipeProcessor( public MediaPipeProcessor(
Context context, String graphName, String inputStreamName, String outputStreamName) { Context context,
boolean useHdr,
String graphName,
String inputStreamName,
String outputStreamName) {
checkState(LOADER.isAvailable()); checkState(LOADER.isAvailable());
// TODO(b/227624622): Confirm whether MediaPipeProcessor could support HDR colors.
checkArgument(!useHdr, "MediaPipeProcessor does not support HDR colors.");
EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext()); EglManager eglManager = new EglManager(EGL14.eglGetCurrentContext());
frameProcessor = frameProcessor =
new FrameProcessor( new FrameProcessor(

View File

@ -16,6 +16,7 @@
package com.google.android.exoplayer2.util; package com.google.android.exoplayer2.util;
import static android.opengl.GLU.gluErrorString; import static android.opengl.GLU.gluErrorString;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import android.content.Context; import android.content.Context;
import android.content.pm.PackageManager; import android.content.pm.PackageManager;
@ -26,6 +27,7 @@ import android.opengl.EGLDisplay;
import android.opengl.EGLSurface; import android.opengl.EGLSurface;
import android.opengl.GLES11Ext; import android.opengl.GLES11Ext;
import android.opengl.GLES20; import android.opengl.GLES20;
import android.opengl.GLES30;
import androidx.annotation.DoNotInline; import androidx.annotation.DoNotInline;
import androidx.annotation.Nullable; import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi; import androidx.annotation.RequiresApi;
@ -488,12 +490,37 @@ public final class GlUtil {
} }
/** /**
* Returns the texture identifier for a newly-allocated texture with the specified dimensions. * Allocates a new RGBA texture with the specified dimensions and color component precision.
* *
* @param width of the new texture in pixels * @param width The width of the new texture in pixels.
* @param height of the new texture in pixels * @param height The height of the new texture in pixels.
* @param useHighPrecisionColorComponents If {@code false}, uses 8-bit unsigned bytes. If {@code
* true}, use 16-bit (half-precision) floating-point.
* @throws GlException If the texture allocation fails.
* @return The texture identifier for the newly-allocated texture.
*/ */
public static int createTexture(int width, int height) throws GlException { public static int createTexture(int width, int height, boolean useHighPrecisionColorComponents)
throws GlException {
// TODO(227624622): Implement a pixel test that confirms 16f has less posterization.
if (useHighPrecisionColorComponents) {
checkState(Util.SDK_INT >= 18, "GLES30 extensions are not supported below API 18.");
return createTexture(width, height, GLES30.GL_RGBA16F, GLES30.GL_HALF_FLOAT);
}
return createTexture(width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE);
}
/**
* Allocates a new RGBA texture with the specified dimensions and color component precision.
*
* @param width The width of the new texture in pixels.
* @param height The height of the new texture in pixels.
* @param internalFormat The number of color components in the texture, as well as their format.
* @param type The data type of the pixel data.
* @throws GlException If the texture allocation fails.
* @return The texture identifier for the newly-allocated texture.
*/
private static int createTexture(int width, int height, int internalFormat, int type)
throws GlException {
assertValidTextureSize(width, height); assertValidTextureSize(width, height);
int texId = generateTexture(); int texId = generateTexture();
bindTexture(GLES20.GL_TEXTURE_2D, texId); bindTexture(GLES20.GL_TEXTURE_2D, texId);
@ -501,12 +528,12 @@ public final class GlUtil {
GLES20.glTexImage2D( GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_2D,
/* level= */ 0, /* level= */ 0,
GLES20.GL_RGBA, internalFormat,
width, width,
height, height,
/* border= */ 0, /* border= */ 0,
GLES20.GL_RGBA, GLES20.GL_RGBA,
GLES20.GL_UNSIGNED_BYTE, type,
byteBuffer); byteBuffer);
checkGlError(); checkGlError();
return texId; return texId;

View File

@ -189,6 +189,7 @@ public class BitmapTestUtil {
public static Bitmap createArgb8888BitmapFromCurrentGlFramebuffer(int width, int height) public static Bitmap createArgb8888BitmapFromCurrentGlFramebuffer(int width, int height)
throws GlUtil.GlException { throws GlUtil.GlException {
ByteBuffer rgba8888Buffer = ByteBuffer.allocateDirect(width * height * 4); ByteBuffer rgba8888Buffer = ByteBuffer.allocateDirect(width * height * 4);
// TODO(b/227624622): Add support for reading HDR bitmaps.
GLES20.glReadPixels( GLES20.glReadPixels(
0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, rgba8888Buffer); 0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, rgba8888Buffer);
GlUtil.checkGlError(); GlUtil.checkGlError();
@ -208,7 +209,10 @@ public class BitmapTestUtil {
* @return The identifier of the newly created texture. * @return The identifier of the newly created texture.
*/ */
public static int createGlTextureFromBitmap(Bitmap bitmap) throws GlUtil.GlException { public static int createGlTextureFromBitmap(Bitmap bitmap) throws GlUtil.GlException {
int texId = GlUtil.createTexture(bitmap.getWidth(), bitmap.getHeight()); // TODO(b/227624622): Add support for reading HDR bitmaps.
int texId =
GlUtil.createTexture(
bitmap.getWidth(), bitmap.getHeight(), /* useHighPrecisionColorComponents= */ false);
// Put the flipped bitmap in the OpenGL texture as the bitmap's positive y-axis points down // Put the flipped bitmap in the OpenGL texture as the bitmap's positive y-axis points down
// while OpenGL's positive y-axis points up. // while OpenGL's positive y-axis points up.
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, flipBitmapVertically(bitmap), 0); GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, flipBitmapVertically(bitmap), 0);

View File

@ -89,7 +89,7 @@ public final class CropPixelTest {
String testId = "drawFrame_noEdits"; String testId = "drawFrame_noEdits";
cropTextureProcessor = cropTextureProcessor =
new Crop(/* left= */ -1, /* right= */ 1, /* bottom= */ -1, /* top= */ 1) new Crop(/* left= */ -1, /* right= */ 1, /* bottom= */ -1, /* top= */ 1)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = cropTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = cropTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
@ -113,7 +113,7 @@ public final class CropPixelTest {
String testId = "drawFrame_cropSmaller"; String testId = "drawFrame_cropSmaller";
cropTextureProcessor = cropTextureProcessor =
new Crop(/* left= */ -.9f, /* right= */ .1f, /* bottom= */ -1f, /* top= */ .5f) new Crop(/* left= */ -.9f, /* right= */ .1f, /* bottom= */ -1f, /* top= */ .5f)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = cropTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = cropTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(CROP_SMALLER_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(CROP_SMALLER_PNG_ASSET_PATH);
@ -137,7 +137,7 @@ public final class CropPixelTest {
String testId = "drawFrame_cropLarger"; String testId = "drawFrame_cropLarger";
cropTextureProcessor = cropTextureProcessor =
new Crop(/* left= */ -2f, /* right= */ 2f, /* bottom= */ -1f, /* top= */ 2f) new Crop(/* left= */ -2f, /* right= */ 2f, /* bottom= */ -1f, /* top= */ 2f)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = cropTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = cropTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(CROP_LARGER_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(CROP_LARGER_PNG_ASSET_PATH);
@ -157,7 +157,9 @@ public final class CropPixelTest {
} }
private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException { private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException {
outputTexId = GlUtil.createTexture(outputWidth, outputHeight); outputTexId =
GlUtil.createTexture(
outputWidth, outputHeight, /* useHighPrecisionColorComponents= */ false);
int frameBuffer = GlUtil.createFboForTexture(outputTexId); int frameBuffer = GlUtil.createFboForTexture(outputTexId);
GlUtil.focusFramebuffer( GlUtil.focusFramebuffer(
checkNotNull(eglDisplay), checkNotNull(eglDisplay),

View File

@ -320,6 +320,9 @@ public final class GlEffectsFrameProcessorPixelTest {
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
} }
// TODO(b/227624622): Add a test for HDR input after BitmapTestUtil can read HDR bitmaps, using
// GlEffectWrapper to ensure usage of intermediate textures.
/** /**
* Set up and prepare the first frame from an input video, as well as relevant test * Set up and prepare the first frame from an input video, as well as relevant test
* infrastructure. The frame will be sent towards the {@link GlEffectsFrameProcessor}, and output * infrastructure. The frame will be sent towards the {@link GlEffectsFrameProcessor}, and output
@ -379,7 +382,7 @@ public final class GlEffectsFrameProcessorPixelTest {
/* streamOffsetUs= */ 0L, /* streamOffsetUs= */ 0L,
effects, effects,
DebugViewProvider.NONE, DebugViewProvider.NONE,
/* enableExperimentalHdrEditing= */ false)); /* useHdr= */ false));
glEffectsFrameProcessor.setInputFrameInfo( glEffectsFrameProcessor.setInputFrameInfo(
new FrameInfo(inputWidth, inputHeight, pixelWidthHeightRatio)); new FrameInfo(inputWidth, inputHeight, pixelWidthHeightRatio));
glEffectsFrameProcessor.registerInputFrame(); glEffectsFrameProcessor.registerInputFrame();
@ -494,9 +497,9 @@ public final class GlEffectsFrameProcessorPixelTest {
} }
@Override @Override
public GlTextureProcessor toGlTextureProcessor(Context context) public GlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException { throws FrameProcessingException {
return effect.toGlTextureProcessor(context); return effect.toGlTextureProcessor(context, useHdr);
} }
} }
} }

View File

@ -72,7 +72,7 @@ public final class MatrixTransformationProcessorPixelTest {
EGLSurface placeholderEglSurface = GlUtil.createPlaceholderEglSurface(eglDisplay); EGLSurface placeholderEglSurface = GlUtil.createPlaceholderEglSurface(eglDisplay);
GlUtil.focusEglSurface(eglDisplay, eglContext, placeholderEglSurface, width, height); GlUtil.focusEglSurface(eglDisplay, eglContext, placeholderEglSurface, width, height);
inputTexId = BitmapTestUtil.createGlTextureFromBitmap(inputBitmap); inputTexId = BitmapTestUtil.createGlTextureFromBitmap(inputBitmap);
outputTexId = GlUtil.createTexture(width, height); outputTexId = GlUtil.createTexture(width, height, /* useHighPrecisionColorComponents= */ false);
int frameBuffer = GlUtil.createFboForTexture(outputTexId); int frameBuffer = GlUtil.createFboForTexture(outputTexId);
GlUtil.focusFramebuffer( GlUtil.focusFramebuffer(
eglDisplay, eglContext, placeholderEglSurface, frameBuffer, width, height); eglDisplay, eglContext, placeholderEglSurface, frameBuffer, width, height);
@ -93,7 +93,10 @@ public final class MatrixTransformationProcessorPixelTest {
String testId = "drawFrame_noEdits"; String testId = "drawFrame_noEdits";
Matrix identityMatrix = new Matrix(); Matrix identityMatrix = new Matrix();
matrixTransformationFrameProcessor = matrixTransformationFrameProcessor =
new MatrixTransformationProcessor(context, (long presentationTimeUs) -> identityMatrix); new MatrixTransformationProcessor(
context,
/* useHdr= */ false,
/* matrixTransformation= */ (long presentationTimeUs) -> identityMatrix);
matrixTransformationFrameProcessor.configure(width, height); matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
@ -117,7 +120,9 @@ public final class MatrixTransformationProcessorPixelTest {
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
matrixTransformationFrameProcessor = matrixTransformationFrameProcessor =
new MatrixTransformationProcessor( new MatrixTransformationProcessor(
context, /* matrixTransformation= */ (long presentationTimeUs) -> translateRightMatrix); context,
/* useHdr= */ false,
/* matrixTransformation= */ (long presentationTimeUs) -> translateRightMatrix);
matrixTransformationFrameProcessor.configure(width, height); matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH);
@ -141,7 +146,9 @@ public final class MatrixTransformationProcessorPixelTest {
scaleNarrowMatrix.postScale(.5f, 1.2f); scaleNarrowMatrix.postScale(.5f, 1.2f);
matrixTransformationFrameProcessor = matrixTransformationFrameProcessor =
new MatrixTransformationProcessor( new MatrixTransformationProcessor(
context, /* matrixTransformation= */ (long presentationTimeUs) -> scaleNarrowMatrix); context,
/* useHdr= */ false,
/* matrixTransformation= */ (long presentationTimeUs) -> scaleNarrowMatrix);
matrixTransformationFrameProcessor.configure(width, height); matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_PNG_ASSET_PATH);
@ -165,7 +172,9 @@ public final class MatrixTransformationProcessorPixelTest {
rotate90Matrix.postRotate(/* degrees= */ 90); rotate90Matrix.postRotate(/* degrees= */ 90);
matrixTransformationFrameProcessor = matrixTransformationFrameProcessor =
new MatrixTransformationProcessor( new MatrixTransformationProcessor(
context, /* matrixTransformation= */ (long presentationTimeUs) -> rotate90Matrix); context,
/* useHdr= */ false,
/* matrixTransformation= */ (long presentationTimeUs) -> rotate90Matrix);
matrixTransformationFrameProcessor.configure(width, height); matrixTransformationFrameProcessor.configure(width, height);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_PNG_ASSET_PATH);
@ -181,4 +190,6 @@ public final class MatrixTransformationProcessorPixelTest {
expectedBitmap, actualBitmap, testId); expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
} }
// TODO(b/227624622): Add a test for HDR input after BitmapTestUtil can read HDR bitmaps.
} }

View File

@ -97,7 +97,8 @@ public final class PresentationPixelTest {
public void drawFrame_noEdits_producesExpectedOutput() throws Exception { public void drawFrame_noEdits_producesExpectedOutput() throws Exception {
String testId = "drawFrame_noEdits"; String testId = "drawFrame_noEdits";
presentationTextureProcessor = presentationTextureProcessor =
Presentation.createForHeight(C.LENGTH_UNSET).toGlTextureProcessor(context); Presentation.createForHeight(C.LENGTH_UNSET)
.toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ORIGINAL_PNG_ASSET_PATH);
@ -122,7 +123,7 @@ public final class PresentationPixelTest {
String testId = "drawFrame_changeAspectRatio_scaleToFit_narrow"; String testId = "drawFrame_changeAspectRatio_scaleToFit_narrow";
presentationTextureProcessor = presentationTextureProcessor =
Presentation.createForAspectRatio(/* aspectRatio= */ 1f, Presentation.LAYOUT_SCALE_TO_FIT) Presentation.createForAspectRatio(/* aspectRatio= */ 1f, Presentation.LAYOUT_SCALE_TO_FIT)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = Bitmap expectedBitmap =
@ -148,7 +149,7 @@ public final class PresentationPixelTest {
String testId = "drawFrame_changeAspectRatio_scaleToFit_wide"; String testId = "drawFrame_changeAspectRatio_scaleToFit_wide";
presentationTextureProcessor = presentationTextureProcessor =
Presentation.createForAspectRatio(/* aspectRatio= */ 2f, Presentation.LAYOUT_SCALE_TO_FIT) Presentation.createForAspectRatio(/* aspectRatio= */ 2f, Presentation.LAYOUT_SCALE_TO_FIT)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = Bitmap expectedBitmap =
@ -175,7 +176,7 @@ public final class PresentationPixelTest {
presentationTextureProcessor = presentationTextureProcessor =
Presentation.createForAspectRatio( Presentation.createForAspectRatio(
/* aspectRatio= */ 1f, Presentation.LAYOUT_SCALE_TO_FIT_WITH_CROP) /* aspectRatio= */ 1f, Presentation.LAYOUT_SCALE_TO_FIT_WITH_CROP)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = Bitmap expectedBitmap =
@ -202,7 +203,7 @@ public final class PresentationPixelTest {
presentationTextureProcessor = presentationTextureProcessor =
Presentation.createForAspectRatio( Presentation.createForAspectRatio(
/* aspectRatio= */ 2f, Presentation.LAYOUT_SCALE_TO_FIT_WITH_CROP) /* aspectRatio= */ 2f, Presentation.LAYOUT_SCALE_TO_FIT_WITH_CROP)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = Bitmap expectedBitmap =
@ -228,7 +229,7 @@ public final class PresentationPixelTest {
String testId = "drawFrame_changeAspectRatio_stretchToFit_narrow"; String testId = "drawFrame_changeAspectRatio_stretchToFit_narrow";
presentationTextureProcessor = presentationTextureProcessor =
Presentation.createForAspectRatio(/* aspectRatio= */ 1f, Presentation.LAYOUT_STRETCH_TO_FIT) Presentation.createForAspectRatio(/* aspectRatio= */ 1f, Presentation.LAYOUT_STRETCH_TO_FIT)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = Bitmap expectedBitmap =
@ -254,7 +255,7 @@ public final class PresentationPixelTest {
String testId = "drawFrame_changeAspectRatio_stretchToFit_wide"; String testId = "drawFrame_changeAspectRatio_stretchToFit_wide";
presentationTextureProcessor = presentationTextureProcessor =
Presentation.createForAspectRatio(/* aspectRatio= */ 2f, Presentation.LAYOUT_STRETCH_TO_FIT) Presentation.createForAspectRatio(/* aspectRatio= */ 2f, Presentation.LAYOUT_STRETCH_TO_FIT)
.toGlTextureProcessor(context); .toGlTextureProcessor(context, /* useHdr= */ false);
Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight); Size outputSize = presentationTextureProcessor.configure(inputWidth, inputHeight);
setupOutputTexture(outputSize.getWidth(), outputSize.getHeight()); setupOutputTexture(outputSize.getWidth(), outputSize.getHeight());
Bitmap expectedBitmap = Bitmap expectedBitmap =
@ -275,7 +276,9 @@ public final class PresentationPixelTest {
} }
private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException { private void setupOutputTexture(int outputWidth, int outputHeight) throws GlUtil.GlException {
outputTexId = GlUtil.createTexture(outputWidth, outputHeight); outputTexId =
GlUtil.createTexture(
outputWidth, outputHeight, /* useHighPrecisionColorComponents= */ false);
int frameBuffer = GlUtil.createFboForTexture(outputTexId); int frameBuffer = GlUtil.createFboForTexture(outputTexId);
GlUtil.focusFramebuffer( GlUtil.focusFramebuffer(
checkNotNull(eglDisplay), checkNotNull(eglDisplay),

View File

@ -82,6 +82,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Nullable @Nullable
private EGLSurface outputEglSurface; private EGLSurface outputEglSurface;
// TODO(b/227624622): Instead of inputting useHdr, input ColorInfo to handle HLG and PQ
// differently.
public FinalMatrixTransformationProcessorWrapper( public FinalMatrixTransformationProcessorWrapper(
Context context, Context context,
EGLDisplay eglDisplay, EGLDisplay eglDisplay,

View File

@ -21,11 +21,18 @@ import android.content.Context;
* Interface for a video frame effect with a {@link GlTextureProcessor} implementation. * Interface for a video frame effect with a {@link GlTextureProcessor} implementation.
* *
* <p>Implementations contain information specifying the effect and can be {@linkplain * <p>Implementations contain information specifying the effect and can be {@linkplain
* #toGlTextureProcessor(Context) converted} to a {@link GlTextureProcessor} which applies the * #toGlTextureProcessor(Context, boolean) converted} to a {@link GlTextureProcessor} which applies
* effect. * the effect.
*/ */
public interface GlEffect { public interface GlEffect {
/** Returns a {@link SingleFrameGlTextureProcessor} that applies the effect. */ /**
GlTextureProcessor toGlTextureProcessor(Context context) throws FrameProcessingException; * Returns a {@link SingleFrameGlTextureProcessor} that applies the effect.
*
* @param context A {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
*/
GlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException;
} }

View File

@ -189,7 +189,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
matrixTransformationListBuilder = new ImmutableList.Builder<>(); matrixTransformationListBuilder = new ImmutableList.Builder<>();
sampleFromExternalTexture = false; sampleFromExternalTexture = false;
} }
textureProcessorListBuilder.add(effect.toGlTextureProcessor(context)); textureProcessorListBuilder.add(effect.toGlTextureProcessor(context, useHdr));
} }
textureProcessorListBuilder.add( textureProcessorListBuilder.add(
new FinalMatrixTransformationProcessorWrapper( new FinalMatrixTransformationProcessorWrapper(

View File

@ -49,8 +49,8 @@ public interface GlMatrixTransformation extends GlEffect {
float[] getGlMatrixArray(long presentationTimeUs); float[] getGlMatrixArray(long presentationTimeUs);
@Override @Override
default SingleFrameGlTextureProcessor toGlTextureProcessor(Context context) default SingleFrameGlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
throws FrameProcessingException { throws FrameProcessingException {
return new MatrixTransformationProcessor(context, this); return new MatrixTransformationProcessor(context, useHdr, /* matrixTransformation= */ this);
} }
} }

View File

@ -97,34 +97,40 @@ import java.util.Arrays;
* Creates a new instance. * Creates a new instance.
* *
* @param context The {@link Context}. * @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @param matrixTransformation A {@link MatrixTransformation} that specifies the transformation * @param matrixTransformation A {@link MatrixTransformation} that specifies the transformation
* matrix to use for each frame. * matrix to use for each frame.
* @throws FrameProcessingException If a problem occurs while reading shader files. * @throws FrameProcessingException If a problem occurs while reading shader files.
*/ */
public MatrixTransformationProcessor(Context context, MatrixTransformation matrixTransformation) public MatrixTransformationProcessor(
Context context, boolean useHdr, MatrixTransformation matrixTransformation)
throws FrameProcessingException { throws FrameProcessingException {
this( this(
context, context,
ImmutableList.of(matrixTransformation), ImmutableList.of(matrixTransformation),
/* sampleFromExternalTexture= */ false, /* sampleFromExternalTexture= */ false,
/* useHdr= */ false); useHdr);
} }
/** /**
* Creates a new instance. * Creates a new instance.
* *
* @param context The {@link Context}. * @param context The {@link Context}.
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
* @param matrixTransformation A {@link GlMatrixTransformation} that specifies the transformation * @param matrixTransformation A {@link GlMatrixTransformation} that specifies the transformation
* matrix to use for each frame. * matrix to use for each frame.
* @throws FrameProcessingException If a problem occurs while reading shader files. * @throws FrameProcessingException If a problem occurs while reading shader files.
*/ */
public MatrixTransformationProcessor(Context context, GlMatrixTransformation matrixTransformation) public MatrixTransformationProcessor(
Context context, boolean useHdr, GlMatrixTransformation matrixTransformation)
throws FrameProcessingException { throws FrameProcessingException {
this( this(
context, context,
ImmutableList.of(matrixTransformation), ImmutableList.of(matrixTransformation),
/* sampleFromExternalTexture= */ false, /* sampleFromExternalTexture= */ false,
/* useHdr= */ false); useHdr);
} }
/** /**
@ -147,6 +153,7 @@ import java.util.Arrays;
boolean sampleFromExternalTexture, boolean sampleFromExternalTexture,
boolean useHdr) boolean useHdr)
throws FrameProcessingException { throws FrameProcessingException {
super(useHdr);
if (sampleFromExternalTexture && useHdr && !GlUtil.isYuvTargetExtensionSupported()) { if (sampleFromExternalTexture && useHdr && !GlUtil.isYuvTargetExtensionSupported()) {
throw new FrameProcessingException( throw new FrameProcessingException(
"The EXT_YUV_target extension is required for HDR editing."); "The EXT_YUV_target extension is required for HDR editing.");

View File

@ -38,6 +38,17 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
private int inputHeight; private int inputHeight;
private @MonotonicNonNull TextureInfo outputTexture; private @MonotonicNonNull TextureInfo outputTexture;
private boolean outputTextureInUse; private boolean outputTextureInUse;
private final boolean useHdr;
/**
* Creates a {@code SingleFrameGlTextureProcessor} instance.
*
* @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
* in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
*/
public SingleFrameGlTextureProcessor(boolean useHdr) {
this.useHdr = useHdr;
}
/** /**
* Configures the texture processor based on the input dimensions. * Configures the texture processor based on the input dimensions.
@ -116,7 +127,7 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
if (outputTexture != null) { if (outputTexture != null) {
GlUtil.deleteTexture(outputTexture.texId); GlUtil.deleteTexture(outputTexture.texId);
} }
int outputTexId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight()); int outputTexId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight(), useHdr);
int outputFboId = GlUtil.createFboForTexture(outputTexId); int outputFboId = GlUtil.createFboForTexture(outputTexId);
outputTexture = outputTexture =
new TextureInfo(outputTexId, outputFboId, outputSize.getWidth(), outputSize.getHeight()); new TextureInfo(outputTexId, outputFboId, outputSize.getWidth(), outputSize.getHeight());