HDR: Implement HLG EOTF and OETF.

This allows us to use BT.2020 RGB linear for intermediate shaders, which also
allows us to re-enable PeriodicVignetteProcessor, which should work properly in
linear color-spaces.

Manually tested by adding a GlEffectsWrapper, and confirming that HLG HDR editing still looks correct.

PiperOrigin-RevId: 462265821
(cherry picked from commit 2f977eeec93be5cc71fda57f80362c1bc639c041)

demos/transformer/src/main/java/androidx/media3/demo/transformer/BitmapOverlayProcessor.java

@@ -67,7 +67,7 @@ import java.util.Locale;
    *
    * @param context The {@link Context}.
    * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
-   *     in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
+   *     in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
    * @throws FrameProcessingException If a problem occurs while reading shader files.
    */
   public BitmapOverlayProcessor(Context context, boolean useHdr) throws FrameProcessingException {

demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java

@@ -53,7 +53,7 @@ import java.io.IOException;
    *
    * @param context The {@link Context}.
    * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
-   *     in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
+   *     in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
    * @param centerX The x-coordinate of the center of the effect.
    * @param centerY The y-coordinate of the center of the effect.
    * @param minInnerRadius The lower bound of the radius that is unaffected by the effect.
@@ -71,7 +71,6 @@ import java.io.IOException;
       float outerRadius)
       throws FrameProcessingException {
     super(useHdr);
-    checkArgument(!useHdr, "PeriodicVignetteProcessor does not support HDR color spaces.");
     checkArgument(minInnerRadius <= maxInnerRadius);
     checkArgument(maxInnerRadius <= outerRadius);
     this.minInnerRadius = minInnerRadius;

demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java

@@ -72,7 +72,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
    *
    * @param context The {@link Context}.
    * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
-   *     in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
+   *     in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
    * @param graphName Name of a MediaPipe graph asset to load.
    * @param inputStreamName Name of the input video stream in the graph.
    * @param outputStreamName Name of the input video stream in the graph.

libraries/transformer/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl

@@ -13,24 +13,62 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// ES 3 fragment shader that samples from an external texture with uTexSampler,
+// ES 3 fragment shader that:
-// copying from this texture to the current output while applying the specified
+// 1. samples HLG BT.2020 YUV from an external texture with uTexSampler, where
-// color transform uColorTransform, which should be a YUV to RGB conversion
+//    the sampler uses the EXT_YUV_target extension specified at
-// matrix. The sampler uses the using the EXT_YUV_target extension:
+//    https://www.khronos.org/registry/OpenGL/extensions/EXT/EXT_YUV_target.txt,
-// https://www.khronos.org/registry/OpenGL/extensions/EXT/EXT_YUV_target.txt.
+// 2. Applies a YUV to RGB conversion using the specified color transform
+//    uColorTransform, yielding HLG BT.2020 RGB,
+// 3. If uApplyHlgOetf is 1, outputs HLG BT.2020 RGB. If 0, outputs
+//    linear BT.2020 RGB for intermediate shaders by applying the HLG OETF.
+// 4. Copies this converted texture color to the current output.
 
 #extension GL_OES_EGL_image_external : require
 #extension GL_EXT_YUV_target : require
 precision mediump float;
 uniform __samplerExternal2DY2YEXT uTexSampler;
+// YUV to RGB conversion matrix.
 uniform mat3 uColorTransform;
+uniform float uApplyHlgOetf;
 in vec2 vTexSamplingCoord;
 out vec4 outColor;
+
+// TODO(b/227624622): Consider using mediump to save precision, if it won't lead
+//  to noticeable quantization errors.
+
+// HLG OETF for one channel.
+highp float hlgOetfSingleChannel(highp float hlgChannel) {
+ // Specification:
+ // https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG
+ // Reference implementation:
+ // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=529-543;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
+  const highp float a = 0.17883277;
+  const highp float b = 0.28466892;
+  const highp float c = 0.55991073;
+
+  return hlgChannel <= 1.0 / 12.0 ? sqrt(3.0 * hlgChannel) :
+      a * log(12.0 * hlgChannel - b) + c;
+}
+
+ // BT.2100-0 HLG OETF. Converts nonlinear relative display light to linear
+ // signal values, both normalized to [0, 1].
+highp vec4 hlgOetf(highp vec4 hlgColor) {
+  return vec4(
+    hlgOetfSingleChannel(hlgColor.r),
+    hlgOetfSingleChannel(hlgColor.g),
+    hlgOetfSingleChannel(hlgColor.b),
+    hlgColor.a
+  );
+}
+
+/** Convert YUV to RGBA. */
+vec4 yuvToRgba(vec3 yuv) {
+  vec3 yuvOffset = vec3(yuv.x - 0.0625, yuv.y - 0.5, yuv.z - 0.5);
+  return vec4(uColorTransform * yuvOffset, 1.0);
+}
+
 void main() {
   vec3 srcYuv = texture(uTexSampler, vTexSamplingCoord).xyz;
-  vec3 yuvOffset;
+  outColor = yuvToRgba(srcYuv);
-  yuvOffset.x = srcYuv.r - 0.0625;
+  outColor = (uApplyHlgOetf == 1.0) ? hlgOetf(outColor) : outColor;
-  yuvOffset.y = srcYuv.g - 0.5;
-  yuvOffset.z = srcYuv.b - 0.5;
-  outColor = vec4(uColorTransform * yuvOffset, 1.0);
 }

libraries/transformer/src/main/assets/shaders/fragment_shader_hlg_eotf_es3.glsl

@@ -0,0 +1,55 @@
+#version 300 es
+// Copyright 2022 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// ES 3 fragment shader that:
+// 1. samples linear BT.2020 RGB from a (non-external) texture with uTexSampler,
+// 2. applies the HLG OETF to yield HLG BT.2020 RGB, and
+// 3. copies this converted texture color to the current output.
+
+precision mediump float;
+uniform sampler2D uTexSampler;
+in vec2 vTexSamplingCoord;
+out vec4 outColor;
+uniform mat3 uColorTransform;
+
+// TODO(b/227624622): Consider using mediump to save precision, if it won't lead
+//  to noticeable quantization.
+// HLG OETF for one channel.
+highp float hlgEotfSingleChannel(highp float linearChannel) {
+  // Specification:
+  // https://www.khronos.org/registry/DataFormat/specs/1.3/dataformat.1.3.inline.html#TRANSFER_HLG
+  // Reference implementation:
+  // https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;l=265-279;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa
+  const highp float a = 0.17883277;
+  const highp float b = 0.28466892;
+  const highp float c = 0.55991073;
+  return linearChannel <= 0.5 ? linearChannel * linearChannel / 3.0 :
+      (b + exp((linearChannel - c) / a)) / 12.0;
+}
+
+// BT.2100-0 HLG EOTF. Converts nonlinear signal values to linear relative
+// display light, both normalized to [0,1].
+highp vec4 hlgEotf(highp vec4 linearColor) {
+  return vec4(
+    hlgEotfSingleChannel(linearColor.r),
+    hlgEotfSingleChannel(linearColor.g),
+    hlgEotfSingleChannel(linearColor.b),
+    linearColor.a
+  );
+}
+
+void main() {
+  outColor = hlgEotf(texture(uTexSampler, vTexSamplingCoord));
+}

libraries/transformer/src/main/java/androidx/media3/transformer/FinalMatrixTransformationProcessorWrapper.java

@@ -251,7 +251,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
     MatrixTransformationProcessor matrixTransformationProcessor =
         new MatrixTransformationProcessor(
-            context, matrixTransformationListBuilder.build(), sampleFromExternalTexture, useHdr);
+            context,
+            matrixTransformationListBuilder.build(),
+            sampleFromExternalTexture,
+            useHdr,
+            /* outputOpticalColors= */ true);
     matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
     Size outputSize = matrixTransformationProcessor.configure(inputWidth, inputHeight);
     checkState(outputSize.getWidth() == outputSurfaceInfo.width);

libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java

@@ -33,8 +33,12 @@ public interface GlEffect {
    *
    * @param context A {@link Context}.
    * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
-   *     in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
+   *     in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
    */
+  // TODO(b/227624622): PQ input files will actually have the incorrect HLG OETF applied, so that
+  // the intermediate color space will be PQ with the HLG OETF applied. This means intermediate
+  // GlEffects affecting color will look incorrect on PQ input. Fix this by implementing proper PQ
+  // OETF / EOTF support.
   GlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
       throws FrameProcessingException;
 }

libraries/transformer/src/main/java/androidx/media3/transformer/GlEffectsFrameProcessor.java

@@ -185,7 +185,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       if (!matrixTransformations.isEmpty() || sampleFromExternalTexture) {
         textureProcessorListBuilder.add(
             new MatrixTransformationProcessor(
-                context, matrixTransformations, sampleFromExternalTexture, useHdr));
+                context,
+                matrixTransformations,
+                sampleFromExternalTexture,
+                useHdr,
+                /* outputOpticalColors= */ false));
         matrixTransformationListBuilder = new ImmutableList.Builder<>();
         sampleFromExternalTexture = false;
       }

libraries/transformer/src/main/java/androidx/media3/transformer/MatrixTransformationProcessor.java

@@ -50,6 +50,8 @@ import java.util.Arrays;
   private static final String VERTEX_SHADER_TRANSFORMATION_ES3_PATH =
       "shaders/vertex_shader_transformation_es3.glsl";
   private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl";
+  private static final String FRAGMENT_SHADER_HLG_EOTF_ES3_PATH =
+      "shaders/fragment_shader_hlg_eotf_es3.glsl";
   private static final String FRAGMENT_SHADER_COPY_EXTERNAL_PATH =
       "shaders/fragment_shader_copy_external_es2.glsl";
   private static final String FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH =
@@ -100,7 +102,7 @@ import java.util.Arrays;
    *
    * @param context The {@link Context}.
    * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
-   *     in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
+   *     in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
    * @param matrixTransformation A {@link MatrixTransformation} that specifies the transformation
    *     matrix to use for each frame.
    * @throws FrameProcessingException If a problem occurs while reading shader files.
@@ -112,7 +114,8 @@ import java.util.Arrays;
         context,
         ImmutableList.of(matrixTransformation),
         /* sampleFromExternalTexture= */ false,
-        useHdr);
+        useHdr,
+        /* outputOpticalColors= */ false);
   }
 
   /**
@@ -120,7 +123,7 @@ import java.util.Arrays;
    *
    * @param context The {@link Context}.
    * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
-   *     in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
+   *     in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
    * @param matrixTransformation A {@link GlMatrixTransformation} that specifies the transformation
    *     matrix to use for each frame.
    * @throws FrameProcessingException If a problem occurs while reading shader files.
@@ -132,7 +135,8 @@ import java.util.Arrays;
         context,
         ImmutableList.of(matrixTransformation),
         /* sampleFromExternalTexture= */ false,
-        useHdr);
+        useHdr,
+        /* outputOpticalColors= */ false);
   }
 
   /**
@@ -146,6 +150,9 @@ import java.util.Arrays;
    *     provide the transformation matrix associated with the external texture.
    * @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code
    *     EXT_YUV_target} OpenGL extension.
+   * @param outputOpticalColors If {@code true} and {@code useHdr} is also {@code true}, outputs a
+   *     non-linear optical, or display light colors, possibly by applying the EOTF (Electro-optical
+   *     transfer function). Otherwise, outputs linear electrical colors.
    * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
    *     operation fails or is unsupported.
    */
@@ -153,12 +160,13 @@ import java.util.Arrays;
       Context context,
       ImmutableList<GlMatrixTransformation> matrixTransformations,
       boolean sampleFromExternalTexture,
-      boolean useHdr)
+      boolean useHdr,
+      boolean outputOpticalColors)
       throws FrameProcessingException {
     super(useHdr);
     if (sampleFromExternalTexture && useHdr && !GlUtil.isYuvTargetExtensionSupported()) {
       throw new FrameProcessingException(
-          "The EXT_YUV_target extension is required for HDR editing.");
+          "The EXT_YUV_target extension is required for HDR editing input.");
     }
 
     this.matrixTransformations = matrixTransformations;
@@ -176,6 +184,11 @@ import java.util.Arrays;
           useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
       fragmentShaderFilePath =
           useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
+    } else if (outputOpticalColors) {
+      vertexShaderFilePath =
+          useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
+      fragmentShaderFilePath =
+          useHdr ? FRAGMENT_SHADER_HLG_EOTF_ES3_PATH : FRAGMENT_SHADER_COPY_PATH;
     } else {
       vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
       fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH;
@@ -190,6 +203,11 @@ import java.util.Arrays;
     if (useHdr && sampleFromExternalTexture) {
       // In HDR editing mode the decoder output is sampled in YUV.
       glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
+      // TODO(b/227624622): Implement PQ, and use an @IntDef to select between HLG, PQ, and no
+      //  transfer function.
+      // Applying the OETF will output a linear signal. Not applying the OETF will output an optical
+      // signal.
+      glProgram.setFloatUniform("uApplyHlgOetf", outputOpticalColors ? 0.0f : 1.0f);
     }
     float[] identityMatrix = new float[16];
     Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0);

libraries/transformer/src/main/java/androidx/media3/transformer/SingleFrameGlTextureProcessor.java

@@ -46,7 +46,7 @@ public abstract class SingleFrameGlTextureProcessor implements GlTextureProcesso
    * Creates a {@code SingleFrameGlTextureProcessor} instance.
    *
    * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be
-   *     in HLG/PQ RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
+   *     in linear RGB BT.2020. If {@code false}, colors will be in gamma RGB BT.709.
    */
   public SingleFrameGlTextureProcessor(boolean useHdr) {
     this.useHdr = useHdr;