remove degammaing: change setSdrWorkingColorSpace default

The second stage of the changes remove the conversion to linear colors in the SDR effects pipeline by default.

also resolves Issue: androidx/media#1050

PiperOrigin-RevId: 630108296

RELEASENOTES.md

@@ -55,6 +55,9 @@
     *   Fix bug where `TimestampWrapper` crashes when used with
         `ExoPlayer#setVideoEffects`
         ([#821](https://github.com/androidx/media/issues/821)).
+    *   Change default SDR color working space from linear colors to electrical
+        BT 709 SDR video. Also provides third option to retain the original
+        colorspace.
 *   Muxers:
 *   IMA extension:
     *   Promote API that is required for apps to play

libraries/common/src/main/java/androidx/media3/common/util/GlProgram.java

@@ -204,6 +204,15 @@ public final class GlProgram {
     checkNotNull(uniformByName.get(name)).setFloats(value);
   }
 
+  /** Sets a {@code float[]} type uniform if {@code name} is present, no-op otherwise. */
+  public void setFloatsUniformIfPresent(String name, float[] value) {
+    @Nullable Uniform uniform = uniformByName.get(name);
+    if (uniform == null) {
+      return;
+    }
+    uniform.setFloats(value);
+  }
+
   /** Binds all attributes and uniforms in the program. */
   public void bindAttributesAndUniforms() throws GlUtil.GlException {
     for (Attribute attribute : attributes) {

libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java

@@ -438,7 +438,13 @@ public final class DefaultVideoFrameProcessorPixelTest {
   @Test
   public void increaseBrightness_matchesGoldenFile() throws Exception {
     videoFrameProcessorTestRunner =
-        getDefaultFrameProcessorTestRunnerBuilder(testId).setEffects(new Brightness(0.5f)).build();
+        getDefaultFrameProcessorTestRunnerBuilder(testId)
+            .setVideoFrameProcessorFactory(
+                new DefaultVideoFrameProcessor.Factory.Builder()
+                    .setSdrWorkingColorSpace(DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR)
+                    .build())
+            .setEffects(new Brightness(0.5f))
+            .build();
     Bitmap expectedBitmap = readBitmap(INCREASE_BRIGHTNESS_PNG_ASSET_PATH);
 
     videoFrameProcessorTestRunner.processFirstFrameAndEnd();
@@ -545,6 +551,10 @@ public final class DefaultVideoFrameProcessorPixelTest {
   public void grayscaleThenIncreaseRedChannel_matchesGoldenFile() throws Exception {
     videoFrameProcessorTestRunner =
         getDefaultFrameProcessorTestRunnerBuilder(testId)
+            .setVideoFrameProcessorFactory(
+                new DefaultVideoFrameProcessor.Factory.Builder()
+                    .setSdrWorkingColorSpace(DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR)
+                    .build())
             .setEffects(
                 RgbFilter.createGrayscaleFilter(),
                 new RgbAdjustment.Builder().setRedScale(3).build())

libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl

@@ -20,9 +20,7 @@
 //    colorspace with the colors transferred to either linear or SMPTE 170M as
 //    requested by uSdrWorkingColorSpace.
 // 3. Applies a 4x4 RGB color matrix to change the pixel colors.
-// 4. Outputs as requested by uOutputColorTransfer. Use COLOR_TRANSFER_LINEAR
+// 4. Outputs as requested by uOutputColorTransfer.
-//    for outputting to intermediate shaders, or COLOR_TRANSFER_SDR_VIDEO to
-//    output electrical colors via an OETF (e.g. to an encoder).
 
 #extension GL_OES_EGL_image_external : require
 precision mediump float;

libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl

@@ -21,9 +21,7 @@
 //    colorspace with the colors transferred to either linear or SMPTE 170M as
 //    requested by uSdrWorkingColorSpace.
 // 3. Applies a 4x4 RGB color matrix to change the pixel colors.
-// 4. Outputs as requested by uOutputColorTransfer. Use COLOR_TRANSFER_LINEAR
+// 4. Outputs as requested by uOutputColorTransfer.
-//    for outputting to intermediate shaders, or COLOR_TRANSFER_SDR_VIDEO to
-//    output electrical colors via an OETF (e.g. to an encoder).
 
 precision mediump float;
 uniform sampler2D uTexSampler;

libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java

@@ -30,7 +30,6 @@ import android.opengl.Matrix;
 import androidx.annotation.RequiresApi;
 import androidx.media3.common.C;
 import androidx.media3.common.ColorInfo;
-import androidx.media3.common.Format;
 import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.VideoFrameProcessor.InputType;
 import androidx.media3.common.util.GlProgram;
@@ -74,6 +73,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       "shaders/vertex_shader_transformation_es3.glsl";
   private static final String FRAGMENT_SHADER_TRANSFORMATION_PATH =
       "shaders/fragment_shader_transformation_es2.glsl";
+  private static final String FRAGMENT_SHADER_COPY_PATH = "shaders/fragment_shader_copy_es2.glsl";
   private static final String FRAGMENT_SHADER_OETF_ES3_PATH =
       "shaders/fragment_shader_oetf_es3.glsl";
   private static final String FRAGMENT_SHADER_TRANSFORMATION_SDR_OETF_ES2_PATH =
@@ -180,14 +180,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       List<RgbMatrix> rgbMatrices,
       boolean useHdr)
       throws VideoFrameProcessingException {
+    String fragmentShaderFilePath =
+        rgbMatrices.isEmpty()
+            // Ensure colors not multiplied by a uRgbMatrix (even the identity) as it can create
+            // color shifts on electrical pq tonemapped content.
+            ? FRAGMENT_SHADER_COPY_PATH
+            : FRAGMENT_SHADER_TRANSFORMATION_PATH;
     GlProgram glProgram =
-        createGlProgram(
+        createGlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, fragmentShaderFilePath);
-            context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_TRANSFORMATION_PATH);
 
-    // TODO: b/263306471 - when default working color space changes to WORKING_COLOR_SPACE_DEFAULT,
+    // No transfer functions needed/applied, because input and output are in the same color space.
-    //  make sure no color transfers are applied in shader.
-
-    // No transfer functions needed, because input and output are both optical colors.
     return new DefaultShaderProgram(
         glProgram,
         ImmutableList.copyOf(matrixTransformations),
@@ -239,6 +241,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
                 : FRAGMENT_SHADER_TRANSFORMATION_SDR_INTERNAL_PATH;
     GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
     if (!isUsingUltraHdr) {
+      checkArgument(
+          isInputTransferHdr
+              || inputColorInfo.colorTransfer == C.COLOR_TRANSFER_SRGB
+              || inputColorInfo.colorTransfer == C.COLOR_TRANSFER_SDR);
       glProgram.setIntUniform("uInputColorTransfer", inputColorInfo.colorTransfer);
     }
     if (isInputTransferHdr) {
@@ -342,6 +348,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
             ? FRAGMENT_SHADER_OETF_ES3_PATH
             : shouldApplyOetf
                 ? FRAGMENT_SHADER_TRANSFORMATION_SDR_OETF_ES2_PATH
+                : rgbMatrices.isEmpty()
+                    // Ensure colors not multiplied by a uRgbMatrix (even the identity) as it can
+                    // create color shifts on electrical pq tonemapped content.
+                    ? FRAGMENT_SHADER_COPY_PATH
                     : FRAGMENT_SHADER_TRANSFORMATION_PATH;
     GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
 
@@ -379,14 +389,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     @C.ColorTransfer int outputColorTransfer = outputColorInfo.colorTransfer;
     if (isInputTransferHdr) {
       // TODO(b/239735341): Add a setBooleanUniform method to GlProgram.
-      checkArgument(outputColorTransfer != Format.NO_VALUE);
       if (outputColorTransfer == C.COLOR_TRANSFER_SDR) {
         // When tone-mapping from HDR to SDR, COLOR_TRANSFER_SDR is interpreted as
         // COLOR_TRANSFER_GAMMA_2_2.
         outputColorTransfer = C.COLOR_TRANSFER_GAMMA_2_2;
       }
+      checkArgument(
+          outputColorTransfer == C.COLOR_TRANSFER_LINEAR
+              || outputColorTransfer == C.COLOR_TRANSFER_GAMMA_2_2
+              || outputColorTransfer == C.COLOR_TRANSFER_ST2084
+              || outputColorTransfer == C.COLOR_TRANSFER_HLG);
       glProgram.setIntUniform("uOutputColorTransfer", outputColorTransfer);
     } else if (isExpandingColorGamut) {
+      checkArgument(
+          outputColorTransfer == C.COLOR_TRANSFER_LINEAR
+              || outputColorTransfer == C.COLOR_TRANSFER_ST2084
+              || outputColorTransfer == C.COLOR_TRANSFER_HLG);
       glProgram.setIntUniform("uOutputColorTransfer", outputColorTransfer);
     } else {
       glProgram.setIntUniform("uSdrWorkingColorSpace", sdrWorkingColorSpace);
@@ -479,7 +497,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       setGainmapSamplerAndUniforms();
       glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
       glProgram.setFloatsUniform("uTransformationMatrix", compositeTransformationMatrixArray);
-      glProgram.setFloatsUniform("uRgbMatrix", compositeRgbMatrixArray);
+      glProgram.setFloatsUniformIfPresent("uRgbMatrix", compositeRgbMatrixArray);
       glProgram.setBufferAttribute(
           "aFramePosition",
           GlUtil.createVertexBuffer(visiblePolygon),

libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java

@@ -140,7 +140,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
 
       /** Creates an instance. */
       public Builder() {
-        sdrWorkingColorSpace = WORKING_COLOR_SPACE_LINEAR;
+        sdrWorkingColorSpace = WORKING_COLOR_SPACE_DEFAULT;
         requireRegisteringAllInputFrames = true;
       }
 
@@ -153,7 +153,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
         requireRegisteringAllInputFrames = !factory.repeatLastRegisteredFrame;
       }
 
-      // TODO: b/263306471 - Change default to WORKING_COLOR_SPACE_DEFAULT.
       /**
        * Sets the {@link WorkingColorSpace} in which frames passed to intermediate effects will be
        * represented.

libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java

@@ -25,6 +25,7 @@ import static androidx.media3.common.util.Util.newSingleThreadScheduledExecutor;
 import static androidx.media3.effect.DebugTraceUtil.EVENT_COMPOSITOR_OUTPUT_TEXTURE_RENDERED;
 import static androidx.media3.effect.DebugTraceUtil.EVENT_VFP_OUTPUT_TEXTURE_RENDERED;
 import static androidx.media3.effect.DebugTraceUtil.logEvent;
+import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 import android.content.Context;
@@ -119,6 +120,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
     // TODO - b/289986435: Support injecting VideoFrameProcessor.Factory.
     videoFrameProcessorFactory =
         new DefaultVideoFrameProcessor.Factory.Builder()
+            .setSdrWorkingColorSpace(WORKING_COLOR_SPACE_LINEAR)
             .setGlObjectsProvider(glObjectsProvider)
             .setExecutorService(sharedExecutorService)
             .build();

libraries/effect/src/main/java/androidx/media3/effect/OverlayEffect.java

@@ -23,6 +23,9 @@ import com.google.common.collect.ImmutableList;
 /**
  * Applies a list of {@link TextureOverlay}s to a frame in FIFO order (the last overlay in the list
  * is displayed on top).
+ *
+ * <p>This effect assumes a non-{@linkplain DefaultVideoFrameProcessor#WORKING_COLOR_SPACE_LINEAR
+ * linear} working color space.
  */
 @UnstableApi
 public final class OverlayEffect implements GlEffect {

libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java

@@ -188,21 +188,7 @@ import com.google.common.collect.ImmutableList;
             .append("  outputColor.a = overlayColor.a + videoColor.a * (1.0 - overlayColor.a);\n")
             .append("  return outputColor;\n")
             .append("}\n")
-            .append("\n")
+            .append("\n");
-            .append("float srgbEotfSingleChannel(float srgb) {\n")
-            .append("  return srgb <= 0.04045 ? srgb / 12.92 : pow((srgb + 0.055) / 1.055, 2.4);\n")
-            .append("}\n")
-            .append("// sRGB EOTF.\n")
-            .append("vec3 applyEotf(const vec3 srgb) {\n")
-            .append("// Reference implementation:\n")
-            .append(
-                "// https://cs.android.com/android/platform/superproject/+/master:frameworks/native/libs/renderengine/gl/ProgramCache.cpp;drc=de09f10aa504fd8066370591a00c9ff1cafbb7fa;l=235\n")
-            .append("  return vec3(\n")
-            .append("    srgbEotfSingleChannel(srgb.r),\n")
-            .append("    srgbEotfSingleChannel(srgb.g),\n")
-            .append("    srgbEotfSingleChannel(srgb.b)\n")
-            .append("  );\n")
-            .append("}\n");
 
     for (int texUnitIndex = 1; texUnitIndex <= numOverlays; texUnitIndex++) {
       shader
@@ -227,14 +213,10 @@ import com.google.common.collect.ImmutableList;
               formatInvariant(
                   "    uOverlayTexSampler%d, vOverlayTexSamplingCoord%d, uOverlayAlphaScale%d);\n",
                   texUnitIndex, texUnitIndex, texUnitIndex))
-          .append(formatInvariant("  vec4 opticalOverlayColor%d = vec4(\n", texUnitIndex))
           .append(
               formatInvariant(
-                  "    applyEotf(electricalOverlayColor%d.rgb), electricalOverlayColor%d.a);\n",
+                  "  fragColor = getMixColor(fragColor, electricalOverlayColor%d);\n",
-                  texUnitIndex, texUnitIndex))
+                  texUnitIndex));
-          .append(
-              formatInvariant(
-                  "  fragColor = getMixColor(fragColor, opticalOverlayColor%d);\n", texUnitIndex));
     }
 
     shader.append("  gl_FragColor = fragColor;\n").append("}\n");

libraries/effect/src/main/java/androidx/media3/effect/RgbFilter.java

@@ -23,7 +23,12 @@ import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.util.UnstableApi;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
-/** Provides common color filters. */
+/**
+ * Provides common color filters.
+ *
+ * <p>This effect assumes a {@linkplain DefaultVideoFrameProcessor#WORKING_COLOR_SPACE_LINEAR
+ * linear} working color space.
+ */
 @UnstableApi
 public final class RgbFilter implements RgbMatrix {
   private static final int COLOR_FILTER_GRAYSCALE_INDEX = 1;

libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java

@@ -860,6 +860,7 @@ public final class DefaultVideoCompositorPixelTest {
       DefaultVideoFrameProcessor.Factory.Builder defaultVideoFrameProcessorFactoryBuilder =
           new DefaultVideoFrameProcessor.Factory.Builder()
               .setGlObjectsProvider(glObjectsProvider)
+              .setSdrWorkingColorSpace(DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR)
               .setTextureOutput(
                   /* textureOutputListener= */ (outputTextureProducer,
                       outputTexture,

libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerMultiSequenceCompositionTest.java

@@ -38,6 +38,7 @@ import androidx.media3.common.util.Size;
 import androidx.media3.common.util.Util;
 import androidx.media3.effect.AlphaScale;
 import androidx.media3.effect.Contrast;
+import androidx.media3.effect.DefaultVideoFrameProcessor;
 import androidx.media3.effect.OverlaySettings;
 import androidx.media3.effect.Presentation;
 import androidx.media3.effect.ScaleAndRotateTransformation;
@@ -103,7 +104,7 @@ public final class TransformerMultiSequenceCompositionTest {
             VideoCompositorSettings.DEFAULT);
 
     ExportTestResult result =
-        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+        new TransformerAndroidTestRunner.Builder(context, getLinearColorSpaceTransformer())
             .build()
             .run(testId, composition);
 
@@ -137,7 +138,7 @@ public final class TransformerMultiSequenceCompositionTest {
             VideoCompositorSettings.DEFAULT);
 
     ExportTestResult result =
-        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+        new TransformerAndroidTestRunner.Builder(context, getLinearColorSpaceTransformer())
             .build()
             .run(testId, composition);
 
@@ -193,7 +194,7 @@ public final class TransformerMultiSequenceCompositionTest {
             pictureInPictureVideoCompositorSettings);
 
     ExportTestResult result =
-        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+        new TransformerAndroidTestRunner.Builder(context, getLinearColorSpaceTransformer())
             .build()
             .run(testId, composition);
 
@@ -202,6 +203,16 @@ public final class TransformerMultiSequenceCompositionTest {
         extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId);
   }
 
+  private Transformer getLinearColorSpaceTransformer() {
+    // Use linear color space for grayscale effects.
+    return new Transformer.Builder(context)
+        .setVideoFrameProcessorFactory(
+            new DefaultVideoFrameProcessor.Factory.Builder()
+                .setSdrWorkingColorSpace(DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR)
+                .build())
+        .build();
+  }
+
   private static EditedMediaItem editedMediaItemByClippingVideo(String uri, List<Effect> effects) {
     return new EditedMediaItem.Builder(
             MediaItem.fromUri(uri)

libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerSequenceEffectTest.java

@@ -43,6 +43,7 @@ import androidx.media3.common.Effect;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.util.Util;
 import androidx.media3.effect.BitmapOverlay;
+import androidx.media3.effect.DefaultVideoFrameProcessor;
 import androidx.media3.effect.OverlayEffect;
 import androidx.media3.effect.Presentation;
 import androidx.media3.effect.RgbFilter;
@@ -154,7 +155,7 @@ public final class TransformerSequenceEffectTest {
                 SINGLE_30_FPS_VIDEO_FRAME_THRESHOLD_MS));
 
     ExportTestResult result =
-        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+        new TransformerAndroidTestRunner.Builder(context, getLinearColorSpaceTransformer())
             .build()
             .run(testId, composition);
 
@@ -177,7 +178,7 @@ public final class TransformerSequenceEffectTest {
             oneFrameFromImage(JPG_PORTRAIT_ASSET_URI_STRING, NO_EFFECT));
 
     ExportTestResult result =
-        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+        new TransformerAndroidTestRunner.Builder(context, getLinearColorSpaceTransformer())
             .build()
             .run(testId, composition);
 
@@ -201,7 +202,7 @@ public final class TransformerSequenceEffectTest {
                 SINGLE_30_FPS_VIDEO_FRAME_THRESHOLD_MS));
 
     ExportTestResult result =
-        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+        new TransformerAndroidTestRunner.Builder(context, getLinearColorSpaceTransformer())
             .build()
             .run(testId, composition);
 
@@ -226,7 +227,7 @@ public final class TransformerSequenceEffectTest {
             clippedVideo(MP4_ASSET_URI_STRING, NO_EFFECT, SINGLE_30_FPS_VIDEO_FRAME_THRESHOLD_MS));
 
     ExportTestResult result =
-        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+        new TransformerAndroidTestRunner.Builder(context, getLinearColorSpaceTransformer())
             .build()
             .run(testId, composition);
 
@@ -251,7 +252,7 @@ public final class TransformerSequenceEffectTest {
             oneFrameFromImage(JPG_ASSET_URI_STRING, NO_EFFECT));
 
     ExportTestResult result =
-        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+        new TransformerAndroidTestRunner.Builder(context, getLinearColorSpaceTransformer())
             .build()
             .run(testId, composition);
 
@@ -260,6 +261,16 @@ public final class TransformerSequenceEffectTest {
         extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId);
   }
 
+  private Transformer getLinearColorSpaceTransformer() {
+    // Use linear color space for grayscale effects.
+    return new Transformer.Builder(context)
+        .setVideoFrameProcessorFactory(
+            new DefaultVideoFrameProcessor.Factory.Builder()
+                .setSdrWorkingColorSpace(DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR)
+                .build())
+        .build();
+  }
+
   private static OverlayEffect createOverlayEffect() throws IOException {
     return new OverlayEffect(
         ImmutableList.of(

libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java

@@ -132,15 +132,7 @@ import org.checkerframework.dataflow.qual.Pure;
         encoderWrapper.getHdrModeAfterFallback() == HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL
             && ColorInfo.isTransferHdr(videoGraphInputColor);
     if (isGlToneMapping) {
-      // For consistency with the Android platform, OpenGL tone mapping outputs colors with
+      videoGraphOutputColor = SDR_BT709_LIMITED;
-      // C.COLOR_TRANSFER_GAMMA_2_2 instead of C.COLOR_TRANSFER_SDR, and outputs this as
-      // C.COLOR_TRANSFER_SDR to the encoder.
-      videoGraphOutputColor =
-          new ColorInfo.Builder()
-              .setColorSpace(C.COLOR_SPACE_BT709)
-              .setColorRange(C.COLOR_RANGE_LIMITED)
-              .setColorTransfer(C.COLOR_TRANSFER_GAMMA_2_2)
-              .build();
     }
 
     try {