support hdr text overlays

adds luminance multiplier to allow the luminance (i.e. brightness) over a text overlay to be scaled

PiperOrigin-RevId: 643047928

RELEASENOTES.md

@@ -25,6 +25,8 @@
 *   Effect:
     *   Remove unused `OverlaySettings.useHdr` since dynamic range of overlay
         and frame must match.
+    *   Add HDR support for `TextOverlay`. Luminance of the text overlay can be
+        adjusted with `OverlaySettings.setHdrLuminanceMultiplier`.
 *   Muxers:
 *   IMA extension:
 *   Session:

libraries/effect/src/main/java/androidx/media3/effect/OverlaySettings.java

@@ -36,6 +36,7 @@ public final class OverlaySettings {
     private Pair<Float, Float> overlayFrameAnchor;
     private Pair<Float, Float> scale;
     private float rotationDegrees;
+    private float hdrLuminanceMultiplier;
 
     /** Creates a new {@link Builder}. */
     public Builder() {
@@ -44,6 +45,7 @@ public final class OverlaySettings {
       overlayFrameAnchor = Pair.create(0f, 0f);
       scale = Pair.create(1f, 1f);
       rotationDegrees = 0f;
+      hdrLuminanceMultiplier = 1f;
     }
 
     private Builder(OverlaySettings overlaySettings) {
@@ -140,10 +142,31 @@ public final class OverlaySettings {
       return this;
     }
 
+    /**
+     * Set the luminance multiplier of an SDR overlay when overlaid on a HDR frame.
+     *
+     * <p>Scales the luminance of the overlay to adjust the output brightness of the overlay on the
+     * frame. The default value is 1, which scales the overlay colors into the standard HDR
+     * luminance within the processing pipeline. Use 0.5 to scale the luminance of the overlay to
+     * SDR range, so that no extra luminance is added.
+     *
+     * <p>Currently only supported on text overlays
+     */
+    @CanIgnoreReturnValue
+    public Builder setHdrLuminanceMultiplier(float hdrLuminanceMultiplier) {
+      this.hdrLuminanceMultiplier = hdrLuminanceMultiplier;
+      return this;
+    }
+
     /** Creates an instance of {@link OverlaySettings}, using defaults if values are unset. */
     public OverlaySettings build() {
       return new OverlaySettings(
-          alphaScale, backgroundFrameAnchor, overlayFrameAnchor, scale, rotationDegrees);
+          alphaScale,
+          backgroundFrameAnchor,
+          overlayFrameAnchor,
+          scale,
+          rotationDegrees,
+          hdrLuminanceMultiplier);
     }
   }
 
@@ -162,17 +185,22 @@ public final class OverlaySettings {
   /** The rotation of the overlay, counter-clockwise. */
   public final float rotationDegrees;
 
+  /** The luminance multiplier of an SDR overlay when overlaid on a HDR frame. */
+  public final float hdrLuminanceMultiplier;
+
   private OverlaySettings(
       float alphaScale,
       Pair<Float, Float> backgroundFrameAnchor,
       Pair<Float, Float> overlayFrameAnchor,
       Pair<Float, Float> scale,
-      float rotationDegrees) {
+      float rotationDegrees,
+      float hdrLuminanceMultiplier) {
     this.alphaScale = alphaScale;
     this.backgroundFrameAnchor = backgroundFrameAnchor;
     this.overlayFrameAnchor = overlayFrameAnchor;
     this.scale = scale;
     this.rotationDegrees = rotationDegrees;
+    this.hdrLuminanceMultiplier = hdrLuminanceMultiplier;
   }
 
   /** Returns a new {@link Builder} initialized with the values of this instance. */

libraries/effect/src/main/java/androidx/media3/effect/OverlayShaderProgram.java

@@ -17,6 +17,7 @@ package androidx.media3.effect;
 
 import static androidx.media3.common.util.Assertions.checkArgument;
 import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.common.util.Assertions.checkState;
 import static androidx.media3.common.util.Util.formatInvariant;
 import static androidx.media3.common.util.Util.loadAsset;
 
@@ -25,6 +26,7 @@ import android.content.Context;
 import android.graphics.Bitmap;
 import android.graphics.Gainmap;
 import android.opengl.GLES20;
+import android.opengl.Matrix;
 import android.util.SparseArray;
 import android.util.SparseIntArray;
 import androidx.annotation.Nullable;
@@ -40,6 +42,14 @@ import java.io.IOException;
 /** Applies zero or more {@link TextureOverlay}s onto each frame. */
 /* package */ final class OverlayShaderProgram extends BaseGlShaderProgram {
 
+  /** Types of HDR overlay. */
+  private static final int HDR_TYPE_ULTRA_HDR = 1;
+
+  private static final int HDR_TYPE_TEXT = 2;
+
+  // The maximum number of samplers allowed in a single GL program is 16.
+  // We use one for every overlay and one for the video.
+  private static final int MAX_OVERLAY_SAMPLERS = 15;
   private static final String ULTRA_HDR_INSERT = "shaders/insert_ultra_hdr.glsl";
   private static final String FRAGMENT_SHADER_METHODS_INSERT =
       "shaders/insert_overlay_fragment_shader_methods.glsl";
@@ -48,7 +58,8 @@ import java.io.IOException;
   private final GlProgram glProgram;
   private final SamplerOverlayMatrixProvider samplerOverlayMatrixProvider;
   private final ImmutableList<TextureOverlay> overlays;
-  private final boolean useHdr;
+
+  @Nullable private final int[] hdrTypes;
   private final SparseArray<Gainmap> lastGainmaps;
   private final SparseIntArray gainmapTexIds;
 
@@ -67,20 +78,14 @@ import java.io.IOException;
       throws VideoFrameProcessingException {
     super(/* useHighPrecisionColorComponents= */ useHdr, /* texturePoolCapacity= */ 1);
     if (useHdr) {
-      // Each UltraHDR overlay uses an extra texture to apply the gainmap to the base in the shader.
+      hdrTypes = findHdrTypes(overlays);
-      checkArgument(
-          overlays.size() <= 7,
-          "OverlayShaderProgram does not support more than 7 HDR overlays in the same instance.");
-      checkArgument(Util.SDK_INT >= 34);
     } else {
-      // The maximum number of samplers allowed in a single GL program is 16.
+      hdrTypes = null;
-      // We use one for every overlay and one for the video.
       checkArgument(
-          overlays.size() <= 15,
+          overlays.size() <= MAX_OVERLAY_SAMPLERS,
           "OverlayShaderProgram does not support more than 15 SDR overlays in the same instance.");
     }
 
-    this.useHdr = useHdr;
     this.overlays = overlays;
     this.samplerOverlayMatrixProvider = new SamplerOverlayMatrixProvider();
     lastGainmaps = new SparseArray<>();
@@ -89,7 +94,7 @@ import java.io.IOException;
       glProgram =
           new GlProgram(
               createVertexShader(overlays.size()),
-              createFragmentShader(context, overlays.size(), useHdr));
+              createFragmentShader(context, overlays.size(), hdrTypes));
     } catch (GlUtil.GlException | IOException e) {
       throw new VideoFrameProcessingException(e);
     }
@@ -119,7 +124,8 @@ import java.io.IOException;
       for (int texUnitIndex = 1; texUnitIndex <= overlays.size(); texUnitIndex++) {
         TextureOverlay overlay = overlays.get(texUnitIndex - 1);
 
-        if (useHdr) {
+        if (hdrTypes != null) {
+          if (hdrTypes[texUnitIndex - 1] == HDR_TYPE_ULTRA_HDR) {
             checkArgument(overlay instanceof BitmapOverlay);
             Bitmap bitmap = ((BitmapOverlay) overlay).getBitmap(presentationTimeUs);
             checkArgument(bitmap.hasGainmap());
@@ -134,8 +140,19 @@ import java.io.IOException;
                 GlUtil.setTexture(gainmapTexIds.get(texUnitIndex), gainmap.getGainmapContents());
               }
               glProgram.setSamplerTexIdUniform(
-                "uGainmapTexSampler" + texUnitIndex, gainmapTexIds.get(texUnitIndex), texUnitIndex);
+                  "uGainmapTexSampler" + texUnitIndex,
-            GainmapUtil.setGainmapUniforms(glProgram, lastGainmaps.get(texUnitIndex), texUnitIndex);
+                  gainmapTexIds.get(texUnitIndex),
+                  texUnitIndex);
+              GainmapUtil.setGainmapUniforms(
+                  glProgram, lastGainmaps.get(texUnitIndex), texUnitIndex);
+            }
+          } else if (hdrTypes[texUnitIndex - 1] == HDR_TYPE_TEXT) {
+            float[] luminanceMatrix = GlUtil.create4x4IdentityMatrix();
+            float multiplier =
+                overlay.getOverlaySettings(presentationTimeUs).hdrLuminanceMultiplier;
+            Matrix.scaleM(luminanceMatrix, /* mOffset= */ 0, multiplier, multiplier, multiplier);
+            glProgram.setFloatsUniform(
+                formatInvariant("uLuminanceMatrix%d", texUnitIndex), luminanceMatrix);
           }
         }
 
@@ -172,7 +189,7 @@ import java.io.IOException;
       glProgram.delete();
       for (int i = 0; i < overlays.size(); i++) {
         overlays.get(i).release();
-        if (useHdr) {
+        if (hdrTypes != null && hdrTypes[i] == HDR_TYPE_ULTRA_HDR) {
           int gainmapTexId = gainmapTexIds.get(i, /* valueIfKeyNotFound= */ C.INDEX_UNSET);
           if (gainmapTexId != C.INDEX_UNSET) {
             GlUtil.deleteTexture(gainmapTexId);
@@ -184,6 +201,32 @@ import java.io.IOException;
     }
   }
 
+  private static int[] findHdrTypes(ImmutableList<TextureOverlay> overlays) {
+    int[] hdrTypes = new int[overlays.size()];
+    int overlaySamplersAvailable = MAX_OVERLAY_SAMPLERS;
+    for (int i = 0; i < overlays.size(); i++) {
+      TextureOverlay overlay = overlays.get(i);
+      if (overlay instanceof TextOverlay) {
+        // TextOverlay must be checked first since they extend BitmapOverlay.
+        hdrTypes[i] = HDR_TYPE_TEXT;
+        overlaySamplersAvailable -= 1;
+      } else if (overlay instanceof BitmapOverlay) {
+        checkState(Util.SDK_INT >= 34);
+        hdrTypes[i] = HDR_TYPE_ULTRA_HDR;
+        // Each UltraHDR overlay uses an extra texture to apply the gainmap to the base in the
+        // shader.
+        overlaySamplersAvailable -= 2;
+      } else {
+        throw new IllegalArgumentException(overlay + " is not supported on HDR content.");
+      }
+      if (overlaySamplersAvailable < 0) {
+        throw new IllegalArgumentException(
+            "Too many HDR overlays in the same OverlayShaderProgram instance.");
+      }
+    }
+    return hdrTypes;
+  }
+
   private static String createVertexShader(int numOverlays) {
     StringBuilder shader =
         new StringBuilder()
@@ -219,8 +262,8 @@ import java.io.IOException;
     return shader.toString();
   }
 
-  private static String createFragmentShader(Context context, int numOverlays, boolean useHdr)
+  private static String createFragmentShader(
-      throws IOException {
+      Context context, int numOverlays, @Nullable int[] hdrTypes) throws IOException {
     StringBuilder shader =
         new StringBuilder()
             .append("#version 100\n")
@@ -231,7 +274,7 @@ import java.io.IOException;
 
     shader.append(loadAsset(context, FRAGMENT_SHADER_METHODS_INSERT));
 
-    if (useHdr) {
+    if (hdrTypes != null) {
       shader.append(loadAsset(context, ULTRA_HDR_INSERT));
     }
 
@@ -241,7 +284,8 @@ import java.io.IOException;
           .append(formatInvariant("uniform float uOverlayAlphaScale%d;\n", texUnitIndex))
           .append(formatInvariant("varying vec2 vOverlayTexSamplingCoord%d;\n", texUnitIndex))
           .append("\n");
-      if (useHdr) {
+      if (hdrTypes != null) {
+        if (hdrTypes[texUnitIndex - 1] == HDR_TYPE_ULTRA_HDR) {
           shader
               .append("// Uniforms for applying the gainmap to the base.\n")
               .append(formatInvariant("uniform sampler2D uGainmapTexSampler%d;\n", texUnitIndex))
@@ -256,6 +300,9 @@ import java.io.IOException;
               .append(formatInvariant("uniform float uDisplayRatioHdr%d;\n", texUnitIndex))
               .append(formatInvariant("uniform float uDisplayRatioSdr%d;\n", texUnitIndex))
               .append("\n");
+        } else if (hdrTypes[texUnitIndex - 1] == HDR_TYPE_TEXT) {
+          shader.append(formatInvariant("uniform mat4 uLuminanceMatrix%d;\n", texUnitIndex));
+        }
       }
     }
 
@@ -276,12 +323,20 @@ import java.io.IOException;
             + "  vec4 opticalBt2020OverlayColor% =\n"
             + "      vec4(scaleHdrLuminance(bt709ToBt2020(opticalBt709Color%)),"
             + "           electricalOverlayColor%.a);";
+    String luminanceApplicationTemplate =
+        "vec4 opticalOverlayColor% = uLuminanceMatrix% * srgbEotf(electricalOverlayColor%);\n";
     for (int texUnitIndex = 1; texUnitIndex <= numOverlays; texUnitIndex++) {
       shader.append(replaceFormatSpecifierWithIndex(eletricalColorTemplate, texUnitIndex));
       String overlayMixColor = "electricalOverlayColor";
-      if (useHdr) {
+      if (hdrTypes != null) {
+        if (hdrTypes[texUnitIndex - 1] == HDR_TYPE_ULTRA_HDR) {
           shader.append(replaceFormatSpecifierWithIndex(gainmapApplicationTemplate, texUnitIndex));
           overlayMixColor = "opticalBt2020OverlayColor";
+        } else if (hdrTypes[texUnitIndex - 1] == HDR_TYPE_TEXT) {
+          shader.append(
+              replaceFormatSpecifierWithIndex(luminanceApplicationTemplate, texUnitIndex));
+          overlayMixColor = "opticalOverlayColor";
+        }
       }
       shader.append(
           formatInvariant(

libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/DefaultVideoFrameProcessorTextureOutputPixelTest.java

@@ -36,7 +36,11 @@ import static com.google.common.truth.Truth.assertThat;
 
 import android.content.Context;
 import android.graphics.Bitmap;
+import android.graphics.Color;
 import android.graphics.Matrix;
+import android.text.Spannable;
+import android.text.SpannableString;
+import android.text.style.ForegroundColorSpan;
 import androidx.media3.common.C;
 import androidx.media3.common.ColorInfo;
 import androidx.media3.common.Effect;
@@ -52,6 +56,8 @@ import androidx.media3.effect.DefaultVideoFrameProcessor;
 import androidx.media3.effect.GaussianBlur;
 import androidx.media3.effect.GlTextureProducer;
 import androidx.media3.effect.OverlayEffect;
+import androidx.media3.effect.OverlaySettings;
+import androidx.media3.effect.TextOverlay;
 import androidx.media3.test.utils.BitmapPixelTestUtil;
 import androidx.media3.test.utils.TextureBitmapReader;
 import androidx.media3.test.utils.VideoFrameProcessorTestRunner;
@@ -64,6 +70,7 @@ import org.json.JSONException;
 import org.junit.After;
 import org.junit.AssumptionViolatedException;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestName;
@@ -105,6 +112,11 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
       "test-generated-goldens/hdr-goldens/ultrahdr_overlay_hlg.png";
   private static final String ULTRA_HDR_OVERLAY_PQ_PNG_ASSET_PATH =
       "test-generated-goldens/hdr-goldens/ultrahdr_overlay_pq.png";
+  private static final String ULTRA_HDR_AND_TEXT_OVERLAY_PNG_ASSET_PATH =
+      "test-generated-goldens/hdr-goldens/ultrahdr_and_text_overlay.png";
+
+  private static final String HDR_TEXT_OVERLAY_PNG_ASSET_PATH =
+      "test-generated-goldens/hdr-goldens/text_overlay.png";
 
   /** Input SDR video of which we only use the first frame. */
   private static final String INPUT_SDR_MP4_ASSET_STRING = "media/mp4/sample.mp4";
@@ -115,6 +127,8 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
   /** Input HLG video of which we only use the first frame. */
   private static final String INPUT_HLG10_MP4_ASSET_STRING = "media/mp4/hlg-1080p.mp4";
 
+  public static final float HDR_PSNR_THRESHOLD = 43.5f;
+
   @Rule public final TestName testName = new TestName();
 
   private String testId;
@@ -251,6 +265,60 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
         .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE);
   }
 
+  @Test
+  @Ignore("TODO: b/344529901 - enable this test when fixed.")
+  public void ultraHdrBitmapAndTextOverlay_hlg10Input_matchesGoldenFile() throws Exception {
+    Context context = getApplicationContext();
+    Format format = MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT;
+    assumeDeviceSupportsUltraHdrEditing();
+    assumeDeviceSupportsHdrEditing(testId, format);
+    assumeFormatsSupported(context, testId, /* inputFormat= */ format, /* outputFormat= */ null);
+    ColorInfo colorInfo = checkNotNull(format.colorInfo);
+    Bitmap inputBitmap = readBitmap(ULTRA_HDR_ASSET_PATH);
+    inputBitmap =
+        Bitmap.createScaledBitmap(
+            inputBitmap,
+            inputBitmap.getWidth() / 8,
+            inputBitmap.getHeight() / 8,
+            /* filter= */ true);
+    BitmapOverlay bitmapOverlay = BitmapOverlay.createStaticBitmapOverlay(inputBitmap);
+    SpannableString overlayText = new SpannableString("W R G B");
+    overlayText.setSpan(
+        new ForegroundColorSpan(Color.WHITE),
+        /* start= */ 0,
+        /* end= */ 1,
+        Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+    overlayText.setSpan(
+        new ForegroundColorSpan(Color.RED),
+        /* start= */ 2,
+        /* end= */ 3,
+        Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+    overlayText.setSpan(
+        new ForegroundColorSpan(Color.GREEN),
+        /* start= */ 4,
+        /* end= */ 5,
+        Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+    overlayText.setSpan(
+        new ForegroundColorSpan(Color.BLUE),
+        /* start= */ 6,
+        /* end= */ 7,
+        Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+    TextOverlay textOverlay =
+        TextOverlay.createStaticTextOverlay(overlayText, new OverlaySettings.Builder().build());
+    videoFrameProcessorTestRunner =
+        getDefaultFrameProcessorTestRunnerBuilder(testId)
+            .setEffects(new OverlayEffect(ImmutableList.of(bitmapOverlay, textOverlay)))
+            .setOutputColorInfo(colorInfo)
+            .setVideoAssetPath(INPUT_HLG10_MP4_ASSET_STRING)
+            .build();
+    Bitmap expectedBitmap = readBitmap(ULTRA_HDR_AND_TEXT_OVERLAY_PNG_ASSET_PATH);
+
+    videoFrameProcessorTestRunner.processFirstFrameAndEnd();
+    Bitmap actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap();
+
+    assertBitmapsAreSimilar(expectedBitmap, actualBitmap, HDR_PSNR_THRESHOLD);
+  }
+
   @Test
   public void ultraHdrBitmapOverlay_hlg10Input_matchesGoldenFile() throws Exception {
     Context context = getApplicationContext();
@@ -333,6 +401,52 @@ public final class DefaultVideoFrameProcessorTextureOutputPixelTest {
         .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_DIFFERENT_DEVICE_FP16);
   }
 
+  @Test
+  public void textOverlay_hdr10Input_matchesGoldenFile() throws Exception {
+    Context context = getApplicationContext();
+    Format format = MP4_ASSET_720P_4_SECOND_HDR10_FORMAT;
+    assumeDeviceSupportsUltraHdrEditing();
+    assumeDeviceSupportsHdrEditing(testId, format);
+    assumeFormatsSupported(context, testId, /* inputFormat= */ format, /* outputFormat= */ null);
+    ColorInfo colorInfo = checkNotNull(format.colorInfo);
+    SpannableString overlayText = new SpannableString("W R G B");
+    overlayText.setSpan(
+        new ForegroundColorSpan(Color.WHITE),
+        /* start= */ 0,
+        /* end= */ 1,
+        Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+    overlayText.setSpan(
+        new ForegroundColorSpan(Color.RED),
+        /* start= */ 2,
+        /* end= */ 3,
+        Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+    overlayText.setSpan(
+        new ForegroundColorSpan(Color.GREEN),
+        /* start= */ 4,
+        /* end= */ 5,
+        Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+    overlayText.setSpan(
+        new ForegroundColorSpan(Color.BLUE),
+        /* start= */ 6,
+        /* end= */ 7,
+        Spannable.SPAN_EXCLUSIVE_EXCLUSIVE);
+    TextOverlay textOverlay =
+        TextOverlay.createStaticTextOverlay(
+            overlayText, new OverlaySettings.Builder().setHdrLuminanceMultiplier(3f).build());
+    videoFrameProcessorTestRunner =
+        getDefaultFrameProcessorTestRunnerBuilder(testId)
+            .setEffects(new OverlayEffect(ImmutableList.of(textOverlay)))
+            .setOutputColorInfo(colorInfo)
+            .setVideoAssetPath(INPUT_PQ_MP4_ASSET_STRING)
+            .build();
+    Bitmap expectedBitmap = readBitmap(HDR_TEXT_OVERLAY_PNG_ASSET_PATH);
+
+    videoFrameProcessorTestRunner.processFirstFrameAndEnd();
+    Bitmap actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap();
+
+    assertBitmapsAreSimilar(expectedBitmap, actualBitmap, HDR_PSNR_THRESHOLD);
+  }
+
   @Test
   public void noEffects_hlg10Input_matchesGoldenFile() throws Exception {
     Context context = getApplicationContext();