From c79f950d26592e556a9b31f41308d22a3129d0f5 Mon Sep 17 00:00:00 2001 From: tofunmi Date: Fri, 2 Feb 2024 07:27:52 -0800 Subject: [PATCH] ultra HDR: send gainmap downstream if HDR output requested removing the useHdr parameter from queueInputBitmap() it suggests we support changing between HDR and SDR within a stream, which we don't support. instead, identifying whether to use HDR from the shaderprogram which is informed by the inputColorInfo when the stream is registered. PiperOrigin-RevId: 603681736 --- .../androidx/media3/common/GlTextureInfo.java | 47 ++++++++++++- .../media3/effect/BitmapTextureManager.java | 67 +++++++++---------- .../effect/DefaultVideoFrameProcessor.java | 3 +- .../media3/effect/TextureManager.java | 6 +- 4 files changed, 79 insertions(+), 44 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/GlTextureInfo.java b/libraries/common/src/main/java/androidx/media3/common/GlTextureInfo.java index dd92eaca52..329f053fca 100644 --- a/libraries/common/src/main/java/androidx/media3/common/GlTextureInfo.java +++ b/libraries/common/src/main/java/androidx/media3/common/GlTextureInfo.java @@ -15,6 +15,10 @@ */ package androidx.media3.common; +import static androidx.media3.common.util.Assertions.checkState; + +import android.graphics.Gainmap; +import androidx.annotation.Nullable; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.UnstableApi; @@ -51,8 +55,17 @@ public final class GlTextureInfo { /** The height of the texture, in pixels, or {@link C#LENGTH_UNSET} if not specified. */ public final int height; + /** The {@link Gainmap} associated to this texture, or {@code null} if not specified. */ + @Nullable public final Gainmap gainmap; + /** - * Creates a new instance. + * The OpenGL texture identifier for a gainmap associated to this contents of {@link #texId}, or + * {@link C#INDEX_UNSET} if not specified. + */ + public final int gainmapTexId; + + /** + * Creates a new instance with {@code null} {@link Gainmap} and unspecified {@code gainmapTexId}. * * @param texId The OpenGL texture identifier, or {@link C#INDEX_UNSET} if not specified. * @param fboId Identifier of a framebuffer object associated with the texture, or {@link @@ -68,6 +81,35 @@ public final class GlTextureInfo { this.rboId = rboId; this.width = width; this.height = height; + this.gainmap = null; + this.gainmapTexId = C.INDEX_UNSET; + } + + /** + * Creates a new instance. + * + * @param texId The OpenGL texture identifier, or {@link C#INDEX_UNSET} if not specified. + * @param fboId Identifier of a framebuffer object associated with the texture, or {@link + * C#INDEX_UNSET} if not specified. + * @param rboId Identifier of a renderbuffer object associated with the texture, or {@link + * C#INDEX_UNSET} if not specified. + * @param width The width of the texture, in pixels, or {@link C#LENGTH_UNSET} if not specified. + * @param height The height of the texture, in pixels, or {@link C#LENGTH_UNSET} if not specified. + * @param gainmap The {@link Gainmap} associated to this texture. + * @param gainmapTexId The OpenGL texture identifier for a gainmap associated with the contents of + * {@link #texId}. + */ + public GlTextureInfo( + int texId, int fboId, int rboId, int width, int height, Gainmap gainmap, int gainmapTexId) { + this.texId = texId; + this.fboId = fboId; + this.rboId = rboId; + this.width = width; + this.height = height; + this.gainmap = gainmap; + checkState( + gainmapTexId != C.INDEX_UNSET, "If gainmap is non-null, the gainmapTexId must be set"); + this.gainmapTexId = gainmapTexId; } /** Releases all information associated with this instance. */ @@ -81,5 +123,8 @@ public final class GlTextureInfo { if (rboId != C.INDEX_UNSET) { GlUtil.deleteRbo(rboId); } + if (gainmapTexId != C.INDEX_UNSET) { + GlUtil.deleteTexture(gainmapTexId); + } } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java index 8ebec19dca..b25af0fb89 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java @@ -21,6 +21,7 @@ import static androidx.media3.common.util.Assertions.checkState; import android.graphics.Bitmap; import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; import androidx.media3.common.FrameInfo; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; @@ -40,24 +41,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @UnstableApi /* package */ final class BitmapTextureManager extends TextureManager { - private static final String UNSUPPORTED_IMAGE_CONFIGURATION = - "Unsupported Image Configuration: No more than 8 bits of precision should be used for each" - + " RGB channel."; - - private final GlObjectsProvider glObjectsProvider; - private @MonotonicNonNull GlShaderProgram shaderProgram; // The queue holds all bitmaps with one or more frames pending to be sent downstream. private final Queue pendingBitmaps; + private final GlObjectsProvider glObjectsProvider; + private @MonotonicNonNull GlShaderProgram shaderProgram; private @MonotonicNonNull GlTextureInfo currentGlTextureInfo; private int downstreamShaderProgramCapacity; - - // TODO - b/262693274: Support HDR. - @SuppressWarnings({"UnusedVariable", "FieldCanBeLocal"}) - private boolean useHdr; - private boolean currentInputStreamEnded; private boolean isNextFrameInTexture; + private boolean useHdr; /** * Creates a new instance. @@ -91,17 +84,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public void queueInputBitmap( - Bitmap inputBitmap, - FrameInfo frameInfo, - TimestampIterator inStreamOffsetsUs, - boolean useHdr) { + Bitmap inputBitmap, FrameInfo frameInfo, TimestampIterator inStreamOffsetsUs) { videoFrameProcessingTaskExecutor.submit( () -> { - setupBitmap(inputBitmap, frameInfo, inStreamOffsetsUs, useHdr); + setupBitmap(inputBitmap, frameInfo, inStreamOffsetsUs); currentInputStreamEnded = false; }); } + @Override + public void setInputFrameInfo(FrameInfo inputFrameInfo) { + this.useHdr = ColorInfo.isTransferHdr(inputFrameInfo.colorInfo); + } + @Override public int getPendingFrameCount() { // Always treat all queued bitmaps as immediately processed. @@ -134,20 +129,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } // Methods that must be called on the GL thread. - private void setupBitmap( - Bitmap bitmap, FrameInfo frameInfo, TimestampIterator inStreamOffsetsUs, boolean useHdr) + private void setupBitmap(Bitmap bitmap, FrameInfo frameInfo, TimestampIterator inStreamOffsetsUs) throws VideoFrameProcessingException { - if (Util.SDK_INT >= 26) { - checkState( - !checkNotNull(bitmap.getConfig()).equals(Bitmap.Config.RGBA_F16), - UNSUPPORTED_IMAGE_CONFIGURATION); - } - if (Util.SDK_INT >= 33) { - checkState( - !checkNotNull(bitmap.getConfig()).equals(Bitmap.Config.RGBA_1010102), - UNSUPPORTED_IMAGE_CONFIGURATION); - } - this.useHdr = useHdr; checkArgument(inStreamOffsetsUs.hasNext(), "Bitmap queued but no timestamps provided."); pendingBitmaps.add(new BitmapFrameSequenceInfo(bitmap, frameInfo, inStreamOffsetsUs)); maybeQueueToShaderProgram(); @@ -222,15 +205,27 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; currentGlTextureInfo.release(); } currentTexId = GlUtil.createTexture(bitmap); + if (useHdr && Util.SDK_INT >= 34 && bitmap.hasGainmap()) { + currentGlTextureInfo = + new GlTextureInfo( + currentTexId, + /* fboId= */ C.INDEX_UNSET, + /* rboId= */ C.INDEX_UNSET, + frameInfo.width, + frameInfo.height, + checkNotNull(bitmap.getGainmap()), + GlUtil.createTexture(bitmap.getGainmap().getGainmapContents())); + } else { + currentGlTextureInfo = + new GlTextureInfo( + currentTexId, + /* fboId= */ C.INDEX_UNSET, + /* rboId= */ C.INDEX_UNSET, + frameInfo.width, + frameInfo.height); + } } catch (GlUtil.GlException e) { throw VideoFrameProcessingException.from(e); } - currentGlTextureInfo = - new GlTextureInfo( - currentTexId, - /* fboId= */ C.INDEX_UNSET, - /* rboId= */ C.INDEX_UNSET, - frameInfo.width, - frameInfo.height); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 08c2d8731c..3bf14f83ed 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -397,8 +397,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { .queueInputBitmap( inputBitmap, new FrameInfo.Builder(frameInfo).setOffsetToAddUs(frameInfo.offsetToAddUs).build(), - timestampIterator, - /* useHdr= */ false); + timestampIterator); return true; } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java index 0505b77cc4..26e9476cac 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/TextureManager.java @@ -79,13 +79,9 @@ import androidx.media3.common.util.TimestampIterator; * @param frameInfo Information about the bitmap being queued. * @param inStreamOffsetsUs The times within the current stream that the bitmap should be shown * at. The timestamps should be monotonically increasing. - * @param useHdr Whether input and/or output colors are HDR. */ public void queueInputBitmap( - Bitmap inputBitmap, - FrameInfo frameInfo, - TimestampIterator inStreamOffsetsUs, - boolean useHdr) { + Bitmap inputBitmap, FrameInfo frameInfo, TimestampIterator inStreamOffsetsUs) { throw new UnsupportedOperationException(); }