From 0b37d860d13df1ac638f5a75c7af32c840e02956 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Wed, 8 Jun 2022 09:44:25 +0000 Subject: [PATCH] Implement default GlTextureProcessor in SingleFrameGlTextureProcessor. SingleFrameGlTextureProcessor is now an abstract class containing a default implementation of the more flexible GlTextureProcessor interface while still exposing the same simple abstract methods for single frame processing it previously did. FrameProcessorChain and GlEffect will be changed to use GlTextureProcessor in follow-ups. PiperOrigin-RevId: 453633000 --- .../BitmapOverlayProcessor.java | 3 +- .../PeriodicVignetteProcessor.java | 3 +- .../demo/transformer/MediaPipeProcessor.java | 3 +- .../android/exoplayer2/util/GlUtil.java | 35 +++++- .../transformer/FrameProcessorChainTest.java | 5 +- .../transformer/ExternalTextureProcessor.java | 3 +- .../transformer/FrameProcessorChain.java | 13 +-- .../MatrixTransformationProcessor.java | 3 +- .../SingleFrameGlTextureProcessor.java | 109 +++++++++++++++--- 9 files changed, 140 insertions(+), 37 deletions(-) diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/BitmapOverlayProcessor.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/BitmapOverlayProcessor.java index 8c8ef80083..17f921cf37 100644 --- a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/BitmapOverlayProcessor.java +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/BitmapOverlayProcessor.java @@ -44,7 +44,7 @@ import java.util.Locale; */ // TODO(b/227625365): Delete this class and use a texture processor from the Transformer library, // once overlaying a bitmap and text is supported in Transformer. -/* package */ final class BitmapOverlayProcessor implements SingleFrameGlTextureProcessor { +/* package */ final class BitmapOverlayProcessor extends SingleFrameGlTextureProcessor { static { GlUtil.glAssertionsEnabled = true; } @@ -147,6 +147,7 @@ import java.util.Locale; @Override public void release() { + super.release(); if (glProgram != null) { glProgram.delete(); } diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/PeriodicVignetteProcessor.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/PeriodicVignetteProcessor.java index 666782e5c5..eb4f97af2e 100644 --- a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/PeriodicVignetteProcessor.java +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/PeriodicVignetteProcessor.java @@ -30,7 +30,7 @@ import java.io.IOException; * A {@link SingleFrameGlTextureProcessor} that periodically dims the frames such that pixels are * darker the further they are away from the frame center. */ -/* package */ final class PeriodicVignetteProcessor implements SingleFrameGlTextureProcessor { +/* package */ final class PeriodicVignetteProcessor extends SingleFrameGlTextureProcessor { static { GlUtil.glAssertionsEnabled = true; } @@ -108,6 +108,7 @@ import java.io.IOException; @Override public void release() { + super.release(); if (glProgram != null) { glProgram.delete(); } diff --git a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java index 8e7badce41..2654980eb6 100644 --- a/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java +++ b/demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java @@ -40,7 +40,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * Runs a MediaPipe graph on input frames. The implementation is currently limited to graphs that * can immediately produce one output frame per input frame. */ -/* package */ final class MediaPipeProcessor implements SingleFrameGlTextureProcessor { +/* package */ final class MediaPipeProcessor extends SingleFrameGlTextureProcessor { private static final LibraryLoader LOADER = new LibraryLoader("mediapipe_jni") { @@ -160,6 +160,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public void release() { + super.release(); checkStateNotNull(frameProcessor).close(); } } diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java b/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java index b32b1200e8..af26525560 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java @@ -342,6 +342,13 @@ public final class GlUtil { } } + /** Fills the pixels in the current output render target with (r=0, g=0, b=0, a=0). */ + public static void clearOutputFrame() { + GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + GlUtil.checkGlError(); + } + /** * Makes the specified {@code eglSurface} the render target, using a viewport of {@code width} by * {@code height} pixels. @@ -368,6 +375,22 @@ public final class GlUtil { Api17.focusRenderTarget(eglDisplay, eglContext, eglSurface, framebuffer, width, height); } + /** + * Makes the specified {@code framebuffer} the render target, using a viewport of {@code width} by + * {@code height} pixels. + * + *

The caller must ensure that there is a current OpenGL context before calling this method. + * + * @param framebuffer The identifier of the framebuffer object to bind as the output render + * target. + * @param width The viewport width, in pixels. + * @param height The viewport height, in pixels. + */ + @RequiresApi(17) + public static void focusFramebufferUsingCurrentContext(int framebuffer, int width, int height) { + Api17.focusFramebufferUsingCurrentContext(framebuffer, width, height); + } + /** * Deletes a GL texture. * @@ -612,14 +635,22 @@ public final class GlUtil { int framebuffer, int width, int height) { + EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext); + checkEglException("Error making context current"); + focusFramebufferUsingCurrentContext(framebuffer, width, height); + } + + @DoNotInline + public static void focusFramebufferUsingCurrentContext(int framebuffer, int width, int height) { + checkEglException( + !Util.areEqual(EGL14.eglGetCurrentContext(), EGL14.EGL_NO_CONTEXT), "No current context"); + int[] boundFramebuffer = new int[1]; GLES20.glGetIntegerv(GLES20.GL_FRAMEBUFFER_BINDING, boundFramebuffer, /* offset= */ 0); if (boundFramebuffer[0] != framebuffer) { GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, framebuffer); } checkGlError(); - EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext); - checkEglException("Error making context current"); GLES20.glViewport(/* x= */ 0, /* y= */ 0, width, height); checkGlError(); } diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainTest.java index 63ddcacd8d..7dbbcb5394 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainTest.java @@ -135,7 +135,7 @@ public final class FrameProcessorChainTest { /* enableExperimentalHdrEditing= */ false); } - private static class FakeTextureProcessor implements SingleFrameGlTextureProcessor { + private static class FakeTextureProcessor extends SingleFrameGlTextureProcessor { private final Size outputSize; @@ -150,8 +150,5 @@ public final class FrameProcessorChainTest { @Override public void drawFrame(int inputTexId, long presentationTimeNs) {} - - @Override - public void release() {} } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExternalTextureProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExternalTextureProcessor.java index f4e2a9911e..4414fd0383 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExternalTextureProcessor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExternalTextureProcessor.java @@ -26,7 +26,7 @@ import com.google.android.exoplayer2.util.GlUtil; import java.io.IOException; /** Copies frames from an external texture and applies color transformations for HDR if needed. */ -/* package */ class ExternalTextureProcessor implements SingleFrameGlTextureProcessor { +/* package */ class ExternalTextureProcessor extends SingleFrameGlTextureProcessor { static { GlUtil.glAssertionsEnabled = true; @@ -115,6 +115,7 @@ import java.io.IOException; @Override public void release() { + super.release(); if (glProgram != null) { glProgram.delete(); } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessorChain.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessorChain.java index 6736b5cbf4..5825dbc1fd 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessorChain.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessorChain.java @@ -29,7 +29,6 @@ import android.opengl.EGLContext; import android.opengl.EGLDisplay; import android.opengl.EGLExt; import android.opengl.EGLSurface; -import android.opengl.GLES20; import android.util.Size; import android.view.Surface; import android.view.SurfaceHolder; @@ -517,12 +516,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputTexture.fboId, outputTexture.width, outputTexture.height); - clearOutputFrame(); + GlUtil.clearOutputFrame(); textureProcessors.get(i).drawFrame(inputTexId, presentationTimeUs); inputTexId = outputTexture.texId; } GlUtil.focusEglSurface(eglDisplay, eglContext, outputEglSurface, outputWidth, outputHeight); - clearOutputFrame(); + GlUtil.clearOutputFrame(); getLast(textureProcessors).drawFrame(inputTexId, presentationTimeUs); EGLExt.eglPresentationTimeANDROID(eglDisplay, outputEglSurface, inputFrameTimeNs); @@ -533,7 +532,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; int finalInputTexId = inputTexId; debugSurfaceViewWrapper.maybeRenderToSurfaceView( () -> { - clearOutputFrame(); + GlUtil.clearOutputFrame(); try { getLast(textureProcessors).drawFrame(finalInputTexId, finalPresentationTimeUs); } catch (FrameProcessingException e) { @@ -553,12 +552,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } - private static void clearOutputFrame() { - GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0); - GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); - GlUtil.checkGlError(); - } - /** * Releases the {@link SingleFrameGlTextureProcessor SingleFrameGlTextureProcessors} and destroys * the OpenGL context. diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformationProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformationProcessor.java index aa6ffc0cdf..1800369c8f 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformationProcessor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformationProcessor.java @@ -39,7 +39,7 @@ import java.util.Arrays; *

The background color of the output frame will be black. */ @SuppressWarnings("FunctionalInterfaceClash") // b/228192298 -/* package */ final class MatrixTransformationProcessor implements SingleFrameGlTextureProcessor { +/* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor { static { GlUtil.glAssertionsEnabled = true; @@ -169,6 +169,7 @@ import java.util.Arrays; @Override public void release() { + super.release(); glProgram.delete(); } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SingleFrameGlTextureProcessor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SingleFrameGlTextureProcessor.java index 03c2930947..a4fd42f750 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SingleFrameGlTextureProcessor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SingleFrameGlTextureProcessor.java @@ -16,36 +16,40 @@ package com.google.android.exoplayer2.transformer; import android.util.Size; +import androidx.annotation.CallSuper; +import com.google.android.exoplayer2.util.GlUtil; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * Manages a GLSL shader program for processing a frame. Implementations generally copy input pixels * into an output frame, with changes to pixels specific to the implementation. * - *

Methods must be called in the following order: - * - *

    - *
  1. {@link #configure(int, int)}, to configure the frame processor based on the input - * dimensions. - *
  2. {@link #drawFrame(int, long)}, to process one frame. - *
  3. {@link #release()}, upon conclusion of processing. - *
+ *

{@code SingleFrameGlTextureProcessor} implementations must produce exactly one output frame + * per input frame with the same presentation timestamp. For more flexibility, implement {@link + * GlTextureProcessor} directly. * *

All methods in this class must be called on the thread that owns the OpenGL context. */ -// TODO(b/227625423): Add GlTextureProcessor interface for async texture processors and make this an -// abstract class with a default implementation of GlTextureProcessor methods. -public interface SingleFrameGlTextureProcessor { +public abstract class SingleFrameGlTextureProcessor implements GlTextureProcessor { + + private @MonotonicNonNull Listener listener; + private int inputWidth; + private int inputHeight; + private @MonotonicNonNull TextureInfo outputTexture; + private boolean outputTextureInUse; /** * Configures the texture processor based on the input dimensions. * - *

This method can be called multiple times. + *

This method must be called before {@linkplain #drawFrame(int,long) drawing} the first frame + * and before drawing subsequent frames with different input dimensions. * * @param inputWidth The input width, in pixels. * @param inputHeight The input height, in pixels. * @return The output {@link Size} of frames processed through {@link #drawFrame(int, long)}. */ - Size configure(int inputWidth, int inputHeight); + public abstract Size configure(int inputWidth, int inputHeight); /** * Draws one frame. @@ -61,8 +65,81 @@ public interface SingleFrameGlTextureProcessor { * @param presentationTimeUs The presentation timestamp of the current frame, in microseconds. * @throws FrameProcessingException If an error occurs while processing or drawing the frame. */ - void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException; + public abstract void drawFrame(int inputTexId, long presentationTimeUs) + throws FrameProcessingException; - /** Releases all resources. */ - void release(); + @Override + public final void setListener(Listener listener) { + this.listener = listener; + } + + @Override + public final boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { + if (outputTextureInUse) { + return false; + } + + try { + if (outputTexture == null + || inputTexture.width != inputWidth + || inputTexture.height != inputHeight) { + configureOutputTexture(inputTexture.width, inputTexture.height); + } + outputTextureInUse = true; + GlUtil.focusFramebufferUsingCurrentContext( + outputTexture.fboId, outputTexture.width, outputTexture.height); + GlUtil.clearOutputFrame(); + drawFrame(inputTexture.texId, presentationTimeUs); + if (listener != null) { + listener.onInputFrameProcessed(inputTexture); + listener.onOutputFrameAvailable(outputTexture, presentationTimeUs); + } + } catch (FrameProcessingException | RuntimeException e) { + if (listener != null) { + listener.onFrameProcessingError( + e instanceof FrameProcessingException + ? (FrameProcessingException) e + : new FrameProcessingException(e)); + } + } + return true; + } + + @EnsuresNonNull("outputTexture") + private void configureOutputTexture(int inputWidth, int inputHeight) { + this.inputWidth = inputWidth; + this.inputHeight = inputHeight; + Size outputSize = configure(inputWidth, inputHeight); + if (outputTexture == null + || outputSize.getWidth() != outputTexture.width + || outputSize.getHeight() != outputTexture.height) { + if (outputTexture != null) { + GlUtil.deleteTexture(outputTexture.texId); + } + int outputTexId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight()); + int outputFboId = GlUtil.createFboForTexture(outputTexId); + outputTexture = + new TextureInfo(outputTexId, outputFboId, outputSize.getWidth(), outputSize.getHeight()); + } + } + + @Override + public final void releaseOutputFrame(TextureInfo outputTexture) { + outputTextureInUse = false; + } + + @Override + public final void signalEndOfInputStream() { + if (listener != null) { + listener.onOutputStreamEnded(); + } + } + + @Override + @CallSuper + public void release() { + if (outputTexture != null) { + GlUtil.deleteTexture(outputTexture.texId); + } + } }