loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement default GlTextureProcessor in SingleFrameGlTextureProcessor.

Browse Source 
SingleFrameGlTextureProcessor is now an abstract class containing a
default implementation of the more flexible GlTextureProcessor interface
while still exposing the same simple abstract methods for single frame
processing it previously did.

FrameProcessorChain and GlEffect will be changed to use
GlTextureProcessor in follow-ups.

PiperOrigin-RevId: 453633000
This commit is contained in:
hschlueter 2022-06-08 09:44:25 +00:00 committed by Marc Baechinger
parent 2c2be2da92
commit 0b37d860d1
9 changed files with 140 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/BitmapOverlayProcessor.java
View File

@ -44,7 +44,7 @@ import java.util.Locale;
*/
// TODO(b/227625365): Delete this class and use a texture processor from the Transformer library,
// once overlaying a bitmap and text is supported in Transformer.
/* package */ final class BitmapOverlayProcessor implements SingleFrameGlTextureProcessor {
/* package */ final class BitmapOverlayProcessor extends SingleFrameGlTextureProcessor {
static {
GlUtil.glAssertionsEnabled = true;
}
@ -147,6 +147,7 @@ import java.util.Locale;
@Override
public void release() {
super.release();
if (glProgram != null) {
glProgram.delete();
}

3
demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/PeriodicVignetteProcessor.java
View File

@ -30,7 +30,7 @@ import java.io.IOException;
* A {@link SingleFrameGlTextureProcessor} that periodically dims the frames such that pixels are
* darker the further they are away from the frame center.
*/
/* package */ final class PeriodicVignetteProcessor implements SingleFrameGlTextureProcessor {
/* package */ final class PeriodicVignetteProcessor extends SingleFrameGlTextureProcessor {
static {
GlUtil.glAssertionsEnabled = true;
}
@ -108,6 +108,7 @@ import java.io.IOException;
@Override
public void release() {
super.release();
if (glProgram != null) {
glProgram.delete();
}

3
demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeProcessor.java
View File

@ -40,7 +40,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* Runs a MediaPipe graph on input frames. The implementation is currently limited to graphs that
* can immediately produce one output frame per input frame.
*/
/* package */ final class MediaPipeProcessor implements SingleFrameGlTextureProcessor {
/* package */ final class MediaPipeProcessor extends SingleFrameGlTextureProcessor {
private static final LibraryLoader LOADER =
new LibraryLoader("mediapipe_jni") {
@ -160,6 +160,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Override
public void release() {
super.release();
checkStateNotNull(frameProcessor).close();
}
}

35
library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java
View File

@ -342,6 +342,13 @@ public final class GlUtil {
}
}
/** Fills the pixels in the current output render target with (r=0, g=0, b=0, a=0). */
public static void clearOutputFrame() {
GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
GlUtil.checkGlError();
}
/**
* Makes the specified {@code eglSurface} the render target, using a viewport of {@code width} by
* {@code height} pixels.
@ -368,6 +375,22 @@ public final class GlUtil {
Api17.focusRenderTarget(eglDisplay, eglContext, eglSurface, framebuffer, width, height);
}
/**
* Makes the specified {@code framebuffer} the render target, using a viewport of {@code width} by
* {@code height} pixels.
*
* <p>The caller must ensure that there is a current OpenGL context before calling this method.
*
* @param framebuffer The identifier of the framebuffer object to bind as the output render
* target.
* @param width The viewport width, in pixels.
* @param height The viewport height, in pixels.
*/
@RequiresApi(17)
public static void focusFramebufferUsingCurrentContext(int framebuffer, int width, int height) {
Api17.focusFramebufferUsingCurrentContext(framebuffer, width, height);
}
/**
* Deletes a GL texture.
*
@ -612,14 +635,22 @@ public final class GlUtil {
int framebuffer,
int width,
int height) {
EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext);
checkEglException("Error making context current");
focusFramebufferUsingCurrentContext(framebuffer, width, height);
}
@DoNotInline
public static void focusFramebufferUsingCurrentContext(int framebuffer, int width, int height) {
checkEglException(
!Util.areEqual(EGL14.eglGetCurrentContext(), EGL14.EGL_NO_CONTEXT), "No current context");
int[] boundFramebuffer = new int[1];
GLES20.glGetIntegerv(GLES20.GL_FRAMEBUFFER_BINDING, boundFramebuffer, /* offset= */ 0);
if (boundFramebuffer[0] != framebuffer) {
GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, framebuffer);
}
checkGlError();
EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext);
checkEglException("Error making context current");
GLES20.glViewport(/* x= */ 0, /* y= */ 0, width, height);
checkGlError();
}

5
library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameProcessorChainTest.java
View File

@ -135,7 +135,7 @@ public final class FrameProcessorChainTest {
/* enableExperimentalHdrEditing= */ false);
}
private static class FakeTextureProcessor implements SingleFrameGlTextureProcessor {
private static class FakeTextureProcessor extends SingleFrameGlTextureProcessor {
private final Size outputSize;
@ -150,8 +150,5 @@ public final class FrameProcessorChainTest {
@Override
public void drawFrame(int inputTexId, long presentationTimeNs) {}
@Override
public void release() {}
}
}

3
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExternalTextureProcessor.java
View File

@ -26,7 +26,7 @@ import com.google.android.exoplayer2.util.GlUtil;
import java.io.IOException;
/** Copies frames from an external texture and applies color transformations for HDR if needed. */
/* package */ class ExternalTextureProcessor implements SingleFrameGlTextureProcessor {
/* package */ class ExternalTextureProcessor extends SingleFrameGlTextureProcessor {
static {
GlUtil.glAssertionsEnabled = true;
@ -115,6 +115,7 @@ import java.io.IOException;
@Override
public void release() {
super.release();
if (glProgram != null) {
glProgram.delete();
}

13
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameProcessorChain.java
View File

@ -29,7 +29,6 @@ import android.opengl.EGLContext;
import android.opengl.EGLDisplay;
import android.opengl.EGLExt;
import android.opengl.EGLSurface;
import android.opengl.GLES20;
import android.util.Size;
import android.view.Surface;
import android.view.SurfaceHolder;
@ -517,12 +516,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputTexture.fboId,
outputTexture.width,
outputTexture.height);
clearOutputFrame();
GlUtil.clearOutputFrame();
textureProcessors.get(i).drawFrame(inputTexId, presentationTimeUs);
inputTexId = outputTexture.texId;
}
GlUtil.focusEglSurface(eglDisplay, eglContext, outputEglSurface, outputWidth, outputHeight);
clearOutputFrame();
GlUtil.clearOutputFrame();
getLast(textureProcessors).drawFrame(inputTexId, presentationTimeUs);
EGLExt.eglPresentationTimeANDROID(eglDisplay, outputEglSurface, inputFrameTimeNs);
@ -533,7 +532,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
int finalInputTexId = inputTexId;
debugSurfaceViewWrapper.maybeRenderToSurfaceView(
() -> {
clearOutputFrame();
GlUtil.clearOutputFrame();
try {
getLast(textureProcessors).drawFrame(finalInputTexId, finalPresentationTimeUs);
} catch (FrameProcessingException e) {
@ -553,12 +552,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
}
private static void clearOutputFrame() {
GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0);
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
GlUtil.checkGlError();
}
/**
* Releases the {@link SingleFrameGlTextureProcessor SingleFrameGlTextureProcessors} and destroys
* the OpenGL context.

3
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MatrixTransformationProcessor.java
View File

@ -39,7 +39,7 @@ import java.util.Arrays;
* <p>The background color of the output frame will be black.
*/
@SuppressWarnings("FunctionalInterfaceClash") // b/228192298
/* package */ final class MatrixTransformationProcessor implements SingleFrameGlTextureProcessor {
/* package */ final class MatrixTransformationProcessor extends SingleFrameGlTextureProcessor {
static {
GlUtil.glAssertionsEnabled = true;
@ -169,6 +169,7 @@ import java.util.Arrays;
@Override
public void release() {
super.release();
glProgram.delete();
}

109
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SingleFrameGlTextureProcessor.java
View File

@ -16,36 +16,40 @@
package com.google.android.exoplayer2.transformer;
import android.util.Size;
import androidx.annotation.CallSuper;
import com.google.android.exoplayer2.util.GlUtil;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* Manages a GLSL shader program for processing a frame. Implementations generally copy input pixels
* into an output frame, with changes to pixels specific to the implementation.
*
* <p>Methods must be called in the following order:
*
* <ol>
* <li>{@link #configure(int, int)}, to configure the frame processor based on the input
* dimensions.
* <li>{@link #drawFrame(int, long)}, to process one frame.
* <li>{@link #release()}, upon conclusion of processing.
* </ol>
* <p>{@code SingleFrameGlTextureProcessor} implementations must produce exactly one output frame
* per input frame with the same presentation timestamp. For more flexibility, implement {@link
* GlTextureProcessor} directly.
*
* <p>All methods in this class must be called on the thread that owns the OpenGL context.
*/
// TODO(b/227625423): Add GlTextureProcessor interface for async texture processors and make this an
// abstract class with a default implementation of GlTextureProcessor methods.
public interface SingleFrameGlTextureProcessor {
public abstract class SingleFrameGlTextureProcessor implements GlTextureProcessor {
private @MonotonicNonNull Listener listener;
private int inputWidth;
private int inputHeight;
private @MonotonicNonNull TextureInfo outputTexture;
private boolean outputTextureInUse;
/**
* Configures the texture processor based on the input dimensions.
*
* <p>This method can be called multiple times.
* <p>This method must be called before {@linkplain #drawFrame(int,long) drawing} the first frame
* and before drawing subsequent frames with different input dimensions.
*
* @param inputWidth The input width, in pixels.
* @param inputHeight The input height, in pixels.
* @return The output {@link Size} of frames processed through {@link #drawFrame(int, long)}.
*/
Size configure(int inputWidth, int inputHeight);
public abstract Size configure(int inputWidth, int inputHeight);
/**
* Draws one frame.
@ -61,8 +65,81 @@ public interface SingleFrameGlTextureProcessor {
* @param presentationTimeUs The presentation timestamp of the current frame, in microseconds.
* @throws FrameProcessingException If an error occurs while processing or drawing the frame.
*/
void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException;
public abstract void drawFrame(int inputTexId, long presentationTimeUs)
throws FrameProcessingException;
/** Releases all resources. */
void release();
@Override
public final void setListener(Listener listener) {
this.listener = listener;
}
@Override
public final boolean maybeQueueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
if (outputTextureInUse) {
return false;
}
try {
if (outputTexture == null
|| inputTexture.width != inputWidth
|| inputTexture.height != inputHeight) {
configureOutputTexture(inputTexture.width, inputTexture.height);
}
outputTextureInUse = true;
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.fboId, outputTexture.width, outputTexture.height);
GlUtil.clearOutputFrame();
drawFrame(inputTexture.texId, presentationTimeUs);
if (listener != null) {
listener.onInputFrameProcessed(inputTexture);
listener.onOutputFrameAvailable(outputTexture, presentationTimeUs);
}
} catch (FrameProcessingException | RuntimeException e) {
if (listener != null) {
listener.onFrameProcessingError(
e instanceof FrameProcessingException
? (FrameProcessingException) e
: new FrameProcessingException(e));
}
}
return true;
}
@EnsuresNonNull("outputTexture")
private void configureOutputTexture(int inputWidth, int inputHeight) {
this.inputWidth = inputWidth;
this.inputHeight = inputHeight;
Size outputSize = configure(inputWidth, inputHeight);
if (outputTexture == null
|| outputSize.getWidth() != outputTexture.width
|| outputSize.getHeight() != outputTexture.height) {
if (outputTexture != null) {
GlUtil.deleteTexture(outputTexture.texId);
}
int outputTexId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight());
int outputFboId = GlUtil.createFboForTexture(outputTexId);
outputTexture =
new TextureInfo(outputTexId, outputFboId, outputSize.getWidth(), outputSize.getHeight());
}
}
@Override
public final void releaseOutputFrame(TextureInfo outputTexture) {
outputTextureInUse = false;
}
@Override
public final void signalEndOfInputStream() {
if (listener != null) {
listener.onOutputStreamEnded();
}
}
@Override
@CallSuper
public void release() {
if (outputTexture != null) {
GlUtil.deleteTexture(outputTexture.texId);
}
}
}