loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compositor: Split out OpenGL drawing logic into inner class.

Browse Source 
Organize logic a bit by splitting logic about how we draw using OpenGL onto a
texture, out from the larger class, which has lots of logic discussing how we select
frames and streams.

No functional change intended, but a few method calls are shuffled around to
simplify things.

PiperOrigin-RevId: 565426225
This commit is contained in:
huangdarwin 2023-09-14 11:23:22 -07:00 committed by Copybara-Service
parent 16b0ea850f
commit c77b8c8479
1 changed files with 113 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

197
libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java
View File

@ -78,16 +78,13 @@ public final class DefaultVideoCompositor implements VideoCompositor {
private static final String THREAD_NAME = "Effect:DefaultVideoCompositor:GlThread";
private static final String TAG = "DefaultVideoCompositor";
private static final String VERTEX_SHADER_PATH = "shaders/vertex_shader_transformation_es2.glsl";
private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_alpha_scale_es2.glsl";
private static final int PRIMARY_INPUT_ID = 0;
private final Context context;
private final VideoCompositor.Listener listener;
private final GlTextureProducer.Listener textureOutputListener;
private final GlObjectsProvider glObjectsProvider;
private final VideoCompositorSettings settings;
private final OverlayMatrixProvider overlayMatrixProvider;
private final CompositorGlProgram compositorGlProgram;
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
@GuardedBy("this")
@ -105,7 +102,6 @@ public final class DefaultVideoCompositor implements VideoCompositor {
// Only used on the GL Thread.
private @MonotonicNonNull EGLContext eglContext;
private @MonotonicNonNull EGLDisplay eglDisplay;
private @MonotonicNonNull GlProgram glProgram;
private @MonotonicNonNull EGLSurface placeholderEglSurface;
/**
@ -122,12 +118,11 @@ public final class DefaultVideoCompositor implements VideoCompositor {
VideoCompositor.Listener listener,
GlTextureProducer.Listener textureOutputListener,
@IntRange(from = 1) int textureOutputCapacity) {
this.context = context;
this.listener = listener;
this.textureOutputListener = textureOutputListener;
this.glObjectsProvider = glObjectsProvider;
this.settings = settings;
this.overlayMatrixProvider = new OverlayMatrixProvider();
this.compositorGlProgram = new CompositorGlProgram(context);
inputSources = new ArrayList<>();
outputTexturePool =
@ -214,7 +209,8 @@ public final class DefaultVideoCompositor implements VideoCompositor {
}
@Override
public void release() {
public synchronized void release() {
checkState(allInputsEnded);
try {
videoFrameProcessingTaskExecutor.release(/* releaseTask= */ this::releaseGlObjects);
} catch (InterruptedException e) {
@ -296,8 +292,6 @@ public final class DefaultVideoCompositor implements VideoCompositor {
return;
}
ensureGlProgramConfigured();
InputFrameInfo primaryInputFrame = framesToComposite.get(PRIMARY_INPUT_ID);
ImmutableList.Builder<Size> inputSizes = new ImmutableList.Builder<>();
@ -313,7 +307,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
long outputPresentationTimestampUs = primaryInputFrame.presentationTimeUs;
outputTextureTimestamps.add(outputPresentationTimestampUs);
drawFrame(framesToComposite, outputTexture);
compositorGlProgram.drawFrame(framesToComposite, outputTexture);
long syncObject = GlUtil.createGlSyncFence();
syncObjects.add(syncObject);
textureOutputListener.onTextureRendered(
@ -403,82 +397,11 @@ public final class DefaultVideoCompositor implements VideoCompositor {
maybeComposite();
}
private void ensureGlProgramConfigured()
throws VideoFrameProcessingException, GlUtil.GlException {
if (glProgram != null) {
return;
}
private void releaseGlObjects() {
try {
glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
glProgram.setBufferAttribute(
"aFramePosition",
GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
glProgram.setFloatsUniform("uTexTransformationMatrix", GlUtil.create4x4IdentityMatrix());
} catch (IOException e) {
throw new VideoFrameProcessingException(e);
}
}
// Enhanced for-loops are discouraged in media3.effect due to short-lived allocations.
@SuppressWarnings("ListReverse")
private void drawFrame(List<InputFrameInfo> framesToComposite, GlTextureInfo outputTexture)
throws GlUtil.GlException {
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.fboId, outputTexture.width, outputTexture.height);
overlayMatrixProvider.configure(new Size(outputTexture.width, outputTexture.height));
GlUtil.clearFocusedBuffers();
GlProgram glProgram = checkNotNull(this.glProgram);
glProgram.use();
// Setup for blending.
GLES20.glEnable(GLES20.GL_BLEND);
// Similar to:
// dst.rgb = src.rgb * src.a + dst.rgb * (1 - src.a)
// dst.a = src.a + dst.a * (1 - src.a)
GLES20.glBlendFuncSeparate(
/* srcRGB= */ GLES20.GL_SRC_ALPHA,
/* dstRGB= */ GLES20.GL_ONE_MINUS_SRC_ALPHA,
/* srcAlpha= */ GLES20.GL_ONE,
/* dstAlpha= */ GLES20.GL_ONE_MINUS_SRC_ALPHA);
GlUtil.checkGlError();
// Draw textures from back to front.
for (int i = framesToComposite.size() - 1; i >= 0; i--) {
blendOntoFocusedTexture(framesToComposite.get(i));
}
GLES20.glDisable(GLES20.GL_BLEND);
GlUtil.checkGlError();
}
private void blendOntoFocusedTexture(InputFrameInfo inputFrameInfo) throws GlUtil.GlException {
GlProgram glProgram = checkNotNull(this.glProgram);
GlTextureInfo inputTexture = inputFrameInfo.texture;
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexture.texId, /* texUnitIndex= */ 0);
float[] transformationMatrix =
overlayMatrixProvider.getTransformationMatrix(
/* overlaySize= */ new Size(inputTexture.width, inputTexture.height),
inputFrameInfo.overlaySettings);
glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrix);
glProgram.setFloatUniform("uAlphaScale", inputFrameInfo.overlaySettings.alphaScale);
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
}
private synchronized void releaseGlObjects() {
try {
checkState(allInputsEnded);
compositorGlProgram.release();
outputTexturePool.deleteAllTextures();
GlUtil.destroyEglSurface(eglDisplay, placeholderEglSurface);
if (glProgram != null) {
glProgram.delete();
}
} catch (GlUtil.GlException e) {
Log.e(TAG, "Error releasing GL resources", e);
} finally {
@ -490,6 +413,112 @@ public final class DefaultVideoCompositor implements VideoCompositor {
}
}
/**
* A wrapper for a {@link GlProgram}, that draws multiple input {@link InputFrameInfo}s onto one
* output {@link GlTextureInfo}.
*
* <p>All methods must be called on a GL thread, unless otherwise stated.
*/
private static final class CompositorGlProgram {
private static final String TAG = "CompositorGlProgram";
private static final String VERTEX_SHADER_PATH =
"shaders/vertex_shader_transformation_es2.glsl";
private static final String FRAGMENT_SHADER_PATH =
"shaders/fragment_shader_alpha_scale_es2.glsl";
private final Context context;
private final OverlayMatrixProvider overlayMatrixProvider;
private @MonotonicNonNull GlProgram glProgram;
/**
* Creates an instance.
*
* <p>May be called on any thread.
*/
public CompositorGlProgram(Context context) {
this.context = context;
this.overlayMatrixProvider = new OverlayMatrixProvider();
}
/** Draws {@link InputFrameInfo}s onto an output {@link GlTextureInfo}. */
// Enhanced for-loops are discouraged in media3.effect due to short-lived allocations.
@SuppressWarnings("ListReverse")
public void drawFrame(List<InputFrameInfo> framesToComposite, GlTextureInfo outputTexture)
throws GlUtil.GlException, VideoFrameProcessingException {
ensureConfigured();
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.fboId, outputTexture.width, outputTexture.height);
overlayMatrixProvider.configure(new Size(outputTexture.width, outputTexture.height));
GlUtil.clearFocusedBuffers();
GlProgram glProgram = checkNotNull(this.glProgram);
glProgram.use();
// Setup for blending.
GLES20.glEnable(GLES20.GL_BLEND);
// Similar to:
// dst.rgb = src.rgb * src.a + dst.rgb * (1 - src.a)
// dst.a = src.a + dst.a * (1 - src.a)
GLES20.glBlendFuncSeparate(
/* srcRGB= */ GLES20.GL_SRC_ALPHA,
/* dstRGB= */ GLES20.GL_ONE_MINUS_SRC_ALPHA,
/* srcAlpha= */ GLES20.GL_ONE,
/* dstAlpha= */ GLES20.GL_ONE_MINUS_SRC_ALPHA);
GlUtil.checkGlError();
// Draw textures from back to front.
for (int i = framesToComposite.size() - 1; i >= 0; i--) {
blendOntoFocusedTexture(framesToComposite.get(i));
}
GLES20.glDisable(GLES20.GL_BLEND);
GlUtil.checkGlError();
}
public void release() {
try {
if (glProgram != null) {
glProgram.delete();
}
} catch (GlUtil.GlException e) {
Log.e(TAG, "Error releasing GL Program", e);
}
}
private void ensureConfigured() throws VideoFrameProcessingException, GlUtil.GlException {
if (glProgram != null) {
return;
}
try {
glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH);
glProgram.setBufferAttribute(
"aFramePosition",
GlUtil.getNormalizedCoordinateBounds(),
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
glProgram.setFloatsUniform("uTexTransformationMatrix", GlUtil.create4x4IdentityMatrix());
} catch (IOException e) {
throw new VideoFrameProcessingException(e);
}
}
private void blendOntoFocusedTexture(InputFrameInfo inputFrameInfo) throws GlUtil.GlException {
GlProgram glProgram = checkNotNull(this.glProgram);
GlTextureInfo inputTexture = inputFrameInfo.texture;
glProgram.setSamplerTexIdUniform("uTexSampler", inputTexture.texId, /* texUnitIndex= */ 0);
float[] transformationMatrix =
overlayMatrixProvider.getTransformationMatrix(
/* overlaySize= */ new Size(inputTexture.width, inputTexture.height),
inputFrameInfo.overlaySettings);
glProgram.setFloatsUniform("uTransformationMatrix", transformationMatrix);
glProgram.setFloatUniform("uAlphaScale", inputFrameInfo.overlaySettings.alphaScale);
glProgram.bindAttributesAndUniforms();
// The four-vertex triangle strip forms a quad.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError();
}
}
/** Holds information on an input source. */
private static final class InputSource {
/**