Move OpenGL usage from VideoSamplePipeline to new OpenGlFrameEditor.

The decoder writes to `OpenGlFrameEditor`'s input `Surface`
and the `OpenGlFrameEditor` writes to the encoder's input `Surface`.

PiperOrigin-RevId: 409931796
This commit is contained in:
hschlueter 2021-11-15 11:07:20 +00:00 committed by Ian Baker
parent c046f40fd0
commit c5904cfb46
2 changed files with 224 additions and 205 deletions

View File

@ -0,0 +1,180 @@
/*
* Copyright 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.opengl.EGL14;
import android.opengl.EGLContext;
import android.opengl.EGLDisplay;
import android.opengl.EGLExt;
import android.opengl.EGLSurface;
import android.opengl.GLES20;
import android.view.Surface;
import androidx.annotation.RequiresApi;
import androidx.media3.common.Format;
import androidx.media3.common.util.GlUtil;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Applies OpenGL transformations to video frames. */
@RequiresApi(18)
/* package */ final class OpenGlFrameEditor {
static {
GlUtil.glAssertionsEnabled = true;
}
public static OpenGlFrameEditor create(
Context context, Format inputFormat, Surface outputSurface) {
EGLDisplay eglDisplay = GlUtil.createEglDisplay();
EGLContext eglContext;
try {
eglContext = GlUtil.createEglContext(eglDisplay);
} catch (GlUtil.UnsupportedEglVersionException e) {
throw new IllegalStateException("EGL version is unsupported", e);
}
EGLSurface eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface);
GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, inputFormat.width, inputFormat.height);
int textureId = GlUtil.createExternalTexture();
GlUtil.Program copyProgram;
try {
copyProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH);
} catch (IOException e) {
throw new IllegalStateException(e);
}
copyProgram.use();
GlUtil.Attribute[] copyAttributes = copyProgram.getAttributes();
checkState(
copyAttributes.length == EXPECTED_NUMBER_OF_ATTRIBUTES,
"Expected program to have " + EXPECTED_NUMBER_OF_ATTRIBUTES + " vertex attributes.");
for (GlUtil.Attribute copyAttribute : copyAttributes) {
if (copyAttribute.name.equals("a_position")) {
copyAttribute.setBuffer(
new float[] {
-1.0f, -1.0f, 0.0f, 1.0f,
1.0f, -1.0f, 0.0f, 1.0f,
-1.0f, 1.0f, 0.0f, 1.0f,
1.0f, 1.0f, 0.0f, 1.0f,
},
/* size= */ 4);
} else if (copyAttribute.name.equals("a_texcoord")) {
copyAttribute.setBuffer(
new float[] {
0.0f, 0.0f, 0.0f, 1.0f,
1.0f, 0.0f, 0.0f, 1.0f,
0.0f, 1.0f, 0.0f, 1.0f,
1.0f, 1.0f, 0.0f, 1.0f,
},
/* size= */ 4);
} else {
throw new IllegalStateException("Unexpected attribute name.");
}
copyAttribute.bind();
}
GlUtil.Uniform[] copyUniforms = copyProgram.getUniforms();
checkState(
copyUniforms.length == EXPECTED_NUMBER_OF_UNIFORMS,
"Expected program to have " + EXPECTED_NUMBER_OF_UNIFORMS + " uniforms.");
GlUtil.@MonotonicNonNull Uniform textureTransformUniform = null;
for (GlUtil.Uniform copyUniform : copyUniforms) {
if (copyUniform.name.equals("tex_sampler")) {
copyUniform.setSamplerTexId(textureId, 0);
copyUniform.bind();
} else if (copyUniform.name.equals("tex_transform")) {
textureTransformUniform = copyUniform;
} else {
throw new IllegalStateException("Unexpected uniform name.");
}
}
return new OpenGlFrameEditor(
eglDisplay, eglContext, eglSurface, textureId, checkNotNull(textureTransformUniform));
}
// Predefined shader values.
private static final String VERTEX_SHADER_FILE_PATH = "shaders/blit_vertex_shader.glsl";
private static final String FRAGMENT_SHADER_FILE_PATH =
"shaders/copy_external_fragment_shader.glsl";
private static final int EXPECTED_NUMBER_OF_ATTRIBUTES = 2;
private static final int EXPECTED_NUMBER_OF_UNIFORMS = 2;
private final float[] textureTransformMatrix;
private final EGLDisplay eglDisplay;
private final EGLContext eglContext;
private final EGLSurface eglSurface;
private final int textureId;
private final SurfaceTexture inputSurfaceTexture;
private final Surface inputSurface;
private final GlUtil.Uniform textureTransformUniform;
private volatile boolean hasInputData;
private OpenGlFrameEditor(
EGLDisplay eglDisplay,
EGLContext eglContext,
EGLSurface eglSurface,
int textureId,
GlUtil.Uniform textureTransformUniform) {
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
this.eglSurface = eglSurface;
this.textureId = textureId;
this.textureTransformUniform = textureTransformUniform;
textureTransformMatrix = new float[16];
inputSurfaceTexture = new SurfaceTexture(textureId);
inputSurfaceTexture.setOnFrameAvailableListener(surfaceTexture -> hasInputData = true);
inputSurface = new Surface(inputSurfaceTexture);
}
/** Releases all resources. */
public void release() {
GlUtil.destroyEglContext(eglDisplay, eglContext);
GlUtil.deleteTexture(textureId);
inputSurfaceTexture.release();
inputSurface.release();
}
/** Informs the editor that there is new input data available for it to process asynchronously. */
public void processData() {
inputSurfaceTexture.updateTexImage();
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
textureTransformUniform.setFloats(textureTransformMatrix);
textureTransformUniform.bind();
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp();
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs);
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
hasInputData = false;
}
/**
* Returns the input {@link Surface} after configuring the editor if it has not previously been
* configured.
*/
public Surface getInputSurface() {
return inputSurface;
}
public boolean hasInputData() {
return hasInputData;
}
}

View File

@ -17,33 +17,18 @@
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.media.MediaCodec;
import android.opengl.EGL14;
import android.opengl.EGLContext;
import android.opengl.EGLDisplay;
import android.opengl.EGLExt;
import android.opengl.EGLSurface;
import android.opengl.GLES20;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.PlaybackException;
import androidx.media3.common.util.GlUtil;
import androidx.media3.decoder.DecoderInputBuffer;
import androidx.media3.exoplayer.ExoPlaybackException;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/**
* Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them.
@ -51,52 +36,24 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@RequiresApi(18)
/* package */ final class VideoSamplePipeline implements SamplePipeline {
static {
GlUtil.glAssertionsEnabled = true;
}
private static final String TAG = "VideoSamplePipeline";
// Predefined shader values.
private static final String VERTEX_SHADER_FILE_PATH = "shaders/blit_vertex_shader.glsl";
private static final String FRAGMENT_SHADER_FILE_PATH =
"shaders/copy_external_fragment_shader.glsl";
private static final int EXPECTED_NUMBER_OF_ATTRIBUTES = 2;
private static final int EXPECTED_NUMBER_OF_UNIFORMS = 2;
private final Context context;
private final int rendererIndex;
private final MediaCodecAdapterWrapper encoder;
private final DecoderInputBuffer encoderOutputBuffer;
private final DecoderInputBuffer decoderInputBuffer;
private final float[] decoderTextureTransformMatrix;
private final Format decoderInputFormat;
private final MediaCodecAdapterWrapper decoder;
private @MonotonicNonNull EGLDisplay eglDisplay;
private @MonotonicNonNull EGLContext eglContext;
private @MonotonicNonNull EGLSurface eglSurface;
private final OpenGlFrameEditor openGlFrameEditor;
private int decoderTextureId;
private @MonotonicNonNull SurfaceTexture decoderSurfaceTexture;
private @MonotonicNonNull Surface decoderSurface;
private @MonotonicNonNull MediaCodecAdapterWrapper decoder;
private volatile boolean isDecoderSurfacePopulated;
private boolean waitingForPopulatedDecoderSurface;
private GlUtil.@MonotonicNonNull Uniform decoderTextureTransformUniform;
public VideoSamplePipeline(
Context context, Format decoderInputFormat, Transformation transformation, int rendererIndex)
throws ExoPlaybackException {
this.decoderInputFormat = decoderInputFormat;
this.rendererIndex = rendererIndex;
this.context = context;
decoderInputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
decoderTextureTransformMatrix = new float[16];
decoderTextureId = GlUtil.TEXTURE_ID_UNSET;
encoderOutputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
@ -114,34 +71,57 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
ImmutableMap.of());
} catch (IOException e) {
// TODO (internal b/184262323): Assign an adequate error code.
throw ExoPlaybackException.createForRenderer(
e,
TAG,
rendererIndex,
decoderInputFormat,
/* rendererFormatSupport= */ C.FORMAT_HANDLED,
/* isRecoverable= */ false,
PlaybackException.ERROR_CODE_UNSPECIFIED);
throw createRendererException(
e, rendererIndex, decoderInputFormat, PlaybackException.ERROR_CODE_UNSPECIFIED);
}
openGlFrameEditor =
OpenGlFrameEditor.create(
context,
decoderInputFormat,
/* outputSurface= */ checkNotNull(encoder.getInputSurface()));
try {
decoder =
MediaCodecAdapterWrapper.createForVideoDecoding(
decoderInputFormat, openGlFrameEditor.getInputSurface());
} catch (IOException e) {
throw createRendererException(
e, rendererIndex, decoderInputFormat, PlaybackException.ERROR_CODE_DECODER_INIT_FAILED);
}
}
@Override
public boolean processData() throws ExoPlaybackException {
ensureOpenGlConfigured();
return !ensureDecoderConfigured() || feedEncoderFromDecoder();
public boolean processData() {
if (decoder.isEnded()) {
return false;
}
if (!openGlFrameEditor.hasInputData()) {
if (!waitingForPopulatedDecoderSurface) {
if (decoder.getOutputBufferInfo() != null) {
decoder.releaseOutputBuffer(/* render= */ true);
waitingForPopulatedDecoderSurface = true;
}
if (decoder.isEnded()) {
encoder.signalEndOfInputStream();
}
}
return false;
}
waitingForPopulatedDecoderSurface = false;
openGlFrameEditor.processData();
return true;
}
@Override
@Nullable
public DecoderInputBuffer dequeueInputBuffer() {
return decoder != null && decoder.maybeDequeueInputBuffer(decoderInputBuffer)
? decoderInputBuffer
: null;
return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null;
}
@Override
public void queueInputBuffer() {
checkStateNotNull(decoder).queueInputBuffer(decoderInputBuffer);
decoder.queueInputBuffer(decoderInputBuffer);
}
@Override
@ -175,154 +155,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@Override
public void release() {
GlUtil.destroyEglContext(eglDisplay, eglContext);
if (decoderTextureId != GlUtil.TEXTURE_ID_UNSET) {
GlUtil.deleteTexture(decoderTextureId);
}
if (decoderSurfaceTexture != null) {
decoderSurfaceTexture.release();
}
if (decoderSurface != null) {
decoderSurface.release();
}
if (decoder != null) {
decoder.release();
}
openGlFrameEditor.release();
decoder.release();
encoder.release();
}
@EnsuresNonNull({"eglDisplay", "eglContext", "eglSurface", "decoderTextureTransformUniform"})
private void ensureOpenGlConfigured() {
if (eglDisplay != null
&& eglContext != null
&& eglSurface != null
&& decoderTextureTransformUniform != null) {
return;
}
eglDisplay = GlUtil.createEglDisplay();
try {
eglContext = GlUtil.createEglContext(eglDisplay);
} catch (GlUtil.UnsupportedEglVersionException e) {
throw new IllegalStateException("EGL version is unsupported", e);
}
eglSurface = GlUtil.getEglSurface(eglDisplay, checkNotNull(encoder.getInputSurface()));
GlUtil.focusSurface(
eglDisplay, eglContext, eglSurface, decoderInputFormat.width, decoderInputFormat.height);
decoderTextureId = GlUtil.createExternalTexture();
GlUtil.Program copyProgram;
try {
copyProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH);
} catch (IOException e) {
throw new IllegalStateException(e);
}
copyProgram.use();
GlUtil.Attribute[] copyAttributes = copyProgram.getAttributes();
checkState(
copyAttributes.length == EXPECTED_NUMBER_OF_ATTRIBUTES,
"Expected program to have " + EXPECTED_NUMBER_OF_ATTRIBUTES + " vertex attributes.");
for (GlUtil.Attribute copyAttribute : copyAttributes) {
if (copyAttribute.name.equals("a_position")) {
copyAttribute.setBuffer(
new float[] {
-1.0f, -1.0f, 0.0f, 1.0f,
1.0f, -1.0f, 0.0f, 1.0f,
-1.0f, 1.0f, 0.0f, 1.0f,
1.0f, 1.0f, 0.0f, 1.0f,
},
/* size= */ 4);
} else if (copyAttribute.name.equals("a_texcoord")) {
copyAttribute.setBuffer(
new float[] {
0.0f, 0.0f, 0.0f, 1.0f,
1.0f, 0.0f, 0.0f, 1.0f,
0.0f, 1.0f, 0.0f, 1.0f,
1.0f, 1.0f, 0.0f, 1.0f,
},
/* size= */ 4);
} else {
throw new IllegalStateException("Unexpected attribute name.");
}
copyAttribute.bind();
}
GlUtil.Uniform[] copyUniforms = copyProgram.getUniforms();
checkState(
copyUniforms.length == EXPECTED_NUMBER_OF_UNIFORMS,
"Expected program to have " + EXPECTED_NUMBER_OF_UNIFORMS + " uniforms.");
for (GlUtil.Uniform copyUniform : copyUniforms) {
if (copyUniform.name.equals("tex_sampler")) {
copyUniform.setSamplerTexId(decoderTextureId, 0);
copyUniform.bind();
} else if (copyUniform.name.equals("tex_transform")) {
decoderTextureTransformUniform = copyUniform;
} else {
throw new IllegalStateException("Unexpected uniform name.");
}
}
checkNotNull(decoderTextureTransformUniform);
}
@EnsuresNonNullIf(
expression = {"decoder", "decoderSurfaceTexture"},
result = true)
private boolean ensureDecoderConfigured() throws ExoPlaybackException {
if (decoder != null && decoderSurfaceTexture != null) {
return true;
}
checkState(decoderTextureId != GlUtil.TEXTURE_ID_UNSET);
decoderSurfaceTexture = new SurfaceTexture(decoderTextureId);
decoderSurfaceTexture.setOnFrameAvailableListener(
surfaceTexture -> isDecoderSurfacePopulated = true);
decoderSurface = new Surface(decoderSurfaceTexture);
try {
decoder = MediaCodecAdapterWrapper.createForVideoDecoding(decoderInputFormat, decoderSurface);
} catch (IOException e) {
throw createRendererException(e, PlaybackException.ERROR_CODE_DECODER_INIT_FAILED);
}
return true;
}
@RequiresNonNull({
"decoder",
"decoderSurfaceTexture",
"decoderTextureTransformUniform",
"eglDisplay",
"eglSurface"
})
private boolean feedEncoderFromDecoder() {
if (decoder.isEnded()) {
return false;
}
if (!isDecoderSurfacePopulated) {
if (!waitingForPopulatedDecoderSurface) {
if (decoder.getOutputBufferInfo() != null) {
decoder.releaseOutputBuffer(/* render= */ true);
waitingForPopulatedDecoderSurface = true;
}
if (decoder.isEnded()) {
encoder.signalEndOfInputStream();
}
}
return false;
}
waitingForPopulatedDecoderSurface = false;
decoderSurfaceTexture.updateTexImage();
decoderSurfaceTexture.getTransformMatrix(decoderTextureTransformMatrix);
decoderTextureTransformUniform.setFloats(decoderTextureTransformMatrix);
decoderTextureTransformUniform.bind();
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
long decoderSurfaceTextureTimestampNs = decoderSurfaceTexture.getTimestamp();
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, decoderSurfaceTextureTimestampNs);
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
isDecoderSurfacePopulated = false;
return true;
}
private ExoPlaybackException createRendererException(Throwable cause, int errorCode) {
private static ExoPlaybackException createRendererException(
Throwable cause, int rendererIndex, Format decoderInputFormat, int errorCode) {
return ExoPlaybackException.createForRenderer(
cause,
TAG,