From c5904cfb46dc2e4d18cd81abebc36fd589238019 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Mon, 15 Nov 2021 11:07:20 +0000 Subject: [PATCH] Move OpenGL usage from VideoSamplePipeline to new OpenGlFrameEditor. The decoder writes to `OpenGlFrameEditor`'s input `Surface` and the `OpenGlFrameEditor` writes to the encoder's input `Surface`. PiperOrigin-RevId: 409931796 --- .../media3/transformer/OpenGlFrameEditor.java | 180 +++++++++++++ .../transformer/VideoSamplePipeline.java | 249 ++++-------------- 2 files changed, 224 insertions(+), 205 deletions(-) create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/OpenGlFrameEditor.java diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/OpenGlFrameEditor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/OpenGlFrameEditor.java new file mode 100644 index 0000000000..c633d562eb --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/OpenGlFrameEditor.java @@ -0,0 +1,180 @@ +/* + * Copyright 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; + +import android.content.Context; +import android.graphics.SurfaceTexture; +import android.opengl.EGL14; +import android.opengl.EGLContext; +import android.opengl.EGLDisplay; +import android.opengl.EGLExt; +import android.opengl.EGLSurface; +import android.opengl.GLES20; +import android.view.Surface; +import androidx.annotation.RequiresApi; +import androidx.media3.common.Format; +import androidx.media3.common.util.GlUtil; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** Applies OpenGL transformations to video frames. */ +@RequiresApi(18) +/* package */ final class OpenGlFrameEditor { + + static { + GlUtil.glAssertionsEnabled = true; + } + + public static OpenGlFrameEditor create( + Context context, Format inputFormat, Surface outputSurface) { + EGLDisplay eglDisplay = GlUtil.createEglDisplay(); + EGLContext eglContext; + try { + eglContext = GlUtil.createEglContext(eglDisplay); + } catch (GlUtil.UnsupportedEglVersionException e) { + throw new IllegalStateException("EGL version is unsupported", e); + } + EGLSurface eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface); + GlUtil.focusSurface(eglDisplay, eglContext, eglSurface, inputFormat.width, inputFormat.height); + int textureId = GlUtil.createExternalTexture(); + GlUtil.Program copyProgram; + try { + copyProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH); + } catch (IOException e) { + throw new IllegalStateException(e); + } + + copyProgram.use(); + GlUtil.Attribute[] copyAttributes = copyProgram.getAttributes(); + checkState( + copyAttributes.length == EXPECTED_NUMBER_OF_ATTRIBUTES, + "Expected program to have " + EXPECTED_NUMBER_OF_ATTRIBUTES + " vertex attributes."); + for (GlUtil.Attribute copyAttribute : copyAttributes) { + if (copyAttribute.name.equals("a_position")) { + copyAttribute.setBuffer( + new float[] { + -1.0f, -1.0f, 0.0f, 1.0f, + 1.0f, -1.0f, 0.0f, 1.0f, + -1.0f, 1.0f, 0.0f, 1.0f, + 1.0f, 1.0f, 0.0f, 1.0f, + }, + /* size= */ 4); + } else if (copyAttribute.name.equals("a_texcoord")) { + copyAttribute.setBuffer( + new float[] { + 0.0f, 0.0f, 0.0f, 1.0f, + 1.0f, 0.0f, 0.0f, 1.0f, + 0.0f, 1.0f, 0.0f, 1.0f, + 1.0f, 1.0f, 0.0f, 1.0f, + }, + /* size= */ 4); + } else { + throw new IllegalStateException("Unexpected attribute name."); + } + copyAttribute.bind(); + } + GlUtil.Uniform[] copyUniforms = copyProgram.getUniforms(); + checkState( + copyUniforms.length == EXPECTED_NUMBER_OF_UNIFORMS, + "Expected program to have " + EXPECTED_NUMBER_OF_UNIFORMS + " uniforms."); + GlUtil.@MonotonicNonNull Uniform textureTransformUniform = null; + for (GlUtil.Uniform copyUniform : copyUniforms) { + if (copyUniform.name.equals("tex_sampler")) { + copyUniform.setSamplerTexId(textureId, 0); + copyUniform.bind(); + } else if (copyUniform.name.equals("tex_transform")) { + textureTransformUniform = copyUniform; + } else { + throw new IllegalStateException("Unexpected uniform name."); + } + } + + return new OpenGlFrameEditor( + eglDisplay, eglContext, eglSurface, textureId, checkNotNull(textureTransformUniform)); + } + + // Predefined shader values. + private static final String VERTEX_SHADER_FILE_PATH = "shaders/blit_vertex_shader.glsl"; + private static final String FRAGMENT_SHADER_FILE_PATH = + "shaders/copy_external_fragment_shader.glsl"; + private static final int EXPECTED_NUMBER_OF_ATTRIBUTES = 2; + private static final int EXPECTED_NUMBER_OF_UNIFORMS = 2; + + private final float[] textureTransformMatrix; + private final EGLDisplay eglDisplay; + private final EGLContext eglContext; + private final EGLSurface eglSurface; + private final int textureId; + private final SurfaceTexture inputSurfaceTexture; + private final Surface inputSurface; + private final GlUtil.Uniform textureTransformUniform; + + private volatile boolean hasInputData; + + private OpenGlFrameEditor( + EGLDisplay eglDisplay, + EGLContext eglContext, + EGLSurface eglSurface, + int textureId, + GlUtil.Uniform textureTransformUniform) { + this.eglDisplay = eglDisplay; + this.eglContext = eglContext; + this.eglSurface = eglSurface; + this.textureId = textureId; + this.textureTransformUniform = textureTransformUniform; + textureTransformMatrix = new float[16]; + inputSurfaceTexture = new SurfaceTexture(textureId); + inputSurfaceTexture.setOnFrameAvailableListener(surfaceTexture -> hasInputData = true); + inputSurface = new Surface(inputSurfaceTexture); + } + + /** Releases all resources. */ + public void release() { + GlUtil.destroyEglContext(eglDisplay, eglContext); + GlUtil.deleteTexture(textureId); + inputSurfaceTexture.release(); + inputSurface.release(); + } + + /** Informs the editor that there is new input data available for it to process asynchronously. */ + public void processData() { + inputSurfaceTexture.updateTexImage(); + inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); + textureTransformUniform.setFloats(textureTransformMatrix); + textureTransformUniform.bind(); + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp(); + EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs); + EGL14.eglSwapBuffers(eglDisplay, eglSurface); + hasInputData = false; + } + + /** + * Returns the input {@link Surface} after configuring the editor if it has not previously been + * configured. + */ + public Surface getInputSurface() { + return inputSurface; + } + + public boolean hasInputData() { + return hasInputData; + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java index 986e20afe5..984b37227a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java @@ -17,33 +17,18 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkState; -import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.content.Context; -import android.graphics.SurfaceTexture; import android.media.MediaCodec; -import android.opengl.EGL14; -import android.opengl.EGLContext; -import android.opengl.EGLDisplay; -import android.opengl.EGLExt; -import android.opengl.EGLSurface; -import android.opengl.GLES20; -import android.view.Surface; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.PlaybackException; -import androidx.media3.common.util.GlUtil; import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.exoplayer.ExoPlaybackException; import com.google.common.collect.ImmutableMap; import java.io.IOException; -import org.checkerframework.checker.nullness.qual.EnsuresNonNull; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; /** * Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them. @@ -51,52 +36,24 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @RequiresApi(18) /* package */ final class VideoSamplePipeline implements SamplePipeline { - static { - GlUtil.glAssertionsEnabled = true; - } - private static final String TAG = "VideoSamplePipeline"; - // Predefined shader values. - private static final String VERTEX_SHADER_FILE_PATH = "shaders/blit_vertex_shader.glsl"; - private static final String FRAGMENT_SHADER_FILE_PATH = - "shaders/copy_external_fragment_shader.glsl"; - private static final int EXPECTED_NUMBER_OF_ATTRIBUTES = 2; - private static final int EXPECTED_NUMBER_OF_UNIFORMS = 2; - - private final Context context; - private final int rendererIndex; - private final MediaCodecAdapterWrapper encoder; private final DecoderInputBuffer encoderOutputBuffer; private final DecoderInputBuffer decoderInputBuffer; - private final float[] decoderTextureTransformMatrix; - private final Format decoderInputFormat; + private final MediaCodecAdapterWrapper decoder; - private @MonotonicNonNull EGLDisplay eglDisplay; - private @MonotonicNonNull EGLContext eglContext; - private @MonotonicNonNull EGLSurface eglSurface; + private final OpenGlFrameEditor openGlFrameEditor; - private int decoderTextureId; - private @MonotonicNonNull SurfaceTexture decoderSurfaceTexture; - private @MonotonicNonNull Surface decoderSurface; - private @MonotonicNonNull MediaCodecAdapterWrapper decoder; - private volatile boolean isDecoderSurfacePopulated; private boolean waitingForPopulatedDecoderSurface; - private GlUtil.@MonotonicNonNull Uniform decoderTextureTransformUniform; public VideoSamplePipeline( Context context, Format decoderInputFormat, Transformation transformation, int rendererIndex) throws ExoPlaybackException { - this.decoderInputFormat = decoderInputFormat; - this.rendererIndex = rendererIndex; - this.context = context; decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); - decoderTextureTransformMatrix = new float[16]; - decoderTextureId = GlUtil.TEXTURE_ID_UNSET; encoderOutputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); @@ -114,34 +71,57 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; ImmutableMap.of()); } catch (IOException e) { // TODO (internal b/184262323): Assign an adequate error code. - throw ExoPlaybackException.createForRenderer( - e, - TAG, - rendererIndex, - decoderInputFormat, - /* rendererFormatSupport= */ C.FORMAT_HANDLED, - /* isRecoverable= */ false, - PlaybackException.ERROR_CODE_UNSPECIFIED); + throw createRendererException( + e, rendererIndex, decoderInputFormat, PlaybackException.ERROR_CODE_UNSPECIFIED); + } + openGlFrameEditor = + OpenGlFrameEditor.create( + context, + decoderInputFormat, + /* outputSurface= */ checkNotNull(encoder.getInputSurface())); + try { + decoder = + MediaCodecAdapterWrapper.createForVideoDecoding( + decoderInputFormat, openGlFrameEditor.getInputSurface()); + } catch (IOException e) { + throw createRendererException( + e, rendererIndex, decoderInputFormat, PlaybackException.ERROR_CODE_DECODER_INIT_FAILED); } } @Override - public boolean processData() throws ExoPlaybackException { - ensureOpenGlConfigured(); - return !ensureDecoderConfigured() || feedEncoderFromDecoder(); + public boolean processData() { + if (decoder.isEnded()) { + return false; + } + + if (!openGlFrameEditor.hasInputData()) { + if (!waitingForPopulatedDecoderSurface) { + if (decoder.getOutputBufferInfo() != null) { + decoder.releaseOutputBuffer(/* render= */ true); + waitingForPopulatedDecoderSurface = true; + } + if (decoder.isEnded()) { + encoder.signalEndOfInputStream(); + } + } + return false; + } + + waitingForPopulatedDecoderSurface = false; + openGlFrameEditor.processData(); + return true; } @Override @Nullable public DecoderInputBuffer dequeueInputBuffer() { - return decoder != null && decoder.maybeDequeueInputBuffer(decoderInputBuffer) - ? decoderInputBuffer - : null; + return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null; } @Override public void queueInputBuffer() { - checkStateNotNull(decoder).queueInputBuffer(decoderInputBuffer); + decoder.queueInputBuffer(decoderInputBuffer); } @Override @@ -175,154 +155,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @Override public void release() { - GlUtil.destroyEglContext(eglDisplay, eglContext); - if (decoderTextureId != GlUtil.TEXTURE_ID_UNSET) { - GlUtil.deleteTexture(decoderTextureId); - } - if (decoderSurfaceTexture != null) { - decoderSurfaceTexture.release(); - } - if (decoderSurface != null) { - decoderSurface.release(); - } - if (decoder != null) { - decoder.release(); - } + openGlFrameEditor.release(); + decoder.release(); encoder.release(); } - @EnsuresNonNull({"eglDisplay", "eglContext", "eglSurface", "decoderTextureTransformUniform"}) - private void ensureOpenGlConfigured() { - if (eglDisplay != null - && eglContext != null - && eglSurface != null - && decoderTextureTransformUniform != null) { - return; - } - - eglDisplay = GlUtil.createEglDisplay(); - try { - eglContext = GlUtil.createEglContext(eglDisplay); - } catch (GlUtil.UnsupportedEglVersionException e) { - throw new IllegalStateException("EGL version is unsupported", e); - } - eglSurface = GlUtil.getEglSurface(eglDisplay, checkNotNull(encoder.getInputSurface())); - GlUtil.focusSurface( - eglDisplay, eglContext, eglSurface, decoderInputFormat.width, decoderInputFormat.height); - decoderTextureId = GlUtil.createExternalTexture(); - GlUtil.Program copyProgram; - try { - copyProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH); - } catch (IOException e) { - throw new IllegalStateException(e); - } - - copyProgram.use(); - GlUtil.Attribute[] copyAttributes = copyProgram.getAttributes(); - checkState( - copyAttributes.length == EXPECTED_NUMBER_OF_ATTRIBUTES, - "Expected program to have " + EXPECTED_NUMBER_OF_ATTRIBUTES + " vertex attributes."); - for (GlUtil.Attribute copyAttribute : copyAttributes) { - if (copyAttribute.name.equals("a_position")) { - copyAttribute.setBuffer( - new float[] { - -1.0f, -1.0f, 0.0f, 1.0f, - 1.0f, -1.0f, 0.0f, 1.0f, - -1.0f, 1.0f, 0.0f, 1.0f, - 1.0f, 1.0f, 0.0f, 1.0f, - }, - /* size= */ 4); - } else if (copyAttribute.name.equals("a_texcoord")) { - copyAttribute.setBuffer( - new float[] { - 0.0f, 0.0f, 0.0f, 1.0f, - 1.0f, 0.0f, 0.0f, 1.0f, - 0.0f, 1.0f, 0.0f, 1.0f, - 1.0f, 1.0f, 0.0f, 1.0f, - }, - /* size= */ 4); - } else { - throw new IllegalStateException("Unexpected attribute name."); - } - copyAttribute.bind(); - } - GlUtil.Uniform[] copyUniforms = copyProgram.getUniforms(); - checkState( - copyUniforms.length == EXPECTED_NUMBER_OF_UNIFORMS, - "Expected program to have " + EXPECTED_NUMBER_OF_UNIFORMS + " uniforms."); - for (GlUtil.Uniform copyUniform : copyUniforms) { - if (copyUniform.name.equals("tex_sampler")) { - copyUniform.setSamplerTexId(decoderTextureId, 0); - copyUniform.bind(); - } else if (copyUniform.name.equals("tex_transform")) { - decoderTextureTransformUniform = copyUniform; - } else { - throw new IllegalStateException("Unexpected uniform name."); - } - } - checkNotNull(decoderTextureTransformUniform); - } - - @EnsuresNonNullIf( - expression = {"decoder", "decoderSurfaceTexture"}, - result = true) - private boolean ensureDecoderConfigured() throws ExoPlaybackException { - if (decoder != null && decoderSurfaceTexture != null) { - return true; - } - - checkState(decoderTextureId != GlUtil.TEXTURE_ID_UNSET); - decoderSurfaceTexture = new SurfaceTexture(decoderTextureId); - decoderSurfaceTexture.setOnFrameAvailableListener( - surfaceTexture -> isDecoderSurfacePopulated = true); - decoderSurface = new Surface(decoderSurfaceTexture); - try { - decoder = MediaCodecAdapterWrapper.createForVideoDecoding(decoderInputFormat, decoderSurface); - } catch (IOException e) { - throw createRendererException(e, PlaybackException.ERROR_CODE_DECODER_INIT_FAILED); - } - return true; - } - - @RequiresNonNull({ - "decoder", - "decoderSurfaceTexture", - "decoderTextureTransformUniform", - "eglDisplay", - "eglSurface" - }) - private boolean feedEncoderFromDecoder() { - if (decoder.isEnded()) { - return false; - } - - if (!isDecoderSurfacePopulated) { - if (!waitingForPopulatedDecoderSurface) { - if (decoder.getOutputBufferInfo() != null) { - decoder.releaseOutputBuffer(/* render= */ true); - waitingForPopulatedDecoderSurface = true; - } - if (decoder.isEnded()) { - encoder.signalEndOfInputStream(); - } - } - return false; - } - - waitingForPopulatedDecoderSurface = false; - decoderSurfaceTexture.updateTexImage(); - decoderSurfaceTexture.getTransformMatrix(decoderTextureTransformMatrix); - decoderTextureTransformUniform.setFloats(decoderTextureTransformMatrix); - decoderTextureTransformUniform.bind(); - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); - long decoderSurfaceTextureTimestampNs = decoderSurfaceTexture.getTimestamp(); - EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, decoderSurfaceTextureTimestampNs); - EGL14.eglSwapBuffers(eglDisplay, eglSurface); - isDecoderSurfacePopulated = false; - return true; - } - - private ExoPlaybackException createRendererException(Throwable cause, int errorCode) { + private static ExoPlaybackException createRendererException( + Throwable cause, int rendererIndex, Format decoderInputFormat, int errorCode) { return ExoPlaybackException.createForRenderer( cause, TAG,