diff --git a/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java b/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java index f20ee3e0c6..b83fe2bf46 100644 --- a/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java +++ b/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java @@ -26,7 +26,6 @@ import android.graphics.Paint; import android.graphics.drawable.BitmapDrawable; import android.opengl.GLES20; import android.opengl.GLUtils; -import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.util.GlUtil; import java.io.IOException; @@ -52,8 +51,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final Canvas overlayCanvas; private GlUtil.@MonotonicNonNull Program program; - @Nullable private GlUtil.Attribute[] attributes; - @Nullable private GlUtil.Uniform[] uniforms; private float bitmapScaleX; private float bitmapScaleY; @@ -88,31 +85,24 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } catch (IOException e) { throw new IllegalStateException(e); } - program.use(); - GlUtil.Attribute[] attributes = program.getAttributes(); - for (GlUtil.Attribute attribute : attributes) { - if (attribute.name.equals("a_position")) { - attribute.setBuffer( - new float[] { - -1, -1, 0, 1, - 1, -1, 0, 1, - -1, 1, 0, 1, - 1, 1, 0, 1 - }, - 4); - } else if (attribute.name.equals("a_texcoord")) { - attribute.setBuffer( - new float[] { - 0, 0, 0, 1, - 1, 0, 0, 1, - 0, 1, 0, 1, - 1, 1, 0, 1 - }, - 4); - } - } - this.attributes = attributes; - this.uniforms = program.getUniforms(); + program.setBufferAttribute( + "a_position", + new float[] { + -1, -1, 0, 1, + 1, -1, 0, 1, + -1, 1, 0, 1, + 1, 1, 0, 1 + }, + 4); + program.setBufferAttribute( + "a_texcoord", + new float[] { + 0, 0, 0, 1, + 1, 0, 0, 1, + 0, 1, 0, 1, + 1, 1, 0, 1 + }, + 4); GLES20.glGenTextures(1, textures, 0); GLES20.glBindTexture(GL10.GL_TEXTURE_2D, textures[0]); GLES20.glTexParameterf(GL10.GL_TEXTURE_2D, GL10.GL_TEXTURE_MIN_FILTER, GL10.GL_NEAREST); @@ -141,36 +131,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; GlUtil.checkGlError(); // Run the shader program. - GlUtil.Uniform[] uniforms = checkNotNull(this.uniforms); - GlUtil.Attribute[] attributes = checkNotNull(this.attributes); - for (GlUtil.Uniform uniform : uniforms) { - switch (uniform.name) { - case "tex_sampler_0": - uniform.setSamplerTexId(frameTexture, /* unit= */ 0); - break; - case "tex_sampler_1": - uniform.setSamplerTexId(textures[0], /* unit= */ 1); - break; - case "scaleX": - uniform.setFloat(bitmapScaleX); - break; - case "scaleY": - uniform.setFloat(bitmapScaleY); - break; - case "tex_transform": - uniform.setFloats(transformMatrix); - break; - default: // fall out - } - } - for (GlUtil.Attribute copyExternalAttribute : attributes) { - copyExternalAttribute.bind(); - } - for (GlUtil.Uniform copyExternalUniform : uniforms) { - copyExternalUniform.bind(); - } + GlUtil.Program program = checkNotNull(this.program); + program.setSamplerTexIdUniform("tex_sampler_0", frameTexture, /* unit= */ 0); + program.setSamplerTexIdUniform("tex_sampler_1", textures[0], /* unit= */ 1); + program.setFloatUniform("scaleX", bitmapScaleX); + program.setFloatUniform("scaleY", bitmapScaleY); + program.setFloatsUniform("tex_transform", transformMatrix); + program.bindAttributesAndUniforms(); GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); GlUtil.checkGlError(); } + + @Override + public void release() { + if (program != null) { + program.delete(); + } + } } diff --git a/demos/gl/src/main/java/androidx/media3/demo/gl/VideoProcessingGLSurfaceView.java b/demos/gl/src/main/java/androidx/media3/demo/gl/VideoProcessingGLSurfaceView.java index 50c07aee24..9c95122d4b 100644 --- a/demos/gl/src/main/java/androidx/media3/demo/gl/VideoProcessingGLSurfaceView.java +++ b/demos/gl/src/main/java/androidx/media3/demo/gl/VideoProcessingGLSurfaceView.java @@ -64,6 +64,9 @@ public final class VideoProcessingGLSurfaceView extends GLSurfaceView { * @param transformMatrix The 4 * 4 transform matrix to be applied to the texture. */ void draw(int frameTexture, long frameTimestampUs, float[] transformMatrix); + + /** Releases any resources associated with this {@link VideoProcessor}. */ + void release(); } private static final int EGL_PROTECTED_CONTENT_EXT = 0x32C0; diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java index afddf6c956..09aa393963 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java @@ -16,6 +16,7 @@ package androidx.media3.common.util; import static android.opengl.GLU.gluErrorString; +import static androidx.media3.common.util.Assertions.checkNotNull; import android.content.Context; import android.content.pm.PackageManager; @@ -37,6 +38,8 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.FloatBuffer; import java.nio.IntBuffer; +import java.util.HashMap; +import java.util.Map; import javax.microedition.khronos.egl.EGL10; /** GL utilities. */ @@ -54,25 +57,20 @@ public final class GlUtil { /** Thrown when the required EGL version is not supported by the device. */ public static final class UnsupportedEglVersionException extends Exception {} - /** GL program. */ + /** + * Represents a GLSL shader program. + * + *

After constructing a program, keep a reference for its lifetime and call {@link #delete()} + * (or release the current GL context) when it's no longer needed. + */ public static final class Program { /** The identifier of a compiled and linked GLSL shader program. */ private final int programId; - /** - * Compiles a GL shader program from vertex and fragment shader GLSL GLES20 code. - * - * @param vertexShaderGlsl The vertex shader program. - * @param fragmentShaderGlsl The fragment shader program. - */ - public Program(String vertexShaderGlsl, String fragmentShaderGlsl) { - programId = GLES20.glCreateProgram(); - checkGlError(); - - // Add the vertex and fragment shaders. - addShader(GLES20.GL_VERTEX_SHADER, vertexShaderGlsl); - addShader(GLES20.GL_FRAGMENT_SHADER, fragmentShaderGlsl); - } + private final Attribute[] attributes; + private final Uniform[] uniforms; + private final Map attributeByName; + private final Map uniformByName; /** * Compiles a GL shader program from vertex and fragment shader GLSL GLES20 code. @@ -87,9 +85,21 @@ public final class GlUtil { this(loadAsset(context, vertexShaderFilePath), loadAsset(context, fragmentShaderFilePath)); } - /** Uses the program. */ - public void use() { - // Link and check for errors. + /** + * Compiles a GL shader program from vertex and fragment shader GLSL GLES20 code. + * + * @param vertexShaderGlsl The vertex shader program. + * @param fragmentShaderGlsl The fragment shader program. + */ + public Program(String vertexShaderGlsl, String fragmentShaderGlsl) { + programId = GLES20.glCreateProgram(); + checkGlError(); + + // Add the vertex and fragment shaders. + addShader(programId, GLES20.GL_VERTEX_SHADER, vertexShaderGlsl); + addShader(programId, GLES20.GL_FRAGMENT_SHADER, fragmentShaderGlsl); + + // Link and use the program, and enumerate attributes/uniforms. GLES20.glLinkProgram(programId); int[] linkStatus = new int[] {GLES20.GL_FALSE}; GLES20.glGetProgramiv(programId, GLES20.GL_LINK_STATUS, linkStatus, 0); @@ -97,14 +107,38 @@ public final class GlUtil { throwGlException( "Unable to link shader program: \n" + GLES20.glGetProgramInfoLog(programId)); } - checkGlError(); - GLES20.glUseProgram(programId); + attributeByName = new HashMap<>(); + int[] attributeCount = new int[1]; + GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_ATTRIBUTES, attributeCount, 0); + attributes = new Attribute[attributeCount[0]]; + for (int i = 0; i < attributeCount[0]; i++) { + Attribute attribute = Attribute.create(programId, i); + attributes[i] = attribute; + attributeByName.put(attribute.name, attribute); + } + uniformByName = new HashMap<>(); + int[] uniformCount = new int[1]; + GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_UNIFORMS, uniformCount, 0); + uniforms = new Uniform[uniformCount[0]]; + for (int i = 0; i < uniformCount[0]; i++) { + Uniform uniform = Uniform.create(programId, i); + uniforms[i] = uniform; + uniformByName.put(uniform.name, uniform); + } + checkGlError(); + } + + /** Uses the program. */ + public void use() { + GLES20.glUseProgram(programId); + checkGlError(); } /** Deletes the program. Deleted programs cannot be used again. */ public void delete() { GLES20.glDeleteProgram(programId); + checkGlError(); } /** @@ -120,234 +154,45 @@ public final class GlUtil { /** Returns the location of an {@link Attribute}. */ private int getAttributeLocation(String attributeName) { - return GLES20.glGetAttribLocation(programId, attributeName); + return GlUtil.getAttributeLocation(programId, attributeName); } /** Returns the location of a {@link Uniform}. */ public int getUniformLocation(String uniformName) { - return GLES20.glGetUniformLocation(programId, uniformName); + return GlUtil.getUniformLocation(programId, uniformName); } - /** Returns the program's {@link Attribute}s. */ - public Attribute[] getAttributes() { - int[] attributeCount = new int[1]; - GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_ATTRIBUTES, attributeCount, 0); - if (attributeCount[0] != 2) { - throw new IllegalStateException("Expected two attributes but found " + attributeCount[0]); + /** Sets a float buffer type attribute. */ + public void setBufferAttribute(String name, float[] values, int size) { + checkNotNull(attributeByName.get(name)).setBuffer(values, size); + } + + /** Sets a texture sampler type uniform. */ + public void setSamplerTexIdUniform(String name, int texId, int unit) { + checkNotNull(uniformByName.get(name)).setSamplerTexId(texId, unit); + } + + /** Sets a float type uniform. */ + public void setFloatUniform(String name, float value) { + checkNotNull(uniformByName.get(name)).setFloat(value); + } + + /** Sets a float array type uniform. */ + public void setFloatsUniform(String name, float[] value) { + checkNotNull(uniformByName.get(name)).setFloats(value); + } + + /** Binds all attributes and uniforms in the program. */ + public void bindAttributesAndUniforms() { + for (Attribute attribute : attributes) { + attribute.bind(); } - - Attribute[] attributes = new Attribute[attributeCount[0]]; - for (int i = 0; i < attributeCount[0]; i++) { - attributes[i] = createAttribute(i); + for (Uniform uniform : uniforms) { + uniform.bind(); } - return attributes; - } - - /** Returns the program's {@link Uniform}s. */ - public Uniform[] getUniforms() { - int[] uniformCount = new int[1]; - GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_UNIFORMS, uniformCount, 0); - - Uniform[] uniforms = new Uniform[uniformCount[0]]; - for (int i = 0; i < uniformCount[0]; i++) { - uniforms[i] = createUniform(i); - } - - return uniforms; - } - - private Attribute createAttribute(int index) { - int[] length = new int[1]; - GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_ATTRIBUTE_MAX_LENGTH, length, 0); - - int[] type = new int[1]; - int[] size = new int[1]; - byte[] nameBytes = new byte[length[0]]; - int[] ignore = new int[1]; - - GLES20.glGetActiveAttrib( - programId, index, length[0], ignore, 0, size, 0, type, 0, nameBytes, 0); - String name = new String(nameBytes, 0, strlen(nameBytes)); - int location = getAttributeLocation(name); - - return new Attribute(name, index, location); - } - - private Uniform createUniform(int index) { - int[] length = new int[1]; - GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_UNIFORM_MAX_LENGTH, length, 0); - - int[] type = new int[1]; - int[] size = new int[1]; - byte[] nameBytes = new byte[length[0]]; - int[] ignore = new int[1]; - - GLES20.glGetActiveUniform( - programId, index, length[0], ignore, 0, size, 0, type, 0, nameBytes, 0); - String name = new String(nameBytes, 0, strlen(nameBytes)); - int location = getUniformLocation(name); - - return new Uniform(name, location, type[0]); - } - - private void addShader(int type, String glsl) { - int shader = GLES20.glCreateShader(type); - GLES20.glShaderSource(shader, glsl); - GLES20.glCompileShader(shader); - - int[] result = new int[] {GLES20.GL_FALSE}; - GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, result, 0); - if (result[0] != GLES20.GL_TRUE) { - throwGlException(GLES20.glGetShaderInfoLog(shader) + ", source: " + glsl); - } - - GLES20.glAttachShader(programId, shader); - GLES20.glDeleteShader(shader); - checkGlError(); } } - /** - * GL attribute, which can be attached to a buffer with {@link Attribute#setBuffer(float[], int)}. - */ - public static final class Attribute { - - /** The name of the attribute in the GLSL sources. */ - public final String name; - - private final int index; - private final int location; - - @Nullable private Buffer buffer; - private int size; - - /* Creates a new Attribute. */ - public Attribute(String name, int index, int location) { - this.name = name; - this.index = index; - this.location = location; - } - - /** - * Configures {@link #bind()} to attach vertices in {@code buffer} (each of size {@code size} - * elements) to this {@link Attribute}. - * - * @param buffer Buffer to bind to this attribute. - * @param size Number of elements per vertex. - */ - public void setBuffer(float[] buffer, int size) { - this.buffer = createBuffer(buffer); - this.size = size; - } - - /** - * Sets the vertex attribute to whatever was attached via {@link #setBuffer(float[], int)}. - * - *

Should be called before each drawing call. - */ - public void bind() { - Buffer buffer = Assertions.checkNotNull(this.buffer, "call setBuffer before bind"); - GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, 0); - GLES20.glVertexAttribPointer( - location, - size, // count - GLES20.GL_FLOAT, // type - false, // normalize - 0, // stride - buffer); - GLES20.glEnableVertexAttribArray(index); - checkGlError(); - } - } - - /** - * GL uniform, which can be attached to a sampler using {@link Uniform#setSamplerTexId(int, int)}. - */ - public static final class Uniform { - - /** The name of the uniform in the GLSL sources. */ - public final String name; - - private final int location; - private final int type; - private final float[] value; - - private int texId; - private int unit; - - /** Creates a new uniform. */ - public Uniform(String name, int location, int type) { - this.name = name; - this.location = location; - this.type = type; - this.value = new float[16]; - } - - /** - * Configures {@link #bind()} to use the specified {@code texId} for this sampler uniform. - * - * @param texId The GL texture identifier from which to sample. - * @param unit The GL texture unit index. - */ - public void setSamplerTexId(int texId, int unit) { - this.texId = texId; - this.unit = unit; - } - - /** Configures {@link #bind()} to use the specified float {@code value} for this uniform. */ - public void setFloat(float value) { - this.value[0] = value; - } - - /** Configures {@link #bind()} to use the specified float[] {@code value} for this uniform. */ - public void setFloats(float[] value) { - System.arraycopy(value, 0, this.value, 0, value.length); - } - - /** - * Sets the uniform to whatever value was passed via {@link #setSamplerTexId(int, int)}, {@link - * #setFloat(float)} or {@link #setFloats(float[])}. - * - *

Should be called before each drawing call. - */ - public void bind() { - if (type == GLES20.GL_FLOAT) { - GLES20.glUniform1fv(location, 1, value, 0); - checkGlError(); - return; - } - - if (type == GLES20.GL_FLOAT_MAT4) { - GLES20.glUniformMatrix4fv(location, 1, false, value, 0); - checkGlError(); - return; - } - - if (texId == 0) { - throw new IllegalStateException("Call setSamplerTexId before bind."); - } - GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + unit); - if (type == GLES11Ext.GL_SAMPLER_EXTERNAL_OES) { - GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, texId); - } else if (type == GLES20.GL_SAMPLER_2D) { - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texId); - } else { - throw new IllegalStateException("Unexpected uniform type: " + type); - } - GLES20.glUniform1i(location, unit); - GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); - GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); - GLES20.glTexParameteri( - GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); - GLES20.glTexParameteri( - GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); - checkGlError(); - } - } - - /** Represents an unset texture ID. */ - public static final int TEXTURE_ID_UNSET = -1; - /** Whether to throw a {@link GlException} in case of an OpenGL error. */ public static boolean glAssertionsEnabled = false; @@ -532,6 +377,30 @@ public final class GlUtil { return texId[0]; } + private static void addShader(int programId, int type, String glsl) { + int shader = GLES20.glCreateShader(type); + GLES20.glShaderSource(shader, glsl); + GLES20.glCompileShader(shader); + + int[] result = new int[] {GLES20.GL_FALSE}; + GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, result, 0); + if (result[0] != GLES20.GL_TRUE) { + throwGlException(GLES20.glGetShaderInfoLog(shader) + ", source: " + glsl); + } + + GLES20.glAttachShader(programId, shader); + GLES20.glDeleteShader(shader); + checkGlError(); + } + + private static int getAttributeLocation(int programId, String attributeName) { + return GLES20.glGetAttribLocation(programId, attributeName); + } + + private static int getUniformLocation(int programId, String uniformName) { + return GLES20.glGetUniformLocation(programId, uniformName); + } + private static void throwGlException(String errorMsg) { Log.e(TAG, errorMsg); if (glAssertionsEnabled) { @@ -555,6 +424,178 @@ public final class GlUtil { return strVal.length; } + /** + * GL attribute, which can be attached to a buffer with {@link Attribute#setBuffer(float[], int)}. + */ + private static final class Attribute { + + /* Returns the attribute at the given index in the program. */ + public static Attribute create(int programId, int index) { + int[] length = new int[1]; + GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_ATTRIBUTE_MAX_LENGTH, length, 0); + + int[] type = new int[1]; + int[] size = new int[1]; + byte[] nameBytes = new byte[length[0]]; + int[] ignore = new int[1]; + + GLES20.glGetActiveAttrib( + programId, index, length[0], ignore, 0, size, 0, type, 0, nameBytes, 0); + String name = new String(nameBytes, 0, strlen(nameBytes)); + int location = getAttributeLocation(programId, name); + + return new Attribute(name, index, location); + } + + /** The name of the attribute in the GLSL sources. */ + public final String name; + + private final int index; + private final int location; + + @Nullable private Buffer buffer; + private int size; + + private Attribute(String name, int index, int location) { + this.name = name; + this.index = index; + this.location = location; + } + + /** + * Configures {@link #bind()} to attach vertices in {@code buffer} (each of size {@code size} + * elements) to this {@link Attribute}. + * + * @param buffer Buffer to bind to this attribute. + * @param size Number of elements per vertex. + */ + public void setBuffer(float[] buffer, int size) { + this.buffer = createBuffer(buffer); + this.size = size; + } + + /** + * Sets the vertex attribute to whatever was attached via {@link #setBuffer(float[], int)}. + * + *

Should be called before each drawing call. + */ + public void bind() { + Buffer buffer = checkNotNull(this.buffer, "call setBuffer before bind"); + GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, 0); + GLES20.glVertexAttribPointer( + location, + size, // count + GLES20.GL_FLOAT, // type + false, // normalize + 0, // stride + buffer); + GLES20.glEnableVertexAttribArray(index); + checkGlError(); + } + } + + /** + * GL uniform, which can be attached to a sampler using {@link Uniform#setSamplerTexId(int, int)}. + */ + private static final class Uniform { + + /** Returns the uniform at the given index in the program. */ + public static Uniform create(int programId, int index) { + int[] length = new int[1]; + GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_UNIFORM_MAX_LENGTH, length, 0); + + int[] type = new int[1]; + int[] size = new int[1]; + byte[] nameBytes = new byte[length[0]]; + int[] ignore = new int[1]; + + GLES20.glGetActiveUniform( + programId, index, length[0], ignore, 0, size, 0, type, 0, nameBytes, 0); + String name = new String(nameBytes, 0, strlen(nameBytes)); + int location = getUniformLocation(programId, name); + + return new Uniform(name, location, type[0]); + } + + /** The name of the uniform in the GLSL sources. */ + public final String name; + + private final int location; + private final int type; + private final float[] value; + + private int texId; + private int unit; + + private Uniform(String name, int location, int type) { + this.name = name; + this.location = location; + this.type = type; + this.value = new float[16]; + } + + /** + * Configures {@link #bind()} to use the specified {@code texId} for this sampler uniform. + * + * @param texId The GL texture identifier from which to sample. + * @param unit The GL texture unit index. + */ + public void setSamplerTexId(int texId, int unit) { + this.texId = texId; + this.unit = unit; + } + + /** Configures {@link #bind()} to use the specified float {@code value} for this uniform. */ + public void setFloat(float value) { + this.value[0] = value; + } + + /** Configures {@link #bind()} to use the specified float[] {@code value} for this uniform. */ + public void setFloats(float[] value) { + System.arraycopy(value, 0, this.value, 0, value.length); + } + + /** + * Sets the uniform to whatever value was passed via {@link #setSamplerTexId(int, int)}, {@link + * #setFloat(float)} or {@link #setFloats(float[])}. + * + *

Should be called before each drawing call. + */ + public void bind() { + if (type == GLES20.GL_FLOAT) { + GLES20.glUniform1fv(location, 1, value, 0); + checkGlError(); + return; + } + + if (type == GLES20.GL_FLOAT_MAT4) { + GLES20.glUniformMatrix4fv(location, 1, false, value, 0); + checkGlError(); + return; + } + + if (texId == 0) { + throw new IllegalStateException("Call setSamplerTexId before bind."); + } + GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + unit); + if (type == GLES11Ext.GL_SAMPLER_EXTERNAL_OES) { + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, texId); + } else if (type == GLES20.GL_SAMPLER_2D) { + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texId); + } else { + throw new IllegalStateException("Unexpected uniform type: " + type); + } + GLES20.glUniform1i(location, unit); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); + GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); + GLES20.glTexParameteri( + GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameteri( + GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); + checkGlError(); + } + } + @RequiresApi(17) private static final class Api17 { private Api17() {} diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/spherical/ProjectionRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/spherical/ProjectionRenderer.java index 8fe10ac16a..03a5283ff6 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/spherical/ProjectionRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/spherical/ProjectionRenderer.java @@ -114,7 +114,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } /** Initializes of the GL components. */ - /* package */ void init() { + public void init() { program = new GlUtil.Program(VERTEX_SHADER, FRAGMENT_SHADER); mvpMatrixHandle = program.getUniformLocation("uMvpMatrix"); uTexMatrixHandle = program.getUniformLocation("uTexMatrix"); @@ -132,7 +132,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param rightEye Whether the right eye view should be drawn. If {@code false}, the left eye view * is drawn. */ - /* package */ void draw(int textureId, float[] mvpMatrix, boolean rightEye) { + public void draw(int textureId, float[] mvpMatrix, boolean rightEye) { MeshData meshData = rightEye ? rightMeshData : leftMeshData; if (meshData == null) { return; @@ -188,7 +188,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } /** Cleans up GL resources. */ - /* package */ void shutdown() { + public void shutdown() { if (program != null) { program.delete(); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/OpenGlFrameEditor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/OpenGlFrameEditor.java index ebf4b2f970..1357491487 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/OpenGlFrameEditor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/OpenGlFrameEditor.java @@ -13,12 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package androidx.media3.transformer; -import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkState; - import android.content.Context; import android.graphics.SurfaceTexture; import android.opengl.EGL14; @@ -31,7 +27,6 @@ import android.view.Surface; import androidx.annotation.RequiresApi; import androidx.media3.common.util.GlUtil; import java.io.IOException; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * OpenGlFrameEditor applies changes to individual video frames using OpenGL. Changes include just @@ -67,73 +62,38 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; int textureId = GlUtil.createExternalTexture(); GlUtil.Program copyProgram; try { + // TODO(internal b/205002913): check the loaded program is consistent with the attributes + // and uniforms expected in the code. copyProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH); } catch (IOException e) { throw new IllegalStateException(e); } - - copyProgram.use(); - GlUtil.Attribute[] copyAttributes = copyProgram.getAttributes(); - checkState( - copyAttributes.length == EXPECTED_NUMBER_OF_ATTRIBUTES, - "Expected program to have " + EXPECTED_NUMBER_OF_ATTRIBUTES + " vertex attributes."); - for (GlUtil.Attribute copyAttribute : copyAttributes) { - if (copyAttribute.name.equals("a_position")) { - copyAttribute.setBuffer( - new float[] { - -1.0f, -1.0f, 0.0f, 1.0f, - 1.0f, -1.0f, 0.0f, 1.0f, - -1.0f, 1.0f, 0.0f, 1.0f, - 1.0f, 1.0f, 0.0f, 1.0f, - }, - /* size= */ 4); - } else if (copyAttribute.name.equals("a_texcoord")) { - copyAttribute.setBuffer( - new float[] { - 0.0f, 0.0f, 0.0f, 1.0f, - 1.0f, 0.0f, 0.0f, 1.0f, - 0.0f, 1.0f, 0.0f, 1.0f, - 1.0f, 1.0f, 0.0f, 1.0f, - }, - /* size= */ 4); - } else { - throw new IllegalStateException("Unexpected attribute name."); - } - copyAttribute.bind(); - } - GlUtil.Uniform[] copyUniforms = copyProgram.getUniforms(); - checkState( - copyUniforms.length == EXPECTED_NUMBER_OF_UNIFORMS, - "Expected program to have " + EXPECTED_NUMBER_OF_UNIFORMS + " uniforms."); - GlUtil.@MonotonicNonNull Uniform textureTransformUniform = null; - for (GlUtil.Uniform copyUniform : copyUniforms) { - if (copyUniform.name.equals("tex_sampler")) { - copyUniform.setSamplerTexId(textureId, 0); - copyUniform.bind(); - } else if (copyUniform.name.equals("tex_transform")) { - textureTransformUniform = copyUniform; - } else { - throw new IllegalStateException("Unexpected uniform name."); - } - } - - return new OpenGlFrameEditor( - eglDisplay, - eglContext, - eglSurface, - textureId, - checkNotNull(textureTransformUniform), - copyProgram, - copyAttributes, - copyUniforms); + copyProgram.setBufferAttribute( + "a_position", + new float[] { + -1.0f, -1.0f, 0.0f, 1.0f, + 1.0f, -1.0f, 0.0f, 1.0f, + -1.0f, 1.0f, 0.0f, 1.0f, + 1.0f, 1.0f, 0.0f, 1.0f, + }, + /* size= */ 4); + copyProgram.setBufferAttribute( + "a_texcoord", + new float[] { + 0.0f, 0.0f, 0.0f, 1.0f, + 1.0f, 0.0f, 0.0f, 1.0f, + 0.0f, 1.0f, 0.0f, 1.0f, + 1.0f, 1.0f, 0.0f, 1.0f, + }, + /* size= */ 4); + copyProgram.setSamplerTexIdUniform("tex_sampler", textureId, /* unit= */ 0); + return new OpenGlFrameEditor(eglDisplay, eglContext, eglSurface, textureId, copyProgram); } // Predefined shader values. private static final String VERTEX_SHADER_FILE_PATH = "shaders/blit_vertex_shader.glsl"; private static final String FRAGMENT_SHADER_FILE_PATH = "shaders/copy_external_fragment_shader.glsl"; - private static final int EXPECTED_NUMBER_OF_ATTRIBUTES = 2; - private static final int EXPECTED_NUMBER_OF_UNIFORMS = 2; private final float[] textureTransformMatrix; private final EGLDisplay eglDisplay; @@ -142,19 +102,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final int textureId; private final SurfaceTexture inputSurfaceTexture; private final Surface inputSurface; - private final GlUtil.Uniform textureTransformUniform; - // TODO(internal: b/206631334): These fields ensure buffers passed to GL are not GC'ed. Implement - // a better way of doing this so they aren't just unused fields. - @SuppressWarnings("unused") private final GlUtil.Program copyProgram; - @SuppressWarnings("unused") - private final GlUtil.Attribute[] copyAttributes; - - @SuppressWarnings("unused") - private final GlUtil.Uniform[] copyUniforms; - private volatile boolean hasInputData; private OpenGlFrameEditor( @@ -162,38 +112,37 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; EGLContext eglContext, EGLSurface eglSurface, int textureId, - GlUtil.Uniform textureTransformUniform, - GlUtil.Program copyProgram, - GlUtil.Attribute[] copyAttributes, - GlUtil.Uniform[] copyUniforms) { + GlUtil.Program copyProgram) { this.eglDisplay = eglDisplay; this.eglContext = eglContext; this.eglSurface = eglSurface; this.textureId = textureId; - this.textureTransformUniform = textureTransformUniform; this.copyProgram = copyProgram; - this.copyAttributes = copyAttributes; - this.copyUniforms = copyUniforms; textureTransformMatrix = new float[16]; inputSurfaceTexture = new SurfaceTexture(textureId); inputSurfaceTexture.setOnFrameAvailableListener(surfaceTexture -> hasInputData = true); inputSurface = new Surface(inputSurfaceTexture); } - /** Releases all resources. */ - public void release() { - GlUtil.destroyEglContext(eglDisplay, eglContext); - GlUtil.deleteTexture(textureId); - inputSurfaceTexture.release(); - inputSurface.release(); + /** Returns the input {@link Surface}. */ + public Surface getInputSurface() { + return inputSurface; } - /** Informs the editor that there is new input data available for it to process asynchronously. */ + /** + * Returns whether there is pending input data that can be processed by calling {@link + * #processData()}. + */ + public boolean hasInputData() { + return hasInputData; + } + + /** Processes pending input data. */ public void processData() { inputSurfaceTexture.updateTexImage(); inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); - textureTransformUniform.setFloats(textureTransformMatrix); - textureTransformUniform.bind(); + copyProgram.setFloatsUniform("tex_transform", textureTransformMatrix); + copyProgram.bindAttributesAndUniforms(); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp(); EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs); @@ -201,15 +150,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; hasInputData = false; } - /** - * Returns the input {@link Surface} after configuring the editor if it has not previously been - * configured. - */ - public Surface getInputSurface() { - return inputSurface; - } - - public boolean hasInputData() { - return hasInputData; + /** Releases all resources. */ + public void release() { + copyProgram.delete(); + GlUtil.deleteTexture(textureId); + GlUtil.destroyEglContext(eglDisplay, eglContext); + inputSurfaceTexture.release(); + inputSurface.release(); } }