Encapsulate attributes and uniforms within Program

Document that apps should retain `GlUtil.Program` while the program is in use,
and keep a reference to attributes/uniforms within the program to make sure
they don't get GC'd causing any allocated buffers passed to GL to become
invalid.

Tested manually by running gldemo and transformer.

PiperOrigin-RevId: 411516894
This commit is contained in:
andrewlewis 2021-11-22 11:44:29 +00:00 committed by Ian Baker
parent 67a0e6d11c
commit 5b22b06ec4
5 changed files with 360 additions and 394 deletions

View File

@ -26,7 +26,6 @@ import android.graphics.Paint;
import android.graphics.drawable.BitmapDrawable; import android.graphics.drawable.BitmapDrawable;
import android.opengl.GLES20; import android.opengl.GLES20;
import android.opengl.GLUtils; import android.opengl.GLUtils;
import androidx.annotation.Nullable;
import androidx.media3.common.C; import androidx.media3.common.C;
import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.GlUtil;
import java.io.IOException; import java.io.IOException;
@ -52,8 +51,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final Canvas overlayCanvas; private final Canvas overlayCanvas;
private GlUtil.@MonotonicNonNull Program program; private GlUtil.@MonotonicNonNull Program program;
@Nullable private GlUtil.Attribute[] attributes;
@Nullable private GlUtil.Uniform[] uniforms;
private float bitmapScaleX; private float bitmapScaleX;
private float bitmapScaleY; private float bitmapScaleY;
@ -88,31 +85,24 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} catch (IOException e) { } catch (IOException e) {
throw new IllegalStateException(e); throw new IllegalStateException(e);
} }
program.use(); program.setBufferAttribute(
GlUtil.Attribute[] attributes = program.getAttributes(); "a_position",
for (GlUtil.Attribute attribute : attributes) { new float[] {
if (attribute.name.equals("a_position")) { -1, -1, 0, 1,
attribute.setBuffer( 1, -1, 0, 1,
new float[] { -1, 1, 0, 1,
-1, -1, 0, 1, 1, 1, 0, 1
1, -1, 0, 1, },
-1, 1, 0, 1, 4);
1, 1, 0, 1 program.setBufferAttribute(
}, "a_texcoord",
4); new float[] {
} else if (attribute.name.equals("a_texcoord")) { 0, 0, 0, 1,
attribute.setBuffer( 1, 0, 0, 1,
new float[] { 0, 1, 0, 1,
0, 0, 0, 1, 1, 1, 0, 1
1, 0, 0, 1, },
0, 1, 0, 1, 4);
1, 1, 0, 1
},
4);
}
}
this.attributes = attributes;
this.uniforms = program.getUniforms();
GLES20.glGenTextures(1, textures, 0); GLES20.glGenTextures(1, textures, 0);
GLES20.glBindTexture(GL10.GL_TEXTURE_2D, textures[0]); GLES20.glBindTexture(GL10.GL_TEXTURE_2D, textures[0]);
GLES20.glTexParameterf(GL10.GL_TEXTURE_2D, GL10.GL_TEXTURE_MIN_FILTER, GL10.GL_NEAREST); GLES20.glTexParameterf(GL10.GL_TEXTURE_2D, GL10.GL_TEXTURE_MIN_FILTER, GL10.GL_NEAREST);
@ -141,36 +131,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
GlUtil.checkGlError(); GlUtil.checkGlError();
// Run the shader program. // Run the shader program.
GlUtil.Uniform[] uniforms = checkNotNull(this.uniforms); GlUtil.Program program = checkNotNull(this.program);
GlUtil.Attribute[] attributes = checkNotNull(this.attributes); program.setSamplerTexIdUniform("tex_sampler_0", frameTexture, /* unit= */ 0);
for (GlUtil.Uniform uniform : uniforms) { program.setSamplerTexIdUniform("tex_sampler_1", textures[0], /* unit= */ 1);
switch (uniform.name) { program.setFloatUniform("scaleX", bitmapScaleX);
case "tex_sampler_0": program.setFloatUniform("scaleY", bitmapScaleY);
uniform.setSamplerTexId(frameTexture, /* unit= */ 0); program.setFloatsUniform("tex_transform", transformMatrix);
break; program.bindAttributesAndUniforms();
case "tex_sampler_1":
uniform.setSamplerTexId(textures[0], /* unit= */ 1);
break;
case "scaleX":
uniform.setFloat(bitmapScaleX);
break;
case "scaleY":
uniform.setFloat(bitmapScaleY);
break;
case "tex_transform":
uniform.setFloats(transformMatrix);
break;
default: // fall out
}
}
for (GlUtil.Attribute copyExternalAttribute : attributes) {
copyExternalAttribute.bind();
}
for (GlUtil.Uniform copyExternalUniform : uniforms) {
copyExternalUniform.bind();
}
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4);
GlUtil.checkGlError(); GlUtil.checkGlError();
} }
@Override
public void release() {
if (program != null) {
program.delete();
}
}
} }

View File

@ -64,6 +64,9 @@ public final class VideoProcessingGLSurfaceView extends GLSurfaceView {
* @param transformMatrix The 4 * 4 transform matrix to be applied to the texture. * @param transformMatrix The 4 * 4 transform matrix to be applied to the texture.
*/ */
void draw(int frameTexture, long frameTimestampUs, float[] transformMatrix); void draw(int frameTexture, long frameTimestampUs, float[] transformMatrix);
/** Releases any resources associated with this {@link VideoProcessor}. */
void release();
} }
private static final int EGL_PROTECTED_CONTENT_EXT = 0x32C0; private static final int EGL_PROTECTED_CONTENT_EXT = 0x32C0;

View File

@ -16,6 +16,7 @@
package androidx.media3.common.util; package androidx.media3.common.util;
import static android.opengl.GLU.gluErrorString; import static android.opengl.GLU.gluErrorString;
import static androidx.media3.common.util.Assertions.checkNotNull;
import android.content.Context; import android.content.Context;
import android.content.pm.PackageManager; import android.content.pm.PackageManager;
@ -37,6 +38,8 @@ import java.nio.ByteBuffer;
import java.nio.ByteOrder; import java.nio.ByteOrder;
import java.nio.FloatBuffer; import java.nio.FloatBuffer;
import java.nio.IntBuffer; import java.nio.IntBuffer;
import java.util.HashMap;
import java.util.Map;
import javax.microedition.khronos.egl.EGL10; import javax.microedition.khronos.egl.EGL10;
/** GL utilities. */ /** GL utilities. */
@ -54,25 +57,20 @@ public final class GlUtil {
/** Thrown when the required EGL version is not supported by the device. */ /** Thrown when the required EGL version is not supported by the device. */
public static final class UnsupportedEglVersionException extends Exception {} public static final class UnsupportedEglVersionException extends Exception {}
/** GL program. */ /**
* Represents a GLSL shader program.
*
* <p>After constructing a program, keep a reference for its lifetime and call {@link #delete()}
* (or release the current GL context) when it's no longer needed.
*/
public static final class Program { public static final class Program {
/** The identifier of a compiled and linked GLSL shader program. */ /** The identifier of a compiled and linked GLSL shader program. */
private final int programId; private final int programId;
/** private final Attribute[] attributes;
* Compiles a GL shader program from vertex and fragment shader GLSL GLES20 code. private final Uniform[] uniforms;
* private final Map<String, Attribute> attributeByName;
* @param vertexShaderGlsl The vertex shader program. private final Map<String, Uniform> uniformByName;
* @param fragmentShaderGlsl The fragment shader program.
*/
public Program(String vertexShaderGlsl, String fragmentShaderGlsl) {
programId = GLES20.glCreateProgram();
checkGlError();
// Add the vertex and fragment shaders.
addShader(GLES20.GL_VERTEX_SHADER, vertexShaderGlsl);
addShader(GLES20.GL_FRAGMENT_SHADER, fragmentShaderGlsl);
}
/** /**
* Compiles a GL shader program from vertex and fragment shader GLSL GLES20 code. * Compiles a GL shader program from vertex and fragment shader GLSL GLES20 code.
@ -87,9 +85,21 @@ public final class GlUtil {
this(loadAsset(context, vertexShaderFilePath), loadAsset(context, fragmentShaderFilePath)); this(loadAsset(context, vertexShaderFilePath), loadAsset(context, fragmentShaderFilePath));
} }
/** Uses the program. */ /**
public void use() { * Compiles a GL shader program from vertex and fragment shader GLSL GLES20 code.
// Link and check for errors. *
* @param vertexShaderGlsl The vertex shader program.
* @param fragmentShaderGlsl The fragment shader program.
*/
public Program(String vertexShaderGlsl, String fragmentShaderGlsl) {
programId = GLES20.glCreateProgram();
checkGlError();
// Add the vertex and fragment shaders.
addShader(programId, GLES20.GL_VERTEX_SHADER, vertexShaderGlsl);
addShader(programId, GLES20.GL_FRAGMENT_SHADER, fragmentShaderGlsl);
// Link and use the program, and enumerate attributes/uniforms.
GLES20.glLinkProgram(programId); GLES20.glLinkProgram(programId);
int[] linkStatus = new int[] {GLES20.GL_FALSE}; int[] linkStatus = new int[] {GLES20.GL_FALSE};
GLES20.glGetProgramiv(programId, GLES20.GL_LINK_STATUS, linkStatus, 0); GLES20.glGetProgramiv(programId, GLES20.GL_LINK_STATUS, linkStatus, 0);
@ -97,14 +107,38 @@ public final class GlUtil {
throwGlException( throwGlException(
"Unable to link shader program: \n" + GLES20.glGetProgramInfoLog(programId)); "Unable to link shader program: \n" + GLES20.glGetProgramInfoLog(programId));
} }
checkGlError();
GLES20.glUseProgram(programId); GLES20.glUseProgram(programId);
attributeByName = new HashMap<>();
int[] attributeCount = new int[1];
GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_ATTRIBUTES, attributeCount, 0);
attributes = new Attribute[attributeCount[0]];
for (int i = 0; i < attributeCount[0]; i++) {
Attribute attribute = Attribute.create(programId, i);
attributes[i] = attribute;
attributeByName.put(attribute.name, attribute);
}
uniformByName = new HashMap<>();
int[] uniformCount = new int[1];
GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_UNIFORMS, uniformCount, 0);
uniforms = new Uniform[uniformCount[0]];
for (int i = 0; i < uniformCount[0]; i++) {
Uniform uniform = Uniform.create(programId, i);
uniforms[i] = uniform;
uniformByName.put(uniform.name, uniform);
}
checkGlError();
}
/** Uses the program. */
public void use() {
GLES20.glUseProgram(programId);
checkGlError();
} }
/** Deletes the program. Deleted programs cannot be used again. */ /** Deletes the program. Deleted programs cannot be used again. */
public void delete() { public void delete() {
GLES20.glDeleteProgram(programId); GLES20.glDeleteProgram(programId);
checkGlError();
} }
/** /**
@ -120,234 +154,45 @@ public final class GlUtil {
/** Returns the location of an {@link Attribute}. */ /** Returns the location of an {@link Attribute}. */
private int getAttributeLocation(String attributeName) { private int getAttributeLocation(String attributeName) {
return GLES20.glGetAttribLocation(programId, attributeName); return GlUtil.getAttributeLocation(programId, attributeName);
} }
/** Returns the location of a {@link Uniform}. */ /** Returns the location of a {@link Uniform}. */
public int getUniformLocation(String uniformName) { public int getUniformLocation(String uniformName) {
return GLES20.glGetUniformLocation(programId, uniformName); return GlUtil.getUniformLocation(programId, uniformName);
} }
/** Returns the program's {@link Attribute}s. */ /** Sets a float buffer type attribute. */
public Attribute[] getAttributes() { public void setBufferAttribute(String name, float[] values, int size) {
int[] attributeCount = new int[1]; checkNotNull(attributeByName.get(name)).setBuffer(values, size);
GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_ATTRIBUTES, attributeCount, 0); }
if (attributeCount[0] != 2) {
throw new IllegalStateException("Expected two attributes but found " + attributeCount[0]); /** Sets a texture sampler type uniform. */
public void setSamplerTexIdUniform(String name, int texId, int unit) {
checkNotNull(uniformByName.get(name)).setSamplerTexId(texId, unit);
}
/** Sets a float type uniform. */
public void setFloatUniform(String name, float value) {
checkNotNull(uniformByName.get(name)).setFloat(value);
}
/** Sets a float array type uniform. */
public void setFloatsUniform(String name, float[] value) {
checkNotNull(uniformByName.get(name)).setFloats(value);
}
/** Binds all attributes and uniforms in the program. */
public void bindAttributesAndUniforms() {
for (Attribute attribute : attributes) {
attribute.bind();
} }
for (Uniform uniform : uniforms) {
Attribute[] attributes = new Attribute[attributeCount[0]]; uniform.bind();
for (int i = 0; i < attributeCount[0]; i++) {
attributes[i] = createAttribute(i);
} }
return attributes;
}
/** Returns the program's {@link Uniform}s. */
public Uniform[] getUniforms() {
int[] uniformCount = new int[1];
GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_UNIFORMS, uniformCount, 0);
Uniform[] uniforms = new Uniform[uniformCount[0]];
for (int i = 0; i < uniformCount[0]; i++) {
uniforms[i] = createUniform(i);
}
return uniforms;
}
private Attribute createAttribute(int index) {
int[] length = new int[1];
GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_ATTRIBUTE_MAX_LENGTH, length, 0);
int[] type = new int[1];
int[] size = new int[1];
byte[] nameBytes = new byte[length[0]];
int[] ignore = new int[1];
GLES20.glGetActiveAttrib(
programId, index, length[0], ignore, 0, size, 0, type, 0, nameBytes, 0);
String name = new String(nameBytes, 0, strlen(nameBytes));
int location = getAttributeLocation(name);
return new Attribute(name, index, location);
}
private Uniform createUniform(int index) {
int[] length = new int[1];
GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_UNIFORM_MAX_LENGTH, length, 0);
int[] type = new int[1];
int[] size = new int[1];
byte[] nameBytes = new byte[length[0]];
int[] ignore = new int[1];
GLES20.glGetActiveUniform(
programId, index, length[0], ignore, 0, size, 0, type, 0, nameBytes, 0);
String name = new String(nameBytes, 0, strlen(nameBytes));
int location = getUniformLocation(name);
return new Uniform(name, location, type[0]);
}
private void addShader(int type, String glsl) {
int shader = GLES20.glCreateShader(type);
GLES20.glShaderSource(shader, glsl);
GLES20.glCompileShader(shader);
int[] result = new int[] {GLES20.GL_FALSE};
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, result, 0);
if (result[0] != GLES20.GL_TRUE) {
throwGlException(GLES20.glGetShaderInfoLog(shader) + ", source: " + glsl);
}
GLES20.glAttachShader(programId, shader);
GLES20.glDeleteShader(shader);
checkGlError();
} }
} }
/**
* GL attribute, which can be attached to a buffer with {@link Attribute#setBuffer(float[], int)}.
*/
public static final class Attribute {
/** The name of the attribute in the GLSL sources. */
public final String name;
private final int index;
private final int location;
@Nullable private Buffer buffer;
private int size;
/* Creates a new Attribute. */
public Attribute(String name, int index, int location) {
this.name = name;
this.index = index;
this.location = location;
}
/**
* Configures {@link #bind()} to attach vertices in {@code buffer} (each of size {@code size}
* elements) to this {@link Attribute}.
*
* @param buffer Buffer to bind to this attribute.
* @param size Number of elements per vertex.
*/
public void setBuffer(float[] buffer, int size) {
this.buffer = createBuffer(buffer);
this.size = size;
}
/**
* Sets the vertex attribute to whatever was attached via {@link #setBuffer(float[], int)}.
*
* <p>Should be called before each drawing call.
*/
public void bind() {
Buffer buffer = Assertions.checkNotNull(this.buffer, "call setBuffer before bind");
GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, 0);
GLES20.glVertexAttribPointer(
location,
size, // count
GLES20.GL_FLOAT, // type
false, // normalize
0, // stride
buffer);
GLES20.glEnableVertexAttribArray(index);
checkGlError();
}
}
/**
* GL uniform, which can be attached to a sampler using {@link Uniform#setSamplerTexId(int, int)}.
*/
public static final class Uniform {
/** The name of the uniform in the GLSL sources. */
public final String name;
private final int location;
private final int type;
private final float[] value;
private int texId;
private int unit;
/** Creates a new uniform. */
public Uniform(String name, int location, int type) {
this.name = name;
this.location = location;
this.type = type;
this.value = new float[16];
}
/**
* Configures {@link #bind()} to use the specified {@code texId} for this sampler uniform.
*
* @param texId The GL texture identifier from which to sample.
* @param unit The GL texture unit index.
*/
public void setSamplerTexId(int texId, int unit) {
this.texId = texId;
this.unit = unit;
}
/** Configures {@link #bind()} to use the specified float {@code value} for this uniform. */
public void setFloat(float value) {
this.value[0] = value;
}
/** Configures {@link #bind()} to use the specified float[] {@code value} for this uniform. */
public void setFloats(float[] value) {
System.arraycopy(value, 0, this.value, 0, value.length);
}
/**
* Sets the uniform to whatever value was passed via {@link #setSamplerTexId(int, int)}, {@link
* #setFloat(float)} or {@link #setFloats(float[])}.
*
* <p>Should be called before each drawing call.
*/
public void bind() {
if (type == GLES20.GL_FLOAT) {
GLES20.glUniform1fv(location, 1, value, 0);
checkGlError();
return;
}
if (type == GLES20.GL_FLOAT_MAT4) {
GLES20.glUniformMatrix4fv(location, 1, false, value, 0);
checkGlError();
return;
}
if (texId == 0) {
throw new IllegalStateException("Call setSamplerTexId before bind.");
}
GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + unit);
if (type == GLES11Ext.GL_SAMPLER_EXTERNAL_OES) {
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, texId);
} else if (type == GLES20.GL_SAMPLER_2D) {
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texId);
} else {
throw new IllegalStateException("Unexpected uniform type: " + type);
}
GLES20.glUniform1i(location, unit);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameteri(
GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(
GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
checkGlError();
}
}
/** Represents an unset texture ID. */
public static final int TEXTURE_ID_UNSET = -1;
/** Whether to throw a {@link GlException} in case of an OpenGL error. */ /** Whether to throw a {@link GlException} in case of an OpenGL error. */
public static boolean glAssertionsEnabled = false; public static boolean glAssertionsEnabled = false;
@ -532,6 +377,30 @@ public final class GlUtil {
return texId[0]; return texId[0];
} }
private static void addShader(int programId, int type, String glsl) {
int shader = GLES20.glCreateShader(type);
GLES20.glShaderSource(shader, glsl);
GLES20.glCompileShader(shader);
int[] result = new int[] {GLES20.GL_FALSE};
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, result, 0);
if (result[0] != GLES20.GL_TRUE) {
throwGlException(GLES20.glGetShaderInfoLog(shader) + ", source: " + glsl);
}
GLES20.glAttachShader(programId, shader);
GLES20.glDeleteShader(shader);
checkGlError();
}
private static int getAttributeLocation(int programId, String attributeName) {
return GLES20.glGetAttribLocation(programId, attributeName);
}
private static int getUniformLocation(int programId, String uniformName) {
return GLES20.glGetUniformLocation(programId, uniformName);
}
private static void throwGlException(String errorMsg) { private static void throwGlException(String errorMsg) {
Log.e(TAG, errorMsg); Log.e(TAG, errorMsg);
if (glAssertionsEnabled) { if (glAssertionsEnabled) {
@ -555,6 +424,178 @@ public final class GlUtil {
return strVal.length; return strVal.length;
} }
/**
* GL attribute, which can be attached to a buffer with {@link Attribute#setBuffer(float[], int)}.
*/
private static final class Attribute {
/* Returns the attribute at the given index in the program. */
public static Attribute create(int programId, int index) {
int[] length = new int[1];
GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_ATTRIBUTE_MAX_LENGTH, length, 0);
int[] type = new int[1];
int[] size = new int[1];
byte[] nameBytes = new byte[length[0]];
int[] ignore = new int[1];
GLES20.glGetActiveAttrib(
programId, index, length[0], ignore, 0, size, 0, type, 0, nameBytes, 0);
String name = new String(nameBytes, 0, strlen(nameBytes));
int location = getAttributeLocation(programId, name);
return new Attribute(name, index, location);
}
/** The name of the attribute in the GLSL sources. */
public final String name;
private final int index;
private final int location;
@Nullable private Buffer buffer;
private int size;
private Attribute(String name, int index, int location) {
this.name = name;
this.index = index;
this.location = location;
}
/**
* Configures {@link #bind()} to attach vertices in {@code buffer} (each of size {@code size}
* elements) to this {@link Attribute}.
*
* @param buffer Buffer to bind to this attribute.
* @param size Number of elements per vertex.
*/
public void setBuffer(float[] buffer, int size) {
this.buffer = createBuffer(buffer);
this.size = size;
}
/**
* Sets the vertex attribute to whatever was attached via {@link #setBuffer(float[], int)}.
*
* <p>Should be called before each drawing call.
*/
public void bind() {
Buffer buffer = checkNotNull(this.buffer, "call setBuffer before bind");
GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, 0);
GLES20.glVertexAttribPointer(
location,
size, // count
GLES20.GL_FLOAT, // type
false, // normalize
0, // stride
buffer);
GLES20.glEnableVertexAttribArray(index);
checkGlError();
}
}
/**
* GL uniform, which can be attached to a sampler using {@link Uniform#setSamplerTexId(int, int)}.
*/
private static final class Uniform {
/** Returns the uniform at the given index in the program. */
public static Uniform create(int programId, int index) {
int[] length = new int[1];
GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_UNIFORM_MAX_LENGTH, length, 0);
int[] type = new int[1];
int[] size = new int[1];
byte[] nameBytes = new byte[length[0]];
int[] ignore = new int[1];
GLES20.glGetActiveUniform(
programId, index, length[0], ignore, 0, size, 0, type, 0, nameBytes, 0);
String name = new String(nameBytes, 0, strlen(nameBytes));
int location = getUniformLocation(programId, name);
return new Uniform(name, location, type[0]);
}
/** The name of the uniform in the GLSL sources. */
public final String name;
private final int location;
private final int type;
private final float[] value;
private int texId;
private int unit;
private Uniform(String name, int location, int type) {
this.name = name;
this.location = location;
this.type = type;
this.value = new float[16];
}
/**
* Configures {@link #bind()} to use the specified {@code texId} for this sampler uniform.
*
* @param texId The GL texture identifier from which to sample.
* @param unit The GL texture unit index.
*/
public void setSamplerTexId(int texId, int unit) {
this.texId = texId;
this.unit = unit;
}
/** Configures {@link #bind()} to use the specified float {@code value} for this uniform. */
public void setFloat(float value) {
this.value[0] = value;
}
/** Configures {@link #bind()} to use the specified float[] {@code value} for this uniform. */
public void setFloats(float[] value) {
System.arraycopy(value, 0, this.value, 0, value.length);
}
/**
* Sets the uniform to whatever value was passed via {@link #setSamplerTexId(int, int)}, {@link
* #setFloat(float)} or {@link #setFloats(float[])}.
*
* <p>Should be called before each drawing call.
*/
public void bind() {
if (type == GLES20.GL_FLOAT) {
GLES20.glUniform1fv(location, 1, value, 0);
checkGlError();
return;
}
if (type == GLES20.GL_FLOAT_MAT4) {
GLES20.glUniformMatrix4fv(location, 1, false, value, 0);
checkGlError();
return;
}
if (texId == 0) {
throw new IllegalStateException("Call setSamplerTexId before bind.");
}
GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + unit);
if (type == GLES11Ext.GL_SAMPLER_EXTERNAL_OES) {
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, texId);
} else if (type == GLES20.GL_SAMPLER_2D) {
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texId);
} else {
throw new IllegalStateException("Unexpected uniform type: " + type);
}
GLES20.glUniform1i(location, unit);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameteri(
GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(
GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
checkGlError();
}
}
@RequiresApi(17) @RequiresApi(17)
private static final class Api17 { private static final class Api17 {
private Api17() {} private Api17() {}

View File

@ -114,7 +114,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
/** Initializes of the GL components. */ /** Initializes of the GL components. */
/* package */ void init() { public void init() {
program = new GlUtil.Program(VERTEX_SHADER, FRAGMENT_SHADER); program = new GlUtil.Program(VERTEX_SHADER, FRAGMENT_SHADER);
mvpMatrixHandle = program.getUniformLocation("uMvpMatrix"); mvpMatrixHandle = program.getUniformLocation("uMvpMatrix");
uTexMatrixHandle = program.getUniformLocation("uTexMatrix"); uTexMatrixHandle = program.getUniformLocation("uTexMatrix");
@ -132,7 +132,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @param rightEye Whether the right eye view should be drawn. If {@code false}, the left eye view * @param rightEye Whether the right eye view should be drawn. If {@code false}, the left eye view
* is drawn. * is drawn.
*/ */
/* package */ void draw(int textureId, float[] mvpMatrix, boolean rightEye) { public void draw(int textureId, float[] mvpMatrix, boolean rightEye) {
MeshData meshData = rightEye ? rightMeshData : leftMeshData; MeshData meshData = rightEye ? rightMeshData : leftMeshData;
if (meshData == null) { if (meshData == null) {
return; return;
@ -188,7 +188,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} }
/** Cleans up GL resources. */ /** Cleans up GL resources. */
/* package */ void shutdown() { public void shutdown() {
if (program != null) { if (program != null) {
program.delete(); program.delete();
} }

View File

@ -13,12 +13,8 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package androidx.media3.transformer; package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import android.content.Context; import android.content.Context;
import android.graphics.SurfaceTexture; import android.graphics.SurfaceTexture;
import android.opengl.EGL14; import android.opengl.EGL14;
@ -31,7 +27,6 @@ import android.view.Surface;
import androidx.annotation.RequiresApi; import androidx.annotation.RequiresApi;
import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.GlUtil;
import java.io.IOException; import java.io.IOException;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** /**
* OpenGlFrameEditor applies changes to individual video frames using OpenGL. Changes include just * OpenGlFrameEditor applies changes to individual video frames using OpenGL. Changes include just
@ -67,73 +62,38 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
int textureId = GlUtil.createExternalTexture(); int textureId = GlUtil.createExternalTexture();
GlUtil.Program copyProgram; GlUtil.Program copyProgram;
try { try {
// TODO(internal b/205002913): check the loaded program is consistent with the attributes
// and uniforms expected in the code.
copyProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH); copyProgram = new GlUtil.Program(context, VERTEX_SHADER_FILE_PATH, FRAGMENT_SHADER_FILE_PATH);
} catch (IOException e) { } catch (IOException e) {
throw new IllegalStateException(e); throw new IllegalStateException(e);
} }
copyProgram.setBufferAttribute(
copyProgram.use(); "a_position",
GlUtil.Attribute[] copyAttributes = copyProgram.getAttributes(); new float[] {
checkState( -1.0f, -1.0f, 0.0f, 1.0f,
copyAttributes.length == EXPECTED_NUMBER_OF_ATTRIBUTES, 1.0f, -1.0f, 0.0f, 1.0f,
"Expected program to have " + EXPECTED_NUMBER_OF_ATTRIBUTES + " vertex attributes."); -1.0f, 1.0f, 0.0f, 1.0f,
for (GlUtil.Attribute copyAttribute : copyAttributes) { 1.0f, 1.0f, 0.0f, 1.0f,
if (copyAttribute.name.equals("a_position")) { },
copyAttribute.setBuffer( /* size= */ 4);
new float[] { copyProgram.setBufferAttribute(
-1.0f, -1.0f, 0.0f, 1.0f, "a_texcoord",
1.0f, -1.0f, 0.0f, 1.0f, new float[] {
-1.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f, 0.0f, 1.0f,
1.0f, 1.0f, 0.0f, 1.0f, 1.0f, 0.0f, 0.0f, 1.0f,
}, 0.0f, 1.0f, 0.0f, 1.0f,
/* size= */ 4); 1.0f, 1.0f, 0.0f, 1.0f,
} else if (copyAttribute.name.equals("a_texcoord")) { },
copyAttribute.setBuffer( /* size= */ 4);
new float[] { copyProgram.setSamplerTexIdUniform("tex_sampler", textureId, /* unit= */ 0);
0.0f, 0.0f, 0.0f, 1.0f, return new OpenGlFrameEditor(eglDisplay, eglContext, eglSurface, textureId, copyProgram);
1.0f, 0.0f, 0.0f, 1.0f,
0.0f, 1.0f, 0.0f, 1.0f,
1.0f, 1.0f, 0.0f, 1.0f,
},
/* size= */ 4);
} else {
throw new IllegalStateException("Unexpected attribute name.");
}
copyAttribute.bind();
}
GlUtil.Uniform[] copyUniforms = copyProgram.getUniforms();
checkState(
copyUniforms.length == EXPECTED_NUMBER_OF_UNIFORMS,
"Expected program to have " + EXPECTED_NUMBER_OF_UNIFORMS + " uniforms.");
GlUtil.@MonotonicNonNull Uniform textureTransformUniform = null;
for (GlUtil.Uniform copyUniform : copyUniforms) {
if (copyUniform.name.equals("tex_sampler")) {
copyUniform.setSamplerTexId(textureId, 0);
copyUniform.bind();
} else if (copyUniform.name.equals("tex_transform")) {
textureTransformUniform = copyUniform;
} else {
throw new IllegalStateException("Unexpected uniform name.");
}
}
return new OpenGlFrameEditor(
eglDisplay,
eglContext,
eglSurface,
textureId,
checkNotNull(textureTransformUniform),
copyProgram,
copyAttributes,
copyUniforms);
} }
// Predefined shader values. // Predefined shader values.
private static final String VERTEX_SHADER_FILE_PATH = "shaders/blit_vertex_shader.glsl"; private static final String VERTEX_SHADER_FILE_PATH = "shaders/blit_vertex_shader.glsl";
private static final String FRAGMENT_SHADER_FILE_PATH = private static final String FRAGMENT_SHADER_FILE_PATH =
"shaders/copy_external_fragment_shader.glsl"; "shaders/copy_external_fragment_shader.glsl";
private static final int EXPECTED_NUMBER_OF_ATTRIBUTES = 2;
private static final int EXPECTED_NUMBER_OF_UNIFORMS = 2;
private final float[] textureTransformMatrix; private final float[] textureTransformMatrix;
private final EGLDisplay eglDisplay; private final EGLDisplay eglDisplay;
@ -142,19 +102,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final int textureId; private final int textureId;
private final SurfaceTexture inputSurfaceTexture; private final SurfaceTexture inputSurfaceTexture;
private final Surface inputSurface; private final Surface inputSurface;
private final GlUtil.Uniform textureTransformUniform;
// TODO(internal: b/206631334): These fields ensure buffers passed to GL are not GC'ed. Implement
// a better way of doing this so they aren't just unused fields.
@SuppressWarnings("unused")
private final GlUtil.Program copyProgram; private final GlUtil.Program copyProgram;
@SuppressWarnings("unused")
private final GlUtil.Attribute[] copyAttributes;
@SuppressWarnings("unused")
private final GlUtil.Uniform[] copyUniforms;
private volatile boolean hasInputData; private volatile boolean hasInputData;
private OpenGlFrameEditor( private OpenGlFrameEditor(
@ -162,38 +112,37 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
EGLContext eglContext, EGLContext eglContext,
EGLSurface eglSurface, EGLSurface eglSurface,
int textureId, int textureId,
GlUtil.Uniform textureTransformUniform, GlUtil.Program copyProgram) {
GlUtil.Program copyProgram,
GlUtil.Attribute[] copyAttributes,
GlUtil.Uniform[] copyUniforms) {
this.eglDisplay = eglDisplay; this.eglDisplay = eglDisplay;
this.eglContext = eglContext; this.eglContext = eglContext;
this.eglSurface = eglSurface; this.eglSurface = eglSurface;
this.textureId = textureId; this.textureId = textureId;
this.textureTransformUniform = textureTransformUniform;
this.copyProgram = copyProgram; this.copyProgram = copyProgram;
this.copyAttributes = copyAttributes;
this.copyUniforms = copyUniforms;
textureTransformMatrix = new float[16]; textureTransformMatrix = new float[16];
inputSurfaceTexture = new SurfaceTexture(textureId); inputSurfaceTexture = new SurfaceTexture(textureId);
inputSurfaceTexture.setOnFrameAvailableListener(surfaceTexture -> hasInputData = true); inputSurfaceTexture.setOnFrameAvailableListener(surfaceTexture -> hasInputData = true);
inputSurface = new Surface(inputSurfaceTexture); inputSurface = new Surface(inputSurfaceTexture);
} }
/** Releases all resources. */ /** Returns the input {@link Surface}. */
public void release() { public Surface getInputSurface() {
GlUtil.destroyEglContext(eglDisplay, eglContext); return inputSurface;
GlUtil.deleteTexture(textureId);
inputSurfaceTexture.release();
inputSurface.release();
} }
/** Informs the editor that there is new input data available for it to process asynchronously. */ /**
* Returns whether there is pending input data that can be processed by calling {@link
* #processData()}.
*/
public boolean hasInputData() {
return hasInputData;
}
/** Processes pending input data. */
public void processData() { public void processData() {
inputSurfaceTexture.updateTexImage(); inputSurfaceTexture.updateTexImage();
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
textureTransformUniform.setFloats(textureTransformMatrix); copyProgram.setFloatsUniform("tex_transform", textureTransformMatrix);
textureTransformUniform.bind(); copyProgram.bindAttributesAndUniforms();
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp(); long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp();
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs); EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs);
@ -201,15 +150,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
hasInputData = false; hasInputData = false;
} }
/** /** Releases all resources. */
* Returns the input {@link Surface} after configuring the editor if it has not previously been public void release() {
* configured. copyProgram.delete();
*/ GlUtil.deleteTexture(textureId);
public Surface getInputSurface() { GlUtil.destroyEglContext(eglDisplay, eglContext);
return inputSurface; inputSurfaceTexture.release();
} inputSurface.release();
public boolean hasInputData() {
return hasInputData;
} }
} }