loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove unnecessary memory allocation in OpenGL

Browse Source 
PiperOrigin-RevId: 579835031
This commit is contained in:
claincly 2023-11-06 07:18:45 -08:00 committed by Copybara-Service
parent 98e9022c0e
commit 414b72619b
8 changed files with 49 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java
View File

@ -23,6 +23,7 @@ import static androidx.media3.common.util.Assertions.checkState;
import android.content.Context;
import android.content.pm.PackageManager;
import android.graphics.Bitmap;
import android.opengl.EGL14;
import android.opengl.EGLConfig;
import android.opengl.EGLContext;
@ -31,6 +32,7 @@ import android.opengl.EGLSurface;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import android.opengl.GLES30;
import android.opengl.GLUtils;
import android.opengl.Matrix;
import androidx.annotation.DoNotInline;
import androidx.annotation.IntRange;
@ -575,8 +577,8 @@ public final class GlUtil {
* @param height The height of the new texture in pixels.
* @param useHighPrecisionColorComponents If {@code false}, uses colors with 8-bit unsigned bytes.
* If {@code true}, use 16-bit (half-precision) floating-point.
* @throws GlException If the texture allocation fails.
* @return The texture identifier for the newly-allocated texture.
* @throws GlException If the texture allocation fails.
*/
public static int createTexture(int width, int height, boolean useHighPrecisionColorComponents)
throws GlException {
@ -588,9 +590,31 @@ public final class GlUtil {
return createTexture(width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE);
}
/**
* Allocates a new texture, initialized with the {@link Bitmap bitmap} data.
*
* <p>The created texture will have the same size as the specified {@link Bitmap}.
*
* @param bitmap The {@link Bitmap} for which the texture is created.
* @return The texture identifier for the newly-allocated texture.
* @throws GlException If the texture allocation fails.
*/
public static int createTexture(Bitmap bitmap) throws GlException {
assertValidTextureSize(bitmap.getWidth(), bitmap.getHeight());
int texId = generateTexture();
bindTexture(GLES20.GL_TEXTURE_2D, texId);
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, bitmap, /* border= */ 0);
checkGlError();
return texId;
}
/**
* Allocates a new RGBA texture with the specified dimensions and color component precision.
*
* <p>The created texture is not zero-initialized. To clear the texture, {@linkplain
* #focusFramebuffer(EGLDisplay, EGLContext, EGLSurface, int, int, int) focus} on the texture and
* {@linkplain #clearFocusedBuffers() clear} its content.
*
* @param width The width of the new texture in pixels.
* @param height The height of the new texture in pixels.
* @param internalFormat The number of color components in the texture, as well as their format.
@ -603,7 +627,6 @@ public final class GlUtil {
assertValidTextureSize(width, height);
int texId = generateTexture();
bindTexture(GLES20.GL_TEXTURE_2D, texId);
ByteBuffer byteBuffer = ByteBuffer.allocateDirect(width * height * 4);
GLES20.glTexImage2D(
GLES20.GL_TEXTURE_2D,
/* level= */ 0,
@ -613,7 +636,7 @@ public final class GlUtil {
/* border= */ 0,
GLES20.GL_RGBA,
type,
byteBuffer);
/* buffer= */ null);
checkGlError();
return texId;
}

5
libraries/effect/src/main/java/androidx/media3/effect/BaseGlShaderProgram.java
View File

@ -121,8 +121,13 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
/**
* Returns {@code true} if the texture buffer should be cleared before calling {@link #drawFrame}
* or {@code false} if it should retain the content of the last drawn frame.
*
* <p>When returning {@code false}, the shader program must clear the texture before first drawing
* to it, because textures are not zero-initialized when created. This can be done by calling
* {@link GlUtil#clearFocusedBuffers()}.
*/
public boolean shouldClearTextureBuffer() {
// TODO - b/309428083: Clear the texture before first use.
return true;
}

14
libraries/effect/src/main/java/androidx/media3/effect/BitmapOverlay.java
View File

@ -20,8 +20,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
import android.content.Context;
import android.graphics.Bitmap;
import android.net.Uri;
import android.opengl.GLES20;
import android.opengl.GLUtils;
import androidx.media3.common.C;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.util.BitmapLoader;
@ -78,17 +76,7 @@ public abstract class BitmapOverlay extends TextureOverlay {
GlUtil.deleteTexture(lastTextureId);
}
lastTextureId =
GlUtil.createTexture(
bitmap.getWidth(),
bitmap.getHeight(),
/* useHighPrecisionColorComponents= */ false);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, lastTextureId);
GLUtils.texImage2D(
GLES20.GL_TEXTURE_2D,
/* level= */ 0,
BitmapUtil.flipBitmapVertically(checkNotNull(lastBitmap)),
/* border= */ 0);
GlUtil.checkGlError();
GlUtil.createTexture(BitmapUtil.flipBitmapVertically(checkNotNull(lastBitmap)));
} catch (GlUtil.GlException e) {
throw new VideoFrameProcessingException(e);
}

11
libraries/effect/src/main/java/androidx/media3/effect/BitmapTextureManager.java
View File

@ -20,8 +20,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import android.graphics.Bitmap;
import android.opengl.GLES20;
import android.opengl.GLUtils;
import androidx.media3.common.C;
import androidx.media3.common.FrameInfo;
import androidx.media3.common.GlObjectsProvider;
@ -55,6 +53,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private @MonotonicNonNull GlTextureInfo currentGlTextureInfo;
private int downstreamShaderProgramCapacity;
// TODO - b/262693274: Support HDR. Currently this variable is not used and will trigger error
// prone warning.
private boolean useHdr;
private boolean currentInputStreamEnded;
private boolean isNextFrameInTexture;
@ -212,12 +212,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
if (currentGlTextureInfo != null) {
currentGlTextureInfo.release();
}
currentTexId =
GlUtil.createTexture(
frameInfo.width, frameInfo.height, /* useHighPrecisionColorComponents= */ useHdr);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, currentTexId);
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, bitmap, /* border= */ 0);
GlUtil.checkGlError();
currentTexId = GlUtil.createTexture(bitmap);
} catch (GlUtil.GlException e) {
throw VideoFrameProcessingException.from(e);
}

13
libraries/effect/src/main/java/androidx/media3/effect/SingleColorLut.java
View File

@ -21,8 +21,6 @@ import static androidx.media3.common.util.Assertions.checkState;
import android.content.Context;
import android.graphics.Bitmap;
import android.opengl.GLES20;
import android.opengl.GLUtils;
import androidx.media3.common.Format;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.util.GlUtil;
@ -154,20 +152,11 @@ public final class SingleColorLut implements ColorLut {
checkState(!useHdr, "HDR is currently not supported.");
try {
lutTextureId = storeLutAsTexture(lut);
lutTextureId = GlUtil.createTexture(lut);
} catch (GlUtil.GlException e) {
throw new VideoFrameProcessingException("Could not store the LUT as a texture.", e);
}
return new ColorLutShaderProgram(context, /* colorLut= */ this, useHdr);
}
private static int storeLutAsTexture(Bitmap bitmap) throws GlUtil.GlException {
int lutTextureId =
GlUtil.createTexture(
bitmap.getWidth(), bitmap.getHeight(), /* useHighPrecisionColorComponents= */ false);
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, bitmap, /* border= */ 0);
GlUtil.checkGlError();
return lutTextureId;
}
}

11
libraries/effect/src/main/java/androidx/media3/effect/ThumbnailStripShaderProgram.java
View File

@ -34,6 +34,7 @@ import java.io.IOException;
private final GlProgram glProgram;
private final ThumbnailStripEffect thumbnailStripEffect;
private boolean clearedGlBuffer;
public ThumbnailStripShaderProgram(
Context context, boolean useHdr, ThumbnailStripEffect thumbnailStripEffect)
@ -68,6 +69,16 @@ import java.io.IOException;
@Override
public void drawFrame(int inputTexId, long presentationTimeUs)
throws VideoFrameProcessingException {
if (!clearedGlBuffer) {
try {
GlUtil.clearFocusedBuffers();
} catch (GlUtil.GlException e) {
throw new VideoFrameProcessingException(e, presentationTimeUs);
}
clearedGlBuffer = true;
}
long targetPresentationTimeUs = Util.msToUs(thumbnailStripEffect.getNextTimestampMs());
// Ignore the frame if there are no more thumbnails to draw or if it's earlier than the target.
if (thumbnailStripEffect.isDone() || presentationTimeUs < targetPresentationTimeUs) {

9
libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java
View File

@ -33,7 +33,6 @@ import android.graphics.PixelFormat;
import android.media.Image;
import android.opengl.GLES20;
import android.opengl.GLES30;
import android.opengl.GLUtils;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.media3.common.util.GlUtil;
@ -500,15 +499,9 @@ public class BitmapPixelTestUtil {
*/
@RequiresApi(17) // #flipBitmapVertically.
public static int createGlTextureFromBitmap(Bitmap bitmap) throws GlUtil.GlException {
int texId =
GlUtil.createTexture(
bitmap.getWidth(), bitmap.getHeight(), /* useHighPrecisionColorComponents= */ false);
// Put the flipped bitmap in the OpenGL texture as the bitmap's positive y-axis points down
// while OpenGL's positive y-axis points up.
GLUtils.texImage2D(
GLES20.GL_TEXTURE_2D, /* level= */ 0, flipBitmapVertically(bitmap), /* border= */ 0);
GlUtil.checkGlError();
return texId;
return GlUtil.createTexture(flipBitmapVertically(bitmap));
}
@RequiresApi(17) // Bitmap#isPremultiplied.

10
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View File

@ -30,8 +30,6 @@ import android.media.Image;
import android.media.MediaFormat;
import android.opengl.EGLContext;
import android.opengl.EGLDisplay;
import android.opengl.GLES20;
import android.opengl.GLUtils;
import android.os.Build;
import android.util.Pair;
import androidx.annotation.Nullable;
@ -589,13 +587,7 @@ public final class AndroidTestUtil {
* <p>Must have a GL context set up.
*/
public static int generateTextureFromBitmap(Bitmap bitmap) throws GlUtil.GlException {
int texId =
GlUtil.createTexture(
bitmap.getWidth(), bitmap.getHeight(), /* useHighPrecisionColorComponents= */ false);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texId);
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, bitmap, /* border= */ 0);
GlUtil.checkGlError();
return texId;
return GlUtil.createTexture(bitmap);
}
/**