loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Effect: Remove GlTextureInfo accessor methods.

Browse Source 
The extra check was a bit excessive, especially as it's called multiple times
per frame.

PiperOrigin-RevId: 545657102
This commit is contained in:
huangdarwin 2023-07-05 14:58:25 +01:00 committed by Rohit Singh
parent 11dec52b6a
commit 7a368b9a11
11 changed files with 54 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
demos/transformer/src/withMediaPipe/java/androidx/media3/demo/transformer/MediaPipeShaderProgram.java
View File

@ -164,8 +164,7 @@ import java.util.concurrent.Future;
public void queueInputFrame(
GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
AppTextureFrame appTextureFrame =
new AppTextureFrame(
inputTexture.getTexId(), inputTexture.getWidth(), inputTexture.getHeight());
new AppTextureFrame(inputTexture.texId, inputTexture.width, inputTexture.height);
// TODO(b/238302213): Handle timestamps restarting from 0 when applying effects to a playlist.
// MediaPipe will fail if the timestamps are not monotonically increasing.
// Also make sure that a MediaPipe graph producing additional frames only starts producing

65
libraries/common/src/main/java/androidx/media3/common/GlTextureInfo.java
View File

@ -15,8 +15,6 @@
*/
package androidx.media3.common;
import static androidx.media3.common.util.Assertions.checkState;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
@ -32,13 +30,26 @@ public final class GlTextureInfo {
/* width= */ C.LENGTH_UNSET,
/* height= */ C.LENGTH_UNSET);
private final int texId;
private final int fboId;
private final int rboId;
private final int width;
private final int height;
/** The OpenGL texture identifier, or {@link C#INDEX_UNSET} if not specified. */
public final int texId;
private boolean isReleased;
/**
* Identifier of a framebuffer object associated with the texture, or {@link C#INDEX_UNSET} if not
* specified.
*/
public final int fboId;
/**
* Identifier of a renderbuffer object attached with the framebuffer, or {@link C#INDEX_UNSET} if
* not specified.
*/
public final int rboId;
/** The width of the texture, in pixels, or {@link C#LENGTH_UNSET} if not specified. */
public final int width;
/** The height of the texture, in pixels, or {@link C#LENGTH_UNSET} if not specified. */
public final int height;
/**
* Creates a new instance.
@ -59,44 +70,8 @@ public final class GlTextureInfo {
this.height = height;
}
/** The OpenGL texture identifier, or {@link C#INDEX_UNSET} if not specified. */
public int getTexId() {
checkState(!isReleased);
return texId;
}
/**
* Identifier of a framebuffer object associated with the texture, or {@link C#INDEX_UNSET} if not
* specified.
*/
public int getFboId() {
checkState(!isReleased);
return fboId;
}
/**
* Identifier of a renderbuffer object attached with the framebuffer, or {@link C#INDEX_UNSET} if
* not specified.
*/
public int getRboId() {
checkState(!isReleased);
return rboId;
}
/** The width of the texture, in pixels, or {@link C#LENGTH_UNSET} if not specified. */
public int getWidth() {
checkState(!isReleased);
return width;
}
/** The height of the texture, in pixels, or {@link C#LENGTH_UNSET} if not specified. */
public int getHeight() {
checkState(!isReleased);
return height;
}
/** Releases all information associated with this instance. */
public void release() throws GlUtil.GlException {
isReleased = true;
if (texId != C.INDEX_UNSET) {
GlUtil.deleteTexture(texId);
}

6
libraries/effect/src/main/java/androidx/media3/effect/BaseGlShaderProgram.java
View File

@ -119,7 +119,7 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
public void queueInputFrame(
GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
try {
Size outputTextureSize = configure(inputTexture.getWidth(), inputTexture.getHeight());
Size outputTextureSize = configure(inputTexture.width, inputTexture.height);
outputTexturePool.ensureConfigured(
glObjectsProvider, outputTextureSize.getWidth(), outputTextureSize.getHeight());
@ -128,9 +128,9 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
// Copy frame to fbo.
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.getFboId(), outputTexture.getWidth(), outputTexture.getHeight());
outputTexture.fboId, outputTexture.width, outputTexture.height);
GlUtil.clearFocusedBuffers();
drawFrame(inputTexture.getTexId(), presentationTimeUs);
drawFrame(inputTexture.texId, presentationTimeUs);
inputListener.onInputFrameProcessed(inputTexture);
outputListener.onOutputFrameAvailable(outputTexture, presentationTimeUs);
} catch (VideoFrameProcessingException | GlUtil.GlException | NoSuchElementException e) {

24
libraries/effect/src/main/java/androidx/media3/effect/DefaultFrameDroppingShaderProgram.java
View File

@ -132,25 +132,23 @@ import androidx.media3.common.util.Size;
GlObjectsProvider glObjectsProvider, GlTextureInfo newTexture, long presentationTimeUs) {
try {
if (previousTexture == null) {
int texId = GlUtil.createTexture(newTexture.getWidth(), newTexture.getHeight(), useHdr);
int texId = GlUtil.createTexture(newTexture.width, newTexture.height, useHdr);
previousTexture =
glObjectsProvider.createBuffersForTexture(
texId, newTexture.getWidth(), newTexture.getHeight());
glObjectsProvider.createBuffersForTexture(texId, newTexture.width, newTexture.height);
}
GlTextureInfo previousTexture = checkNotNull(this.previousTexture);
if (previousTexture.getHeight() != newTexture.getHeight()
|| previousTexture.getWidth() != newTexture.getWidth()) {
if (previousTexture.height != newTexture.height
|| previousTexture.width != newTexture.width) {
previousTexture.release();
int texId = GlUtil.createTexture(newTexture.getWidth(), newTexture.getHeight(), useHdr);
int texId = GlUtil.createTexture(newTexture.width, newTexture.height, useHdr);
previousTexture =
glObjectsProvider.createBuffersForTexture(
texId, newTexture.getWidth(), newTexture.getHeight());
glObjectsProvider.createBuffersForTexture(texId, newTexture.width, newTexture.height);
}
GlUtil.focusFramebufferUsingCurrentContext(
previousTexture.getFboId(), previousTexture.getWidth(), previousTexture.getHeight());
previousTexture.fboId, previousTexture.width, previousTexture.height);
GlUtil.clearFocusedBuffers();
drawFrame(newTexture.getTexId(), presentationTimeUs);
drawFrame(newTexture.texId, presentationTimeUs);
previousPresentationTimeUs = presentationTimeUs;
this.previousTexture = previousTexture;
} catch (VideoFrameProcessingException | GlUtil.GlException e) {
@ -174,7 +172,7 @@ import androidx.media3.common.util.Size;
private void queuePreviousFrame(GlObjectsProvider glObjectsProvider) {
try {
GlTextureInfo previousTexture = checkNotNull(this.previousTexture);
Size outputTextureSize = configure(previousTexture.getWidth(), previousTexture.getHeight());
Size outputTextureSize = configure(previousTexture.width, previousTexture.height);
outputTexturePool.ensureConfigured(
glObjectsProvider, outputTextureSize.getWidth(), outputTextureSize.getHeight());
@ -183,10 +181,10 @@ import androidx.media3.common.util.Size;
// Copy frame to fbo.
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.getFboId(), outputTexture.getWidth(), outputTexture.getHeight());
outputTexture.fboId, outputTexture.width, outputTexture.height);
GlUtil.clearFocusedBuffers();
drawFrame(previousTexture.getTexId(), previousPresentationTimeUs);
drawFrame(previousTexture.texId, previousPresentationTimeUs);
getOutputListener().onOutputFrameAvailable(outputTexture, previousPresentationTimeUs);
lastQueuedPresentationTimeUs = previousPresentationTimeUs;
} catch (VideoFrameProcessingException | GlUtil.GlException e) {

13
libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
View File

@ -328,8 +328,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long renderTimeNs) {
try {
if (renderTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME
|| !ensureConfigured(
glObjectsProvider, inputTexture.getWidth(), inputTexture.getHeight())) {
|| !ensureConfigured(glObjectsProvider, inputTexture.width, inputTexture.height)) {
inputListener.onInputFrameProcessed(inputTexture);
return; // Drop frames when requested, or there is no output surface and output texture.
}
@ -365,7 +364,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputSurfaceInfo.width,
outputSurfaceInfo.height);
GlUtil.clearFocusedBuffers();
defaultShaderProgram.drawFrame(inputTexture.getTexId(), presentationTimeUs);
defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
EGLExt.eglPresentationTimeANDROID(
eglDisplay,
@ -382,9 +381,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
GlTextureInfo outputTexture = outputTexturePool.useTexture();
outputTextureTimestamps.add(presentationTimeUs);
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.getFboId(), outputTexture.getWidth(), outputTexture.getHeight());
outputTexture.fboId, outputTexture.width, outputTexture.height);
GlUtil.clearFocusedBuffers();
checkNotNull(defaultShaderProgram).drawFrame(inputTexture.getTexId(), presentationTimeUs);
checkNotNull(defaultShaderProgram).drawFrame(inputTexture.texId, presentationTimeUs);
long syncObject = GlUtil.createGlSyncFence();
syncObjects.add(syncObject);
checkNotNull(textureOutputListener)
@ -534,10 +533,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer();
defaultShaderProgram.setOutputColorTransfer(
debugSurfaceViewWrapper.outputColorTransfer);
defaultShaderProgram.drawFrame(inputTexture.getTexId(), presentationTimeUs);
defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
defaultShaderProgram.setOutputColorTransfer(configuredColorTransfer);
} else {
defaultShaderProgram.drawFrame(inputTexture.getTexId(), presentationTimeUs);
defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
}
},
glObjectsProvider);

2
libraries/effect/src/main/java/androidx/media3/effect/TexIdTextureManager.java
View File

@ -69,7 +69,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
videoFrameProcessingTaskExecutor.submit(
() ->
checkNotNull(frameProcessedListener)
.onInputFrameProcessed(inputTexture.getTexId(), GlUtil.createGlSyncFence()));
.onInputFrameProcessed(inputTexture.texId, GlUtil.createGlSyncFence()));
}
@Override

8
libraries/effect/src/main/java/androidx/media3/effect/TexturePool.java
View File

@ -77,7 +77,7 @@ import java.util.Queue;
return;
}
GlTextureInfo texture = getIteratorToAllTextures().next();
if (texture.getWidth() != width || texture.getHeight() != height) {
if (texture.width != width || texture.height != height) {
deleteAllTextures();
createTextures(glObjectsProvider, width, height);
}
@ -100,8 +100,6 @@ import java.util.Queue;
* <p>Throws {@link IllegalStateException} if {@code textureInfo} isn't in use.
*/
public void freeTexture(GlTextureInfo textureInfo) {
// TODO(b/262694346): Check before adding to freeTexture, that this texture wasn't released
// already.
checkState(inUseTextures.contains(textureInfo));
inUseTextures.remove(textureInfo);
freeTextures.add(textureInfo);
@ -113,8 +111,6 @@ import java.util.Queue;
* <p>Throws {@link IllegalStateException} if there's no textures in use to free.
*/
public void freeTexture() {
// TODO(b/262694346): Check before adding to freeTexture, that this texture wasn't released
// already.
checkState(!inUseTextures.isEmpty());
GlTextureInfo texture = inUseTextures.remove();
freeTextures.add(texture);
@ -122,8 +118,6 @@ import java.util.Queue;
/** Free all in-use textures. */
public void freeAllTextures() {
// TODO(b/262694346): Check before adding to freeTexture, that this texture wasn't released
// already.
freeTextures.addAll(inUseTextures);
inUseTextures.clear();
}

14
libraries/effect/src/main/java/androidx/media3/effect/VideoCompositor.java
View File

@ -132,11 +132,11 @@ public final class VideoCompositor {
// * Allow different frame dimensions.
InputFrameInfo inputFrame1 = framesToComposite.get(0);
InputFrameInfo inputFrame2 = framesToComposite.get(1);
checkState(inputFrame1.texture.getWidth() == inputFrame2.texture.getWidth());
checkState(inputFrame1.texture.getHeight() == inputFrame2.texture.getHeight());
checkState(inputFrame1.texture.width == inputFrame2.texture.width);
checkState(inputFrame1.texture.height == inputFrame2.texture.height);
try {
outputTexturePool.ensureConfigured(
glObjectsProvider, inputFrame1.texture.getWidth(), inputFrame1.texture.getHeight());
glObjectsProvider, inputFrame1.texture.width, inputFrame1.texture.height);
GlTextureInfo outputTexture = outputTexturePool.useTexture();
drawFrame(inputFrame1.texture, inputFrame2.texture, outputTexture);
@ -177,15 +177,13 @@ public final class VideoCompositor {
GlTextureInfo inputTexture1, GlTextureInfo inputTexture2, GlTextureInfo outputTexture)
throws GlUtil.GlException {
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.getFboId(), outputTexture.getWidth(), outputTexture.getHeight());
outputTexture.fboId, outputTexture.width, outputTexture.height);
GlUtil.clearFocusedBuffers();
GlProgram glProgram = checkNotNull(this.glProgram);
glProgram.use();
glProgram.setSamplerTexIdUniform(
"uTexSampler1", inputTexture1.getTexId(), /* texUnitIndex= */ 0);
glProgram.setSamplerTexIdUniform(
"uTexSampler2", inputTexture2.getTexId(), /* texUnitIndex= */ 1);
glProgram.setSamplerTexIdUniform("uTexSampler1", inputTexture1.texId, /* texUnitIndex= */ 0);
glProgram.setSamplerTexIdUniform("uTexSampler2", inputTexture2.texId, /* texUnitIndex= */ 1);
glProgram.setFloatsUniform("uTexTransformationMatrix", GlUtil.create4x4IdentityMatrix());
glProgram.setFloatsUniform("uTransformationMatrix", GlUtil.create4x4IdentityMatrix());

4
libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java
View File

@ -353,7 +353,7 @@ public final class VideoFrameProcessorTestRunner {
public void queueInputTexture(GlTextureInfo inputTexture, long pts) {
videoFrameProcessor.setInputFrameInfo(
new FrameInfo.Builder(inputTexture.getWidth(), inputTexture.getHeight())
new FrameInfo.Builder(inputTexture.width, inputTexture.height)
.setPixelWidthHeightRatio(pixelWidthHeightRatio)
.build());
videoFrameProcessor.setOnInputFrameProcessedListener(
@ -365,7 +365,7 @@ public final class VideoFrameProcessorTestRunner {
throw new VideoFrameProcessingException(e);
}
});
videoFrameProcessor.queueInputTexture(inputTexture.getTexId(), pts);
videoFrameProcessor.queueInputTexture(inputTexture.texId, pts);
}
/** {@link #endFrameProcessing(long)} with {@link #VIDEO_FRAME_PROCESSING_WAIT_MS} applied. */

4
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TextureBitmapReader.java
View File

@ -73,10 +73,10 @@ public final class TextureBitmapReader implements VideoFrameProcessorTestRunner.
throws VideoFrameProcessingException {
try {
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.getFboId(), outputTexture.getWidth(), outputTexture.getHeight());
outputTexture.fboId, outputTexture.width, outputTexture.height);
outputBitmap =
createBitmapFromCurrentGlFrameBuffer(
outputTexture.getWidth(), outputTexture.getHeight(), useHighPrecisionColorComponents);
outputTexture.width, outputTexture.height, useHighPrecisionColorComponents);
outputTimestampsToBitmaps.put(presentationTimeUs, outputBitmap);
} catch (GlUtil.GlException e) {
throw new VideoFrameProcessingException(e);

2
libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoCompositorPixelTest.java
View File

@ -128,7 +128,7 @@ public final class VideoCompositorPixelTest {
}
compositedOutputBitmap.set(
BitmapPixelTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(
outputTexture.getWidth(), outputTexture.getHeight()));
outputTexture.width, outputTexture.height));
} catch (GlUtil.GlException e) {
throw VideoFrameProcessingException.from(e);
} finally {