loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Effect: Make TexturePool and use in FinalWrapper.

Browse Source 
Have the FinalShaderProgramWrapper / VideoFrameProcessor texture
output access textures provided through a texture pool, that
recycles used textures.

Also, add the TexturePool interface to generally re-use textures.

PiperOrigin-RevId: 532754377
This commit is contained in:
huangdarwin 2023-05-17 13:17:51 +01:00 committed by Ian Baker
parent e0d6f67dd9
commit 94efcd7917
5 changed files with 196 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
libraries/common/src/main/java/androidx/media3/common/GlTextureInfo.java
View File

@ -20,6 +20,7 @@ import androidx.media3.common.util.UnstableApi;
/** Contains information describing an OpenGL texture. */
@UnstableApi
public final class GlTextureInfo {
// TODO: b/262694346 - Add a release() method for GlTextureInfo.
/** A {@link GlTextureInfo} instance with all fields unset. */
public static final GlTextureInfo UNSET =

4
libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java
View File

@ -558,8 +558,8 @@ public final class GlUtil {
*
* @param width The width of the new texture in pixels.
* @param height The height of the new texture in pixels.
* @param useHighPrecisionColorComponents If {@code false}, uses 8-bit unsigned bytes. If {@code
* true}, use 16-bit (half-precision) floating-point.
* @param useHighPrecisionColorComponents If {@code false}, uses colors with 8-bit unsigned bytes.
* If {@code true}, use 16-bit (half-precision) floating-point.
* @throws GlException If the texture allocation fails.
* @return The texture identifier for the newly-allocated texture.
*/

95
libraries/effect/src/main/java/androidx/media3/effect/BaseGlShaderProgram.java
View File

@ -24,10 +24,7 @@ import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.Size;
import androidx.media3.common.util.UnstableApi;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.MoreExecutors;
import java.util.ArrayDeque;
import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.concurrent.Executor;
@ -47,13 +44,7 @@ import java.util.concurrent.Executor;
*/
@UnstableApi
public abstract class BaseGlShaderProgram implements GlShaderProgram {
private final ArrayDeque<GlTextureInfo> freeOutputTextures;
private final ArrayDeque<GlTextureInfo> inUseOutputTextures;
private final int texturePoolCapacity;
private final boolean useHdr;
private GlObjectsProvider glObjectsProvider;
private final TexturePool outputTexturePool;
protected InputListener inputListener;
private OutputListener outputListener;
private ErrorListener errorListener;
@ -69,11 +60,8 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
* texture cache, the size should be the number of textures to cache.
*/
public BaseGlShaderProgram(boolean useHdr, int texturePoolCapacity) {
freeOutputTextures = new ArrayDeque<>(texturePoolCapacity);
inUseOutputTextures = new ArrayDeque<>(texturePoolCapacity);
this.useHdr = useHdr;
this.texturePoolCapacity = texturePoolCapacity;
glObjectsProvider = GlObjectsProvider.DEFAULT;
outputTexturePool =
new TexturePool(/* useHighPrecisionColorComponents= */ useHdr, texturePoolCapacity);
inputListener = new InputListener() {};
outputListener = new OutputListener() {};
errorListener = (frameProcessingException) -> {};
@ -114,15 +102,7 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
@Override
public void setInputListener(InputListener inputListener) {
this.inputListener = inputListener;
int numberOfFreeFramesToNotify;
if (getIteratorToAllTextures().hasNext()) {
// The frame buffers have already been allocated.
numberOfFreeFramesToNotify = freeOutputTextures.size();
} else {
// Defer frame buffer allocation to when queueing input frames.
numberOfFreeFramesToNotify = texturePoolCapacity;
}
for (int i = 0; i < numberOfFreeFramesToNotify; i++) {
for (int i = 0; i < outputTexturePool.freeTextureCount(); i++) {
inputListener.onReadyToAcceptInputFrame();
}
}
@ -143,22 +123,19 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
checkState(
!frameProcessingStarted,
"The GlObjectsProvider cannot be set after frame processing has started.");
this.glObjectsProvider = glObjectsProvider;
outputTexturePool.setGlObjectsProvider(glObjectsProvider);
}
@Override
public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) {
try {
configureAllOutputTextures(inputTexture.width, inputTexture.height);
checkState(
!freeOutputTextures.isEmpty(),
"The GlShaderProgram does not currently accept input frames. Release prior output frames"
+ " first.");
Size outputTextureSize = configure(inputTexture.width, inputTexture.height);
outputTexturePool.ensureConfigured(
outputTextureSize.getWidth(), outputTextureSize.getHeight());
frameProcessingStarted = true;
// Focus on the next free buffer.
GlTextureInfo outputTexture = freeOutputTextures.remove();
inUseOutputTextures.add(outputTexture);
GlTextureInfo outputTexture = outputTexturePool.useTexture();
// Copy frame to fbo.
GlUtil.focusFramebufferUsingCurrentContext(
@ -176,9 +153,7 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
@Override
public void releaseOutputFrame(GlTextureInfo outputTexture) {
frameProcessingStarted = true;
checkState(inUseOutputTextures.contains(outputTexture));
inUseOutputTextures.remove(outputTexture);
freeOutputTextures.add(outputTexture);
outputTexturePool.freeTexture(outputTexture);
inputListener.onReadyToAcceptInputFrame();
}
@ -192,10 +167,9 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
@CallSuper
public void flush() {
frameProcessingStarted = true;
freeOutputTextures.addAll(inUseOutputTextures);
inUseOutputTextures.clear();
outputTexturePool.freeAllTextures();
inputListener.onFlush();
for (int i = 0; i < freeOutputTextures.size(); i++) {
for (int i = 0; i < outputTexturePool.capacity(); i++) {
inputListener.onReadyToAcceptInputFrame();
}
}
@ -205,52 +179,9 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram {
public void release() throws VideoFrameProcessingException {
frameProcessingStarted = true;
try {
deleteAllOutputTextures();
outputTexturePool.deleteAllTextures();
} catch (GlUtil.GlException e) {
throw new VideoFrameProcessingException(e);
}
}
private void configureAllOutputTextures(int inputWidth, int inputHeight)
throws GlUtil.GlException, VideoFrameProcessingException {
Iterator<GlTextureInfo> allTextures = getIteratorToAllTextures();
if (!allTextures.hasNext()) {
createAllOutputTextures(inputWidth, inputHeight);
return;
}
GlTextureInfo outputGlTextureInfo = allTextures.next();
if (outputGlTextureInfo.width != inputWidth || outputGlTextureInfo.height != inputHeight) {
deleteAllOutputTextures();
createAllOutputTextures(inputWidth, inputHeight);
}
}
private void createAllOutputTextures(int width, int height)
throws GlUtil.GlException, VideoFrameProcessingException {
checkState(freeOutputTextures.isEmpty());
checkState(inUseOutputTextures.isEmpty());
Size outputSize = configure(width, height);
for (int i = 0; i < texturePoolCapacity; i++) {
int outputTexId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight(), useHdr);
GlTextureInfo outputTexture =
glObjectsProvider.createBuffersForTexture(
outputTexId, outputSize.getWidth(), outputSize.getHeight());
freeOutputTextures.add(outputTexture);
}
}
private void deleteAllOutputTextures() throws GlUtil.GlException {
Iterator<GlTextureInfo> allTextures = getIteratorToAllTextures();
while (allTextures.hasNext()) {
GlTextureInfo textureInfo = allTextures.next();
GlUtil.deleteTexture(textureInfo.texId);
GlUtil.deleteFbo(textureInfo.fboId);
}
freeOutputTextures.clear();
inUseOutputTextures.clear();
}
private Iterator<GlTextureInfo> getIteratorToAllTextures() {
return Iterables.concat(freeOutputTextures, inUseOutputTextures).iterator();
}
}

52
libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
View File

@ -44,6 +44,7 @@ import androidx.media3.common.util.Log;
import androidx.media3.common.util.Size;
import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList;
import java.util.ArrayDeque;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.Executor;
@ -87,9 +88,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final Executor videoFrameProcessorListenerExecutor;
private final VideoFrameProcessor.Listener videoFrameProcessorListener;
private final Queue<Pair<GlTextureInfo, Long>> availableFrames;
private final Queue<Pair<GlTextureInfo, Long>> outputTextures;
private final TexturePool outputTexturePool;
private final Queue<Long> outputTextureTimestamps; // Synchronized with outputTexturePool.
@Nullable private final DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener;
private final int textureOutputCapacity;
private int inputWidth;
private int inputHeight;
@ -143,11 +144,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.videoFrameProcessorListener = videoFrameProcessorListener;
this.glObjectsProvider = glObjectsProvider;
this.textureOutputListener = textureOutputListener;
this.textureOutputCapacity = textureOutputCapacity;
inputListener = new InputListener() {};
availableFrames = new ConcurrentLinkedQueue<>();
outputTextures = new ConcurrentLinkedQueue<>();
boolean useHighPrecisionColorComponents = ColorInfo.isTransferHdr(outputColorInfo);
outputTexturePool = new TexturePool(useHighPrecisionColorComponents, textureOutputCapacity);
outputTextureTimestamps = new ArrayDeque<>(textureOutputCapacity);
}
@Override
@ -156,6 +159,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
!frameProcessingStarted,
"The GlObjectsProvider cannot be set after frame processing has started.");
this.glObjectsProvider = glObjectsProvider;
outputTexturePool.setGlObjectsProvider(glObjectsProvider);
}
@Override
@ -207,7 +211,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
availableFrames.add(Pair.create(inputTexture, presentationTimeUs));
}
} else {
checkState(outputTextures.size() < textureOutputCapacity);
checkState(outputTexturePool.freeTextureCount() > 0);
renderFrame(inputTexture, presentationTimeUs, /* renderTimeNs= */ presentationTimeUs * 1000);
}
maybeOnReadyToAcceptInputFrame();
@ -219,16 +223,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
throw new UnsupportedOperationException();
}
public void releaseOutputFrame(long presentationTimeUs) throws VideoFrameProcessingException {
while (!outputTextures.isEmpty()
&& checkNotNull(outputTextures.peek()).second <= presentationTimeUs) {
GlTextureInfo outputTexture = outputTextures.remove().first;
try {
GlUtil.deleteTexture(outputTexture.texId);
GlUtil.deleteFbo(outputTexture.fboId);
} catch (GlUtil.GlException exception) {
throw new VideoFrameProcessingException(exception);
}
public void releaseOutputFrame(long presentationTimeUs) {
while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity()
&& checkNotNull(outputTextureTimestamps.peek()) <= presentationTimeUs) {
outputTexturePool.freeTexture();
outputTextureTimestamps.remove();
maybeOnReadyToAcceptInputFrame();
}
}
@ -261,11 +260,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
defaultShaderProgram.release();
}
try {
while (!outputTextures.isEmpty()) {
GlTextureInfo outputTexture = outputTextures.remove().first;
GlUtil.deleteTexture(outputTexture.texId);
GlUtil.deleteFbo(outputTexture.fboId);
}
outputTexturePool.deleteAllTextures();
GlUtil.destroyEglSurface(eglDisplay, outputEglSurface);
} catch (GlUtil.GlException e) {
throw new VideoFrameProcessingException(e);
@ -304,7 +299,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
private void maybeOnReadyToAcceptInputFrame() {
if (textureOutputListener == null || outputTextures.size() < textureOutputCapacity) {
if (textureOutputListener == null || outputTexturePool.freeTextureCount() > 0) {
inputListener.onReadyToAcceptInputFrame();
}
}
@ -363,15 +358,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private void renderFrameToOutputTexture(GlTextureInfo inputTexture, long presentationTimeUs)
throws GlUtil.GlException, VideoFrameProcessingException {
// TODO(b/262694346): Use a texture pool instead of creating a new texture on every frame.
int outputTexId =
GlUtil.createTexture(
outputWidth,
outputHeight,
/* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(outputColorInfo));
GlTextureInfo outputTexture =
glObjectsProvider.createBuffersForTexture(outputTexId, outputWidth, outputHeight);
GlTextureInfo outputTexture = outputTexturePool.useTexture();
outputTextureTimestamps.add(presentationTimeUs);
GlUtil.focusFramebufferUsingCurrentContext(
outputTexture.fboId, outputTexture.width, outputTexture.height);
GlUtil.clearOutputFrame();
@ -381,7 +369,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// glFinish. Consider removing glFinish and requiring onTextureRendered to handle
// synchronization.
GLES20.glFinish();
outputTextures.add(Pair.create(outputTexture, presentationTimeUs));
checkNotNull(textureOutputListener).onTextureRendered(outputTexture, presentationTimeUs);
}
@ -445,6 +432,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// Frames are only rendered automatically when outputting to an encoder.
/* isEncoderInputSurface= */ renderFramesAutomatically);
}
if (textureOutputListener != null) {
outputTexturePool.ensureConfigured(outputWidth, outputHeight);
}
@Nullable
SurfaceView debugSurfaceView =

159
libraries/effect/src/main/java/androidx/media3/effect/TexturePool.java Normal file
View File

@ -0,0 +1,159 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.effect;
import static androidx.media3.common.util.Assertions.checkState;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.util.GlUtil;
import com.google.common.collect.Iterables;
import java.util.ArrayDeque;
import java.util.Iterator;
import java.util.Queue;
/** Holds {@code capacity} textures, to re-use textures. */
/* package */ final class TexturePool {
private final Queue<GlTextureInfo> freeTextures;
private final Queue<GlTextureInfo> inUseTextures;
private final int capacity;
private final boolean useHighPrecisionColorComponents;
private GlObjectsProvider glObjectsProvider;
/**
* Creates a {@code TexturePool} instance.
*
* @param useHighPrecisionColorComponents If {@code false}, uses colors with 8-bit unsigned bytes.
* If {@code true}, use 16-bit (half-precision) floating-point.
* @param capacity The capacity of the texture pool.
*/
public TexturePool(boolean useHighPrecisionColorComponents, int capacity) {
this.capacity = capacity;
this.useHighPrecisionColorComponents = useHighPrecisionColorComponents;
freeTextures = new ArrayDeque<>(capacity);
inUseTextures = new ArrayDeque<>(capacity);
glObjectsProvider = new DefaultGlObjectsProvider(/* sharedEglContext= */ null);
}
/** Sets the {@link GlObjectsProvider}. */
public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) {
checkState(!isConfigured());
this.glObjectsProvider = glObjectsProvider;
}
/** Returns whether the instance has been {@linkplain #ensureConfigured configured}. */
public boolean isConfigured() {
return getIteratorToAllTextures().hasNext();
}
/** Returns the {@code capacity} of the instance. */
public int capacity() {
return capacity;
}
/** Returns the number of free textures available to {@link #useTexture}. */
public int freeTextureCount() {
if (!isConfigured()) {
return capacity;
}
return freeTextures.size();
}
/**
* Ensures that this instance is configured with the {@code width} and {@code height}.
*
* <p>Reconfigures backing textures as needed.
*/
public void ensureConfigured(int width, int height) throws GlUtil.GlException {
if (!isConfigured()) {
createTextures(width, height);
return;
}
GlTextureInfo texture = getIteratorToAllTextures().next();
if (texture.width != width || texture.height != height) {
deleteAllTextures();
createTextures(width, height);
}
}
/** Returns a {@link GlTextureInfo} and marks it as in-use. */
public GlTextureInfo useTexture() {
if (freeTextures.isEmpty()) {
throw new IllegalStateException(
"Textures are all in use. Please release in-use textures before calling useTexture.");
}
GlTextureInfo texture = freeTextures.remove();
inUseTextures.add(texture);
return texture;
}
/**
* Frees the texture represented by {@code textureInfo}.
*
* <p>Throws {@link IllegalStateException} if {@code textureInfo} isn't in use.
*/
public void freeTexture(GlTextureInfo textureInfo) {
checkState(inUseTextures.contains(textureInfo));
inUseTextures.remove(textureInfo);
freeTextures.add(textureInfo);
}
/**
* Frees the oldest in-use texture.
*
* <p>Throws {@link IllegalStateException} if there's no textures in use to free.
*/
public void freeTexture() {
checkState(!inUseTextures.isEmpty());
GlTextureInfo texture = inUseTextures.remove();
freeTextures.add(texture);
}
/** Free all in-use textures. */
public void freeAllTextures() {
freeTextures.addAll(inUseTextures);
inUseTextures.clear();
}
/** Deletes all textures. */
public void deleteAllTextures() throws GlUtil.GlException {
Iterator<GlTextureInfo> allTextures = getIteratorToAllTextures();
while (allTextures.hasNext()) {
GlTextureInfo textureInfo = allTextures.next();
GlUtil.deleteTexture(textureInfo.texId);
GlUtil.deleteFbo(textureInfo.fboId);
}
freeTextures.clear();
inUseTextures.clear();
}
private void createTextures(int width, int height) throws GlUtil.GlException {
checkState(freeTextures.isEmpty());
checkState(inUseTextures.isEmpty());
for (int i = 0; i < capacity; i++) {
int texId = GlUtil.createTexture(width, height, useHighPrecisionColorComponents);
GlTextureInfo texture = glObjectsProvider.createBuffersForTexture(texId, width, height);
freeTextures.add(texture);
}
}
private Iterator<GlTextureInfo> getIteratorToAllTextures() {
return Iterables.concat(freeTextures, inUseTextures).iterator();
}
}