From 94efcd7917ad5c175aac52280ee82f054d24a953 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Wed, 17 May 2023 13:17:51 +0100 Subject: [PATCH] Effect: Make TexturePool and use in FinalWrapper. Have the FinalShaderProgramWrapper / VideoFrameProcessor texture output access textures provided through a texture pool, that recycles used textures. Also, add the TexturePool interface to generally re-use textures. PiperOrigin-RevId: 532754377 --- .../androidx/media3/common/GlTextureInfo.java | 1 + .../androidx/media3/common/util/GlUtil.java | 4 +- .../media3/effect/BaseGlShaderProgram.java | 95 ++--------- .../effect/FinalShaderProgramWrapper.java | 52 +++--- .../androidx/media3/effect/TexturePool.java | 159 ++++++++++++++++++ 5 files changed, 196 insertions(+), 115 deletions(-) create mode 100644 libraries/effect/src/main/java/androidx/media3/effect/TexturePool.java diff --git a/libraries/common/src/main/java/androidx/media3/common/GlTextureInfo.java b/libraries/common/src/main/java/androidx/media3/common/GlTextureInfo.java index 7857932cd6..f073e9d137 100644 --- a/libraries/common/src/main/java/androidx/media3/common/GlTextureInfo.java +++ b/libraries/common/src/main/java/androidx/media3/common/GlTextureInfo.java @@ -20,6 +20,7 @@ import androidx.media3.common.util.UnstableApi; /** Contains information describing an OpenGL texture. */ @UnstableApi public final class GlTextureInfo { + // TODO: b/262694346 - Add a release() method for GlTextureInfo. /** A {@link GlTextureInfo} instance with all fields unset. */ public static final GlTextureInfo UNSET = diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java index c7e3de1a6f..e20da85644 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java @@ -558,8 +558,8 @@ public final class GlUtil { * * @param width The width of the new texture in pixels. * @param height The height of the new texture in pixels. - * @param useHighPrecisionColorComponents If {@code false}, uses 8-bit unsigned bytes. If {@code - * true}, use 16-bit (half-precision) floating-point. + * @param useHighPrecisionColorComponents If {@code false}, uses colors with 8-bit unsigned bytes. + * If {@code true}, use 16-bit (half-precision) floating-point. * @throws GlException If the texture allocation fails. * @return The texture identifier for the newly-allocated texture. */ diff --git a/libraries/effect/src/main/java/androidx/media3/effect/BaseGlShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/BaseGlShaderProgram.java index ab7554658a..3ceb2924ec 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/BaseGlShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/BaseGlShaderProgram.java @@ -24,10 +24,7 @@ import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Size; import androidx.media3.common.util.UnstableApi; -import com.google.common.collect.Iterables; import com.google.common.util.concurrent.MoreExecutors; -import java.util.ArrayDeque; -import java.util.Iterator; import java.util.NoSuchElementException; import java.util.concurrent.Executor; @@ -47,13 +44,7 @@ import java.util.concurrent.Executor; */ @UnstableApi public abstract class BaseGlShaderProgram implements GlShaderProgram { - - private final ArrayDeque freeOutputTextures; - private final ArrayDeque inUseOutputTextures; - private final int texturePoolCapacity; - private final boolean useHdr; - - private GlObjectsProvider glObjectsProvider; + private final TexturePool outputTexturePool; protected InputListener inputListener; private OutputListener outputListener; private ErrorListener errorListener; @@ -69,11 +60,8 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram { * texture cache, the size should be the number of textures to cache. */ public BaseGlShaderProgram(boolean useHdr, int texturePoolCapacity) { - freeOutputTextures = new ArrayDeque<>(texturePoolCapacity); - inUseOutputTextures = new ArrayDeque<>(texturePoolCapacity); - this.useHdr = useHdr; - this.texturePoolCapacity = texturePoolCapacity; - glObjectsProvider = GlObjectsProvider.DEFAULT; + outputTexturePool = + new TexturePool(/* useHighPrecisionColorComponents= */ useHdr, texturePoolCapacity); inputListener = new InputListener() {}; outputListener = new OutputListener() {}; errorListener = (frameProcessingException) -> {}; @@ -114,15 +102,7 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram { @Override public void setInputListener(InputListener inputListener) { this.inputListener = inputListener; - int numberOfFreeFramesToNotify; - if (getIteratorToAllTextures().hasNext()) { - // The frame buffers have already been allocated. - numberOfFreeFramesToNotify = freeOutputTextures.size(); - } else { - // Defer frame buffer allocation to when queueing input frames. - numberOfFreeFramesToNotify = texturePoolCapacity; - } - for (int i = 0; i < numberOfFreeFramesToNotify; i++) { + for (int i = 0; i < outputTexturePool.freeTextureCount(); i++) { inputListener.onReadyToAcceptInputFrame(); } } @@ -143,22 +123,19 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram { checkState( !frameProcessingStarted, "The GlObjectsProvider cannot be set after frame processing has started."); - this.glObjectsProvider = glObjectsProvider; + outputTexturePool.setGlObjectsProvider(glObjectsProvider); } @Override public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) { try { - configureAllOutputTextures(inputTexture.width, inputTexture.height); - checkState( - !freeOutputTextures.isEmpty(), - "The GlShaderProgram does not currently accept input frames. Release prior output frames" - + " first."); + Size outputTextureSize = configure(inputTexture.width, inputTexture.height); + outputTexturePool.ensureConfigured( + outputTextureSize.getWidth(), outputTextureSize.getHeight()); frameProcessingStarted = true; // Focus on the next free buffer. - GlTextureInfo outputTexture = freeOutputTextures.remove(); - inUseOutputTextures.add(outputTexture); + GlTextureInfo outputTexture = outputTexturePool.useTexture(); // Copy frame to fbo. GlUtil.focusFramebufferUsingCurrentContext( @@ -176,9 +153,7 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram { @Override public void releaseOutputFrame(GlTextureInfo outputTexture) { frameProcessingStarted = true; - checkState(inUseOutputTextures.contains(outputTexture)); - inUseOutputTextures.remove(outputTexture); - freeOutputTextures.add(outputTexture); + outputTexturePool.freeTexture(outputTexture); inputListener.onReadyToAcceptInputFrame(); } @@ -192,10 +167,9 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram { @CallSuper public void flush() { frameProcessingStarted = true; - freeOutputTextures.addAll(inUseOutputTextures); - inUseOutputTextures.clear(); + outputTexturePool.freeAllTextures(); inputListener.onFlush(); - for (int i = 0; i < freeOutputTextures.size(); i++) { + for (int i = 0; i < outputTexturePool.capacity(); i++) { inputListener.onReadyToAcceptInputFrame(); } } @@ -205,52 +179,9 @@ public abstract class BaseGlShaderProgram implements GlShaderProgram { public void release() throws VideoFrameProcessingException { frameProcessingStarted = true; try { - deleteAllOutputTextures(); + outputTexturePool.deleteAllTextures(); } catch (GlUtil.GlException e) { throw new VideoFrameProcessingException(e); } } - - private void configureAllOutputTextures(int inputWidth, int inputHeight) - throws GlUtil.GlException, VideoFrameProcessingException { - Iterator allTextures = getIteratorToAllTextures(); - if (!allTextures.hasNext()) { - createAllOutputTextures(inputWidth, inputHeight); - return; - } - GlTextureInfo outputGlTextureInfo = allTextures.next(); - if (outputGlTextureInfo.width != inputWidth || outputGlTextureInfo.height != inputHeight) { - deleteAllOutputTextures(); - createAllOutputTextures(inputWidth, inputHeight); - } - } - - private void createAllOutputTextures(int width, int height) - throws GlUtil.GlException, VideoFrameProcessingException { - checkState(freeOutputTextures.isEmpty()); - checkState(inUseOutputTextures.isEmpty()); - Size outputSize = configure(width, height); - for (int i = 0; i < texturePoolCapacity; i++) { - int outputTexId = GlUtil.createTexture(outputSize.getWidth(), outputSize.getHeight(), useHdr); - GlTextureInfo outputTexture = - glObjectsProvider.createBuffersForTexture( - outputTexId, outputSize.getWidth(), outputSize.getHeight()); - freeOutputTextures.add(outputTexture); - } - } - - private void deleteAllOutputTextures() throws GlUtil.GlException { - Iterator allTextures = getIteratorToAllTextures(); - while (allTextures.hasNext()) { - GlTextureInfo textureInfo = allTextures.next(); - GlUtil.deleteTexture(textureInfo.texId); - GlUtil.deleteFbo(textureInfo.fboId); - } - freeOutputTextures.clear(); - inUseOutputTextures.clear(); - } - - private Iterator getIteratorToAllTextures() { - return Iterables.concat(freeOutputTextures, inUseOutputTextures).iterator(); - } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java index 16eb525558..d78696cb99 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java @@ -44,6 +44,7 @@ import androidx.media3.common.util.Log; import androidx.media3.common.util.Size; import androidx.media3.common.util.Util; import com.google.common.collect.ImmutableList; +import java.util.ArrayDeque; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Executor; @@ -87,9 +88,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final Executor videoFrameProcessorListenerExecutor; private final VideoFrameProcessor.Listener videoFrameProcessorListener; private final Queue> availableFrames; - private final Queue> outputTextures; + private final TexturePool outputTexturePool; + private final Queue outputTextureTimestamps; // Synchronized with outputTexturePool. @Nullable private final DefaultVideoFrameProcessor.TextureOutputListener textureOutputListener; - private final int textureOutputCapacity; private int inputWidth; private int inputHeight; @@ -143,11 +144,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.videoFrameProcessorListener = videoFrameProcessorListener; this.glObjectsProvider = glObjectsProvider; this.textureOutputListener = textureOutputListener; - this.textureOutputCapacity = textureOutputCapacity; inputListener = new InputListener() {}; availableFrames = new ConcurrentLinkedQueue<>(); - outputTextures = new ConcurrentLinkedQueue<>(); + + boolean useHighPrecisionColorComponents = ColorInfo.isTransferHdr(outputColorInfo); + outputTexturePool = new TexturePool(useHighPrecisionColorComponents, textureOutputCapacity); + outputTextureTimestamps = new ArrayDeque<>(textureOutputCapacity); } @Override @@ -156,6 +159,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; !frameProcessingStarted, "The GlObjectsProvider cannot be set after frame processing has started."); this.glObjectsProvider = glObjectsProvider; + outputTexturePool.setGlObjectsProvider(glObjectsProvider); } @Override @@ -207,7 +211,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; availableFrames.add(Pair.create(inputTexture, presentationTimeUs)); } } else { - checkState(outputTextures.size() < textureOutputCapacity); + checkState(outputTexturePool.freeTextureCount() > 0); renderFrame(inputTexture, presentationTimeUs, /* renderTimeNs= */ presentationTimeUs * 1000); } maybeOnReadyToAcceptInputFrame(); @@ -219,16 +223,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throw new UnsupportedOperationException(); } - public void releaseOutputFrame(long presentationTimeUs) throws VideoFrameProcessingException { - while (!outputTextures.isEmpty() - && checkNotNull(outputTextures.peek()).second <= presentationTimeUs) { - GlTextureInfo outputTexture = outputTextures.remove().first; - try { - GlUtil.deleteTexture(outputTexture.texId); - GlUtil.deleteFbo(outputTexture.fboId); - } catch (GlUtil.GlException exception) { - throw new VideoFrameProcessingException(exception); - } + public void releaseOutputFrame(long presentationTimeUs) { + while (outputTexturePool.freeTextureCount() < outputTexturePool.capacity() + && checkNotNull(outputTextureTimestamps.peek()) <= presentationTimeUs) { + outputTexturePool.freeTexture(); + outputTextureTimestamps.remove(); maybeOnReadyToAcceptInputFrame(); } } @@ -261,11 +260,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; defaultShaderProgram.release(); } try { - while (!outputTextures.isEmpty()) { - GlTextureInfo outputTexture = outputTextures.remove().first; - GlUtil.deleteTexture(outputTexture.texId); - GlUtil.deleteFbo(outputTexture.fboId); - } + outputTexturePool.deleteAllTextures(); GlUtil.destroyEglSurface(eglDisplay, outputEglSurface); } catch (GlUtil.GlException e) { throw new VideoFrameProcessingException(e); @@ -304,7 +299,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } private void maybeOnReadyToAcceptInputFrame() { - if (textureOutputListener == null || outputTextures.size() < textureOutputCapacity) { + if (textureOutputListener == null || outputTexturePool.freeTextureCount() > 0) { inputListener.onReadyToAcceptInputFrame(); } } @@ -363,15 +358,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private void renderFrameToOutputTexture(GlTextureInfo inputTexture, long presentationTimeUs) throws GlUtil.GlException, VideoFrameProcessingException { - // TODO(b/262694346): Use a texture pool instead of creating a new texture on every frame. - int outputTexId = - GlUtil.createTexture( - outputWidth, - outputHeight, - /* useHighPrecisionColorComponents= */ ColorInfo.isTransferHdr(outputColorInfo)); - GlTextureInfo outputTexture = - glObjectsProvider.createBuffersForTexture(outputTexId, outputWidth, outputHeight); - + GlTextureInfo outputTexture = outputTexturePool.useTexture(); + outputTextureTimestamps.add(presentationTimeUs); GlUtil.focusFramebufferUsingCurrentContext( outputTexture.fboId, outputTexture.width, outputTexture.height); GlUtil.clearOutputFrame(); @@ -381,7 +369,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // glFinish. Consider removing glFinish and requiring onTextureRendered to handle // synchronization. GLES20.glFinish(); - outputTextures.add(Pair.create(outputTexture, presentationTimeUs)); checkNotNull(textureOutputListener).onTextureRendered(outputTexture, presentationTimeUs); } @@ -445,6 +432,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // Frames are only rendered automatically when outputting to an encoder. /* isEncoderInputSurface= */ renderFramesAutomatically); } + if (textureOutputListener != null) { + outputTexturePool.ensureConfigured(outputWidth, outputHeight); + } @Nullable SurfaceView debugSurfaceView = diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TexturePool.java b/libraries/effect/src/main/java/androidx/media3/effect/TexturePool.java new file mode 100644 index 0000000000..88cb54a2ed --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/TexturePool.java @@ -0,0 +1,159 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkState; + +import androidx.media3.common.GlObjectsProvider; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.util.GlUtil; +import com.google.common.collect.Iterables; +import java.util.ArrayDeque; +import java.util.Iterator; +import java.util.Queue; + +/** Holds {@code capacity} textures, to re-use textures. */ +/* package */ final class TexturePool { + private final Queue freeTextures; + private final Queue inUseTextures; + private final int capacity; + private final boolean useHighPrecisionColorComponents; + + private GlObjectsProvider glObjectsProvider; + + /** + * Creates a {@code TexturePool} instance. + * + * @param useHighPrecisionColorComponents If {@code false}, uses colors with 8-bit unsigned bytes. + * If {@code true}, use 16-bit (half-precision) floating-point. + * @param capacity The capacity of the texture pool. + */ + public TexturePool(boolean useHighPrecisionColorComponents, int capacity) { + this.capacity = capacity; + this.useHighPrecisionColorComponents = useHighPrecisionColorComponents; + + freeTextures = new ArrayDeque<>(capacity); + inUseTextures = new ArrayDeque<>(capacity); + + glObjectsProvider = new DefaultGlObjectsProvider(/* sharedEglContext= */ null); + } + + /** Sets the {@link GlObjectsProvider}. */ + public void setGlObjectsProvider(GlObjectsProvider glObjectsProvider) { + checkState(!isConfigured()); + this.glObjectsProvider = glObjectsProvider; + } + + /** Returns whether the instance has been {@linkplain #ensureConfigured configured}. */ + public boolean isConfigured() { + return getIteratorToAllTextures().hasNext(); + } + + /** Returns the {@code capacity} of the instance. */ + public int capacity() { + return capacity; + } + + /** Returns the number of free textures available to {@link #useTexture}. */ + public int freeTextureCount() { + if (!isConfigured()) { + return capacity; + } + return freeTextures.size(); + } + + /** + * Ensures that this instance is configured with the {@code width} and {@code height}. + * + *

Reconfigures backing textures as needed. + */ + public void ensureConfigured(int width, int height) throws GlUtil.GlException { + if (!isConfigured()) { + createTextures(width, height); + return; + } + GlTextureInfo texture = getIteratorToAllTextures().next(); + if (texture.width != width || texture.height != height) { + deleteAllTextures(); + createTextures(width, height); + } + } + + /** Returns a {@link GlTextureInfo} and marks it as in-use. */ + public GlTextureInfo useTexture() { + if (freeTextures.isEmpty()) { + throw new IllegalStateException( + "Textures are all in use. Please release in-use textures before calling useTexture."); + } + GlTextureInfo texture = freeTextures.remove(); + inUseTextures.add(texture); + return texture; + } + + /** + * Frees the texture represented by {@code textureInfo}. + * + *

Throws {@link IllegalStateException} if {@code textureInfo} isn't in use. + */ + public void freeTexture(GlTextureInfo textureInfo) { + checkState(inUseTextures.contains(textureInfo)); + inUseTextures.remove(textureInfo); + freeTextures.add(textureInfo); + } + + /** + * Frees the oldest in-use texture. + * + *

Throws {@link IllegalStateException} if there's no textures in use to free. + */ + public void freeTexture() { + checkState(!inUseTextures.isEmpty()); + GlTextureInfo texture = inUseTextures.remove(); + freeTextures.add(texture); + } + + /** Free all in-use textures. */ + public void freeAllTextures() { + freeTextures.addAll(inUseTextures); + inUseTextures.clear(); + } + + /** Deletes all textures. */ + public void deleteAllTextures() throws GlUtil.GlException { + Iterator allTextures = getIteratorToAllTextures(); + while (allTextures.hasNext()) { + GlTextureInfo textureInfo = allTextures.next(); + GlUtil.deleteTexture(textureInfo.texId); + GlUtil.deleteFbo(textureInfo.fboId); + } + freeTextures.clear(); + inUseTextures.clear(); + } + + private void createTextures(int width, int height) throws GlUtil.GlException { + checkState(freeTextures.isEmpty()); + checkState(inUseTextures.isEmpty()); + for (int i = 0; i < capacity; i++) { + int texId = GlUtil.createTexture(width, height, useHighPrecisionColorComponents); + GlTextureInfo texture = glObjectsProvider.createBuffersForTexture(texId, width, height); + freeTextures.add(texture); + } + } + + private Iterator getIteratorToAllTextures() { + return Iterables.concat(freeTextures, inUseTextures).iterator(); + } +}