diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java index dcf231e6f9..66da72cc58 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java @@ -23,6 +23,7 @@ import static androidx.media3.common.util.Assertions.checkState; import android.content.Context; import android.content.pm.PackageManager; import android.graphics.Bitmap; +import android.graphics.Rect; import android.opengl.EGL14; import android.opengl.EGLConfig; import android.opengl.EGLContext; @@ -824,6 +825,43 @@ public final class GlUtil { checkGlError(); } + /** + * Copies the pixels from {@code readFboId} into {@code drawFboId}. Requires OpenGL ES 3.0. + * + *

When the input pixel region (given by {@code readRect}) doesn't have the same size as the + * output region (given by {@code drawRect}), this method uses {@link GLES20#GL_LINEAR} filtering + * to scale the image contents. + * + * @param readFboId The framebuffer object to read from. + * @param readRect The rectangular region of {@code readFboId} to read from. + * @param drawFboId The framebuffer object to draw into. + * @param drawRect The rectangular region of {@code drawFboId} to draw into. + */ + public static void blitFrameBuffer(int readFboId, Rect readRect, int drawFboId, Rect drawRect) + throws GlException { + int[] boundFramebuffer = new int[1]; + GLES20.glGetIntegerv(GLES20.GL_FRAMEBUFFER_BINDING, boundFramebuffer, /* offset= */ 0); + checkGlError(); + GLES30.glBindFramebuffer(GLES30.GL_READ_FRAMEBUFFER, readFboId); + checkGlError(); + GLES30.glBindFramebuffer(GLES30.GL_DRAW_FRAMEBUFFER, drawFboId); + checkGlError(); + GLES30.glBlitFramebuffer( + readRect.left, + readRect.top, + readRect.right, + readRect.bottom, + drawRect.left, + drawRect.top, + drawRect.right, + drawRect.bottom, + GLES30.GL_COLOR_BUFFER_BIT, + GLES30.GL_LINEAR); + checkGlError(); + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, /* framebuffer= */ boundFramebuffer[0]); + checkGlError(); + } + /** * Throws a {@link GlException} with the given message if {@code expression} evaluates to {@code * false}. diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/QueuingGlShaderProgramTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/QueuingGlShaderProgramTest.java new file mode 100644 index 0000000000..0af36dc595 --- /dev/null +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/QueuingGlShaderProgramTest.java @@ -0,0 +1,179 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.effect.EffectsTestUtil.generateAndProcessFrames; +import static androidx.media3.effect.EffectsTestUtil.getAndAssertOutputBitmaps; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.util.concurrent.Futures.immediateFuture; + +import android.content.Context; +import android.graphics.Color; +import android.text.Spannable; +import android.text.SpannableString; +import android.text.style.AbsoluteSizeSpan; +import android.text.style.ForegroundColorSpan; +import android.text.style.TypefaceSpan; +import android.util.Pair; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.util.Consumer; +import androidx.media3.test.utils.TextureBitmapReader; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Future; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +/** Tests for {@link QueuingGlShaderProgram}. */ +@RunWith(AndroidJUnit4.class) +public class QueuingGlShaderProgramTest { + @Rule public final TestName testName = new TestName(); + + private static final String ASSET_PATH = "test-generated-goldens/QueuingGlShaderProgramTest"; + + private static final int BLANK_FRAME_WIDTH = 100; + private static final int BLANK_FRAME_HEIGHT = 50; + private static final Consumer TEXT_SPAN_CONSUMER = + (text) -> { + text.setSpan( + new ForegroundColorSpan(Color.BLACK), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new AbsoluteSizeSpan(/* size= */ 24), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + text.setSpan( + new TypefaceSpan(/* family= */ "sans-serif"), + /* start= */ 0, + text.length(), + Spannable.SPAN_EXCLUSIVE_EXCLUSIVE); + }; + + private @MonotonicNonNull TextureBitmapReader textureBitmapReader; + private String testId; + + @Before + public void setUp() { + textureBitmapReader = new TextureBitmapReader(); + testId = testName.getMethodName(); + } + + @Test + public void queuingGlShaderProgram_withQueueSizeOne_outputsFramesInOrder() throws Exception { + List> events = new ArrayList<>(); + ImmutableList frameTimesUs = ImmutableList.of(0L, 333_333L, 666_667L); + ImmutableList actualPresentationTimesUs = + generateAndProcessFrames( + BLANK_FRAME_WIDTH, + BLANK_FRAME_HEIGHT, + frameTimesUs, + new TestGlEffect(events, /* queueSize= */ 1), + textureBitmapReader, + TEXT_SPAN_CONSUMER); + + assertThat(actualPresentationTimesUs).containsExactlyElementsIn(frameTimesUs).inOrder(); + assertThat(events) + .containsExactly( + Pair.create("queueInputFrame", 0L), + Pair.create("finishProcessingAndBlend", 0L), + Pair.create("queueInputFrame", 333_333L), + Pair.create("finishProcessingAndBlend", 333_333L), + Pair.create("queueInputFrame", 666_667L), + Pair.create("finishProcessingAndBlend", 666_667L)) + .inOrder(); + + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); + } + + @Test + public void queuingGlShaderProgram_withQueueSizeTwo_outputsFramesInOrder() throws Exception { + List> events = new ArrayList<>(); + ImmutableList frameTimesUs = ImmutableList.of(0L, 333_333L, 666_667L); + ImmutableList actualPresentationTimesUs = + generateAndProcessFrames( + BLANK_FRAME_WIDTH, + BLANK_FRAME_HEIGHT, + frameTimesUs, + new TestGlEffect(events, /* queueSize= */ 2), + textureBitmapReader, + TEXT_SPAN_CONSUMER); + + assertThat(actualPresentationTimesUs).containsExactlyElementsIn(frameTimesUs).inOrder(); + assertThat(events) + .containsExactly( + Pair.create("queueInputFrame", 0L), + Pair.create("queueInputFrame", 333_333L), + Pair.create("finishProcessingAndBlend", 0L), + Pair.create("queueInputFrame", 666_667L), + Pair.create("finishProcessingAndBlend", 333_333L), + Pair.create("finishProcessingAndBlend", 666_667L)) + .inOrder(); + getAndAssertOutputBitmaps(textureBitmapReader, actualPresentationTimesUs, testId, ASSET_PATH); + } + + private static class TestGlEffect implements GlEffect { + + private final List> events; + private final int queueSize; + + TestGlEffect(List> events, int queueSize) { + this.events = events; + this.queueSize = queueSize; + } + + @Override + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) { + return new QueuingGlShaderProgram<>( + /* useHighPrecisionColorComponents= */ useHdr, + queueSize, + new NoOpConcurrentEffect(events)); + } + } + + private static class NoOpConcurrentEffect + implements QueuingGlShaderProgram.ConcurrentEffect { + private final List> events; + + NoOpConcurrentEffect(List> events) { + this.events = events; + } + + @Override + public Future queueInputFrame(GlTextureInfo textureInfo, long presentationTimeUs) { + checkState(textureInfo.width == BLANK_FRAME_WIDTH); + checkState(textureInfo.height == BLANK_FRAME_HEIGHT); + events.add(Pair.create("queueInputFrame", presentationTimeUs)); + return immediateFuture(presentationTimeUs); + } + + @Override + public void finishProcessingAndBlend( + GlTextureInfo outputFrame, long presentationTimeUs, Long result) { + checkState(result == presentationTimeUs); + events.add(Pair.create("finishProcessingAndBlend", presentationTimeUs)); + } + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java new file mode 100644 index 0000000000..3473ebfa12 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java @@ -0,0 +1,302 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkState; + +import android.graphics.Rect; +import androidx.annotation.CallSuper; +import androidx.annotation.IntRange; +import androidx.media3.common.C; +import androidx.media3.common.GlObjectsProvider; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.UnstableApi; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.MoreExecutors; +import java.util.ArrayDeque; +import java.util.Queue; +import java.util.concurrent.Executor; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +/** + * An implementation of {@link GlShaderProgram} that enables {@linkplain ConcurrentEffect + * asynchronous} processing of video frames outside the current OpenGL context without processor + * stalls. + * + *

Data Dependencies and Processor Stalls

+ * + * Sharing image data between GPU and a {@link ConcurrentEffect} running on another processor + * creates a data dependency. The GPU must finish processing the frame before the data can be + * {@linkplain ConcurrentEffect#queueInputFrame submitted} to the other processor. And the other + * processor must finish processing the image data before any modifications can be {@linkplain + * ConcurrentEffect#finishProcessingAndBlend drawn} back to the main video stream. + * + *

If we force a synchronization and data transfer (e.g. via {@link + * android.opengl.GLES20#glReadPixels}) too early a processor would stall without any work + * available. + * + *

To keep multiple processors busy, {@code QueuingGlShaderProgram} maintains a queue of frames + * that are being processed by the provided {@link ConcurrentEffect}. The queue pipelines the + * processing stages and allows one frame to be processed on the GPU, while at the same time another + * frame is processed by the {@link ConcurrentEffect}. The size of the queue is configurable on + * construction, and should be large enough to compensate for the time required to execute the + * {@linkplain ConcurrentEffect asynchronous effect}, and any data transfer that is required between + * the processors. + * + *

The output frame {@link GlTextureInfo} produced by this class contains a copy of the + * {@linkplain #queueInputFrame input frame}, unless the frame contents were modified by the {@link + * ConcurrentEffect}. + * + *

All methods in this class must be called on the thread that owns the OpenGL context. + * + * @param An intermediate type used by {@link ConcurrentEffect} implementations. + */ +@UnstableApi +/* package */ final class QueuingGlShaderProgram implements GlShaderProgram { + + private static final long PROCESSING_TIMEOUT_MS = 500_000L; + + /** A concurrent effect that is applied by the {@link QueuingGlShaderProgram}. */ + public interface ConcurrentEffect { + /** + * Submits a frame to be processed by the concurrent effect. + * + *

The {@linkplain GlTextureInfo textureInfo} will hold the image data corresponding to the + * frame at {@code presentationTimeUs}. The image data will not be modified until the returned + * {@link Future} {@linkplain Future#isDone() completes} or {@linkplain Future#isCancelled() is + * cancelled}. + * + *

The {@linkplain GlTextureInfo textureInfo} will have a valid {@linkplain + * GlTextureInfo#fboId framebuffer object}. + * + *

This method will be called on the thread that owns the OpenGL context. + * + * @param textureInfo The texture info of the current frame. + * @param presentationTimeUs The presentation timestamp of the input frame, in microseconds. + * @return A {@link Future} representing pending completion of the task. + */ + Future queueInputFrame(GlTextureInfo textureInfo, long presentationTimeUs); + + /** + * Finishes processing the frame at {@code presentationTimeUs}. This method optionally allows + * the instance to draw an overlay or blend with the {@linkplain GlTextureInfo output frame}. + * + *

The {@linkplain GlTextureInfo outputFrame} contains the image data corresponding to the + * frame at {@code presentationTimeUs} when this method is invoked. + * + *

This method will be called on the thread that owns the OpenGL context. + * + * @param outputFrame The texture info of the frame. + * @param presentationTimeUs The presentation timestamp of the frame, in microseconds. + * @param result The result of the asynchronous computation in {@link #queueInputFrame}. + */ + void finishProcessingAndBlend(GlTextureInfo outputFrame, long presentationTimeUs, T result) + throws VideoFrameProcessingException; + } + + private final ConcurrentEffect concurrentEffect; + private final TexturePool outputTexturePool; + private final Queue> frameQueue; + private InputListener inputListener; + private OutputListener outputListener; + private ErrorListener errorListener; + private Executor errorListenerExecutor; + private int inputWidth; + private int inputHeight; + + /** + * Creates a {@code QueuingGlShaderProgram} instance. + * + * @param useHighPrecisionColorComponents If {@code false}, uses colors with 8-bit unsigned bytes. + * If {@code true}, use 16-bit (half-precision) floating-point. + * @param queueSize The number of frames to buffer before producing output, and also the capacity + * of the texture pool. + * @param concurrentEffect The asynchronous effect to apply to each frame. + */ + public QueuingGlShaderProgram( + boolean useHighPrecisionColorComponents, + @IntRange(from = 1) int queueSize, + ConcurrentEffect concurrentEffect) { + checkArgument(queueSize > 0); + this.concurrentEffect = concurrentEffect; + frameQueue = new ArrayDeque<>(queueSize); + outputTexturePool = new TexturePool(useHighPrecisionColorComponents, queueSize); + inputListener = new InputListener() {}; + outputListener = new OutputListener() {}; + errorListener = (frameProcessingException) -> {}; + errorListenerExecutor = MoreExecutors.directExecutor(); + inputWidth = C.LENGTH_UNSET; + inputHeight = C.LENGTH_UNSET; + } + + @Override + public void setInputListener(InputListener inputListener) { + this.inputListener = inputListener; + for (int i = 0; i < outputTexturePool.freeTextureCount(); i++) { + inputListener.onReadyToAcceptInputFrame(); + } + } + + @Override + public void setOutputListener(OutputListener outputListener) { + this.outputListener = outputListener; + } + + @Override + public void setErrorListener(Executor errorListenerExecutor, ErrorListener errorListener) { + this.errorListenerExecutor = errorListenerExecutor; + this.errorListener = errorListener; + } + + @Override + public void queueInputFrame( + GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { + try { + if (inputWidth != inputTexture.width + || inputHeight != inputTexture.height + || !outputTexturePool.isConfigured()) { + inputWidth = inputTexture.width; + inputHeight = inputTexture.height; + outputTexturePool.ensureConfigured(glObjectsProvider, inputWidth, inputHeight); + } + + // Focus on the next free buffer. + GlTextureInfo outputTexture = outputTexturePool.useTexture(); + + // Copy frame from inputTexture fbo to outputTexture fbo. + checkState(inputTexture.fboId != C.INDEX_UNSET); + GlUtil.blitFrameBuffer( + inputTexture.fboId, + new Rect(/* left= */ 0, /* top= */ 0, /* right= */ inputWidth, /* bottom= */ inputHeight), + outputTexture.fboId, + new Rect( + /* left= */ 0, /* top= */ 0, /* right= */ inputWidth, /* bottom= */ inputHeight)); + + Future task = concurrentEffect.queueInputFrame(outputTexture, presentationTimeUs); + frameQueue.add(new TimedTextureInfo(outputTexture, presentationTimeUs, task)); + + inputListener.onInputFrameProcessed(inputTexture); + + if (frameQueue.size() == outputTexturePool.capacity()) { + checkState(outputOneFrame()); + } + } catch (GlUtil.GlException e) { + onError(e); + } + } + + @Override + public void releaseOutputFrame(GlTextureInfo outputTexture) { + if (!outputTexturePool.isUsingTexture(outputTexture)) { + // This allows us to ignore outputTexture instances not associated with this + // GlShaderProgram instance. This may happen if a GlShaderProgram is introduced into + // the GlShaderProgram chain after frames already exist in the pipeline. + // TODO - b/320481157: Consider removing this if condition and disallowing disconnecting a + // GlShaderProgram while it still has in-use frames. + return; + } + outputTexturePool.freeTexture(outputTexture); + inputListener.onReadyToAcceptInputFrame(); + } + + @Override + public void signalEndOfCurrentInputStream() { + while (outputOneFrame()) {} + outputListener.onCurrentOutputStreamEnded(); + } + + @Override + @CallSuper + public void flush() { + cancelProcessingOfPendingFrames(); + outputTexturePool.freeAllTextures(); + inputListener.onFlush(); + for (int i = 0; i < outputTexturePool.capacity(); i++) { + inputListener.onReadyToAcceptInputFrame(); + } + } + + @Override + @CallSuper + public void release() throws VideoFrameProcessingException { + try { + cancelProcessingOfPendingFrames(); + outputTexturePool.deleteAllTextures(); + } catch (GlUtil.GlException e) { + throw new VideoFrameProcessingException(e); + } + } + + /** + * Outputs one frame from {@link #frameQueue}. + * + *

Returns {@code false} if no more frames are available for output. + */ + private boolean outputOneFrame() { + TimedTextureInfo timedTextureInfo = frameQueue.poll(); + if (timedTextureInfo == null) { + return false; + } + try { + T result = + Futures.getChecked( + timedTextureInfo.task, + VideoFrameProcessingException.class, + PROCESSING_TIMEOUT_MS, + TimeUnit.MILLISECONDS); + GlUtil.focusFramebufferUsingCurrentContext( + timedTextureInfo.textureInfo.fboId, + timedTextureInfo.textureInfo.width, + timedTextureInfo.textureInfo.height); + concurrentEffect.finishProcessingAndBlend( + timedTextureInfo.textureInfo, timedTextureInfo.presentationTimeUs, result); + outputListener.onOutputFrameAvailable( + timedTextureInfo.textureInfo, timedTextureInfo.presentationTimeUs); + return true; + } catch (GlUtil.GlException | VideoFrameProcessingException e) { + onError(e); + return false; + } + } + + private void cancelProcessingOfPendingFrames() { + TimedTextureInfo timedTextureInfo; + while ((timedTextureInfo = frameQueue.poll()) != null) { + timedTextureInfo.task.cancel(/* mayInterruptIfRunning= */ false); + } + } + + private void onError(Exception e) { + errorListenerExecutor.execute( + () -> errorListener.onError(VideoFrameProcessingException.from(e))); + } + + private static class TimedTextureInfo { + final GlTextureInfo textureInfo; + final long presentationTimeUs; + final Future task; + + TimedTextureInfo(GlTextureInfo textureInfo, long presentationTimeUs, Future task) { + this.textureInfo = textureInfo; + this.presentationTimeUs = presentationTimeUs; + this.task = task; + } + } +} diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/QueuingGlShaderProgramTest/pts_0.png b/libraries/test_data/src/test/assets/test-generated-goldens/QueuingGlShaderProgramTest/pts_0.png new file mode 100644 index 0000000000..4ffb8030fd Binary files /dev/null and b/libraries/test_data/src/test/assets/test-generated-goldens/QueuingGlShaderProgramTest/pts_0.png differ diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/QueuingGlShaderProgramTest/pts_333333.png b/libraries/test_data/src/test/assets/test-generated-goldens/QueuingGlShaderProgramTest/pts_333333.png new file mode 100644 index 0000000000..c4faffefe2 Binary files /dev/null and b/libraries/test_data/src/test/assets/test-generated-goldens/QueuingGlShaderProgramTest/pts_333333.png differ diff --git a/libraries/test_data/src/test/assets/test-generated-goldens/QueuingGlShaderProgramTest/pts_666667.png b/libraries/test_data/src/test/assets/test-generated-goldens/QueuingGlShaderProgramTest/pts_666667.png new file mode 100644 index 0000000000..cd7cfdcbd1 Binary files /dev/null and b/libraries/test_data/src/test/assets/test-generated-goldens/QueuingGlShaderProgramTest/pts_666667.png differ