From 4c3ac818732c7a05f860ec13e36f0580cbc2fc35 Mon Sep 17 00:00:00 2001 From: claincly Date: Fri, 21 Feb 2025 10:08:59 -0800 Subject: [PATCH] Add a class wrapping GlTextureInfo and presentation time The grouping is used in some places and will be used in dynamic effect update PiperOrigin-RevId: 729569353 --- .../media3/effect/DefaultVideoCompositor.java | 34 +++++++-------- .../effect/FinalShaderProgramWrapper.java | 11 +++-- .../effect/FrameConsumptionManager.java | 17 ++++---- .../effect/MultipleInputVideoGraph.java | 17 ++------ .../media3/effect/QueuingGlShaderProgram.java | 42 ++++++++++--------- .../media3/effect/TimedGlTextureInfo.java | 33 +++++++++++++++ 6 files changed, 87 insertions(+), 67 deletions(-) create mode 100644 libraries/effect/src/main/java/androidx/media3/effect/TimedGlTextureInfo.java diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java index ed70566027..b8715a3c62 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java @@ -201,8 +201,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { InputFrameInfo inputFrameInfo = new InputFrameInfo( textureProducer, - inputTexture, - presentationTimeUs, + new TimedGlTextureInfo(inputTexture, presentationTimeUs), settings.getOverlaySettings(inputIndex, presentationTimeUs)); inputSource.frameInfos.add(inputFrameInfo); @@ -260,13 +259,15 @@ public final class DefaultVideoCompositor implements VideoCompositor { // nextTimestampToComposite. @Nullable InputFrameInfo nextPrimaryFrame = primaryInputSource.frameInfos.peek(); long nextTimestampToComposite = - nextPrimaryFrame != null ? nextPrimaryFrame.presentationTimeUs : C.TIME_UNSET; + nextPrimaryFrame != null + ? nextPrimaryFrame.timedGlTextureInfo.presentationTimeUs + : C.TIME_UNSET; int numberOfSecondaryFramesBeforeOrAtNextTargetTimestamp = Iterables.size( Iterables.filter( secondaryInputSource.frameInfos, - frame -> frame.presentationTimeUs <= nextTimestampToComposite)); + frame -> frame.timedGlTextureInfo.presentationTimeUs <= nextTimestampToComposite)); releaseFrames( secondaryInputSource, /* numberOfFramesToRelease= */ max( @@ -277,7 +278,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { for (int i = 0; i < numberOfFramesToRelease; i++) { InputFrameInfo frameInfoToRelease = inputSource.frameInfos.remove(); frameInfoToRelease.textureProducer.releaseOutputTexture( - frameInfoToRelease.presentationTimeUs); + frameInfoToRelease.timedGlTextureInfo.presentationTimeUs); } } @@ -302,7 +303,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { ImmutableList.Builder inputSizes = new ImmutableList.Builder<>(); for (int i = 0; i < framesToComposite.size(); i++) { - GlTextureInfo texture = framesToComposite.get(i).texture; + GlTextureInfo texture = framesToComposite.get(i).timedGlTextureInfo.glTextureInfo; inputSizes.add(new Size(texture.width, texture.height)); } Size outputSize = settings.getOutputSize(inputSizes.build()); @@ -310,7 +311,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { glObjectsProvider, outputSize.getWidth(), outputSize.getHeight()); GlTextureInfo outputTexture = outputTexturePool.useTexture(); - long outputPresentationTimestampUs = primaryInputFrame.presentationTimeUs; + long outputPresentationTimestampUs = primaryInputFrame.timedGlTextureInfo.presentationTimeUs; outputTextureTimestamps.add(outputPresentationTimestampUs); compositorGlProgram.drawFrame(framesToComposite, outputTexture); @@ -369,16 +370,18 @@ public final class DefaultVideoCompositor implements VideoCompositor { Iterator frameInfosIterator = secondaryInputSource.frameInfos.iterator(); while (frameInfosIterator.hasNext()) { InputFrameInfo candidateFrame = frameInfosIterator.next(); - long candidateTimestampUs = candidateFrame.presentationTimeUs; + long candidateTimestampUs = candidateFrame.timedGlTextureInfo.presentationTimeUs; long candidateAbsDistance = - abs(candidateTimestampUs - primaryFrameToComposite.presentationTimeUs); + abs( + candidateTimestampUs + - primaryFrameToComposite.timedGlTextureInfo.presentationTimeUs); if (candidateAbsDistance < minTimeDiffFromPrimaryUs) { minTimeDiffFromPrimaryUs = candidateAbsDistance; secondaryFrameToComposite = candidateFrame; } - if (candidateTimestampUs > primaryFrameToComposite.presentationTimeUs + if (candidateTimestampUs > primaryFrameToComposite.timedGlTextureInfo.presentationTimeUs || (!frameInfosIterator.hasNext() && secondaryInputSource.isInputEnded)) { framesToComposite.add(checkNotNull(secondaryFrameToComposite)); break; @@ -503,7 +506,7 @@ public final class DefaultVideoCompositor implements VideoCompositor { private void blendOntoFocusedTexture(InputFrameInfo inputFrameInfo) throws GlUtil.GlException { GlProgram glProgram = checkNotNull(this.glProgram); - GlTextureInfo inputTexture = inputFrameInfo.texture; + GlTextureInfo inputTexture = inputFrameInfo.timedGlTextureInfo.glTextureInfo; glProgram.setSamplerTexIdUniform("uTexSampler", inputTexture.texId, /* texUnitIndex= */ 0); float[] transformationMatrix = overlayMatrixProvider.getTransformationMatrix( @@ -537,18 +540,15 @@ public final class DefaultVideoCompositor implements VideoCompositor { /** Holds information on a frame and how to release it. */ private static final class InputFrameInfo { public final GlTextureProducer textureProducer; - public final GlTextureInfo texture; - public final long presentationTimeUs; + public final TimedGlTextureInfo timedGlTextureInfo; public final OverlaySettings overlaySettings; public InputFrameInfo( GlTextureProducer textureProducer, - GlTextureInfo texture, - long presentationTimeUs, + TimedGlTextureInfo timedGlTextureInfo, OverlaySettings overlaySettings) { this.textureProducer = textureProducer; - this.texture = texture; - this.presentationTimeUs = presentationTimeUs; + this.timedGlTextureInfo = timedGlTextureInfo; this.overlaySettings = overlaySettings; } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java index 8b4868a922..60f02f3187 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java @@ -28,7 +28,6 @@ import android.opengl.EGLContext; import android.opengl.EGLDisplay; import android.opengl.EGLExt; import android.opengl.EGLSurface; -import android.util.Pair; import android.view.Surface; import androidx.annotation.Nullable; import androidx.media3.common.C; @@ -86,7 +85,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; private final Executor videoFrameProcessorListenerExecutor; private final VideoFrameProcessor.Listener videoFrameProcessorListener; - private final Queue> availableFrames; + private final Queue availableFrames; private final TexturePool outputTexturePool; private final LongArrayQueue outputTextureTimestamps; // Synchronized with outputTexturePool. private final LongArrayQueue syncObjects; @@ -221,7 +220,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; presentationTimeUs, /* renderTimeNs= */ presentationTimeUs * 1000); } else { - availableFrames.add(Pair.create(inputTexture, presentationTimeUs)); + availableFrames.add(new TimedGlTextureInfo(inputTexture, presentationTimeUs)); } inputListener.onReadyToAcceptInputFrame(); } else { @@ -307,11 +306,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return; } checkState(!renderFramesAutomatically); - Pair oldestAvailableFrame = availableFrames.remove(); + TimedGlTextureInfo oldestAvailableFrame = availableFrames.remove(); renderFrame( glObjectsProvider, - /* inputTexture= */ oldestAvailableFrame.first, - /* presentationTimeUs= */ oldestAvailableFrame.second, + oldestAvailableFrame.glTextureInfo, + oldestAvailableFrame.presentationTimeUs, renderTimeNs); if (availableFrames.isEmpty() && isInputStreamEndedWithPendingAvailableFrames) { checkNotNull(onInputStreamProcessedListener).onInputStreamProcessed(); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FrameConsumptionManager.java b/libraries/effect/src/main/java/androidx/media3/effect/FrameConsumptionManager.java index 51141e0d1e..2216841075 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FrameConsumptionManager.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FrameConsumptionManager.java @@ -15,7 +15,6 @@ */ package androidx.media3.effect; -import android.util.Pair; import androidx.annotation.GuardedBy; import androidx.annotation.Nullable; import androidx.media3.common.C; @@ -39,7 +38,7 @@ import java.util.Queue; private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; @GuardedBy("this") - private final Queue> availableFrames; + private final Queue availableFrames; @GuardedBy("this") private int consumingGlShaderProgramInputCapacity; @@ -63,7 +62,7 @@ import java.util.Queue; @Override public synchronized void onReadyToAcceptInputFrame() { - @Nullable Pair pendingFrame = availableFrames.poll(); + @Nullable TimedGlTextureInfo pendingFrame = availableFrames.poll(); if (pendingFrame == null) { consumingGlShaderProgramInputCapacity++; return; @@ -72,11 +71,9 @@ import java.util.Queue; videoFrameProcessingTaskExecutor.submit( () -> consumingGlShaderProgram.queueInputFrame( - glObjectsProvider, - /* inputTexture= */ pendingFrame.first, - /* presentationTimeUs= */ pendingFrame.second)); - @Nullable Pair nextPendingFrame = availableFrames.peek(); - if (nextPendingFrame != null && nextPendingFrame.second == C.TIME_END_OF_SOURCE) { + glObjectsProvider, pendingFrame.glTextureInfo, pendingFrame.presentationTimeUs)); + @Nullable TimedGlTextureInfo nextPendingFrame = availableFrames.peek(); + if (nextPendingFrame != null && nextPendingFrame.presentationTimeUs == C.TIME_END_OF_SOURCE) { videoFrameProcessingTaskExecutor.submit( consumingGlShaderProgram::signalEndOfCurrentInputStream); availableFrames.remove(); @@ -97,7 +94,7 @@ import java.util.Queue; glObjectsProvider, inputTexture, presentationTimeUs)); consumingGlShaderProgramInputCapacity--; } else { - availableFrames.add(Pair.create(inputTexture, presentationTimeUs)); + availableFrames.add(new TimedGlTextureInfo(inputTexture, presentationTimeUs)); } } @@ -107,7 +104,7 @@ import java.util.Queue; */ public synchronized void signalEndOfCurrentStream() { if (!availableFrames.isEmpty()) { - availableFrames.add(Pair.create(GlTextureInfo.UNSET, C.TIME_END_OF_SOURCE)); + availableFrames.add(new TimedGlTextureInfo(GlTextureInfo.UNSET, C.TIME_END_OF_SOURCE)); } else { videoFrameProcessingTaskExecutor.submit( consumingGlShaderProgram::signalEndOfCurrentInputStream); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java index ed61e73046..3ce329e250 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java @@ -87,7 +87,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { private final ExecutorService sharedExecutorService; private final DefaultVideoFrameProcessor.Factory videoFrameProcessorFactory; - private final Queue compositorOutputTextures; + private final Queue compositorOutputTextures; private final SparseArray compositorOutputTextureReleases; private final long initialTimestampOffsetUs; @@ -363,8 +363,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { DebugTraceUtil.logEvent( COMPONENT_COMPOSITOR, EVENT_OUTPUT_TEXTURE_RENDERED, presentationTimeUs); - compositorOutputTextures.add( - new CompositorOutputTextureInfo(outputTexture, presentationTimeUs)); + compositorOutputTextures.add(new TimedGlTextureInfo(outputTexture, presentationTimeUs)); compositorOutputTextureReleases.put( outputTexture.texId, new CompositorOutputTextureRelease(textureProducer, presentationTimeUs)); @@ -421,7 +420,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { return; } - @Nullable CompositorOutputTextureInfo outputTexture = compositorOutputTextures.peek(); + @Nullable TimedGlTextureInfo outputTexture = compositorOutputTextures.peek(); if (outputTexture == null) { return; } @@ -446,16 +445,6 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { : VideoFrameProcessingException.from(e))); } - private static final class CompositorOutputTextureInfo { - public final GlTextureInfo glTextureInfo; - public final long presentationTimeUs; - - private CompositorOutputTextureInfo(GlTextureInfo glTextureInfo, long presentationTimeUs) { - this.glTextureInfo = glTextureInfo; - this.presentationTimeUs = presentationTimeUs; - } - } - private static final class CompositorOutputTextureRelease { private final GlTextureProducer textureProducer; private final long presentationTimeUs; diff --git a/libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java index b544ebb968..1f4884e616 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/QueuingGlShaderProgram.java @@ -142,7 +142,7 @@ import java.util.concurrent.TimeUnit; private final ConcurrentEffect concurrentEffect; private final TexturePool outputTexturePool; - private final Queue> frameQueue; + private final Queue> frameQueue; private InputListener inputListener; private OutputListener outputListener; private ErrorListener errorListener; @@ -226,7 +226,8 @@ import java.util.concurrent.TimeUnit; Future task = concurrentEffect.queueInputFrame(glObjectsProvider, outputTexture, presentationTimeUs); - frameQueue.add(new TimedTextureInfo(outputTexture, presentationTimeUs, task)); + frameQueue.add( + new QueuedFrame(new TimedGlTextureInfo(outputTexture, presentationTimeUs), task)); inputListener.onInputFrameProcessed(inputTexture); @@ -297,25 +298,28 @@ import java.util.concurrent.TimeUnit; *

Returns {@code false} if no more frames are available for output. */ private boolean outputOneFrame() { - TimedTextureInfo timedTextureInfo = frameQueue.poll(); - if (timedTextureInfo == null) { + QueuedFrame queuedFrame = frameQueue.poll(); + if (queuedFrame == null) { return false; } try { T result = Futures.getChecked( - timedTextureInfo.task, + queuedFrame.task, VideoFrameProcessingException.class, PROCESSING_TIMEOUT_MS, TimeUnit.MILLISECONDS); GlUtil.focusFramebufferUsingCurrentContext( - timedTextureInfo.textureInfo.fboId, - timedTextureInfo.textureInfo.width, - timedTextureInfo.textureInfo.height); + queuedFrame.timedGlTextureInfo.glTextureInfo.fboId, + queuedFrame.timedGlTextureInfo.glTextureInfo.width, + queuedFrame.timedGlTextureInfo.glTextureInfo.height); concurrentEffect.finishProcessingAndBlend( - timedTextureInfo.textureInfo, timedTextureInfo.presentationTimeUs, result); + queuedFrame.timedGlTextureInfo.glTextureInfo, + queuedFrame.timedGlTextureInfo.presentationTimeUs, + result); outputListener.onOutputFrameAvailable( - timedTextureInfo.textureInfo, timedTextureInfo.presentationTimeUs); + queuedFrame.timedGlTextureInfo.glTextureInfo, + queuedFrame.timedGlTextureInfo.presentationTimeUs); return true; } catch (GlUtil.GlException | VideoFrameProcessingException e) { onError(e); @@ -324,9 +328,9 @@ import java.util.concurrent.TimeUnit; } private void cancelProcessingOfPendingFrames() { - TimedTextureInfo timedTextureInfo; - while ((timedTextureInfo = frameQueue.poll()) != null) { - timedTextureInfo.task.cancel(/* mayInterruptIfRunning= */ false); + QueuedFrame queuedFrame; + while ((queuedFrame = frameQueue.poll()) != null) { + queuedFrame.task.cancel(/* mayInterruptIfRunning= */ false); } } @@ -335,14 +339,12 @@ import java.util.concurrent.TimeUnit; () -> errorListener.onError(VideoFrameProcessingException.from(e))); } - private static class TimedTextureInfo { - final GlTextureInfo textureInfo; - final long presentationTimeUs; - final Future task; + private static final class QueuedFrame { + public final TimedGlTextureInfo timedGlTextureInfo; + public final Future task; - TimedTextureInfo(GlTextureInfo textureInfo, long presentationTimeUs, Future task) { - this.textureInfo = textureInfo; - this.presentationTimeUs = presentationTimeUs; + public QueuedFrame(TimedGlTextureInfo timedGlTextureInfo, Future task) { + this.timedGlTextureInfo = timedGlTextureInfo; this.task = task; } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TimedGlTextureInfo.java b/libraries/effect/src/main/java/androidx/media3/effect/TimedGlTextureInfo.java new file mode 100644 index 0000000000..4876c51549 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/TimedGlTextureInfo.java @@ -0,0 +1,33 @@ +/* + * Copyright 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import androidx.media3.common.GlTextureInfo; + +/* package */ final class TimedGlTextureInfo { + + /** The {@link GlTextureInfo}. */ + public final GlTextureInfo glTextureInfo; + + /** The designated presentation time with the texture, in microseconds. */ + public final long presentationTimeUs; + + /** Creates a new instance. */ + public TimedGlTextureInfo(GlTextureInfo glTextureInfo, long presentationTimeUs) { + this.glTextureInfo = glTextureInfo; + this.presentationTimeUs = presentationTimeUs; + } +}