diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index 90289943b6..408ce167cb 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -280,6 +280,8 @@ public interface VideoFrameProcessor { /** * Updates an {@linkplain Listener#onOutputFrameAvailableForRendering available frame} with the * modified effects. + * + *

This method can be called from any thread. */ void redraw(); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index 23bcb3acc3..ccf63f375b 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.common.util.GlUtil.getDefaultEglDisplay; +import static androidx.media3.common.util.Util.castNonNull; import static androidx.media3.effect.DebugTraceUtil.COMPONENT_VFP; import static androidx.media3.effect.DebugTraceUtil.EVENT_RECEIVE_END_OF_ALL_INPUT; import static androidx.media3.effect.DebugTraceUtil.EVENT_REGISTER_NEW_INPUT_STREAM; @@ -157,6 +158,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private @MonotonicNonNull GlObjectsProvider glObjectsProvider; private GlTextureProducer.@MonotonicNonNull Listener textureOutputListener; private int textureOutputCapacity; + private boolean enableReplayableCache; private boolean requireRegisteringAllInputFrames; private boolean experimentalAdjustSurfaceTextureTransformationMatrix; private boolean experimentalRepeatInputBitmapWithoutResampling; @@ -175,6 +177,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { glObjectsProvider = factory.glObjectsProvider; textureOutputListener = factory.textureOutputListener; textureOutputCapacity = factory.textureOutputCapacity; + enableReplayableCache = factory.enableReplayableCache; requireRegisteringAllInputFrames = !factory.repeatLastRegisteredFrame; experimentalAdjustSurfaceTextureTransformationMatrix = factory.experimentalAdjustSurfaceTextureTransformationMatrix; @@ -265,6 +268,22 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { return this; } + /** + * Sets whether to use a frame cache to {@link DefaultVideoFrameProcessor#redraw} frames. + * + *

The default value is {@code false}, in this case calling {@link + * VideoFrameProcessor#redraw} throws {@link UnsupportedOperationException}. + * + *

Using a frame cache enables precise redrawing, but increases resource and power usages. + * + * @param enableReplayableCache Whether to use a frame cache. + */ + @CanIgnoreReturnValue + public Builder setEnableReplayableCache(boolean enableReplayableCache) { + this.enableReplayableCache = enableReplayableCache; + return this; + } + /** * Sets texture output settings. * @@ -340,6 +359,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { executorService, textureOutputListener, textureOutputCapacity, + enableReplayableCache, experimentalAdjustSurfaceTextureTransformationMatrix, experimentalRepeatInputBitmapWithoutResampling); } @@ -351,6 +371,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { @Nullable private final ExecutorService executorService; @Nullable private final GlTextureProducer.Listener textureOutputListener; private final int textureOutputCapacity; + private final boolean enableReplayableCache; private final boolean experimentalAdjustSurfaceTextureTransformationMatrix; private final boolean experimentalRepeatInputBitmapWithoutResampling; @@ -361,6 +382,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { @Nullable ExecutorService executorService, @Nullable GlTextureProducer.Listener textureOutputListener, int textureOutputCapacity, + boolean enableReplayableCache, boolean experimentalAdjustSurfaceTextureTransformationMatrix, boolean experimentalRepeatInputBitmapWithoutResampling) { this.sdrWorkingColorSpace = sdrWorkingColorSpace; @@ -369,6 +391,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { this.executorService = executorService; this.textureOutputListener = textureOutputListener; this.textureOutputCapacity = textureOutputCapacity; + this.enableReplayableCache = enableReplayableCache; this.experimentalAdjustSurfaceTextureTransformationMatrix = experimentalAdjustSurfaceTextureTransformationMatrix; this.experimentalRepeatInputBitmapWithoutResampling = @@ -437,6 +460,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { listener, instanceGlObjectsProvider, shouldReleaseGlObjectsProvider, + enableReplayableCache, textureOutputListener, textureOutputCapacity, repeatLastRegisteredFrame, @@ -492,6 +516,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private final Object lock; private final ColorInfo outputColorInfo; private final DebugViewProvider debugViewProvider; + @Nullable private final ReplayableFrameCacheGlShaderProgram frameCache; private volatile @MonotonicNonNull FrameInfo nextInputFrameInfo; private volatile boolean inputStreamEnded; @@ -508,7 +533,8 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { FinalShaderProgramWrapper finalShaderProgramWrapper, boolean renderFramesAutomatically, ColorInfo outputColorInfo, - DebugViewProvider debugViewProvider) { + DebugViewProvider debugViewProvider, + @Nullable ReplayableFrameCacheGlShaderProgram frameCache) { this.context = context; this.glObjectsProvider = glObjectsProvider; this.shouldReleaseGlObjectsProvider = shouldReleaseGlObjectsProvider; @@ -521,18 +547,30 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { this.activeEffects = new ArrayList<>(); this.lock = new Object(); this.outputColorInfo = outputColorInfo; + this.frameCache = frameCache; this.debugViewProvider = debugViewProvider; this.finalShaderProgramWrapper = finalShaderProgramWrapper; this.intermediateGlShaderPrograms = new ArrayList<>(); this.inputStreamRegisteredCondition = new ConditionVariable(); inputStreamRegisteredCondition.open(); this.finalShaderProgramWrapper.setListener( - () -> { - if (inputStreamEnded) { - listenerExecutor.execute(listener::onEnded); - DebugTraceUtil.logEvent(COMPONENT_VFP, EVENT_SIGNAL_ENDED, C.TIME_END_OF_SOURCE); - } else { - submitPendingInputStream(); + new FinalShaderProgramWrapper.Listener() { + @Override + public void onInputStreamProcessed() { + if (inputStreamEnded) { + listenerExecutor.execute(listener::onEnded); + DebugTraceUtil.logEvent(COMPONENT_VFP, EVENT_SIGNAL_ENDED, C.TIME_END_OF_SOURCE); + } else { + DefaultVideoFrameProcessor.this.submitPendingInputStream(); + } + } + + @Override + public void onFrameRendered(long presentationTimeUs) { + if (frameCache == null) { + return; + } + frameCache.onFrameRendered(presentationTimeUs); } }); } @@ -624,9 +662,31 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { return inputSwitcher.getInputSurface(); } + /** + * {@inheritDoc} + * + *

{@code DefaultVideoFrameProcessor} keeps track of the redraw requests received. If a call to + * redraw is made when another redraw request is ongoing, the new request will be performed later + * when the ongoing redraw completes, and this method will return immediately. + */ @Override public void redraw() { - throw new UnsupportedOperationException(); + if (frameCache == null) { + throw new UnsupportedOperationException( + "Replaying when enableReplayableCache is set to false"); + } + // TODO: b/391109644 - Call listener method in VideoFrameMetadataListener and debounce + // accordingly. + if (frameCache.isEmpty()) { + // Don't redraw right after flush, because the frame cache is also be flushed and it's empty. + return; + } + videoFrameProcessingTaskExecutor.submit( + () -> { + finalShaderProgramWrapper.prepareToRedraw( + castNonNull(frameCache).getReplayFramePresentationTimeUs()); + frameCache.replayFrame(); + }); } /** @@ -853,6 +913,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { Listener listener, GlObjectsProvider glObjectsProvider, boolean shouldReleaseGlObjectsProvider, + boolean enableReplayableCache, @Nullable GlTextureProducer.Listener textureOutputListener, int textureOutputCapacity, boolean repeatLastRegisteredFrame, @@ -860,8 +921,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { boolean experimentalRepeatInputBitmapWithoutResampling) throws GlUtil.GlException, VideoFrameProcessingException { EGLDisplay eglDisplay = getDefaultEglDisplay(); + boolean isOutputTransferHdr = ColorInfo.isTransferHdr(outputColorInfo); int[] configAttributes = - ColorInfo.isTransferHdr(outputColorInfo) + isOutputTransferHdr ? GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_1010102 : GlUtil.EGL_CONFIG_ATTRIBUTES_RGBA_8888; Pair eglContextAndPlaceholderSurface = @@ -874,7 +936,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { .setHdrStaticInfo(null) .build(); ColorInfo intermediateColorInfo = - ColorInfo.isTransferHdr(outputColorInfo) + isOutputTransferHdr ? linearColorInfo : sdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR ? linearColorInfo @@ -919,7 +981,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { finalShaderProgramWrapper, renderFramesAutomatically, outputColorInfo, - debugViewProvider); + debugViewProvider, + enableReplayableCache + ? new ReplayableFrameCacheGlShaderProgram(context, /* useHdr= */ isOutputTransferHdr) + : null); } /** @@ -1042,18 +1107,25 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { if (forceReconfigure || !activeEffects.equals(inputStreamInfo.effects)) { if (!intermediateGlShaderPrograms.isEmpty()) { - for (int i = 0; i < intermediateGlShaderPrograms.size(); i++) { + // If frameCache is present, it's the first item in the list, skip releasing it. + int startIndex = frameCache == null ? 0 : 1; + for (int i = startIndex; i < intermediateGlShaderPrograms.size(); i++) { intermediateGlShaderPrograms.get(i).release(); } intermediateGlShaderPrograms.clear(); } + if (frameCache != null) { + intermediateGlShaderPrograms.add(frameCache); + } + ImmutableList.Builder effectsListBuilder = new ImmutableList.Builder().addAll(inputStreamInfo.effects); if (debugViewProvider != DebugViewProvider.NONE) { effectsListBuilder.add(new DebugViewEffect(debugViewProvider, outputColorInfo)); } - // The GlShaderPrograms that should be inserted in between InputSwitcher and + + // The GlShaderPrograms that should be inserted in between the frame cache and // FinalShaderProgramWrapper. intermediateGlShaderPrograms.addAll( createGlShaderPrograms( diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java index 83c6442b0b..62e3754e2e 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java @@ -41,6 +41,7 @@ import androidx.media3.common.util.GlUtil; import androidx.media3.common.util.Log; import androidx.media3.common.util.LongArrayQueue; import androidx.media3.common.util.Size; +import androidx.media3.common.util.SystemClock; import androidx.media3.effect.DefaultVideoFrameProcessor.WorkingColorSpace; import com.google.common.collect.ImmutableList; import java.util.ArrayList; @@ -72,6 +73,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * #signalEndOfCurrentInputStream()}. */ void onInputStreamProcessed(); + + /** Called when a frame is rendered to the output surface. */ + void onFrameRendered(long presentationTimeUs); } private static final String TAG = "FinalShaderWrapper"; @@ -112,6 +116,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private boolean outputSurfaceInfoChanged; @Nullable private SurfaceInfo outputSurfaceInfo; + private long redrawFramePresentationTimeUs; + /** Wraps the {@link Surface} in {@link #outputSurfaceInfo}. */ @Nullable private EGLSurface outputEglSurface; @@ -149,6 +155,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputTexturePool = new TexturePool(useHighPrecisionColorComponents, textureOutputCapacity); outputTextureTimestamps = new LongArrayQueue(textureOutputCapacity); syncObjects = new LongArrayQueue(textureOutputCapacity); + redrawFramePresentationTimeUs = C.TIME_UNSET; } // GlTextureProducer interface. Can be called on any thread. @@ -213,8 +220,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public void queueInputFrame( GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { videoFrameProcessingTaskExecutor.verifyVideoFrameProcessingThread(); - videoFrameProcessorListenerExecutor.execute( - () -> videoFrameProcessorListener.onOutputFrameAvailableForRendering(presentationTimeUs)); + if (!isWaitingForRedrawFrame()) { + // Don't report output available when redrawing - the redrawn frames are released immediately. + videoFrameProcessorListenerExecutor.execute( + () -> videoFrameProcessorListener.onOutputFrameAvailableForRendering(presentationTimeUs)); + } + if (textureOutputListener == null) { if (renderFramesAutomatically) { renderFrame( @@ -224,6 +235,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* renderTimeNs= */ presentationTimeUs * 1000); } else { availableFrames.add(new TimedGlTextureInfo(inputTexture, presentationTimeUs)); + if (isWaitingForRedrawFrame()) { + if (presentationTimeUs == redrawFramePresentationTimeUs) { + redrawFramePresentationTimeUs = C.TIME_UNSET; + renderFrame( + glObjectsProvider, + inputTexture, + presentationTimeUs, + /* renderTimeNs= */ SystemClock.DEFAULT.nanoTime()); + availableFrames.clear(); + } else { + // Skip other frames when waiting for the replay frame to arrive, so that the producer + // can continue processing, but keep it in the availableFrames for the player to call + // renderFrame. + inputListener.onInputFrameProcessed(inputTexture); + } + } } inputListener.onReadyToAcceptInputFrame(); } else { @@ -309,6 +336,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return; } checkState(!renderFramesAutomatically); + if (availableFrames.isEmpty()) { + // This only happens with redrawn frame. The available output frame notification on the player + // side runs on another thread and when redrawing rapidly, the player could receive an output + // frame from a previous redraw. + return; + } + TimedGlTextureInfo oldestAvailableFrame = availableFrames.remove(); renderFrame( glObjectsProvider, @@ -337,6 +371,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } + /* package */ void prepareToRedraw(long redrawFramePresentationTimeUs) { + this.redrawFramePresentationTimeUs = redrawFramePresentationTimeUs; + for (int i = 0; i < availableFrames.size(); i++) { + TimedGlTextureInfo availableFrame = availableFrames.remove(); + inputListener.onInputFrameProcessed(availableFrame.glTextureInfo); + } + } + /** Must be called on the GL thread. */ private void setOutputSurfaceInfoInternal(@Nullable SurfaceInfo outputSurfaceInfo) { if (textureOutputListener != null) { @@ -391,6 +433,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } + private boolean isWaitingForRedrawFrame() { + return redrawFramePresentationTimeUs != C.TIME_UNSET; + } + private void renderFrame( GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, @@ -398,8 +444,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; long renderTimeNs) { try { if (renderTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME - || !ensureConfigured(glObjectsProvider, inputTexture.width, inputTexture.height)) { + || !ensureConfigured(glObjectsProvider, inputTexture.width, inputTexture.height) + || (isWaitingForRedrawFrame() && presentationTimeUs != redrawFramePresentationTimeUs)) { inputListener.onInputFrameProcessed(inputTexture); + if (renderTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME) { + checkNotNull(listener).onFrameRendered(presentationTimeUs); + } return; // Drop frames when requested, or there is no output surface and output texture. } if (outputSurfaceInfo != null) { @@ -445,6 +495,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; EGLExt.eglPresentationTimeANDROID(eglDisplay, outputEglSurface, eglPresentationTimeNs); EGL14.eglSwapBuffers(eglDisplay, outputEglSurface); + checkNotNull(listener).onFrameRendered(presentationTimeUs); DebugTraceUtil.logEvent(COMPONENT_VFP, EVENT_RENDERED_TO_OUTPUT_SURFACE, presentationTimeUs); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/ReplayableFrameCacheGlShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/ReplayableFrameCacheGlShaderProgram.java new file mode 100644 index 0000000000..b06cd110a1 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/ReplayableFrameCacheGlShaderProgram.java @@ -0,0 +1,124 @@ +/* + * Copyright 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; + +import android.content.Context; +import androidx.media3.common.C; +import androidx.media3.common.GlObjectsProvider; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.VideoFrameProcessingException; + +/** + * A shader program that caches the input frames, and {@linkplain #replayFrame replays} the oldest + * input frame when instructed. + */ +/* package */ final class ReplayableFrameCacheGlShaderProgram extends FrameCacheGlShaderProgram { + private static final int CAPACITY = 2; + private static final int REPLAY_FRAME_INDEX = 0; + private static final int REGULAR_FRAME_INDEX = 1; + + // Use a manually managed array to be more efficient than List add/remove methods. + private final TimedGlTextureInfo[] cachedFrames; + private int cacheSize; + + public ReplayableFrameCacheGlShaderProgram(Context context, boolean useHdr) + throws VideoFrameProcessingException { + super(context, CAPACITY, useHdr); + cachedFrames = new TimedGlTextureInfo[CAPACITY]; + } + + @Override + public void queueInputFrame( + GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { + checkState(cacheSize < CAPACITY); + super.queueInputFrame(glObjectsProvider, inputTexture, presentationTimeUs); + cachedFrames[cacheSize++] = + new TimedGlTextureInfo( + checkNotNull(outputTexturePool.getMostRecentlyUsedTexture()), presentationTimeUs); + } + + @Override + public void releaseOutputFrame(GlTextureInfo outputTexture) { + // Do nothing here as this method will be called as soon as the output frame is queued into the + // subsequent shader program. This class only releases output frame based on rendering event + // from the FinalShaderProgramWrapper. See onFrameRendered(). + } + + @Override + public void flush() { + cacheSize = 0; + super.flush(); + } + + /** Returns whether there is no cached frame. */ + public boolean isEmpty() { + return cacheSize == 0; + } + + /** + * Returns the presentation time of the frame that will be replayed, if {@link #replayFrame()} is + * called. + */ + public long getReplayFramePresentationTimeUs() { + if (isEmpty()) { + return C.TIME_UNSET; + } + return cachedFrames[REPLAY_FRAME_INDEX].presentationTimeUs; + } + + /** + * Replays the frame from cache, with the {@linkplain #getReplayFramePresentationTimeUs replay + * timestamp}. + */ + public void replayFrame() { + if (isEmpty()) { + return; + } + + // Get the oldest frame that is queued. + TimedGlTextureInfo oldestFrame = cachedFrames[REPLAY_FRAME_INDEX]; + getOutputListener() + .onOutputFrameAvailable(oldestFrame.glTextureInfo, oldestFrame.presentationTimeUs); + + // Queue the subsequent frame also to keep the player's output frame queue full. + if (cacheSize > 1) { + TimedGlTextureInfo secondOldestFrame = cachedFrames[REGULAR_FRAME_INDEX]; + getOutputListener() + .onOutputFrameAvailable( + secondOldestFrame.glTextureInfo, secondOldestFrame.presentationTimeUs); + } + } + + /** Removes a frame from the cache when a frame of the {@code presentationTimeUs} is rendered. */ + public void onFrameRendered(long presentationTimeUs) { + // Cache needs to be full when capacity is two, only release frame n when frame n+1 is released. + if (cacheSize < CAPACITY + || presentationTimeUs < cachedFrames[REGULAR_FRAME_INDEX].presentationTimeUs) { + return; + } + + // Evict the oldest frame. + TimedGlTextureInfo cachedFrame = cachedFrames[REPLAY_FRAME_INDEX]; + cachedFrames[REPLAY_FRAME_INDEX] = cachedFrames[REGULAR_FRAME_INDEX]; + cacheSize--; + + // Release the texture, this also calls readyToAcceptInput. + super.releaseOutputFrame(cachedFrame.glTextureInfo); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java index 639a0056d7..511662ceea 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java @@ -278,7 +278,7 @@ public class SingleInputVideoGraph implements VideoGraph { @Override public void redraw() { - throw new UnsupportedOperationException(); + checkStateNotNull(videoFrameProcessor).redraw(); } @Override diff --git a/libraries/effect/src/main/java/androidx/media3/effect/TexturePool.java b/libraries/effect/src/main/java/androidx/media3/effect/TexturePool.java index f7d125c2ff..ff382f45ab 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/TexturePool.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/TexturePool.java @@ -17,18 +17,19 @@ package androidx.media3.effect; import static androidx.media3.common.util.Assertions.checkState; +import androidx.annotation.Nullable; import androidx.media3.common.GlObjectsProvider; import androidx.media3.common.GlTextureInfo; import androidx.media3.common.util.GlUtil; import com.google.common.collect.Iterables; import java.util.ArrayDeque; +import java.util.Deque; import java.util.Iterator; -import java.util.Queue; /** Holds {@code capacity} textures, to re-use textures. */ /* package */ final class TexturePool { - private final Queue freeTextures; - private final Queue inUseTextures; + private final Deque freeTextures; + private final Deque inUseTextures; private final int capacity; private final boolean useHighPrecisionColorComponents; @@ -94,6 +95,15 @@ import java.util.Queue; return texture; } + /** Returns the {@link GlTextureInfo} that is most recently {@linkplain #useTexture used}. */ + @Nullable + public GlTextureInfo getMostRecentlyUsedTexture() { + if (inUseTextures.isEmpty()) { + return null; + } + return inUseTextures.getLast(); + } + /** * Frees the texture represented by {@code textureInfo}. * diff --git a/libraries/exoplayer/proguard-rules.txt b/libraries/exoplayer/proguard-rules.txt index 91685e1af6..a9fd1b6ee2 100644 --- a/libraries/exoplayer/proguard-rules.txt +++ b/libraries/exoplayer/proguard-rules.txt @@ -78,4 +78,5 @@ -dontnote androidx.media3.effect.DefaultVideoFrameProcessor$Factory$Builder -keepclasseswithmembers class androidx.media3.effect.DefaultVideoFrameProcessor$Factory$Builder { androidx.media3.effect.DefaultVideoFrameProcessor$Factory build(); + androidx.media3.effect.DefaultVideoFrameProcessor$Factory$Builder setEnableReplayableCache(boolean); } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 3c4aef377a..493aad7d66 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -870,9 +870,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer if (!hasSetVideoSink) { if (videoEffects != null && videoSink == null) { PlaybackVideoGraphWrapper playbackVideoGraphWrapper = - new PlaybackVideoGraphWrapper.Builder(context, videoFrameReleaseControl) - .setClock(getClock()) - .build(); + createPlaybackVideoGraphWrapper(context, videoFrameReleaseControl); playbackVideoGraphWrapper.setTotalVideoInputCount(1); videoSink = playbackVideoGraphWrapper.getSink(/* inputIndex= */ 0); } @@ -946,6 +944,15 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer } } + /** Creates a {@link PlaybackVideoGraphWrapper} instance. */ + protected PlaybackVideoGraphWrapper createPlaybackVideoGraphWrapper( + Context context, VideoFrameReleaseControl videoFrameReleaseControl) { + // TODO: b/391109644 - Add a more explicit API to enable replaying. + return new PlaybackVideoGraphWrapper.Builder(context, videoFrameReleaseControl) + .setClock(getClock()) + .build(); + } + @Override public void enableMayRenderStartOfStream() { if (videoSink != null) { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java index a3cd3acb3e..0a3f16023e 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/PlaybackVideoGraphWrapper.java @@ -62,6 +62,7 @@ import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import java.lang.reflect.Method; import java.util.List; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.Executor; @@ -128,6 +129,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video private Clock clock; private boolean requestOpenGlToneMapping; private boolean built; + private boolean enableReplayableCache; /** Creates a builder. */ public Builder(Context context, VideoFrameReleaseControl videoFrameReleaseControl) { @@ -222,6 +224,21 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video return this; } + /** + * Sets whether to enable replayable cache. + * + *

By default, the replayable cache is not enabled. Enable it to achieve accurate effect + * update, at the cost of using more power and computing resources. + * + * @param enableReplayableCache Whether replayable cache is enabled. + * @return This builder, for convenience. + */ + @CanIgnoreReturnValue + public Builder setEnableReplayableCache(boolean enableReplayableCache) { + this.enableReplayableCache = enableReplayableCache; + return this; + } + /** * Builds the {@link PlaybackVideoGraphWrapper}. * @@ -233,7 +250,8 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video if (videoGraphFactory == null) { if (videoFrameProcessorFactory == null) { - videoFrameProcessorFactory = new ReflectiveDefaultVideoFrameProcessorFactory(); + videoFrameProcessorFactory = + new ReflectiveDefaultVideoFrameProcessorFactory(enableReplayableCache); } videoGraphFactory = new ReflectiveSingleInputVideoGraphFactory(videoFrameProcessorFactory); } @@ -696,6 +714,7 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video @Override public void redraw() { checkState(isInitialized()); + PlaybackVideoGraphWrapper.this.flush(/* resetPosition= */ false); checkNotNull(videoGraph).redraw(); } @@ -1059,29 +1078,25 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video */ private static final class ReflectiveDefaultVideoFrameProcessorFactory implements VideoFrameProcessor.Factory { - private static final Supplier - VIDEO_FRAME_PROCESSOR_FACTORY_SUPPLIER = - Suppliers.memoize( - () -> { - try { - // LINT.IfChange - Class defaultVideoFrameProcessorFactoryBuilderClass = - Class.forName( - "androidx.media3.effect.DefaultVideoFrameProcessor$Factory$Builder"); - Object builder = - defaultVideoFrameProcessorFactoryBuilderClass - .getConstructor() - .newInstance(); - return (VideoFrameProcessor.Factory) - checkNotNull( - defaultVideoFrameProcessorFactoryBuilderClass - .getMethod("build") - .invoke(builder)); - // LINT.ThenChange(../../../../../../../proguard-rules.txt) - } catch (Exception e) { - throw new IllegalStateException(e); - } - }); + + private static final Supplier> DEFAULT_VIDEO_FRAME_PROCESSOR_FACTORY_BUILDER_CLASS = + Suppliers.memoize( + () -> { + try { + // LINT.IfChange + return Class.forName( + "androidx.media3.effect.DefaultVideoFrameProcessor$Factory$Builder"); + // LINT.ThenChange(../../../../../../../proguard-rules.txt) + } catch (Exception e) { + throw new IllegalStateException(e); + } + }); + + private final boolean enableReplayableCache; + + public ReflectiveDefaultVideoFrameProcessorFactory(boolean enableReplayableCache) { + this.enableReplayableCache = enableReplayableCache; + } @Override public VideoFrameProcessor create( @@ -1092,15 +1107,31 @@ public final class PlaybackVideoGraphWrapper implements VideoSinkProvider, Video Executor listenerExecutor, VideoFrameProcessor.Listener listener) throws VideoFrameProcessingException { - return VIDEO_FRAME_PROCESSOR_FACTORY_SUPPLIER - .get() - .create( - context, - debugViewProvider, - outputColorInfo, - renderFramesAutomatically, - listenerExecutor, - listener); + try { + Class defaultVideoFrameProcessorFactoryBuilderClass = + DEFAULT_VIDEO_FRAME_PROCESSOR_FACTORY_BUILDER_CLASS.get(); + Object builder = + defaultVideoFrameProcessorFactoryBuilderClass.getConstructor().newInstance(); + Method setUseReplayableCacheMethod = + defaultVideoFrameProcessorFactoryBuilderClass.getMethod( + "setEnableReplayableCache", boolean.class); + setUseReplayableCacheMethod.invoke(builder, enableReplayableCache); + VideoFrameProcessor.Factory factory = + (VideoFrameProcessor.Factory) + checkNotNull( + defaultVideoFrameProcessorFactoryBuilderClass + .getMethod("build") + .invoke(builder)); + return factory.create( + context, + debugViewProvider, + outputColorInfo, + renderFramesAutomatically, + listenerExecutor, + listener); + } catch (Exception e) { + throw new VideoFrameProcessingException(e); + } } } }