From f5a6ecdda1cbff67de320615e1eccf4be31b4db0 Mon Sep 17 00:00:00 2001 From: christosts Date: Fri, 11 Aug 2023 11:59:50 +0000 Subject: [PATCH] Implement the CompositingVideoSinkProvider This is the first iteration for the CompositingVideoSinkProvider which is basically a copy of MediaCodecVideoRenderer's VieoFrameProcessorManager. The MediaCodecVideoRenderer now instantiates a CompositingVideoSinkProvider instead of a VieoFrameProcessorManager. PiperOrigin-RevId: 555903928 --- .../video/CompositingVideoSinkProvider.java | 643 +++++++++++++ .../video/MediaCodecVideoRenderer.java | 846 ++++-------------- .../media3/exoplayer/video/VideoSink.java | 42 +- .../CompositingVideoSinkProviderTest.java | 169 ++++ 4 files changed, 1041 insertions(+), 659 deletions(-) create mode 100644 libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java create mode 100644 libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java new file mode 100644 index 0000000000..cdfc279988 --- /dev/null +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java @@ -0,0 +1,643 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.video; + +import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Assertions.checkStateNotNull; + +import android.content.Context; +import android.graphics.Bitmap; +import android.os.Handler; +import android.util.Pair; +import android.view.Surface; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.DebugViewProvider; +import androidx.media3.common.Effect; +import androidx.media3.common.Format; +import androidx.media3.common.FrameInfo; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.SurfaceInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.VideoSize; +import androidx.media3.common.util.Size; +import androidx.media3.common.util.TimedValueQueue; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.util.Util; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executor; +import org.checkerframework.checker.initialization.qual.Initialized; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** Handles composition of video sinks. */ +@UnstableApi +/* package */ final class CompositingVideoSinkProvider { + + private final Context context; + private final VideoFrameProcessor.Factory videoFrameProcessorFactory; + private final VideoSink.RenderControl renderControl; + + @Nullable private VideoSinkImpl videoSinkImpl; + @Nullable private List videoEffects; + @Nullable private VideoFrameMetadataListener videoFrameMetadataListener; + private boolean released; + + /** Creates a new instance. */ + public CompositingVideoSinkProvider( + Context context, + VideoFrameProcessor.Factory videoFrameProcessorFactory, + VideoSink.RenderControl renderControl) { + this.context = context; + this.videoFrameProcessorFactory = videoFrameProcessorFactory; + this.renderControl = renderControl; + } + + /** + * Initializes the provider for video frame processing. Can be called up to one time and only + * after video effects are {@linkplain #setVideoEffects(List) set}. + * + * @param sourceFormat The format of the compressed video. + * @throws VideoSink.VideoSinkException If enabling the provider failed. + */ + public void initialize(Format sourceFormat) throws VideoSink.VideoSinkException { + checkState(!released && videoSinkImpl == null); + checkStateNotNull(videoEffects); + + try { + videoSinkImpl = + new VideoSinkImpl(context, videoFrameProcessorFactory, renderControl, sourceFormat); + if (videoFrameMetadataListener != null) { + videoSinkImpl.setVideoFrameMetadataListener(videoFrameMetadataListener); + } + } catch (VideoFrameProcessingException e) { + throw new VideoSink.VideoSinkException(e, sourceFormat); + } + } + + /** Returns whether this provider is initialized for frame processing. */ + public boolean isInitialized() { + return videoSinkImpl != null; + } + + /** Releases the sink provider. */ + public void release() { + if (released) { + return; + } + + if (videoSinkImpl != null) { + videoSinkImpl.release(); + videoSinkImpl = null; + } + released = true; + } + + /** Returns a {@link VideoSink} to forward video frames for processing. */ + public VideoSink getSink() { + return checkStateNotNull(videoSinkImpl); + } + + /** Sets video effects on this provider. */ + public void setVideoEffects(List videoEffects) { + this.videoEffects = videoEffects; + if (isInitialized()) { + checkStateNotNull(videoSinkImpl).setVideoEffects(videoEffects); + } + } + + /** + * Sets the offset, in microseconds, that is added to the video frames presentation timestamps + * from the player. + * + *

Must be called after the sink provider is {@linkplain #initialize(Format) initialized}. + */ + public void setStreamOffsetUs(long streamOffsetUs) { + checkStateNotNull(videoSinkImpl).setStreamOffsetUs(streamOffsetUs); + } + + /** + * Sets the output surface info. + * + *

Must be called after the sink provider is {@linkplain #initialize(Format) initialized}. + */ + public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) { + checkStateNotNull(videoSinkImpl).setOutputSurfaceInfo(outputSurface, outputResolution); + } + + /** + * Clears the set output surface info. + * + *

Must be called after the sink provider is {@linkplain #initialize(Format) initialized}. + */ + public void clearOutputSurfaceInfo() { + checkStateNotNull(videoSinkImpl).clearOutputSurfaceInfo(); + } + + /** Sets a {@link VideoFrameMetadataListener} which is used in the returned {@link VideoSink}. */ + public void setVideoFrameMetadataListener(VideoFrameMetadataListener videoFrameMetadataListener) { + this.videoFrameMetadataListener = videoFrameMetadataListener; + if (isInitialized()) { + checkStateNotNull(videoSinkImpl).setVideoFrameMetadataListener(videoFrameMetadataListener); + } + } + + private static final class VideoSinkImpl implements VideoSink, VideoFrameProcessor.Listener { + + private final Context context; + private final RenderControl renderControl; + private final VideoFrameProcessor videoFrameProcessor; + // TODO b/293447478 - Use a queue for primitive longs to avoid the cost of boxing to Long. + private final ArrayDeque processedFramesBufferTimestampsUs; + private final TimedValueQueue streamOffsets; + private final TimedValueQueue videoSizeChanges; + private final Handler handler; + private final int videoFrameProcessorMaxPendingFrameCount; + private final ArrayList videoEffects; + @Nullable private final Effect rotationEffect; + + private VideoSink.@MonotonicNonNull Listener listener; + private @MonotonicNonNull Executor listenerExecutor; + @Nullable private VideoFrameMetadataListener videoFrameMetadataListener; + @Nullable private Format inputFormat; + @Nullable private Pair currentSurfaceAndSize; + + /** + * Whether the last frame of the current stream is decoded and registered to {@link + * VideoFrameProcessor}. + */ + private boolean registeredLastFrame; + + /** + * Whether the last frame of the current stream is processed by the {@link VideoFrameProcessor}. + */ + private boolean processedLastFrame; + + /** Whether the last frame of the current stream is released to the output {@link Surface}. */ + private boolean releasedLastFrame; + + private long lastCodecBufferPresentationTimestampUs; + private VideoSize processedFrameSize; + private VideoSize reportedVideoSize; + private boolean pendingVideoSizeChange; + private boolean renderedFirstFrame; + private long inputStreamOffsetUs; + private boolean pendingInputStreamOffsetChange; + private long outputStreamOffsetUs; + private float playbackSpeed; + + // TODO b/292111083 - Remove the field and trigger the callback on every video size change. + private boolean onVideoSizeChangedCalled; + + /** Creates a new instance. */ + public VideoSinkImpl( + Context context, + VideoFrameProcessor.Factory videoFrameProcessorFactory, + RenderControl renderControl, + Format sourceFormat) + throws VideoFrameProcessingException { + this.context = context; + this.renderControl = renderControl; + processedFramesBufferTimestampsUs = new ArrayDeque<>(); + streamOffsets = new TimedValueQueue<>(); + videoSizeChanges = new TimedValueQueue<>(); + // TODO b/226330223 - Investigate increasing frame count when frame dropping is + // allowed. + // TODO b/278234847 - Evaluate whether limiting frame count when frame dropping is not allowed + // reduces decoder timeouts, and consider restoring. + videoFrameProcessorMaxPendingFrameCount = + Util.getMaxPendingFramesCountForMediaCodecDecoders(context); + lastCodecBufferPresentationTimestampUs = C.TIME_UNSET; + processedFrameSize = VideoSize.UNKNOWN; + reportedVideoSize = VideoSize.UNKNOWN; + playbackSpeed = 1f; + + // Playback thread handler. + handler = Util.createHandlerForCurrentLooper(); + + Pair inputAndOutputColorInfos = + experimentalGetVideoFrameProcessorColorConfiguration(sourceFormat.colorInfo); + + @SuppressWarnings("nullness:assignment") + @Initialized + VideoSinkImpl thisRef = this; + videoFrameProcessor = + videoFrameProcessorFactory.create( + context, + DebugViewProvider.NONE, + inputAndOutputColorInfos.first, + inputAndOutputColorInfos.second, + /* renderFramesAutomatically= */ false, + /* listenerExecutor= */ handler::post, + thisRef); + if (currentSurfaceAndSize != null) { + Size outputSurfaceSize = currentSurfaceAndSize.second; + videoFrameProcessor.setOutputSurfaceInfo( + new SurfaceInfo( + currentSurfaceAndSize.first, + outputSurfaceSize.getWidth(), + outputSurfaceSize.getHeight())); + } + videoEffects = new ArrayList<>(); + // MediaCodec applies rotation after API 21 + rotationEffect = + Util.SDK_INT < 21 && sourceFormat.rotationDegrees != 0 + ? ScaleAndRotateAccessor.createRotationEffect(sourceFormat.rotationDegrees) + : null; + } + + // VideoSink impl + + @Override + public void flush() { + videoFrameProcessor.flush(); + processedFramesBufferTimestampsUs.clear(); + streamOffsets.clear(); + handler.removeCallbacksAndMessages(/* token= */ null); + renderedFirstFrame = false; + if (registeredLastFrame) { + registeredLastFrame = false; + processedLastFrame = false; + releasedLastFrame = false; + } + } + + @Override + public boolean isReady() { + return renderedFirstFrame; + } + + @Override + public boolean isEnded() { + return releasedLastFrame; + } + + @Override + public void registerInputStream(@InputType int inputType, Format format) { + if (inputType != INPUT_TYPE_SURFACE) { + throw new UnsupportedOperationException("Unsupported input type " + inputType); + } + this.inputFormat = format; + maybeRegisterInputStream(); + + if (registeredLastFrame) { + registeredLastFrame = false; + processedLastFrame = false; + releasedLastFrame = false; + } + } + + @Override + public void setListener(Listener listener, Executor executor) { + if (Util.areEqual(this.listener, listener)) { + checkState(Util.areEqual(listenerExecutor, executor)); + return; + } + this.listener = listener; + this.listenerExecutor = executor; + } + + @Override + public boolean isFrameDropAllowedOnInput() { + return Util.isFrameDropAllowedOnSurfaceInput(context); + } + + @Override + public Surface getInputSurface() { + return videoFrameProcessor.getInputSurface(); + } + + @Override + public long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame) { + checkState(videoFrameProcessorMaxPendingFrameCount != C.LENGTH_UNSET); + if (videoFrameProcessor.getPendingInputFrameCount() + >= videoFrameProcessorMaxPendingFrameCount) { + return C.TIME_UNSET; + } + videoFrameProcessor.registerInputFrame(); + // The sink takes in frames with monotonically increasing, non-offset frame + // timestamps. That is, with two ten-second long videos, the first frame of the second video + // should bear a timestamp of 10s seen from VideoFrameProcessor; while in ExoPlayer, the + // timestamp of the said frame would be 0s, but the streamOffset is incremented 10s to include + // the duration of the first video. Thus this correction is need to correct for the different + // handling of presentation timestamps in ExoPlayer and VideoFrameProcessor. + long bufferPresentationTimeUs = framePresentationTimeUs + inputStreamOffsetUs; + if (pendingInputStreamOffsetChange) { + streamOffsets.add(bufferPresentationTimeUs, inputStreamOffsetUs); + pendingInputStreamOffsetChange = false; + } + if (isLastFrame) { + registeredLastFrame = true; + lastCodecBufferPresentationTimestampUs = bufferPresentationTimeUs; + } + return bufferPresentationTimeUs * 1000; + } + + @Override + public boolean queueBitmap(Bitmap inputBitmap, long durationUs, float frameRate) { + throw new UnsupportedOperationException(); + } + + @Override + public void render(long positionUs, long elapsedRealtimeUs) { + while (!processedFramesBufferTimestampsUs.isEmpty()) { + long bufferPresentationTimeUs = checkNotNull(processedFramesBufferTimestampsUs.peek()); + // check whether this buffer comes with a new stream offset. + if (maybeUpdateOutputStreamOffset(bufferPresentationTimeUs)) { + renderedFirstFrame = false; + } + long framePresentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs; + boolean isLastFrame = processedLastFrame && processedFramesBufferTimestampsUs.size() == 1; + long frameRenderTimeNs = + renderControl.getFrameRenderTimeNs( + bufferPresentationTimeUs, positionUs, elapsedRealtimeUs, playbackSpeed); + if (frameRenderTimeNs == RenderControl.RENDER_TIME_TRY_AGAIN_LATER) { + return; + } else if (framePresentationTimeUs == RenderControl.RENDER_TIME_DROP) { + // TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush + // VideoFrameProcessor input frames in this case. + releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame); + continue; + } + renderControl.onNextFrame(bufferPresentationTimeUs); + if (videoFrameMetadataListener != null) { + videoFrameMetadataListener.onVideoFrameAboutToBeRendered( + framePresentationTimeUs, + frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY + ? System.nanoTime() + : frameRenderTimeNs, + checkNotNull(inputFormat), + /* mediaFormat= */ null); + } + releaseProcessedFrameInternal( + frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY + ? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY + : frameRenderTimeNs, + isLastFrame); + + maybeNotifyVideoSizeChanged(bufferPresentationTimeUs); + } + } + + @Override + public void setPlaybackSpeed(float speed) { + checkArgument(speed >= 0.0); + this.playbackSpeed = speed; + } + + // VideoFrameProcessor.Listener impl + + @Override + public void onOutputSizeChanged(int width, int height) { + VideoSize newVideoSize = new VideoSize(width, height); + if (!processedFrameSize.equals(newVideoSize)) { + processedFrameSize = newVideoSize; + pendingVideoSizeChange = true; + } + } + + @Override + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + if (pendingVideoSizeChange) { + videoSizeChanges.add(presentationTimeUs, processedFrameSize); + pendingVideoSizeChange = false; + } + if (registeredLastFrame) { + checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET); + } + processedFramesBufferTimestampsUs.add(presentationTimeUs); + // TODO b/257464707 - Support extensively modified media. + if (registeredLastFrame && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) { + processedLastFrame = true; + } + } + + @Override + public void onError(VideoFrameProcessingException exception) { + if (listener == null || listenerExecutor == null) { + return; + } + listenerExecutor.execute( + () -> { + if (listener != null) { + listener.onError( + /* videoSink= */ this, + new VideoSink.VideoSinkException( + exception, + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_RAW) + .setWidth(processedFrameSize.width) + .setHeight(processedFrameSize.height) + .build())); + } + }); + } + + @Override + public void onEnded() { + throw new IllegalStateException(); + } + + // Other methods + + public void release() { + videoFrameProcessor.release(); + handler.removeCallbacksAndMessages(/* token= */ null); + streamOffsets.clear(); + processedFramesBufferTimestampsUs.clear(); + renderedFirstFrame = false; + } + + /** Sets the {@linkplain Effect video effects}. */ + public void setVideoEffects(List videoEffects) { + this.videoEffects.clear(); + this.videoEffects.addAll(videoEffects); + maybeRegisterInputStream(); + } + + public void setStreamOffsetUs(long streamOffsetUs) { + pendingInputStreamOffsetChange = inputStreamOffsetUs != streamOffsetUs; + inputStreamOffsetUs = streamOffsetUs; + } + + public void setVideoFrameMetadataListener( + VideoFrameMetadataListener videoFrameMetadataListener) { + this.videoFrameMetadataListener = videoFrameMetadataListener; + } + + private void maybeRegisterInputStream() { + if (inputFormat == null) { + return; + } + + ArrayList effects = new ArrayList<>(); + if (rotationEffect != null) { + effects.add(rotationEffect); + } + effects.addAll(videoEffects); + Format inputFormat = checkNotNull(this.inputFormat); + videoFrameProcessor.registerInputStream( + VideoFrameProcessor.INPUT_TYPE_SURFACE, + effects, + new FrameInfo.Builder(inputFormat.width, inputFormat.height) + .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio) + .build()); + } + + /** + * Returns a {@link Pair} of {@linkplain ColorInfo input color} and {@linkplain ColorInfo output + * color} to configure the {@code VideoFrameProcessor}. + */ + private static Pair experimentalGetVideoFrameProcessorColorConfiguration( + @Nullable ColorInfo inputColorInfo) { + // TODO b/279163661 - Remove this method after VideoFrameProcessor supports texture ID + // input/output. + // explicit check for nullness + if (inputColorInfo == null || !ColorInfo.isTransferHdr(inputColorInfo)) { + return Pair.create(ColorInfo.SDR_BT709_LIMITED, ColorInfo.SDR_BT709_LIMITED); + } + + if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG) { + // SurfaceView only supports BT2020 PQ input, converting HLG to PQ. + return Pair.create( + inputColorInfo, + inputColorInfo.buildUpon().setColorTransfer(C.COLOR_TRANSFER_ST2084).build()); + } + + return Pair.create(inputColorInfo, inputColorInfo); + } + + /** + * Sets the output surface info. + * + * @param outputSurface The {@link Surface} to which {@link VideoFrameProcessor} outputs. + * @param outputResolution The {@link Size} of the output resolution. + */ + public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) { + if (currentSurfaceAndSize != null + && currentSurfaceAndSize.first.equals(outputSurface) + && currentSurfaceAndSize.second.equals(outputResolution)) { + return; + } + renderedFirstFrame = + currentSurfaceAndSize == null || currentSurfaceAndSize.first.equals(outputSurface); + currentSurfaceAndSize = Pair.create(outputSurface, outputResolution); + videoFrameProcessor.setOutputSurfaceInfo( + new SurfaceInfo( + outputSurface, outputResolution.getWidth(), outputResolution.getHeight())); + } + + /** Clears the set output surface info. */ + public void clearOutputSurfaceInfo() { + videoFrameProcessor.setOutputSurfaceInfo(null); + currentSurfaceAndSize = null; + renderedFirstFrame = false; + } + + private boolean maybeUpdateOutputStreamOffset(long bufferPresentationTimeUs) { + boolean updatedOffset = false; + @Nullable Long newOutputStreamOffsetUs = streamOffsets.pollFloor(bufferPresentationTimeUs); + if (newOutputStreamOffsetUs != null && newOutputStreamOffsetUs != outputStreamOffsetUs) { + outputStreamOffsetUs = newOutputStreamOffsetUs; + updatedOffset = true; + } + return updatedOffset; + } + + private void releaseProcessedFrameInternal(long releaseTimeNs, boolean isLastFrame) { + videoFrameProcessor.renderOutputFrame(releaseTimeNs); + processedFramesBufferTimestampsUs.remove(); + if (releaseTimeNs != VideoFrameProcessor.DROP_OUTPUT_FRAME) { + renderControl.onFrameRendered(); + if (!renderedFirstFrame) { + if (listener != null) { + checkNotNull(listenerExecutor) + .execute(() -> checkNotNull(listener).onFirstFrameRendered(this)); + } + renderedFirstFrame = true; + } + } + if (isLastFrame) { + releasedLastFrame = true; + } + } + + private void maybeNotifyVideoSizeChanged(long bufferPresentationTimeUs) { + if (onVideoSizeChangedCalled || listener == null) { + return; + } + + @Nullable VideoSize videoSize = videoSizeChanges.pollFloor(bufferPresentationTimeUs); + if (videoSize == null) { + return; + } + + if (!videoSize.equals(VideoSize.UNKNOWN) && !videoSize.equals(reportedVideoSize)) { + reportedVideoSize = videoSize; + checkNotNull(listenerExecutor) + .execute(() -> checkNotNull(listener).onVideoSizeChanged(this, videoSize)); + } + onVideoSizeChangedCalled = true; + } + + private static final class ScaleAndRotateAccessor { + private static @MonotonicNonNull Constructor + scaleAndRotateTransformationBuilderConstructor; + private static @MonotonicNonNull Method setRotationMethod; + private static @MonotonicNonNull Method buildScaleAndRotateTransformationMethod; + + public static Effect createRotationEffect(float rotationDegrees) { + try { + prepare(); + Object builder = scaleAndRotateTransformationBuilderConstructor.newInstance(); + setRotationMethod.invoke(builder, rotationDegrees); + return (Effect) checkNotNull(buildScaleAndRotateTransformationMethod.invoke(builder)); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + + @EnsuresNonNull({ + "scaleAndRotateTransformationBuilderConstructor", + "setRotationMethod", + "buildScaleAndRotateTransformationMethod" + }) + private static void prepare() throws NoSuchMethodException, ClassNotFoundException { + if (scaleAndRotateTransformationBuilderConstructor == null + || setRotationMethod == null + || buildScaleAndRotateTransformationMethod == null) { + // TODO: b/284964524 - Add LINT and proguard checks for media3.effect reflection. + Class scaleAndRotateTransformationBuilderClass = + Class.forName("androidx.media3.effect.ScaleAndRotateTransformation$Builder"); + scaleAndRotateTransformationBuilderConstructor = + scaleAndRotateTransformationBuilderClass.getConstructor(); + setRotationMethod = + scaleAndRotateTransformationBuilderClass.getMethod("setRotationDegrees", float.class); + buildScaleAndRotateTransformationMethod = + scaleAndRotateTransformationBuilderClass.getMethod("build"); + } + } + } + } +} diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 1957db36bc..17f7c7edaf 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -18,7 +18,6 @@ package androidx.media3.exoplayer.video; import static android.view.Display.DEFAULT_DISPLAY; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; -import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.common.util.Util.msToUs; import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_MAX_INPUT_SIZE_EXCEEDED; import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_VIDEO_MAX_RESOLUTION_EXCEEDED; @@ -53,10 +52,8 @@ import androidx.media3.common.DebugViewProvider; import androidx.media3.common.DrmInitData; import androidx.media3.common.Effect; import androidx.media3.common.Format; -import androidx.media3.common.FrameInfo; import androidx.media3.common.MimeTypes; import androidx.media3.common.PlaybackException; -import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoSize; @@ -87,17 +84,10 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener.EventDispatche import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; -import com.google.errorprone.annotations.CanIgnoreReturnValue; -import java.lang.reflect.Constructor; -import java.lang.reflect.Method; +import com.google.common.util.concurrent.MoreExecutors; import java.nio.ByteBuffer; -import java.util.ArrayDeque; import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executor; -import org.checkerframework.checker.initialization.qual.UnderInitialization; -import org.checkerframework.checker.nullness.qual.EnsuresNonNull; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * Decodes and renders video using {@link MediaCodec}. @@ -123,7 +113,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * */ @UnstableApi -public class MediaCodecVideoRenderer extends MediaCodecRenderer { +public class MediaCodecVideoRenderer extends MediaCodecRenderer implements VideoSink.RenderControl { private static final String TAG = "MediaCodecVideoRenderer"; private static final String KEY_CROP_LEFT = "crop-left"; @@ -147,13 +137,16 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { /** The minimum input buffer size for HEVC. */ private static final int HEVC_MAX_INPUT_SIZE_THRESHOLD = 2 * 1024 * 1024; + /** The maximum earliest time, in microseconds, to release a frame on the surface. */ + private static final long MAX_EARLY_US_THRESHOLD = 50_000; + private static boolean evaluatedDeviceNeedsSetOutputSurfaceWorkaround; private static boolean deviceNeedsSetOutputSurfaceWorkaround; private final Context context; private final VideoFrameReleaseHelper frameReleaseHelper; + private final CompositingVideoSinkProvider videoSinkProvider; private final EventDispatcher eventDispatcher; - private final VideoFrameProcessorManager videoFrameProcessorManager; private final long allowedJoiningTimeMs; private final int maxDroppedFramesToNotify; private final boolean deviceNeedsNoPostProcessWorkaround; @@ -161,7 +154,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { private CodecMaxValues codecMaxValues; private boolean codecNeedsSetOutputSurfaceWorkaround; private boolean codecHandlesHdr10PlusOutOfBandMetadata; - @Nullable private Surface displaySurface; @Nullable private PlaceholderSurface placeholderSurface; private boolean haveReportedFirstFrameRenderedForCurrentSurface; @@ -178,14 +170,16 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { private long totalVideoFrameProcessingOffsetUs; private int videoFrameProcessingOffsetCount; private long lastFrameReleaseTimeNs; - private VideoSize decodedVideoSize; @Nullable private VideoSize reportedVideoSize; + private boolean hasEffects; + private boolean hasInitializedPlayback; private boolean tunneling; private int tunnelingAudioSessionId; /* package */ @Nullable OnFrameRenderedListenerV23 tunnelingOnFrameRenderedListener; @Nullable private VideoFrameMetadataListener frameMetadataListener; + @Nullable private VideoSink videoSink; /** * @param context A context. @@ -402,9 +396,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { this.context = context.getApplicationContext(); frameReleaseHelper = new VideoFrameReleaseHelper(this.context); eventDispatcher = new EventDispatcher(eventHandler, eventListener); - videoFrameProcessorManager = - new VideoFrameProcessorManager( - videoFrameProcessorFactory, frameReleaseHelper, /* renderer= */ this); + videoSinkProvider = + new CompositingVideoSinkProvider( + context, videoFrameProcessorFactory, /* renderControl= */ this); deviceNeedsNoPostProcessWorkaround = deviceNeedsNoPostProcessWorkaround(); joiningDeadlineMs = C.TIME_UNSET; scalingMode = C.VIDEO_SCALING_MODE_DEFAULT; @@ -527,6 +521,49 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { format); } + // RenderControl implementation + + @Override + public long getFrameRenderTimeNs( + long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed) { + long earlyUs = + calculateEarlyTimeUs( + positionUs, + elapsedRealtimeUs, + presentationTimeUs, + getState() == STATE_STARTED, + playbackSpeed, + getClock()); + if (isBufferLate(earlyUs)) { + return VideoSink.RenderControl.RENDER_TIME_DROP; + } + if (shouldForceRender(positionUs, earlyUs)) { + return VideoSink.RenderControl.RENDER_TIME_IMMEDIATELY; + } + + if (getState() != STATE_STARTED + || positionUs == initialPositionUs + || earlyUs > MAX_EARLY_US_THRESHOLD) { + return VideoSink.RenderControl.RENDER_TIME_TRY_AGAIN_LATER; + } + // Compute the buffer's desired release time in nanoseconds. + long unadjustedFrameReleaseTimeNs = getClock().nanoTime() + (earlyUs * 1000); + // Apply a timestamp adjustment, if there is one. + return frameReleaseHelper.adjustReleaseTime(unadjustedFrameReleaseTimeNs); + } + + @Override + public void onNextFrame(long presentationTimeUs) { + frameReleaseHelper.onNextFrame(presentationTimeUs); + } + + @Override + public void onFrameRendered() { + lastRenderRealtimeUs = Util.msToUs(getClock().elapsedRealtime()); + } + + // Other methods + /** * Returns a list of decoders that can decode media in the specified format, in the priority order * specified by the {@link MediaCodecSelector}. Note that since the {@link MediaCodecSelector} @@ -616,9 +653,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { @Override protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException { super.onPositionReset(positionUs, joining); - if (videoFrameProcessorManager.isEnabled()) { - videoFrameProcessorManager.flush(); + if (videoSink != null) { + videoSink.flush(); } + + if (videoSinkProvider.isInitialized()) { + videoSinkProvider.setStreamOffsetUs(getOutputStreamOffsetUs()); + } + lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); frameReleaseHelper.onPositionReset(); lastBufferPresentationTimeUs = C.TIME_UNSET; @@ -633,17 +675,13 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { @Override public boolean isEnded() { - boolean isEnded = super.isEnded(); - if (videoFrameProcessorManager.isEnabled()) { - isEnded &= videoFrameProcessorManager.releasedLastFrame(); - } - return isEnded; + return super.isEnded() && (videoSink == null || videoSink.isEnded()); } @Override public boolean isReady() { if (super.isReady() - && (!videoFrameProcessorManager.isEnabled() || videoFrameProcessorManager.isReady()) + && (videoSink == null || videoSink.isReady()) && (firstFrameState == C.FIRST_FRAME_RENDERED || (placeholderSurface != null && displaySurface == placeholderSurface) || getCodec() == null @@ -705,15 +743,21 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { try { super.onReset(); } finally { - if (videoFrameProcessorManager.isEnabled()) { - videoFrameProcessorManager.reset(); - } + hasInitializedPlayback = false; if (placeholderSurface != null) { releasePlaceholderSurface(); } } } + @Override + protected void onRelease() { + super.onRelease(); + if (videoSinkProvider.isInitialized()) { + videoSinkProvider.release(); + } + } + @Override public void handleMessage(@MessageType int messageType, @Nullable Object message) throws ExoPlaybackException { @@ -733,6 +777,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { break; case MSG_SET_VIDEO_FRAME_METADATA_LISTENER: frameMetadataListener = (VideoFrameMetadataListener) message; + videoSinkProvider.setVideoFrameMetadataListener(frameMetadataListener); break; case MSG_SET_AUDIO_SESSION_ID: int tunnelingAudioSessionId = (int) message; @@ -746,14 +791,16 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { case MSG_SET_VIDEO_EFFECTS: @SuppressWarnings("unchecked") List videoEffects = (List) checkNotNull(message); - videoFrameProcessorManager.setVideoEffects(videoEffects); + videoSinkProvider.setVideoEffects(videoEffects); + hasEffects = true; break; case MSG_SET_VIDEO_OUTPUT_RESOLUTION: Size outputResolution = (Size) checkNotNull(message); - if (outputResolution.getWidth() != 0 + if (videoSinkProvider.isInitialized() + && outputResolution.getWidth() != 0 && outputResolution.getHeight() != 0 && displaySurface != null) { - videoFrameProcessorManager.setOutputSurfaceInfo(displaySurface, outputResolution); + videoSinkProvider.setOutputSurfaceInfo(displaySurface, outputResolution); } break; case MSG_SET_AUDIO_ATTRIBUTES: @@ -792,7 +839,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { @State int state = getState(); @Nullable MediaCodecAdapter codec = getCodec(); - if (codec != null && !videoFrameProcessorManager.isEnabled()) { + if (codec != null && videoSinkProvider.isInitialized()) { if (Util.SDK_INT >= 23 && displaySurface != null && !codecNeedsSetOutputSurfaceWorkaround) { setOutputSurfaceV23(codec, displaySurface); } else { @@ -809,17 +856,16 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { // Set joining deadline to report MediaCodecVideoRenderer is ready. setJoiningDeadlineMs(); } - // When VideoFrameProcessorManager is enabled, set VideoFrameProcessorManager's display - // surface and an unknown size. - if (videoFrameProcessorManager.isEnabled()) { - videoFrameProcessorManager.setOutputSurfaceInfo(displaySurface, Size.UNKNOWN); + // When effects previewing is enabled, set display surface and an unknown size. + if (videoSinkProvider.isInitialized()) { + videoSinkProvider.setOutputSurfaceInfo(displaySurface, Size.UNKNOWN); } } else { // The display surface has been removed. clearReportedVideoSize(); lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED); - if (videoFrameProcessorManager.isEnabled()) { - videoFrameProcessorManager.clearOutputSurfaceInfo(); + if (videoSinkProvider.isInitialized()) { + videoSinkProvider.clearOutputSurfaceInfo(); } } } else if (displaySurface != null && displaySurface != placeholderSurface) { @@ -871,21 +917,22 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } displaySurface = placeholderSurface; } - - if (videoFrameProcessorManager.isEnabled()) { - mediaFormat = videoFrameProcessorManager.amendMediaFormatKeys(mediaFormat); - } - + maybeSetKeyAllowFrameDrop(mediaFormat); return MediaCodecAdapter.Configuration.createForVideoDecoding( codecInfo, mediaFormat, format, - videoFrameProcessorManager.isEnabled() - ? videoFrameProcessorManager.getInputSurface() - : displaySurface, + videoSink != null ? videoSink.getInputSurface() : displaySurface, crypto); } + @SuppressWarnings("InlinedApi") // VideoSink will check the API level + private void maybeSetKeyAllowFrameDrop(MediaFormat mediaFormat) { + if (videoSink != null && !videoSink.isFrameDropAllowedOnInput()) { + mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0); + } + } + @Override protected DecoderReuseEvaluation canReuseCodec( MediaCodecInfo codecInfo, Format oldFormat, Format newFormat) { @@ -911,8 +958,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { @Override public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { super.render(positionUs, elapsedRealtimeUs); - if (videoFrameProcessorManager.isEnabled()) { - videoFrameProcessorManager.releaseProcessedFrames(positionUs, elapsedRealtimeUs); + if (videoSink != null) { + videoSink.render(positionUs, elapsedRealtimeUs); } } @@ -928,6 +975,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { throws ExoPlaybackException { super.setPlaybackSpeed(currentPlaybackSpeed, targetPlaybackSpeed); frameReleaseHelper.onPlaybackSpeed(currentPlaybackSpeed); + if (videoSink != null) { + videoSink.setPlaybackSpeed(currentPlaybackSpeed); + } } /** @@ -1019,9 +1069,51 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { @CallSuper @Override protected void onReadyToInitializeCodec(Format format) throws ExoPlaybackException { - if (!videoFrameProcessorManager.isEnabled()) { - videoFrameProcessorManager.maybeEnable(format, getOutputStreamOffsetUs(), getClock()); + // We only enable effects preview on the first time a codec is initialized and if effects are + // already set. We do not enable effects mid-playback. For effects to be enabled after + // playback has started, the renderer needs to be reset first. + if (hasEffects && !hasInitializedPlayback && !videoSinkProvider.isInitialized()) { + try { + videoSinkProvider.initialize(format); + videoSinkProvider.setStreamOffsetUs(getOutputStreamOffsetUs()); + if (frameMetadataListener != null) { + videoSinkProvider.setVideoFrameMetadataListener(frameMetadataListener); + } + } catch (VideoSink.VideoSinkException e) { + throw createRendererException( + e, format, PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSOR_INIT_FAILED); + } } + + if (videoSink == null && videoSinkProvider.isInitialized()) { + videoSink = videoSinkProvider.getSink(); + videoSink.setListener( + new VideoSink.Listener() { + @Override + public void onFirstFrameRendered(VideoSink videoSink) { + maybeNotifyRenderedFirstFrame(); + } + + @Override + public void onVideoSizeChanged(VideoSink videoSink, VideoSize videoSize) { + maybeNotifyVideoSizeChanged(videoSize); + } + + @Override + public void onError( + VideoSink videoSink, VideoSink.VideoSinkException videoSinkException) { + setPendingPlaybackException( + createRendererException( + videoSinkException, + videoSinkException.format, + PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED)); + } + }, + // Pass a direct executor since the callback handling involves posting on the app looper + // again, so there's no need to do two hops. + MoreExecutors.directExecutor()); + } + hasInitializedPlayback = true; } @Override @@ -1037,7 +1129,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { if (Util.SDK_INT >= 23 && tunneling) { tunnelingOnFrameRenderedListener = new OnFrameRenderedListenerV23(checkNotNull(getCodec())); } - videoFrameProcessorManager.onCodecInitialized(name); } @Override @@ -1125,16 +1216,17 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { height = rotatedHeight; pixelWidthHeightRatio = 1 / pixelWidthHeightRatio; } - } else if (!videoFrameProcessorManager.isEnabled()) { - // Neither the codec nor the VideoFrameProcessor applies the rotation. + } else if (videoSink == null) { + // Neither the codec nor the video sink applies the rotation. unappliedRotationDegrees = format.rotationDegrees; } decodedVideoSize = new VideoSize(width, height, unappliedRotationDegrees, pixelWidthHeightRatio); frameReleaseHelper.onFormatChanged(format.frameRate); - if (videoFrameProcessorManager.isEnabled()) { - videoFrameProcessorManager.setInputFormat( + if (videoSink != null) { + videoSink.registerInputStream( + /* inputType= */ VideoSink.INPUT_TYPE_SURFACE, format .buildUpon() .setWidth(width) @@ -1197,7 +1289,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } if (bufferPresentationTimeUs != lastBufferPresentationTimeUs) { - if (!videoFrameProcessorManager.isEnabled()) { + if (videoSink == null) { frameReleaseHelper.onNextFrame(bufferPresentationTimeUs); } // else, update the frameReleaseHelper when releasing the processed frames. this.lastBufferPresentationTimeUs = bufferPresentationTimeUs; @@ -1211,16 +1303,15 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return true; } - // Note: Use of double rather than float is intentional for accuracy in the calculations below. boolean isStarted = getState() == STATE_STARTED; - long elapsedRealtimeNowUs = msToUs(getClock().elapsedRealtime()); long earlyUs = calculateEarlyTimeUs( positionUs, elapsedRealtimeUs, - elapsedRealtimeNowUs, bufferPresentationTimeUs, - isStarted); + isStarted, + getPlaybackSpeed(), + getClock()); if (displaySurface == placeholderSurface) { // Skip frames in sync with playback, so we'll be at the right frame if the mode changes. @@ -1232,20 +1323,21 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return false; } + if (videoSink != null) { + videoSink.render(positionUs, elapsedRealtimeUs); + long releaseTimeNs = videoSink.registerInputFrame(presentationTimeUs, isLastBuffer); + if (releaseTimeNs == C.TIME_UNSET) { + return false; + } + renderOutputBuffer(codec, bufferIndex, presentationTimeUs, releaseTimeNs); + return true; + } + boolean forceRenderOutputBuffer = shouldForceRender(positionUs, earlyUs); if (forceRenderOutputBuffer) { - boolean notifyFrameMetaDataListener; - if (videoFrameProcessorManager.isEnabled()) { - notifyFrameMetaDataListener = false; - if (!videoFrameProcessorManager.maybeRegisterFrame( - format, presentationTimeUs, isLastBuffer)) { - return false; - } - } else { - notifyFrameMetaDataListener = true; - } - renderOutputBufferNow( - codec, format, bufferIndex, presentationTimeUs, notifyFrameMetaDataListener); + long releaseTimeNs = getClock().nanoTime(); + notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); + renderOutputBuffer(codec, bufferIndex, presentationTimeUs, releaseTimeNs); updateVideoFrameProcessingOffsetCounters(earlyUs); return true; } @@ -1257,13 +1349,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { // Compute the buffer's desired release time in nanoseconds. long systemTimeNs = getClock().nanoTime(); long unadjustedFrameReleaseTimeNs = systemTimeNs + (earlyUs * 1000); - // Apply a timestamp adjustment, if there is one. long adjustedReleaseTimeNs = frameReleaseHelper.adjustReleaseTime(unadjustedFrameReleaseTimeNs); - if (!videoFrameProcessorManager.isEnabled()) { - earlyUs = (adjustedReleaseTimeNs - systemTimeNs) / 1000; - } // else, use the unadjusted earlyUs in previewing use cases. - + earlyUs = (adjustedReleaseTimeNs - systemTimeNs) / 1000; boolean treatDroppedBuffersAsSkipped = joiningDeadlineMs != C.TIME_UNSET; if (shouldDropBuffersToKeyframe(earlyUs, elapsedRealtimeUs, isLastBuffer) && maybeDropBuffersToKeyframe(positionUs, treatDroppedBuffersAsSkipped)) { @@ -1278,23 +1366,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return true; } - if (videoFrameProcessorManager.isEnabled()) { - videoFrameProcessorManager.releaseProcessedFrames(positionUs, elapsedRealtimeUs); - if (videoFrameProcessorManager.maybeRegisterFrame(format, presentationTimeUs, isLastBuffer)) { - renderOutputBufferNow( - codec, - format, - bufferIndex, - presentationTimeUs, - /* notifyFrameMetadataListener= */ false); - return true; - } - return false; - } - if (Util.SDK_INT >= 21) { // Let the underlying framework time the release. - if (earlyUs < 50000) { + if (earlyUs < MAX_EARLY_US_THRESHOLD) { if (shouldSkipBuffersWithIdenticalReleaseTime() && adjustedReleaseTimeNs == lastFrameReleaseTimeNs) { // This frame should be displayed on the same vsync with the previous released frame. We @@ -1365,29 +1439,28 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { * iteration of the rendering loop. * @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, measured at the * start of the current iteration of the rendering loop. - * @param elapsedRealtimeNowUs {@link SystemClock#elapsedRealtime()} in microseconds, measured - * before calling this method. * @param bufferPresentationTimeUs The presentation time of the output buffer in microseconds, * with {@linkplain #getOutputStreamOffsetUs() stream offset added}. * @param isStarted Whether the playback is in {@link #STATE_STARTED}. + * @param playbackSpeed The current playback speed. + * @param clock The {@link Clock} used by the renderer. * @return The calculated early time, in microseconds. */ - private long calculateEarlyTimeUs( + private static long calculateEarlyTimeUs( long positionUs, long elapsedRealtimeUs, - long elapsedRealtimeNowUs, long bufferPresentationTimeUs, - boolean isStarted) { - // Note: Use of double rather than float is intentional for accuracy in the calculations below. - double playbackSpeed = getPlaybackSpeed(); - + boolean isStarted, + float playbackSpeed, + Clock clock) { // Calculate how early we are. In other words, the realtime duration that needs to elapse whilst // the renderer is started before the frame should be rendered. A negative value means that // we're already late. - long earlyUs = (long) ((bufferPresentationTimeUs - positionUs) / playbackSpeed); + // Note: Use of double rather than float is intentional for accuracy in the calculations below. + long earlyUs = (long) ((bufferPresentationTimeUs - positionUs) / (double) playbackSpeed); if (isStarted) { // Account for the elapsed time since the start of this iteration of the rendering loop. - earlyUs -= elapsedRealtimeNowUs - elapsedRealtimeUs; + earlyUs -= Util.msToUs(clock.elapsedRealtime()) - elapsedRealtimeUs; } return earlyUs; @@ -1428,6 +1501,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { protected void onProcessedStreamChange() { super.onProcessedStreamChange(); lowerFirstFrameState(C.FIRST_FRAME_NOT_RENDERED_AFTER_STREAM_CHANGE); + if (videoSinkProvider.isInitialized()) { + videoSinkProvider.setStreamOffsetUs(getOutputStreamOffsetUs()); + } } /** @@ -1538,8 +1614,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { droppedSourceBufferCount, /* droppedDecoderBufferCount= */ buffersInCodecCount); } flushOrReinitializeCodec(); - if (videoFrameProcessorManager.isEnabled()) { - videoFrameProcessorManager.flush(); + if (videoSink != null) { + videoSink.flush(); } return true; } @@ -1577,57 +1653,17 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { videoFrameProcessingOffsetCount++; } - /** - * Returns a {@link Pair} of {@linkplain ColorInfo input color} and {@linkplain ColorInfo output - * color} to configure the {@code VideoFrameProcessor}. - */ - protected Pair experimentalGetVideoFrameProcessorColorConfiguration( - @Nullable ColorInfo inputColorInfo) { - // TODO(b/279163661) Remove this method after VideoFrameProcessor supports texture ID - // input/output. - if (!ColorInfo.isTransferHdr(inputColorInfo)) { - return Pair.create(ColorInfo.SDR_BT709_LIMITED, ColorInfo.SDR_BT709_LIMITED); - } - - if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG) { - // SurfaceView only supports BT2020 PQ input, converting HLG to PQ. - return Pair.create( - inputColorInfo, - inputColorInfo.buildUpon().setColorTransfer(C.COLOR_TRANSFER_ST2084).build()); - } - - return Pair.create(inputColorInfo, inputColorInfo); - } - /** * Renders the output buffer with the specified index now. * * @param codec The codec that owns the output buffer. - * @param format The {@link Format} associated with the buffer. * @param index The index of the output buffer to drop. * @param presentationTimeUs The presentation time of the output buffer, in microseconds. - * @param notifyFrameMetadataListener Whether to notify the {@link VideoFrameMetadataListener}. + * @param releaseTimeNs The release timestamp that needs to be associated with this buffer, in + * nanoseconds. */ - private void renderOutputBufferNow( - MediaCodecAdapter codec, - Format format, - int index, - long presentationTimeUs, - boolean notifyFrameMetadataListener) { - // In previewing mode, use the presentation time as release time so that the SurfaceTexture is - // accompanied by the rendered frame's presentation time. Setting a realtime based release time - // is only relevant when rendering to a SurfaceView (that is when not using VideoFrameProcessor) - // for better frame release. In previewing mode MediaCodec renders to VideoFrameProcessor's - // input surface, which is not a SurfaceView. - long releaseTimeNs = - videoFrameProcessorManager.isEnabled() - ? videoFrameProcessorManager.getCorrectedFramePresentationTimeUs( - presentationTimeUs, getOutputStreamOffsetUs()) - * 1000 - : getClock().nanoTime(); - if (notifyFrameMetadataListener) { - notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); - } + private void renderOutputBuffer( + MediaCodecAdapter codec, int index, long presentationTimeUs, long releaseTimeNs) { if (Util.SDK_INT >= 21) { renderOutputBufferV21(codec, index, presentationTimeUs, releaseTimeNs); } else { @@ -1639,10 +1675,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { * Renders the output buffer with the specified index. This method is only called if the platform * API version of the device is less than 21. * - *

When video frame processing is {@linkplain VideoFrameProcessorManager#isEnabled()} enabled}, - * this method renders to {@link VideoFrameProcessorManager}'s {@linkplain - * VideoFrameProcessorManager#getInputSurface() input surface}. - * * @param codec The codec that owns the output buffer. * @param index The index of the output buffer to drop. * @param presentationTimeUs The presentation time of the output buffer, in microseconds. @@ -1653,7 +1685,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { TraceUtil.endSection(); decoderCounters.renderedOutputBufferCount++; consecutiveDroppedFrameCount = 0; - if (!videoFrameProcessorManager.isEnabled()) { + if (videoSink == null) { lastRenderRealtimeUs = msToUs(getClock().elapsedRealtime()); maybeNotifyVideoSizeChanged(decodedVideoSize); maybeNotifyRenderedFirstFrame(); @@ -1664,10 +1696,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { * Renders the output buffer with the specified index. This method is only called if the platform * API version of the device is 21 or later. * - *

When video frame processing is {@linkplain VideoFrameProcessorManager#isEnabled()} enabled}, - * this method renders to {@link VideoFrameProcessorManager}'s {@linkplain - * VideoFrameProcessorManager#getInputSurface() input surface}. - * * @param codec The codec that owns the output buffer. * @param index The index of the output buffer to drop. * @param presentationTimeUs The presentation time of the output buffer, in microseconds. @@ -1681,7 +1709,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { TraceUtil.endSection(); decoderCounters.renderedOutputBufferCount++; consecutiveDroppedFrameCount = 0; - if (!videoFrameProcessorManager.isEnabled()) { + if (videoSink == null) { lastRenderRealtimeUs = msToUs(getClock().elapsedRealtime()); maybeNotifyVideoSizeChanged(decodedVideoSize); maybeNotifyRenderedFirstFrame(); @@ -1726,7 +1754,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } } - /* package */ void maybeNotifyRenderedFirstFrame() { + private void maybeNotifyRenderedFirstFrame() { if (firstFrameState != C.FIRST_FRAME_RENDERED) { firstFrameState = C.FIRST_FRAME_RENDERED; eventDispatcher.renderedFirstFrame(displaySurface); @@ -1938,504 +1966,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return new MediaCodecVideoDecoderException(cause, codecInfo, displaySurface); } - /** Manages {@link VideoFrameProcessor} interactions. */ - private static final class VideoFrameProcessorManager { - - /** The threshold for releasing a processed frame. */ - private static final long EARLY_THRESHOLD_US = 50_000; - - private final VideoFrameReleaseHelper frameReleaseHelper; - // TODO(b/238302341) Consider removing the reference to the containing class and make this class - // non-static. - private final MediaCodecVideoRenderer renderer; - private final ArrayDeque processedFramesTimestampsUs; - private final ArrayDeque> pendingFrameFormats; - private final VideoFrameProcessor.Factory videoFrameProcessorFactory; - - private @MonotonicNonNull Handler handler; - @Nullable private VideoFrameProcessor videoFrameProcessor; - @Nullable private CopyOnWriteArrayList videoEffects; - @Nullable private Format inputFormat; - - /** - * The current frame {@link Format} and the earliest presentationTimeUs that associates to it. - */ - private @MonotonicNonNull Pair currentFrameFormat; - - private @MonotonicNonNull Clock clock; - @Nullable private Pair currentSurfaceAndSize; - - private int videoFrameProcessorMaxPendingFrameCount; - private boolean canEnableFrameProcessing; - - /** - * Whether the last frame of the current stream is decoded and registered to {@link - * VideoFrameProcessor}. - */ - private boolean registeredLastFrame; - - /** - * Whether the last frame of the current stream is processed by the {@link VideoFrameProcessor}. - */ - private boolean processedLastFrame; - - /** Whether the last frame of the current stream is released to the output {@link Surface}. */ - private boolean releasedLastFrame; - - private long lastCodecBufferPresentationTimestampUs; - private VideoSize processedFrameSize; - private boolean pendingOutputSizeChange; - - /** The presentation time, after which the listener should be notified about the size change. */ - private long pendingOutputSizeChangeNotificationTimeUs; - - private long initialStreamOffsetUs; - - /** Creates a new instance. */ - public VideoFrameProcessorManager( - VideoFrameProcessor.Factory videoFrameProcessorFactory, - VideoFrameReleaseHelper frameReleaseHelper, - @UnderInitialization MediaCodecVideoRenderer renderer) { - this.videoFrameProcessorFactory = videoFrameProcessorFactory; - this.frameReleaseHelper = frameReleaseHelper; - this.renderer = renderer; - processedFramesTimestampsUs = new ArrayDeque<>(); - pendingFrameFormats = new ArrayDeque<>(); - videoFrameProcessorMaxPendingFrameCount = C.LENGTH_UNSET; - canEnableFrameProcessing = true; - lastCodecBufferPresentationTimestampUs = C.TIME_UNSET; - processedFrameSize = VideoSize.UNKNOWN; - pendingOutputSizeChangeNotificationTimeUs = C.TIME_UNSET; - initialStreamOffsetUs = C.TIME_UNSET; - } - - /** Sets the {@linkplain Effect video effects}. */ - public void setVideoEffects(List videoEffects) { - if (this.videoEffects == null) { - this.videoEffects = new CopyOnWriteArrayList<>(videoEffects); - return; - } - this.videoEffects.clear(); - this.videoEffects.addAll(videoEffects); - } - - /** Returns whether video frame processing is enabled. */ - public boolean isEnabled() { - return videoFrameProcessor != null; - } - - /** Returns whether {@code VideoFrameProcessorManager} is ready to accept input frames. */ - public boolean isReady() { - return currentSurfaceAndSize == null || !currentSurfaceAndSize.second.equals(Size.UNKNOWN); - } - - /** - * Whether the {@link VideoFrameProcessor} has released the last frame in the current stream. - */ - public boolean releasedLastFrame() { - return releasedLastFrame; - } - - /** - * Flushes the {@link VideoFrameProcessor}. - * - *

Caller must ensure video frame processing {@linkplain #isEnabled() is enabled} before - * calling this method. - */ - public void flush() { - checkStateNotNull(videoFrameProcessor); - videoFrameProcessor.flush(); - processedFramesTimestampsUs.clear(); - handler.removeCallbacksAndMessages(/* token= */ null); - - if (registeredLastFrame) { - registeredLastFrame = false; - processedLastFrame = false; - releasedLastFrame = false; - } - } - - /** - * Tries to enable video frame processing. - * - *

Caller must ensure video frame processing {@linkplain #isEnabled() is not enabled} before - * calling this method. - * - * @param inputFormat The {@link Format} that is input into the {@link VideoFrameProcessor}. - * @return Whether video frame processing is enabled. - * @throws ExoPlaybackException When enabling the {@link VideoFrameProcessor} failed. - */ - @CanIgnoreReturnValue - public boolean maybeEnable(Format inputFormat, long initialStreamOffsetUs, Clock clock) - throws ExoPlaybackException { - checkState(!isEnabled()); - if (!canEnableFrameProcessing) { - return false; - } - if (videoEffects == null) { - canEnableFrameProcessing = false; - return false; - } - - // Playback thread handler. - handler = Util.createHandlerForCurrentLooper(); - this.clock = clock; - - Pair inputAndOutputColorInfos = - renderer.experimentalGetVideoFrameProcessorColorConfiguration(inputFormat.colorInfo); - try { - // TODO(b/243036513): Set rotation in setInputFormat() after supporting changing effects. - if (!codecAppliesRotation() && inputFormat.rotationDegrees != 0) { - // Insert as the first effect as if the decoder has applied the rotation. - videoEffects.add( - /* index= */ 0, - ScaleAndRotateAccessor.createRotationEffect(inputFormat.rotationDegrees)); - } - - videoFrameProcessor = - videoFrameProcessorFactory.create( - renderer.context, - DebugViewProvider.NONE, - inputAndOutputColorInfos.first, - inputAndOutputColorInfos.second, - /* renderFramesAutomatically= */ false, - /* listenerExecutor= */ handler::post, - new VideoFrameProcessor.Listener() { - @Override - public void onOutputSizeChanged(int width, int height) { - @Nullable Format inputFormat = VideoFrameProcessorManager.this.inputFormat; - checkStateNotNull(inputFormat); - // TODO(b/264889146): Handle Effect that changes output size based on pts. - processedFrameSize = - new VideoSize( - width, - height, - // VideoFrameProcessor is configured to produce rotation free - // frames. - /* unappliedRotationDegrees= */ 0, - // VideoFrameProcessor always outputs pixelWidthHeightRatio 1. - /* pixelWidthHeightRatio= */ 1.f); - pendingOutputSizeChange = true; - } - - @Override - public void onOutputFrameAvailableForRendering(long presentationTimeUs) { - if (registeredLastFrame) { - checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET); - } - processedFramesTimestampsUs.add(presentationTimeUs); - // TODO(b/257464707) Support extensively modified media. - if (registeredLastFrame - && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) { - processedLastFrame = true; - } - if (pendingOutputSizeChange) { - // Report the size change on releasing this frame. - pendingOutputSizeChange = false; - pendingOutputSizeChangeNotificationTimeUs = presentationTimeUs; - } - } - - @Override - public void onError(VideoFrameProcessingException exception) { - renderer.setPendingPlaybackException( - renderer.createRendererException( - exception, - inputFormat, - PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED)); - } - - @Override - public void onEnded() { - throw new IllegalStateException(); - } - }); - - this.initialStreamOffsetUs = initialStreamOffsetUs; - } catch (Exception e) { - throw renderer.createRendererException( - e, inputFormat, PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSOR_INIT_FAILED); - } - - if (currentSurfaceAndSize != null) { - Size outputSurfaceSize = currentSurfaceAndSize.second; - videoFrameProcessor.setOutputSurfaceInfo( - new SurfaceInfo( - currentSurfaceAndSize.first, - outputSurfaceSize.getWidth(), - outputSurfaceSize.getHeight())); - } - - return true; - } - - public long getCorrectedFramePresentationTimeUs( - long framePresentationTimeUs, long currentStreamOffsetUs) { - // VideoFrameProcessor takes in frames with monotonically increasing, non-offset frame - // timestamps. That is, with two ten-second long videos, the first frame of the second video - // should bear a timestamp of 10s seen from VideoFrameProcessor; while in ExoPlayer, the - // timestamp of the said frame would be 0s, but the streamOffset is incremented 10s to include - // the duration of the first video. Thus this correction is need to correct for the different - // handling of presentation timestamps in ExoPlayer and VideoFrameProcessor. - checkState(initialStreamOffsetUs != C.TIME_UNSET); - return framePresentationTimeUs + currentStreamOffsetUs - initialStreamOffsetUs; - } - - /** - * Returns the {@linkplain VideoFrameProcessor#getInputSurface input surface} of the {@link - * VideoFrameProcessor}. - * - *

Caller must ensure the {@code VideoFrameProcessorManager} {@link #isEnabled()} before - * calling this method. - */ - public Surface getInputSurface() { - return checkNotNull(videoFrameProcessor).getInputSurface(); - } - - /** - * Sets the output surface info. - * - * @param outputSurface The {@link Surface} to which {@link VideoFrameProcessor} outputs. - * @param outputResolution The {@link Size} of the output resolution. - */ - public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) { - if (currentSurfaceAndSize != null - && currentSurfaceAndSize.first.equals(outputSurface) - && currentSurfaceAndSize.second.equals(outputResolution)) { - return; - } - currentSurfaceAndSize = Pair.create(outputSurface, outputResolution); - if (isEnabled()) { - checkNotNull(videoFrameProcessor) - .setOutputSurfaceInfo( - new SurfaceInfo( - outputSurface, outputResolution.getWidth(), outputResolution.getHeight())); - } - } - - /** - * Clears the set output surface info. - * - *

Caller must ensure the {@code VideoFrameProcessorManager} {@link #isEnabled()} before - * calling this method. - */ - public void clearOutputSurfaceInfo() { - checkNotNull(videoFrameProcessor).setOutputSurfaceInfo(null); - currentSurfaceAndSize = null; - } - - /** - * Sets the input surface info. - * - *

Caller must ensure the {@code VideoFrameProcessorManager} {@link #isEnabled()} before - * calling this method. - */ - public void setInputFormat(Format inputFormat) { - checkNotNull(videoFrameProcessor) - .registerInputStream( - VideoFrameProcessor.INPUT_TYPE_SURFACE, - checkNotNull(videoEffects), - new FrameInfo.Builder(inputFormat.width, inputFormat.height) - .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio) - .build()); - this.inputFormat = inputFormat; - - if (registeredLastFrame) { - registeredLastFrame = false; - processedLastFrame = false; - releasedLastFrame = false; - } - } - - /** Sets the necessary {@link MediaFormat} keys for video frame processing. */ - @SuppressWarnings("InlinedApi") - public MediaFormat amendMediaFormatKeys(MediaFormat mediaFormat) { - if (Util.SDK_INT >= 29 - && renderer.context.getApplicationContext().getApplicationInfo().targetSdkVersion >= 29) { - mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0); - } - return mediaFormat; - } - - /** - * Must be called when the codec is initialized. - * - *

Sets the {@code videoFrameProcessorMaxPendingFrameCount} based on the {@code codecName}. - */ - public void onCodecInitialized(String codecName) { - videoFrameProcessorMaxPendingFrameCount = - Util.getMaxPendingFramesCountForMediaCodecDecoders(renderer.context); - } - - /** - * Tries to {@linkplain VideoFrameProcessor#registerInputFrame register an input frame}. - * - *

Caller must ensure the {@code VideoFrameProcessorManager} {@link #isEnabled()} before - * calling this method. - * - * @param format The {@link Format} associated with the frame. - * @param isLastBuffer Whether the buffer is the last from the decoder to register. - * @return Whether {@link MediaCodec} should render the frame to {@link VideoFrameProcessor}. - */ - public boolean maybeRegisterFrame( - Format format, long presentationTimestampUs, boolean isLastBuffer) { - checkStateNotNull(videoFrameProcessor); - checkState(videoFrameProcessorMaxPendingFrameCount != C.LENGTH_UNSET); - - if (videoFrameProcessor.getPendingInputFrameCount() - < videoFrameProcessorMaxPendingFrameCount) { - videoFrameProcessor.registerInputFrame(); - - if (currentFrameFormat == null) { - currentFrameFormat = Pair.create(presentationTimestampUs, format); - } else if (!Util.areEqual(format, currentFrameFormat.second)) { - // TODO(b/258213806) Remove format comparison for better performance. - pendingFrameFormats.add(Pair.create(presentationTimestampUs, format)); - } - - if (isLastBuffer) { - registeredLastFrame = true; - lastCodecBufferPresentationTimestampUs = presentationTimestampUs; - } - return true; - } - return false; - } - - /** - * Releases the processed frames to the {@linkplain #setOutputSurfaceInfo output surface}. - * - *

Caller must ensure the {@code VideoFrameProcessorManager} {@link #isEnabled()} before - * calling this method. - */ - public void releaseProcessedFrames(long positionUs, long elapsedRealtimeUs) { - checkStateNotNull(videoFrameProcessor); - while (!processedFramesTimestampsUs.isEmpty()) { - boolean isStarted = renderer.getState() == STATE_STARTED; - long framePresentationTimeUs = checkNotNull(processedFramesTimestampsUs.peek()); - long bufferPresentationTimeUs = framePresentationTimeUs + initialStreamOffsetUs; - long earlyUs = - renderer.calculateEarlyTimeUs( - positionUs, - elapsedRealtimeUs, - msToUs(clock.elapsedRealtime()), - bufferPresentationTimeUs, - isStarted); - - boolean isLastFrame = processedLastFrame && processedFramesTimestampsUs.size() == 1; - boolean shouldReleaseFrameImmediately = renderer.shouldForceRender(positionUs, earlyUs); - if (shouldReleaseFrameImmediately) { - releaseProcessedFrameInternal( - VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY, isLastFrame); - break; - } else if (!isStarted || positionUs == renderer.initialPositionUs) { - return; - } - - // Only release frames that are reasonably close to presentation. - // This way frameReleaseHelper.onNextFrame() is called only once for each frame. - if (earlyUs > EARLY_THRESHOLD_US) { - break; - } - - frameReleaseHelper.onNextFrame(bufferPresentationTimeUs); - long systemNanoTime = checkNotNull(clock).nanoTime(); - long unadjustedFrameReleaseTimeNs = systemNanoTime + earlyUs * 1000; - long adjustedFrameReleaseTimeNs = - frameReleaseHelper.adjustReleaseTime(unadjustedFrameReleaseTimeNs); - earlyUs = (adjustedFrameReleaseTimeNs - systemNanoTime) / 1000; - - // TODO(b/238302341) Handle very late buffers and drop to key frame. Need to flush - // VideoFrameProcessor input frames in this case. - if (renderer.shouldDropOutputBuffer(earlyUs, elapsedRealtimeUs, isLastFrame)) { - releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame); - continue; - } - - if (!pendingFrameFormats.isEmpty() - && bufferPresentationTimeUs > pendingFrameFormats.peek().first) { - currentFrameFormat = pendingFrameFormats.remove(); - } - renderer.notifyFrameMetadataListener( - framePresentationTimeUs, adjustedFrameReleaseTimeNs, currentFrameFormat.second); - if (pendingOutputSizeChangeNotificationTimeUs >= bufferPresentationTimeUs) { - pendingOutputSizeChangeNotificationTimeUs = C.TIME_UNSET; - renderer.maybeNotifyVideoSizeChanged(processedFrameSize); - } - releaseProcessedFrameInternal(adjustedFrameReleaseTimeNs, isLastFrame); - } - } - - /** - * Releases the resources. - * - *

Caller must ensure video frame processing {@linkplain #isEnabled() is not enabled} before - * calling this method. - */ - public void reset() { - checkNotNull(videoFrameProcessor).release(); - videoFrameProcessor = null; - if (handler != null) { - handler.removeCallbacksAndMessages(/* token= */ null); - } - if (videoEffects != null) { - videoEffects.clear(); - } - processedFramesTimestampsUs.clear(); - canEnableFrameProcessing = true; - } - - private void releaseProcessedFrameInternal(long releaseTimeNs, boolean isLastFrame) { - // VideoFrameProcessor renders to its output surface using - // VideoFrameProcessor.renderOutputFrame, to release the MediaCodecVideoRenderer frame. - checkStateNotNull(videoFrameProcessor); - videoFrameProcessor.renderOutputFrame(releaseTimeNs); - processedFramesTimestampsUs.remove(); - renderer.lastRenderRealtimeUs = msToUs(clock.elapsedRealtime()); - if (releaseTimeNs != VideoFrameProcessor.DROP_OUTPUT_FRAME) { - renderer.maybeNotifyRenderedFirstFrame(); - } - if (isLastFrame) { - releasedLastFrame = true; - } - } - - private static final class ScaleAndRotateAccessor { - private static @MonotonicNonNull Constructor - scaleAndRotateTransformationBuilderConstructor; - private static @MonotonicNonNull Method setRotationMethod; - private static @MonotonicNonNull Method buildScaleAndRotateTransformationMethod; - - public static Effect createRotationEffect(float rotationDegrees) throws Exception { - prepare(); - Object builder = scaleAndRotateTransformationBuilderConstructor.newInstance(); - setRotationMethod.invoke(builder, rotationDegrees); - return (Effect) checkNotNull(buildScaleAndRotateTransformationMethod.invoke(builder)); - } - - @EnsuresNonNull({ - "scaleAndRotateTransformationBuilderConstructor", - "setRotationMethod", - "buildScaleAndRotateTransformationMethod" - }) - private static void prepare() throws Exception { - if (scaleAndRotateTransformationBuilderConstructor == null - || setRotationMethod == null - || buildScaleAndRotateTransformationMethod == null) { - // TODO: b/284964524- Add LINT and proguard checks for media3.effect reflection. - Class scaleAndRotateTransformationBuilderClass = - Class.forName("androidx.media3.effect.ScaleAndRotateTransformation$Builder"); - scaleAndRotateTransformationBuilderConstructor = - scaleAndRotateTransformationBuilderClass.getConstructor(); - setRotationMethod = - scaleAndRotateTransformationBuilderClass.getMethod("setRotationDegrees", float.class); - buildScaleAndRotateTransformationMethod = - scaleAndRotateTransformationBuilderClass.getMethod("build"); - } - } - } - } - /** * Delays reflection for loading a {@linkplain VideoFrameProcessor.Factory * DefaultVideoFrameProcessor} instance. diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java index 11bbc7447f..d25a77feb5 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSink.java @@ -33,7 +33,7 @@ import java.util.concurrent.Executor; /** A sink that consumes decoded video frames. */ @UnstableApi -/*package */ interface VideoSink { +/* package */ interface VideoSink { /** Thrown by {@link VideoSink} implementations. */ final class VideoSinkException extends Exception { @@ -61,6 +61,46 @@ import java.util.concurrent.Executor; void onError(VideoSink videoSink, VideoSinkException videoSinkException); } + /** Controls the rendering of video frames. */ + interface RenderControl { + /** Signals a frame must be rendered immediately. */ + long RENDER_TIME_IMMEDIATELY = -1; + + /** Signals a frame must be dropped. */ + long RENDER_TIME_DROP = -2; + + /** Signals that a frame should not be rendered yet. */ + long RENDER_TIME_TRY_AGAIN_LATER = -3; + + /** + * Returns the render timestamp, in nanoseconds, associated with this video frames or one of the + * {@code RENDER_TIME_} constants if the frame must be rendered immediately, dropped or not + * rendered yet. + * + * @param presentationTimeUs The presentation time of the video frame, in microseconds. + * @param positionUs The current playback position, in microseconds. + * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, + * taken approximately at the time the playback position was {@code positionUs}. + * @param playbackSpeed The current playback speed. + * @return The render timestamp, in nanoseconds, associated with this frame, or one of the + * {@code RENDER_TIME_} constants if the frame must be rendered immediately, dropped or not + * rendered yet. + */ + long getFrameRenderTimeNs( + long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed); + + /** + * Informs the rendering control that a video frame will be rendered. Call this method before + * rendering a frame. + * + * @param presentationTimeUs The frame's presentation time, in microseconds. + */ + void onNextFrame(long presentationTimeUs); + + /** Informs the rendering control that a video frame was rendered. */ + void onFrameRendered(); + } + /** * Specifies how the input frames are made available to the video sink. One of {@link * #INPUT_TYPE_SURFACE} or {@link #INPUT_TYPE_BITMAP}. diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java new file mode 100644 index 0000000000..b512583775 --- /dev/null +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java @@ -0,0 +1,169 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.video; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import android.content.Context; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.DebugViewProvider; +import androidx.media3.common.Format; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.VideoFrameProcessor; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import java.util.concurrent.Executor; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; + +/** Unit test for {@link CompositingVideoSinkProvider}. */ +@RunWith(AndroidJUnit4.class) +public final class CompositingVideoSinkProviderTest { + + @Test + public void initialize() throws VideoSink.VideoSinkException { + CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); + provider.setVideoEffects(ImmutableList.of()); + + provider.initialize(new Format.Builder().build()); + + assertThat(provider.isInitialized()).isTrue(); + } + + @Test + public void initialize_withoutEffects_throws() { + CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); + + assertThrows( + IllegalStateException.class, + () -> provider.initialize(new Format.Builder().setWidth(640).setHeight(480).build())); + } + + @Test + public void initialize_calledTwice_throws() throws VideoSink.VideoSinkException { + CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); + provider.setVideoEffects(ImmutableList.of()); + provider.initialize(new Format.Builder().build()); + + assertThrows( + IllegalStateException.class, () -> provider.initialize(new Format.Builder().build())); + } + + @Test + public void initialize_afterRelease_throws() throws VideoSink.VideoSinkException { + CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); + provider.setVideoEffects(ImmutableList.of()); + Format format = new Format.Builder().build(); + + provider.initialize(format); + provider.release(); + + assertThrows(IllegalStateException.class, () -> provider.initialize(format)); + } + + @Test + public void registerInputStream_withInputTypeBitmap_throws() throws VideoSink.VideoSinkException { + CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); + provider.setVideoEffects(ImmutableList.of()); + provider.initialize(new Format.Builder().build()); + VideoSink videoSink = provider.getSink(); + + assertThrows( + UnsupportedOperationException.class, + () -> + videoSink.registerInputStream( + VideoSink.INPUT_TYPE_BITMAP, new Format.Builder().build())); + } + + @Test + public void setOutputStreamOffsetUs_frameReleaseTimesAreAdjusted() + throws VideoSink.VideoSinkException { + CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); + provider.setVideoEffects(ImmutableList.of()); + provider.initialize(new Format.Builder().build()); + VideoSink videoSink = provider.getSink(); + videoSink.registerInputStream( + VideoSink.INPUT_TYPE_SURFACE, new Format.Builder().setWidth(640).setHeight(480).build()); + + assertThat(videoSink.registerInputFrame(/* framePresentationTimeUs= */ 0, false)).isEqualTo(0); + provider.setStreamOffsetUs(1_000); + assertThat(videoSink.registerInputFrame(/* framePresentationTimeUs= */ 0, false)) + .isEqualTo(1_000_000); + provider.setStreamOffsetUs(2_000); + assertThat(videoSink.registerInputFrame(/* framePresentationTimeUs= */ 0, false)) + .isEqualTo(2_000_000); + } + + @Test + public void setListener_calledTwiceWithDifferentExecutor_throws() + throws VideoSink.VideoSinkException { + CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider(); + provider.setVideoEffects(ImmutableList.of()); + provider.initialize(new Format.Builder().build()); + VideoSink videoSink = provider.getSink(); + VideoSink.Listener listener = Mockito.mock(VideoSink.Listener.class); + + videoSink.setListener(listener, /* executor= */ command -> {}); + + assertThrows( + IllegalStateException.class, + () -> videoSink.setListener(listener, /* executor= */ command -> {})); + } + + private static CompositingVideoSinkProvider createCompositingVideoSinkProvider() { + VideoFrameProcessor.Factory factory = new TestVideoFrameProcessorFactory(); + VideoSink.RenderControl renderControl = new TestRenderControl(); + return new CompositingVideoSinkProvider( + ApplicationProvider.getApplicationContext(), factory, renderControl); + } + + private static class TestVideoFrameProcessorFactory implements VideoFrameProcessor.Factory { + // Using a mock but we don't assert mock interactions. If needed to assert interactions, we + // should a fake instead. + private final VideoFrameProcessor videoFrameProcessor = Mockito.mock(VideoFrameProcessor.class); + + @Override + public VideoFrameProcessor create( + Context context, + DebugViewProvider debugViewProvider, + ColorInfo inputColorInfo, + ColorInfo outputColorInfo, + boolean renderFramesAutomatically, + Executor listenerExecutor, + VideoFrameProcessor.Listener listener) + throws VideoFrameProcessingException { + return videoFrameProcessor; + } + } + + private static class TestRenderControl implements VideoSink.RenderControl { + + @Override + public long getFrameRenderTimeNs( + long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed) { + return presentationTimeUs; + } + + @Override + public void onNextFrame(long presentationTimeUs) {} + + @Override + public void onFrameRendered() {} + } +}