Implement the CompositingVideoSinkProvider
This is the first iteration for the CompositingVideoSinkProvider which is basically a copy of MediaCodecVideoRenderer's VieoFrameProcessorManager. The MediaCodecVideoRenderer now instantiates a CompositingVideoSinkProvider instead of a VieoFrameProcessorManager. PiperOrigin-RevId: 555903928
This commit is contained in:
christosts
Tianyi Feng
parent
171e1a1e42
commit
f5a6ecdda1
@ -0,0 +1,643 @@
|
||||
/*
|
||||
* Copyright 2023 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package androidx.media3.exoplayer.video;
|
||||
|
||||
import static androidx.media3.common.util.Assertions.checkArgument;
|
||||
import static androidx.media3.common.util.Assertions.checkNotNull;
|
||||
import static androidx.media3.common.util.Assertions.checkState;
|
||||
import static androidx.media3.common.util.Assertions.checkStateNotNull;
|
||||
|
||||
import android.content.Context;
|
||||
import android.graphics.Bitmap;
|
||||
import android.os.Handler;
|
||||
import android.util.Pair;
|
||||
import android.view.Surface;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.ColorInfo;
|
||||
import androidx.media3.common.DebugViewProvider;
|
||||
import androidx.media3.common.Effect;
|
||||
import androidx.media3.common.Format;
|
||||
import androidx.media3.common.FrameInfo;
|
||||
import androidx.media3.common.MimeTypes;
|
||||
import androidx.media3.common.SurfaceInfo;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.VideoFrameProcessor;
|
||||
import androidx.media3.common.VideoSize;
|
||||
import androidx.media3.common.util.Size;
|
||||
import androidx.media3.common.util.TimedValueQueue;
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
import androidx.media3.common.util.Util;
|
||||
import java.lang.reflect.Constructor;
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.Executor;
|
||||
import org.checkerframework.checker.initialization.qual.Initialized;
|
||||
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
/** Handles composition of video sinks. */
|
||||
@UnstableApi
|
||||
/* package */ final class CompositingVideoSinkProvider {
|
||||
|
||||
private final Context context;
|
||||
private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
|
||||
private final VideoSink.RenderControl renderControl;
|
||||
|
||||
@Nullable private VideoSinkImpl videoSinkImpl;
|
||||
@Nullable private List<Effect> videoEffects;
|
||||
@Nullable private VideoFrameMetadataListener videoFrameMetadataListener;
|
||||
private boolean released;
|
||||
|
||||
/** Creates a new instance. */
|
||||
public CompositingVideoSinkProvider(
|
||||
Context context,
|
||||
VideoFrameProcessor.Factory videoFrameProcessorFactory,
|
||||
VideoSink.RenderControl renderControl) {
|
||||
this.context = context;
|
||||
this.videoFrameProcessorFactory = videoFrameProcessorFactory;
|
||||
this.renderControl = renderControl;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the provider for video frame processing. Can be called up to one time and only
|
||||
* after video effects are {@linkplain #setVideoEffects(List) set}.
|
||||
*
|
||||
* @param sourceFormat The format of the compressed video.
|
||||
* @throws VideoSink.VideoSinkException If enabling the provider failed.
|
||||
*/
|
||||
public void initialize(Format sourceFormat) throws VideoSink.VideoSinkException {
|
||||
checkState(!released && videoSinkImpl == null);
|
||||
checkStateNotNull(videoEffects);
|
||||
|
||||
try {
|
||||
videoSinkImpl =
|
||||
new VideoSinkImpl(context, videoFrameProcessorFactory, renderControl, sourceFormat);
|
||||
if (videoFrameMetadataListener != null) {
|
||||
videoSinkImpl.setVideoFrameMetadataListener(videoFrameMetadataListener);
|
||||
}
|
||||
} catch (VideoFrameProcessingException e) {
|
||||
throw new VideoSink.VideoSinkException(e, sourceFormat);
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns whether this provider is initialized for frame processing. */
|
||||
public boolean isInitialized() {
|
||||
return videoSinkImpl != null;
|
||||
}
|
||||
|
||||
/** Releases the sink provider. */
|
||||
public void release() {
|
||||
if (released) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (videoSinkImpl != null) {
|
||||
videoSinkImpl.release();
|
||||
videoSinkImpl = null;
|
||||
}
|
||||
released = true;
|
||||
}
|
||||
|
||||
/** Returns a {@link VideoSink} to forward video frames for processing. */
|
||||
public VideoSink getSink() {
|
||||
return checkStateNotNull(videoSinkImpl);
|
||||
}
|
||||
|
||||
/** Sets video effects on this provider. */
|
||||
public void setVideoEffects(List<Effect> videoEffects) {
|
||||
this.videoEffects = videoEffects;
|
||||
if (isInitialized()) {
|
||||
checkStateNotNull(videoSinkImpl).setVideoEffects(videoEffects);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the offset, in microseconds, that is added to the video frames presentation timestamps
|
||||
* from the player.
|
||||
*
|
||||
* <p>Must be called after the sink provider is {@linkplain #initialize(Format) initialized}.
|
||||
*/
|
||||
public void setStreamOffsetUs(long streamOffsetUs) {
|
||||
checkStateNotNull(videoSinkImpl).setStreamOffsetUs(streamOffsetUs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the output surface info.
|
||||
*
|
||||
* <p>Must be called after the sink provider is {@linkplain #initialize(Format) initialized}.
|
||||
*/
|
||||
public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) {
|
||||
checkStateNotNull(videoSinkImpl).setOutputSurfaceInfo(outputSurface, outputResolution);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears the set output surface info.
|
||||
*
|
||||
* <p>Must be called after the sink provider is {@linkplain #initialize(Format) initialized}.
|
||||
*/
|
||||
public void clearOutputSurfaceInfo() {
|
||||
checkStateNotNull(videoSinkImpl).clearOutputSurfaceInfo();
|
||||
}
|
||||
|
||||
/** Sets a {@link VideoFrameMetadataListener} which is used in the returned {@link VideoSink}. */
|
||||
public void setVideoFrameMetadataListener(VideoFrameMetadataListener videoFrameMetadataListener) {
|
||||
this.videoFrameMetadataListener = videoFrameMetadataListener;
|
||||
if (isInitialized()) {
|
||||
checkStateNotNull(videoSinkImpl).setVideoFrameMetadataListener(videoFrameMetadataListener);
|
||||
}
|
||||
}
|
||||
|
||||
private static final class VideoSinkImpl implements VideoSink, VideoFrameProcessor.Listener {
|
||||
|
||||
private final Context context;
|
||||
private final RenderControl renderControl;
|
||||
private final VideoFrameProcessor videoFrameProcessor;
|
||||
// TODO b/293447478 - Use a queue for primitive longs to avoid the cost of boxing to Long.
|
||||
private final ArrayDeque<Long> processedFramesBufferTimestampsUs;
|
||||
private final TimedValueQueue<Long> streamOffsets;
|
||||
private final TimedValueQueue<VideoSize> videoSizeChanges;
|
||||
private final Handler handler;
|
||||
private final int videoFrameProcessorMaxPendingFrameCount;
|
||||
private final ArrayList<Effect> videoEffects;
|
||||
@Nullable private final Effect rotationEffect;
|
||||
|
||||
private VideoSink.@MonotonicNonNull Listener listener;
|
||||
private @MonotonicNonNull Executor listenerExecutor;
|
||||
@Nullable private VideoFrameMetadataListener videoFrameMetadataListener;
|
||||
@Nullable private Format inputFormat;
|
||||
@Nullable private Pair<Surface, Size> currentSurfaceAndSize;
|
||||
|
||||
/**
|
||||
* Whether the last frame of the current stream is decoded and registered to {@link
|
||||
* VideoFrameProcessor}.
|
||||
*/
|
||||
private boolean registeredLastFrame;
|
||||
|
||||
/**
|
||||
* Whether the last frame of the current stream is processed by the {@link VideoFrameProcessor}.
|
||||
*/
|
||||
private boolean processedLastFrame;
|
||||
|
||||
/** Whether the last frame of the current stream is released to the output {@link Surface}. */
|
||||
private boolean releasedLastFrame;
|
||||
|
||||
private long lastCodecBufferPresentationTimestampUs;
|
||||
private VideoSize processedFrameSize;
|
||||
private VideoSize reportedVideoSize;
|
||||
private boolean pendingVideoSizeChange;
|
||||
private boolean renderedFirstFrame;
|
||||
private long inputStreamOffsetUs;
|
||||
private boolean pendingInputStreamOffsetChange;
|
||||
private long outputStreamOffsetUs;
|
||||
private float playbackSpeed;
|
||||
|
||||
// TODO b/292111083 - Remove the field and trigger the callback on every video size change.
|
||||
private boolean onVideoSizeChangedCalled;
|
||||
|
||||
/** Creates a new instance. */
|
||||
public VideoSinkImpl(
|
||||
Context context,
|
||||
VideoFrameProcessor.Factory videoFrameProcessorFactory,
|
||||
RenderControl renderControl,
|
||||
Format sourceFormat)
|
||||
throws VideoFrameProcessingException {
|
||||
this.context = context;
|
||||
this.renderControl = renderControl;
|
||||
processedFramesBufferTimestampsUs = new ArrayDeque<>();
|
||||
streamOffsets = new TimedValueQueue<>();
|
||||
videoSizeChanges = new TimedValueQueue<>();
|
||||
// TODO b/226330223 - Investigate increasing frame count when frame dropping is
|
||||
// allowed.
|
||||
// TODO b/278234847 - Evaluate whether limiting frame count when frame dropping is not allowed
|
||||
// reduces decoder timeouts, and consider restoring.
|
||||
videoFrameProcessorMaxPendingFrameCount =
|
||||
Util.getMaxPendingFramesCountForMediaCodecDecoders(context);
|
||||
lastCodecBufferPresentationTimestampUs = C.TIME_UNSET;
|
||||
processedFrameSize = VideoSize.UNKNOWN;
|
||||
reportedVideoSize = VideoSize.UNKNOWN;
|
||||
playbackSpeed = 1f;
|
||||
|
||||
// Playback thread handler.
|
||||
handler = Util.createHandlerForCurrentLooper();
|
||||
|
||||
Pair<ColorInfo, ColorInfo> inputAndOutputColorInfos =
|
||||
experimentalGetVideoFrameProcessorColorConfiguration(sourceFormat.colorInfo);
|
||||
|
||||
@SuppressWarnings("nullness:assignment")
|
||||
@Initialized
|
||||
VideoSinkImpl thisRef = this;
|
||||
videoFrameProcessor =
|
||||
videoFrameProcessorFactory.create(
|
||||
context,
|
||||
DebugViewProvider.NONE,
|
||||
inputAndOutputColorInfos.first,
|
||||
inputAndOutputColorInfos.second,
|
||||
/* renderFramesAutomatically= */ false,
|
||||
/* listenerExecutor= */ handler::post,
|
||||
thisRef);
|
||||
if (currentSurfaceAndSize != null) {
|
||||
Size outputSurfaceSize = currentSurfaceAndSize.second;
|
||||
videoFrameProcessor.setOutputSurfaceInfo(
|
||||
new SurfaceInfo(
|
||||
currentSurfaceAndSize.first,
|
||||
outputSurfaceSize.getWidth(),
|
||||
outputSurfaceSize.getHeight()));
|
||||
}
|
||||
videoEffects = new ArrayList<>();
|
||||
// MediaCodec applies rotation after API 21
|
||||
rotationEffect =
|
||||
Util.SDK_INT < 21 && sourceFormat.rotationDegrees != 0
|
||||
? ScaleAndRotateAccessor.createRotationEffect(sourceFormat.rotationDegrees)
|
||||
: null;
|
||||
}
|
||||
|
||||
// VideoSink impl
|
||||
|
||||
@Override
|
||||
public void flush() {
|
||||
videoFrameProcessor.flush();
|
||||
processedFramesBufferTimestampsUs.clear();
|
||||
streamOffsets.clear();
|
||||
handler.removeCallbacksAndMessages(/* token= */ null);
|
||||
renderedFirstFrame = false;
|
||||
if (registeredLastFrame) {
|
||||
registeredLastFrame = false;
|
||||
processedLastFrame = false;
|
||||
releasedLastFrame = false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isReady() {
|
||||
return renderedFirstFrame;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEnded() {
|
||||
return releasedLastFrame;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void registerInputStream(@InputType int inputType, Format format) {
|
||||
if (inputType != INPUT_TYPE_SURFACE) {
|
||||
throw new UnsupportedOperationException("Unsupported input type " + inputType);
|
||||
}
|
||||
this.inputFormat = format;
|
||||
maybeRegisterInputStream();
|
||||
|
||||
if (registeredLastFrame) {
|
||||
registeredLastFrame = false;
|
||||
processedLastFrame = false;
|
||||
releasedLastFrame = false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setListener(Listener listener, Executor executor) {
|
||||
if (Util.areEqual(this.listener, listener)) {
|
||||
checkState(Util.areEqual(listenerExecutor, executor));
|
||||
return;
|
||||
}
|
||||
this.listener = listener;
|
||||
this.listenerExecutor = executor;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isFrameDropAllowedOnInput() {
|
||||
return Util.isFrameDropAllowedOnSurfaceInput(context);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Surface getInputSurface() {
|
||||
return videoFrameProcessor.getInputSurface();
|
||||
}
|
||||
|
||||
@Override
|
||||
public long registerInputFrame(long framePresentationTimeUs, boolean isLastFrame) {
|
||||
checkState(videoFrameProcessorMaxPendingFrameCount != C.LENGTH_UNSET);
|
||||
if (videoFrameProcessor.getPendingInputFrameCount()
|
||||
>= videoFrameProcessorMaxPendingFrameCount) {
|
||||
return C.TIME_UNSET;
|
||||
}
|
||||
videoFrameProcessor.registerInputFrame();
|
||||
// The sink takes in frames with monotonically increasing, non-offset frame
|
||||
// timestamps. That is, with two ten-second long videos, the first frame of the second video
|
||||
// should bear a timestamp of 10s seen from VideoFrameProcessor; while in ExoPlayer, the
|
||||
// timestamp of the said frame would be 0s, but the streamOffset is incremented 10s to include
|
||||
// the duration of the first video. Thus this correction is need to correct for the different
|
||||
// handling of presentation timestamps in ExoPlayer and VideoFrameProcessor.
|
||||
long bufferPresentationTimeUs = framePresentationTimeUs + inputStreamOffsetUs;
|
||||
if (pendingInputStreamOffsetChange) {
|
||||
streamOffsets.add(bufferPresentationTimeUs, inputStreamOffsetUs);
|
||||
pendingInputStreamOffsetChange = false;
|
||||
}
|
||||
if (isLastFrame) {
|
||||
registeredLastFrame = true;
|
||||
lastCodecBufferPresentationTimestampUs = bufferPresentationTimeUs;
|
||||
}
|
||||
return bufferPresentationTimeUs * 1000;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean queueBitmap(Bitmap inputBitmap, long durationUs, float frameRate) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void render(long positionUs, long elapsedRealtimeUs) {
|
||||
while (!processedFramesBufferTimestampsUs.isEmpty()) {
|
||||
long bufferPresentationTimeUs = checkNotNull(processedFramesBufferTimestampsUs.peek());
|
||||
// check whether this buffer comes with a new stream offset.
|
||||
if (maybeUpdateOutputStreamOffset(bufferPresentationTimeUs)) {
|
||||
renderedFirstFrame = false;
|
||||
}
|
||||
long framePresentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs;
|
||||
boolean isLastFrame = processedLastFrame && processedFramesBufferTimestampsUs.size() == 1;
|
||||
long frameRenderTimeNs =
|
||||
renderControl.getFrameRenderTimeNs(
|
||||
bufferPresentationTimeUs, positionUs, elapsedRealtimeUs, playbackSpeed);
|
||||
if (frameRenderTimeNs == RenderControl.RENDER_TIME_TRY_AGAIN_LATER) {
|
||||
return;
|
||||
} else if (framePresentationTimeUs == RenderControl.RENDER_TIME_DROP) {
|
||||
// TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush
|
||||
// VideoFrameProcessor input frames in this case.
|
||||
releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame);
|
||||
continue;
|
||||
}
|
||||
renderControl.onNextFrame(bufferPresentationTimeUs);
|
||||
if (videoFrameMetadataListener != null) {
|
||||
videoFrameMetadataListener.onVideoFrameAboutToBeRendered(
|
||||
framePresentationTimeUs,
|
||||
frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY
|
||||
? System.nanoTime()
|
||||
: frameRenderTimeNs,
|
||||
checkNotNull(inputFormat),
|
||||
/* mediaFormat= */ null);
|
||||
}
|
||||
releaseProcessedFrameInternal(
|
||||
frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY
|
||||
? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY
|
||||
: frameRenderTimeNs,
|
||||
isLastFrame);
|
||||
|
||||
maybeNotifyVideoSizeChanged(bufferPresentationTimeUs);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setPlaybackSpeed(float speed) {
|
||||
checkArgument(speed >= 0.0);
|
||||
this.playbackSpeed = speed;
|
||||
}
|
||||
|
||||
// VideoFrameProcessor.Listener impl
|
||||
|
||||
@Override
|
||||
public void onOutputSizeChanged(int width, int height) {
|
||||
VideoSize newVideoSize = new VideoSize(width, height);
|
||||
if (!processedFrameSize.equals(newVideoSize)) {
|
||||
processedFrameSize = newVideoSize;
|
||||
pendingVideoSizeChange = true;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onOutputFrameAvailableForRendering(long presentationTimeUs) {
|
||||
if (pendingVideoSizeChange) {
|
||||
videoSizeChanges.add(presentationTimeUs, processedFrameSize);
|
||||
pendingVideoSizeChange = false;
|
||||
}
|
||||
if (registeredLastFrame) {
|
||||
checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET);
|
||||
}
|
||||
processedFramesBufferTimestampsUs.add(presentationTimeUs);
|
||||
// TODO b/257464707 - Support extensively modified media.
|
||||
if (registeredLastFrame && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) {
|
||||
processedLastFrame = true;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(VideoFrameProcessingException exception) {
|
||||
if (listener == null || listenerExecutor == null) {
|
||||
return;
|
||||
}
|
||||
listenerExecutor.execute(
|
||||
() -> {
|
||||
if (listener != null) {
|
||||
listener.onError(
|
||||
/* videoSink= */ this,
|
||||
new VideoSink.VideoSinkException(
|
||||
exception,
|
||||
new Format.Builder()
|
||||
.setSampleMimeType(MimeTypes.VIDEO_RAW)
|
||||
.setWidth(processedFrameSize.width)
|
||||
.setHeight(processedFrameSize.height)
|
||||
.build()));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onEnded() {
|
||||
throw new IllegalStateException();
|
||||
}
|
||||
|
||||
// Other methods
|
||||
|
||||
public void release() {
|
||||
videoFrameProcessor.release();
|
||||
handler.removeCallbacksAndMessages(/* token= */ null);
|
||||
streamOffsets.clear();
|
||||
processedFramesBufferTimestampsUs.clear();
|
||||
renderedFirstFrame = false;
|
||||
}
|
||||
|
||||
/** Sets the {@linkplain Effect video effects}. */
|
||||
public void setVideoEffects(List<Effect> videoEffects) {
|
||||
this.videoEffects.clear();
|
||||
this.videoEffects.addAll(videoEffects);
|
||||
maybeRegisterInputStream();
|
||||
}
|
||||
|
||||
public void setStreamOffsetUs(long streamOffsetUs) {
|
||||
pendingInputStreamOffsetChange = inputStreamOffsetUs != streamOffsetUs;
|
||||
inputStreamOffsetUs = streamOffsetUs;
|
||||
}
|
||||
|
||||
public void setVideoFrameMetadataListener(
|
||||
VideoFrameMetadataListener videoFrameMetadataListener) {
|
||||
this.videoFrameMetadataListener = videoFrameMetadataListener;
|
||||
}
|
||||
|
||||
private void maybeRegisterInputStream() {
|
||||
if (inputFormat == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
ArrayList<Effect> effects = new ArrayList<>();
|
||||
if (rotationEffect != null) {
|
||||
effects.add(rotationEffect);
|
||||
}
|
||||
effects.addAll(videoEffects);
|
||||
Format inputFormat = checkNotNull(this.inputFormat);
|
||||
videoFrameProcessor.registerInputStream(
|
||||
VideoFrameProcessor.INPUT_TYPE_SURFACE,
|
||||
effects,
|
||||
new FrameInfo.Builder(inputFormat.width, inputFormat.height)
|
||||
.setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio)
|
||||
.build());
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link Pair} of {@linkplain ColorInfo input color} and {@linkplain ColorInfo output
|
||||
* color} to configure the {@code VideoFrameProcessor}.
|
||||
*/
|
||||
private static Pair<ColorInfo, ColorInfo> experimentalGetVideoFrameProcessorColorConfiguration(
|
||||
@Nullable ColorInfo inputColorInfo) {
|
||||
// TODO b/279163661 - Remove this method after VideoFrameProcessor supports texture ID
|
||||
// input/output.
|
||||
// explicit check for nullness
|
||||
if (inputColorInfo == null || !ColorInfo.isTransferHdr(inputColorInfo)) {
|
||||
return Pair.create(ColorInfo.SDR_BT709_LIMITED, ColorInfo.SDR_BT709_LIMITED);
|
||||
}
|
||||
|
||||
if (inputColorInfo.colorTransfer == C.COLOR_TRANSFER_HLG) {
|
||||
// SurfaceView only supports BT2020 PQ input, converting HLG to PQ.
|
||||
return Pair.create(
|
||||
inputColorInfo,
|
||||
inputColorInfo.buildUpon().setColorTransfer(C.COLOR_TRANSFER_ST2084).build());
|
||||
}
|
||||
|
||||
return Pair.create(inputColorInfo, inputColorInfo);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the output surface info.
|
||||
*
|
||||
* @param outputSurface The {@link Surface} to which {@link VideoFrameProcessor} outputs.
|
||||
* @param outputResolution The {@link Size} of the output resolution.
|
||||
*/
|
||||
public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) {
|
||||
if (currentSurfaceAndSize != null
|
||||
&& currentSurfaceAndSize.first.equals(outputSurface)
|
||||
&& currentSurfaceAndSize.second.equals(outputResolution)) {
|
||||
return;
|
||||
}
|
||||
renderedFirstFrame =
|
||||
currentSurfaceAndSize == null || currentSurfaceAndSize.first.equals(outputSurface);
|
||||
currentSurfaceAndSize = Pair.create(outputSurface, outputResolution);
|
||||
videoFrameProcessor.setOutputSurfaceInfo(
|
||||
new SurfaceInfo(
|
||||
outputSurface, outputResolution.getWidth(), outputResolution.getHeight()));
|
||||
}
|
||||
|
||||
/** Clears the set output surface info. */
|
||||
public void clearOutputSurfaceInfo() {
|
||||
videoFrameProcessor.setOutputSurfaceInfo(null);
|
||||
currentSurfaceAndSize = null;
|
||||
renderedFirstFrame = false;
|
||||
}
|
||||
|
||||
private boolean maybeUpdateOutputStreamOffset(long bufferPresentationTimeUs) {
|
||||
boolean updatedOffset = false;
|
||||
@Nullable Long newOutputStreamOffsetUs = streamOffsets.pollFloor(bufferPresentationTimeUs);
|
||||
if (newOutputStreamOffsetUs != null && newOutputStreamOffsetUs != outputStreamOffsetUs) {
|
||||
outputStreamOffsetUs = newOutputStreamOffsetUs;
|
||||
updatedOffset = true;
|
||||
}
|
||||
return updatedOffset;
|
||||
}
|
||||
|
||||
private void releaseProcessedFrameInternal(long releaseTimeNs, boolean isLastFrame) {
|
||||
videoFrameProcessor.renderOutputFrame(releaseTimeNs);
|
||||
processedFramesBufferTimestampsUs.remove();
|
||||
if (releaseTimeNs != VideoFrameProcessor.DROP_OUTPUT_FRAME) {
|
||||
renderControl.onFrameRendered();
|
||||
if (!renderedFirstFrame) {
|
||||
if (listener != null) {
|
||||
checkNotNull(listenerExecutor)
|
||||
.execute(() -> checkNotNull(listener).onFirstFrameRendered(this));
|
||||
}
|
||||
renderedFirstFrame = true;
|
||||
}
|
||||
}
|
||||
if (isLastFrame) {
|
||||
releasedLastFrame = true;
|
||||
}
|
||||
}
|
||||
|
||||
private void maybeNotifyVideoSizeChanged(long bufferPresentationTimeUs) {
|
||||
if (onVideoSizeChangedCalled || listener == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
@Nullable VideoSize videoSize = videoSizeChanges.pollFloor(bufferPresentationTimeUs);
|
||||
if (videoSize == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!videoSize.equals(VideoSize.UNKNOWN) && !videoSize.equals(reportedVideoSize)) {
|
||||
reportedVideoSize = videoSize;
|
||||
checkNotNull(listenerExecutor)
|
||||
.execute(() -> checkNotNull(listener).onVideoSizeChanged(this, videoSize));
|
||||
}
|
||||
onVideoSizeChangedCalled = true;
|
||||
}
|
||||
|
||||
private static final class ScaleAndRotateAccessor {
|
||||
private static @MonotonicNonNull Constructor<?>
|
||||
scaleAndRotateTransformationBuilderConstructor;
|
||||
private static @MonotonicNonNull Method setRotationMethod;
|
||||
private static @MonotonicNonNull Method buildScaleAndRotateTransformationMethod;
|
||||
|
||||
public static Effect createRotationEffect(float rotationDegrees) {
|
||||
try {
|
||||
prepare();
|
||||
Object builder = scaleAndRotateTransformationBuilderConstructor.newInstance();
|
||||
setRotationMethod.invoke(builder, rotationDegrees);
|
||||
return (Effect) checkNotNull(buildScaleAndRotateTransformationMethod.invoke(builder));
|
||||
} catch (Exception e) {
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@EnsuresNonNull({
|
||||
"scaleAndRotateTransformationBuilderConstructor",
|
||||
"setRotationMethod",
|
||||
"buildScaleAndRotateTransformationMethod"
|
||||
})
|
||||
private static void prepare() throws NoSuchMethodException, ClassNotFoundException {
|
||||
if (scaleAndRotateTransformationBuilderConstructor == null
|
||||
|| setRotationMethod == null
|
||||
|| buildScaleAndRotateTransformationMethod == null) {
|
||||
// TODO: b/284964524 - Add LINT and proguard checks for media3.effect reflection.
|
||||
Class<?> scaleAndRotateTransformationBuilderClass =
|
||||
Class.forName("androidx.media3.effect.ScaleAndRotateTransformation$Builder");
|
||||
scaleAndRotateTransformationBuilderConstructor =
|
||||
scaleAndRotateTransformationBuilderClass.getConstructor();
|
||||
setRotationMethod =
|
||||
scaleAndRotateTransformationBuilderClass.getMethod("setRotationDegrees", float.class);
|
||||
buildScaleAndRotateTransformationMethod =
|
||||
scaleAndRotateTransformationBuilderClass.getMethod("build");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -33,7 +33,7 @@ import java.util.concurrent.Executor;
|
||||
|
||||
/** A sink that consumes decoded video frames. */
|
||||
@UnstableApi
|
||||
/*package */ interface VideoSink {
|
||||
/* package */ interface VideoSink {
|
||||
|
||||
/** Thrown by {@link VideoSink} implementations. */
|
||||
final class VideoSinkException extends Exception {
|
||||
@ -61,6 +61,46 @@ import java.util.concurrent.Executor;
|
||||
void onError(VideoSink videoSink, VideoSinkException videoSinkException);
|
||||
}
|
||||
|
||||
/** Controls the rendering of video frames. */
|
||||
interface RenderControl {
|
||||
/** Signals a frame must be rendered immediately. */
|
||||
long RENDER_TIME_IMMEDIATELY = -1;
|
||||
|
||||
/** Signals a frame must be dropped. */
|
||||
long RENDER_TIME_DROP = -2;
|
||||
|
||||
/** Signals that a frame should not be rendered yet. */
|
||||
long RENDER_TIME_TRY_AGAIN_LATER = -3;
|
||||
|
||||
/**
|
||||
* Returns the render timestamp, in nanoseconds, associated with this video frames or one of the
|
||||
* {@code RENDER_TIME_} constants if the frame must be rendered immediately, dropped or not
|
||||
* rendered yet.
|
||||
*
|
||||
* @param presentationTimeUs The presentation time of the video frame, in microseconds.
|
||||
* @param positionUs The current playback position, in microseconds.
|
||||
* @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds,
|
||||
* taken approximately at the time the playback position was {@code positionUs}.
|
||||
* @param playbackSpeed The current playback speed.
|
||||
* @return The render timestamp, in nanoseconds, associated with this frame, or one of the
|
||||
* {@code RENDER_TIME_} constants if the frame must be rendered immediately, dropped or not
|
||||
* rendered yet.
|
||||
*/
|
||||
long getFrameRenderTimeNs(
|
||||
long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed);
|
||||
|
||||
/**
|
||||
* Informs the rendering control that a video frame will be rendered. Call this method before
|
||||
* rendering a frame.
|
||||
*
|
||||
* @param presentationTimeUs The frame's presentation time, in microseconds.
|
||||
*/
|
||||
void onNextFrame(long presentationTimeUs);
|
||||
|
||||
/** Informs the rendering control that a video frame was rendered. */
|
||||
void onFrameRendered();
|
||||
}
|
||||
|
||||
/**
|
||||
* Specifies how the input frames are made available to the video sink. One of {@link
|
||||
* #INPUT_TYPE_SURFACE} or {@link #INPUT_TYPE_BITMAP}.
|
||||
|
||||
@ -0,0 +1,169 @@
|
||||
/*
|
||||
* Copyright 2023 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package androidx.media3.exoplayer.video;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static org.junit.Assert.assertThrows;
|
||||
|
||||
import android.content.Context;
|
||||
import androidx.media3.common.ColorInfo;
|
||||
import androidx.media3.common.DebugViewProvider;
|
||||
import androidx.media3.common.Format;
|
||||
import androidx.media3.common.VideoFrameProcessingException;
|
||||
import androidx.media3.common.VideoFrameProcessor;
|
||||
import androidx.test.core.app.ApplicationProvider;
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import java.util.concurrent.Executor;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
/** Unit test for {@link CompositingVideoSinkProvider}. */
|
||||
@RunWith(AndroidJUnit4.class)
|
||||
public final class CompositingVideoSinkProviderTest {
|
||||
|
||||
@Test
|
||||
public void initialize() throws VideoSink.VideoSinkException {
|
||||
CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider();
|
||||
provider.setVideoEffects(ImmutableList.of());
|
||||
|
||||
provider.initialize(new Format.Builder().build());
|
||||
|
||||
assertThat(provider.isInitialized()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void initialize_withoutEffects_throws() {
|
||||
CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider();
|
||||
|
||||
assertThrows(
|
||||
IllegalStateException.class,
|
||||
() -> provider.initialize(new Format.Builder().setWidth(640).setHeight(480).build()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void initialize_calledTwice_throws() throws VideoSink.VideoSinkException {
|
||||
CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider();
|
||||
provider.setVideoEffects(ImmutableList.of());
|
||||
provider.initialize(new Format.Builder().build());
|
||||
|
||||
assertThrows(
|
||||
IllegalStateException.class, () -> provider.initialize(new Format.Builder().build()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void initialize_afterRelease_throws() throws VideoSink.VideoSinkException {
|
||||
CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider();
|
||||
provider.setVideoEffects(ImmutableList.of());
|
||||
Format format = new Format.Builder().build();
|
||||
|
||||
provider.initialize(format);
|
||||
provider.release();
|
||||
|
||||
assertThrows(IllegalStateException.class, () -> provider.initialize(format));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void registerInputStream_withInputTypeBitmap_throws() throws VideoSink.VideoSinkException {
|
||||
CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider();
|
||||
provider.setVideoEffects(ImmutableList.of());
|
||||
provider.initialize(new Format.Builder().build());
|
||||
VideoSink videoSink = provider.getSink();
|
||||
|
||||
assertThrows(
|
||||
UnsupportedOperationException.class,
|
||||
() ->
|
||||
videoSink.registerInputStream(
|
||||
VideoSink.INPUT_TYPE_BITMAP, new Format.Builder().build()));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void setOutputStreamOffsetUs_frameReleaseTimesAreAdjusted()
|
||||
throws VideoSink.VideoSinkException {
|
||||
CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider();
|
||||
provider.setVideoEffects(ImmutableList.of());
|
||||
provider.initialize(new Format.Builder().build());
|
||||
VideoSink videoSink = provider.getSink();
|
||||
videoSink.registerInputStream(
|
||||
VideoSink.INPUT_TYPE_SURFACE, new Format.Builder().setWidth(640).setHeight(480).build());
|
||||
|
||||
assertThat(videoSink.registerInputFrame(/* framePresentationTimeUs= */ 0, false)).isEqualTo(0);
|
||||
provider.setStreamOffsetUs(1_000);
|
||||
assertThat(videoSink.registerInputFrame(/* framePresentationTimeUs= */ 0, false))
|
||||
.isEqualTo(1_000_000);
|
||||
provider.setStreamOffsetUs(2_000);
|
||||
assertThat(videoSink.registerInputFrame(/* framePresentationTimeUs= */ 0, false))
|
||||
.isEqualTo(2_000_000);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void setListener_calledTwiceWithDifferentExecutor_throws()
|
||||
throws VideoSink.VideoSinkException {
|
||||
CompositingVideoSinkProvider provider = createCompositingVideoSinkProvider();
|
||||
provider.setVideoEffects(ImmutableList.of());
|
||||
provider.initialize(new Format.Builder().build());
|
||||
VideoSink videoSink = provider.getSink();
|
||||
VideoSink.Listener listener = Mockito.mock(VideoSink.Listener.class);
|
||||
|
||||
videoSink.setListener(listener, /* executor= */ command -> {});
|
||||
|
||||
assertThrows(
|
||||
IllegalStateException.class,
|
||||
() -> videoSink.setListener(listener, /* executor= */ command -> {}));
|
||||
}
|
||||
|
||||
private static CompositingVideoSinkProvider createCompositingVideoSinkProvider() {
|
||||
VideoFrameProcessor.Factory factory = new TestVideoFrameProcessorFactory();
|
||||
VideoSink.RenderControl renderControl = new TestRenderControl();
|
||||
return new CompositingVideoSinkProvider(
|
||||
ApplicationProvider.getApplicationContext(), factory, renderControl);
|
||||
}
|
||||
|
||||
private static class TestVideoFrameProcessorFactory implements VideoFrameProcessor.Factory {
|
||||
// Using a mock but we don't assert mock interactions. If needed to assert interactions, we
|
||||
// should a fake instead.
|
||||
private final VideoFrameProcessor videoFrameProcessor = Mockito.mock(VideoFrameProcessor.class);
|
||||
|
||||
@Override
|
||||
public VideoFrameProcessor create(
|
||||
Context context,
|
||||
DebugViewProvider debugViewProvider,
|
||||
ColorInfo inputColorInfo,
|
||||
ColorInfo outputColorInfo,
|
||||
boolean renderFramesAutomatically,
|
||||
Executor listenerExecutor,
|
||||
VideoFrameProcessor.Listener listener)
|
||||
throws VideoFrameProcessingException {
|
||||
return videoFrameProcessor;
|
||||
}
|
||||
}
|
||||
|
||||
private static class TestRenderControl implements VideoSink.RenderControl {
|
||||
|
||||
@Override
|
||||
public long getFrameRenderTimeNs(
|
||||
long presentationTimeUs, long positionUs, long elapsedRealtimeUs, float playbackSpeed) {
|
||||
return presentationTimeUs;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onNextFrame(long presentationTimeUs) {}
|
||||
|
||||
@Override
|
||||
public void onFrameRendered() {}
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user