loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ExoPlayer: set videoFrameProcessorFactory in MCVR ctor.

Browse Source 
Also, move to using a Supplier<?> to load a singleton value.

PiperOrigin-RevId: 536733480
This commit is contained in:
huangdarwin 2023-05-31 16:16:24 +00:00 committed by Tofunmi Adigun-Hameed
parent 108000834b
commit 20f90cb1bf
2 changed files with 169 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
RELEASENOTES.md
View File

@ -81,6 +81,8 @@ This release includes the following changes since
size with `MediaCodecVideoRenderer` has a width and height of 0 when size with `MediaCodecVideoRenderer` has a width and height of 0 when
`Player.getCurrentTracks` does not support video, or the size of the `Player.getCurrentTracks` does not support video, or the size of the
supported video track is not yet determined. supported video track is not yet determined.
* Allow `MediaCodecVideoRenderer` to use a custom
`VideoFrameProcessor.Factory`.
* IMA extension: * IMA extension:
* Enable multi-period live DASH streams for DAI. Please note that the * Enable multi-period live DASH streams for DAI. Please note that the
current implementation does not yet support seeking in live streams current implementation does not yet support seeking in live streams

248
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View File

@ -82,6 +82,8 @@ import androidx.media3.exoplayer.mediacodec.MediaCodecSelector;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil.DecoderQueryException; import androidx.media3.exoplayer.mediacodec.MediaCodecUtil.DecoderQueryException;
import androidx.media3.exoplayer.video.VideoRendererEventListener.EventDispatcher; import androidx.media3.exoplayer.video.VideoRendererEventListener.EventDispatcher;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.lang.reflect.Constructor; import java.lang.reflect.Constructor;
@ -90,6 +92,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayDeque; import java.util.ArrayDeque;
import java.util.List; import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Executor;
import org.checkerframework.checker.initialization.qual.UnderInitialization; import org.checkerframework.checker.initialization.qual.UnderInitialization;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull; import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@ -298,7 +301,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
@Nullable Handler eventHandler, @Nullable Handler eventHandler,
@Nullable VideoRendererEventListener eventListener, @Nullable VideoRendererEventListener eventListener,
int maxDroppedFramesToNotify) { int maxDroppedFramesToNotify) {
this( this(
context, context,
codecAdapterFactory, codecAdapterFactory,
@ -342,6 +344,53 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
@Nullable VideoRendererEventListener eventListener, @Nullable VideoRendererEventListener eventListener,
int maxDroppedFramesToNotify, int maxDroppedFramesToNotify,
float assumedMinimumCodecOperatingRate) { float assumedMinimumCodecOperatingRate) {
this(
context,
codecAdapterFactory,
mediaCodecSelector,
allowedJoiningTimeMs,
enableDecoderFallback,
eventHandler,
eventListener,
maxDroppedFramesToNotify,
assumedMinimumCodecOperatingRate,
new ReflectiveDefaultVideoFrameProcessorFactory());
}
/**
* Creates a new instance.
*
* @param context A context.
* @param codecAdapterFactory The {@link MediaCodecAdapter.Factory} used to create {@link
* MediaCodecAdapter} instances.
* @param mediaCodecSelector A decoder selector.
* @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer
* can attempt to seamlessly join an ongoing playback.
* @param enableDecoderFallback Whether to enable fallback to lower-priority decoders if decoder
* initialization fails. This may result in using a decoder that is slower/less efficient than
* the primary decoder.
* @param eventHandler A handler to use when delivering events to {@code eventListener}. May be
* null if delivery of events is not required.
* @param eventListener A listener of events. May be null if delivery of events is not required.
* @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between
* invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}.
* @param assumedMinimumCodecOperatingRate A codec operating rate that all codecs instantiated by
* this renderer are assumed to meet implicitly (i.e. without the operating rate being set
* explicitly using {@link MediaFormat#KEY_OPERATING_RATE}).
* @param videoFrameProcessorFactory The {@link VideoFrameProcessor.Factory} applied on video
* output. {@code null} means a default implementation will be applied.
*/
public MediaCodecVideoRenderer(
Context context,
MediaCodecAdapter.Factory codecAdapterFactory,
MediaCodecSelector mediaCodecSelector,
long allowedJoiningTimeMs,
boolean enableDecoderFallback,
@Nullable Handler eventHandler,
@Nullable VideoRendererEventListener eventListener,
int maxDroppedFramesToNotify,
float assumedMinimumCodecOperatingRate,
VideoFrameProcessor.Factory videoFrameProcessorFactory) {
super( super(
C.TRACK_TYPE_VIDEO, C.TRACK_TYPE_VIDEO,
codecAdapterFactory, codecAdapterFactory,
@ -354,7 +403,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
frameReleaseHelper = new VideoFrameReleaseHelper(this.context); frameReleaseHelper = new VideoFrameReleaseHelper(this.context);
eventDispatcher = new EventDispatcher(eventHandler, eventListener); eventDispatcher = new EventDispatcher(eventHandler, eventListener);
videoFrameProcessorManager = videoFrameProcessorManager =
new VideoFrameProcessorManager(frameReleaseHelper, /* renderer= */ this); new VideoFrameProcessorManager(
videoFrameProcessorFactory, frameReleaseHelper, /* renderer= */ this);
deviceNeedsNoPostProcessWorkaround = deviceNeedsNoPostProcessWorkaround(); deviceNeedsNoPostProcessWorkaround = deviceNeedsNoPostProcessWorkaround();
joiningDeadlineMs = C.TIME_UNSET; joiningDeadlineMs = C.TIME_UNSET;
scalingMode = C.VIDEO_SCALING_MODE_DEFAULT; scalingMode = C.VIDEO_SCALING_MODE_DEFAULT;
@ -1874,6 +1924,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
private final MediaCodecVideoRenderer renderer; private final MediaCodecVideoRenderer renderer;
private final ArrayDeque<Long> processedFramesTimestampsUs; private final ArrayDeque<Long> processedFramesTimestampsUs;
private final ArrayDeque<Pair<Long, Format>> pendingFrameFormats; private final ArrayDeque<Pair<Long, Format>> pendingFrameFormats;
private final VideoFrameProcessor.Factory videoFrameProcessorFactory;
private @MonotonicNonNull Handler handler; private @MonotonicNonNull Handler handler;
@Nullable private VideoFrameProcessor videoFrameProcessor; @Nullable private VideoFrameProcessor videoFrameProcessor;
@ -1913,8 +1964,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
/** Creates a new instance. */ /** Creates a new instance. */
public VideoFrameProcessorManager( public VideoFrameProcessorManager(
VideoFrameProcessor.Factory videoFrameProcessorFactory,
VideoFrameReleaseHelper frameReleaseHelper, VideoFrameReleaseHelper frameReleaseHelper,
@UnderInitialization MediaCodecVideoRenderer renderer) { @UnderInitialization MediaCodecVideoRenderer renderer) {
this.videoFrameProcessorFactory = videoFrameProcessorFactory;
this.frameReleaseHelper = frameReleaseHelper; this.frameReleaseHelper = frameReleaseHelper;
this.renderer = renderer; this.renderer = renderer;
processedFramesTimestampsUs = new ArrayDeque<>(); processedFramesTimestampsUs = new ArrayDeque<>();
@ -2006,69 +2059,68 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
// Insert as the first effect as if the decoder has applied the rotation. // Insert as the first effect as if the decoder has applied the rotation.
videoEffects.add( videoEffects.add(
/* index= */ 0, /* index= */ 0,
VideoFrameProcessorAccessor.createRotationEffect(inputFormat.rotationDegrees)); ScaleAndRotateAccessor.createRotationEffect(inputFormat.rotationDegrees));
} }
videoFrameProcessor = videoFrameProcessor =
VideoFrameProcessorAccessor.getFrameProcessorFactory() videoFrameProcessorFactory.create(
.create( renderer.context,
renderer.context, checkNotNull(videoEffects),
checkNotNull(videoEffects), DebugViewProvider.NONE,
DebugViewProvider.NONE, inputAndOutputColorInfos.first,
inputAndOutputColorInfos.first, inputAndOutputColorInfos.second,
inputAndOutputColorInfos.second, /* renderFramesAutomatically= */ false,
/* renderFramesAutomatically= */ false, /* listenerExecutor= */ handler::post,
/* listenerExecutor= */ handler::post, new VideoFrameProcessor.Listener() {
new VideoFrameProcessor.Listener() { @Override
@Override public void onOutputSizeChanged(int width, int height) {
public void onOutputSizeChanged(int width, int height) { @Nullable Format inputFormat = VideoFrameProcessorManager.this.inputFormat;
@Nullable Format inputFormat = VideoFrameProcessorManager.this.inputFormat; checkStateNotNull(inputFormat);
checkStateNotNull(inputFormat); // TODO(b/264889146): Handle Effect that changes output size based on pts.
// TODO(b/264889146): Handle Effect that changes output size based on pts. processedFrameSize =
processedFrameSize = new VideoSize(
new VideoSize( width,
width, height,
height, // VideoFrameProcessor is configured to produce rotation free
// VideoFrameProcessor is configured to produce rotation free // frames.
// frames. /* unappliedRotationDegrees= */ 0,
/* unappliedRotationDegrees= */ 0, // VideoFrameProcessor always outputs pixelWidthHeightRatio 1.
// VideoFrameProcessor always outputs pixelWidthHeightRatio 1. /* pixelWidthHeightRatio= */ 1.f);
/* pixelWidthHeightRatio= */ 1.f); pendingOutputSizeChange = true;
pendingOutputSizeChange = true; }
}
@Override @Override
public void onOutputFrameAvailableForRendering(long presentationTimeUs) { public void onOutputFrameAvailableForRendering(long presentationTimeUs) {
if (registeredLastFrame) { if (registeredLastFrame) {
checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET); checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET);
} }
processedFramesTimestampsUs.add(presentationTimeUs); processedFramesTimestampsUs.add(presentationTimeUs);
// TODO(b/257464707) Support extensively modified media. // TODO(b/257464707) Support extensively modified media.
if (registeredLastFrame if (registeredLastFrame
&& presentationTimeUs >= lastCodecBufferPresentationTimestampUs) { && presentationTimeUs >= lastCodecBufferPresentationTimestampUs) {
processedLastFrame = true; processedLastFrame = true;
} }
if (pendingOutputSizeChange) { if (pendingOutputSizeChange) {
// Report the size change on releasing this frame. // Report the size change on releasing this frame.
pendingOutputSizeChange = false; pendingOutputSizeChange = false;
pendingOutputSizeChangeNotificationTimeUs = presentationTimeUs; pendingOutputSizeChangeNotificationTimeUs = presentationTimeUs;
} }
} }
@Override @Override
public void onError(VideoFrameProcessingException exception) { public void onError(VideoFrameProcessingException exception) {
renderer.setPendingPlaybackException( renderer.setPendingPlaybackException(
renderer.createRendererException( renderer.createRendererException(
exception, exception,
inputFormat, inputFormat,
PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED)); PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED));
} }
@Override @Override
public void onEnded() { public void onEnded() {
throw new IllegalStateException(); throw new IllegalStateException();
} }
}); });
videoFrameProcessor.registerInputStream(VideoFrameProcessor.INPUT_TYPE_SURFACE); videoFrameProcessor.registerInputStream(VideoFrameProcessor.INPUT_TYPE_SURFACE);
this.initialStreamOffsetUs = initialStreamOffsetUs; this.initialStreamOffsetUs = initialStreamOffsetUs;
} catch (Exception e) { } catch (Exception e) {
@ -2318,13 +2370,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
} }
} }
private static final class VideoFrameProcessorAccessor { private static final class ScaleAndRotateAccessor {
private static @MonotonicNonNull Constructor<?> private static @MonotonicNonNull Constructor<?>
scaleAndRotateTransformationBuilderConstructor; scaleAndRotateTransformationBuilderConstructor;
private static @MonotonicNonNull Method setRotationMethod; private static @MonotonicNonNull Method setRotationMethod;
private static @MonotonicNonNull Method buildScaleAndRotateTransformationMethod; private static @MonotonicNonNull Method buildScaleAndRotateTransformationMethod;
private static @MonotonicNonNull Constructor<?> videoFrameProcessorFactoryBuilderConstructor;
private static @MonotonicNonNull Method buildVideoFrameProcessorFactoryMethod;
public static Effect createRotationEffect(float rotationDegrees) throws Exception { public static Effect createRotationEffect(float rotationDegrees) throws Exception {
prepare(); prepare();
@ -2333,24 +2383,16 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
return (Effect) checkNotNull(buildScaleAndRotateTransformationMethod.invoke(builder)); return (Effect) checkNotNull(buildScaleAndRotateTransformationMethod.invoke(builder));
} }
public static VideoFrameProcessor.Factory getFrameProcessorFactory() throws Exception {
prepare();
Object builder = videoFrameProcessorFactoryBuilderConstructor.newInstance();
return (VideoFrameProcessor.Factory)
checkNotNull(buildVideoFrameProcessorFactoryMethod.invoke(builder));
}
@EnsuresNonNull({ @EnsuresNonNull({
"scaleAndRotateTransformationBuilderConstructor", "scaleAndRotateTransformationBuilderConstructor",
"setRotationMethod", "setRotationMethod",
"buildScaleAndRotateTransformationMethod", "buildScaleAndRotateTransformationMethod"
"videoFrameProcessorFactoryBuilderConstructor",
"buildVideoFrameProcessorFactoryMethod"
}) })
private static void prepare() throws Exception { private static void prepare() throws Exception {
if (scaleAndRotateTransformationBuilderConstructor == null if (scaleAndRotateTransformationBuilderConstructor == null
|| setRotationMethod == null || setRotationMethod == null
|| buildScaleAndRotateTransformationMethod == null) { || buildScaleAndRotateTransformationMethod == null) {
// TODO: b/284964524- Add LINT and proguard checks for media3.effect reflection.
Class<?> scaleAndRotateTransformationBuilderClass = Class<?> scaleAndRotateTransformationBuilderClass =
Class.forName("androidx.media3.effect.ScaleAndRotateTransformation$Builder"); Class.forName("androidx.media3.effect.ScaleAndRotateTransformation$Builder");
scaleAndRotateTransformationBuilderConstructor = scaleAndRotateTransformationBuilderConstructor =
@ -2360,19 +2402,63 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
buildScaleAndRotateTransformationMethod = buildScaleAndRotateTransformationMethod =
scaleAndRotateTransformationBuilderClass.getMethod("build"); scaleAndRotateTransformationBuilderClass.getMethod("build");
} }
if (videoFrameProcessorFactoryBuilderConstructor == null
|| buildVideoFrameProcessorFactoryMethod == null) {
Class<?> videoFrameProcessorFactoryBuilderClass =
Class.forName("androidx.media3.effect.DefaultVideoFrameProcessor$Factory$Builder");
videoFrameProcessorFactoryBuilderConstructor =
videoFrameProcessorFactoryBuilderClass.getConstructor();
buildVideoFrameProcessorFactoryMethod =
videoFrameProcessorFactoryBuilderClass.getMethod("build");
}
} }
} }
} }
/**
* Delays reflection for loading a {@linkplain VideoFrameProcessor.Factory
* DefaultVideoFrameProcessor} instance.
*/
private static final class ReflectiveDefaultVideoFrameProcessorFactory
implements VideoFrameProcessor.Factory {
private static final Supplier<VideoFrameProcessor.Factory>
VIDEO_FRAME_PROCESSOR_FACTORY_SUPPLIER =
Suppliers.memoize(
() -> {
try {
// TODO: b/284964524- Add LINT and proguard checks for media3.effect reflection.
Class<?> defaultVideoFrameProcessorFactoryBuilderClass =
Class.forName(
"androidx.media3.effect.DefaultVideoFrameProcessor$Factory$Builder");
Object builder =
defaultVideoFrameProcessorFactoryBuilderClass
.getConstructor()
.newInstance();
return (VideoFrameProcessor.Factory)
checkNotNull(
defaultVideoFrameProcessorFactoryBuilderClass
.getMethod("build")
.invoke(builder));
} catch (Exception e) {
throw new IllegalStateException(e);
}
});
@Override
public VideoFrameProcessor create(
Context context,
List<Effect> effects,
DebugViewProvider debugViewProvider,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean renderFramesAutomatically,
Executor listenerExecutor,
VideoFrameProcessor.Listener listener)
throws VideoFrameProcessingException {
return VIDEO_FRAME_PROCESSOR_FACTORY_SUPPLIER
.get()
.create(
context,
effects,
debugViewProvider,
inputColorInfo,
outputColorInfo,
renderFramesAutomatically,
listenerExecutor,
listener);
}
}
/** /**
* Returns a maximum video size to use when configuring a codec for {@code format} in a way that * Returns a maximum video size to use when configuring a codec for {@code format} in a way that
* will allow possible adaptation to other compatible formats that are expected to have the same * will allow possible adaptation to other compatible formats that are expected to have the same