loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-05-16 20:19:57 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Create frame processor in MCVR

Browse Source 
PiperOrigin-RevId: 495368262
This commit is contained in:
claincly 2022-12-14 19:24:26 +00:00 committed by Ian Baker
parent 91557ac9d4
commit 4398af3a4d
2 changed files with 247 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java
View File

@ -1097,6 +1097,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
if (codecOperatingRate <= assumedMinimumCodecOperatingRate) {
codecOperatingRate = CODEC_OPERATING_RATE_UNSET;
}
onReadyToInitializeCodec(inputFormat);
codecInitializingTimestamp = SystemClock.elapsedRealtime();
MediaCodecAdapter.Configuration configuration =
getMediaCodecConfiguration(codecInfo, inputFormat, crypto, codecOperatingRate);
@ -1381,6 +1382,22 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
return true;
}
/**
* Called when ready to initialize the {@link MediaCodecAdapter}.
*
* <p>This method is called just before the renderer obtains the {@linkplain
* #getMediaCodecConfiguration configuration} for the {@link MediaCodecAdapter} and creates the
* adapter via the passed in {@link MediaCodecAdapter.Factory}.
*
* <p>The default implementation is a no-op.
*
* @param format The {@link Format} for which the codec is being configured.
* @throws ExoPlaybackException If an error occurs preparing for initializing the codec.
*/
protected void onReadyToInitializeCodec(Format format) throws ExoPlaybackException {
// Do nothing.
}
/**
* Called when a {@link MediaCodec} has been created and configured.
*

231
library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java
View File

@ -50,6 +50,7 @@ import com.google.android.exoplayer2.ExoPlaybackException;
import com.google.android.exoplayer2.ExoPlayer;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.FormatHolder;
import com.google.android.exoplayer2.PlaybackException;
import com.google.android.exoplayer2.PlayerMessage.Target;
import com.google.android.exoplayer2.RendererCapabilities;
import com.google.android.exoplayer2.decoder.DecoderCounters;
@ -64,15 +65,27 @@ import com.google.android.exoplayer2.mediacodec.MediaCodecRenderer;
import com.google.android.exoplayer2.mediacodec.MediaCodecSelector;
import com.google.android.exoplayer2.mediacodec.MediaCodecUtil;
import com.google.android.exoplayer2.mediacodec.MediaCodecUtil.DecoderQueryException;
import com.google.android.exoplayer2.util.DebugViewProvider;
import com.google.android.exoplayer2.util.Effect;
import com.google.android.exoplayer2.util.FrameInfo;
import com.google.android.exoplayer2.util.FrameProcessingException;
import com.google.android.exoplayer2.util.FrameProcessor;
import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.MediaFormatUtil;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.Size;
import com.google.android.exoplayer2.util.SurfaceInfo;
import com.google.android.exoplayer2.util.TraceUtil;
import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.VideoRendererEventListener.EventDispatcher;
import com.google.common.collect.ImmutableList;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.nio.ByteBuffer;
import java.util.ArrayDeque;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import org.checkerframework.checker.initialization.qual.UnderInitialization;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* Decodes and renders video using {@link MediaCodec}.
@ -84,6 +97,8 @@ import java.util.List;
* <li>Message with type {@link #MSG_SET_VIDEO_OUTPUT} to set the output. The message payload
* should be the target {@link Surface}, or null to clear the output. Other non-null payloads
* have the effect of clearing the output.
* <li>Message with type {@link #MSG_SET_VIDEO_OUTPUT_RESOLUTION} to set the output resolution.
* The message payload should be the output resolution in {@link Size}.
* <li>Message with type {@link #MSG_SET_SCALING_MODE} to set the video scaling mode. The message
* payload should be one of the integer scaling modes in {@link C.VideoScalingMode}. Note that
* the scaling mode only applies if the {@link Surface} targeted by this renderer is owned by
@ -125,6 +140,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
private final Context context;
private final VideoFrameReleaseHelper frameReleaseHelper;
private final EventDispatcher eventDispatcher;
private final FrameProcessorManager frameProcessorManager;
private final long allowedJoiningTimeMs;
private final int maxDroppedFramesToNotify;
private final boolean deviceNeedsNoPostProcessWorkaround;
@ -328,6 +344,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
this.context = context.getApplicationContext();
frameReleaseHelper = new VideoFrameReleaseHelper(this.context);
eventDispatcher = new EventDispatcher(eventHandler, eventListener);
frameProcessorManager = new FrameProcessorManager(this);
deviceNeedsNoPostProcessWorkaround = deviceNeedsNoPostProcessWorkaround();
joiningDeadlineMs = C.TIME_UNSET;
scalingMode = C.VIDEO_SCALING_MODE_DEFAULT;
@ -612,6 +629,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
try {
super.onReset();
} finally {
if (frameProcessorManager.isEnabled()) {
frameProcessorManager.reset();
}
if (placeholderSurface != null) {
releasePlaceholderSurface();
}
@ -647,6 +667,12 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
}
}
break;
case MSG_SET_VIDEO_OUTPUT_RESOLUTION:
Size outputResolution = (Size) checkNotNull(message);
if (displaySurface != null && frameProcessorManager.isEnabled()) {
frameProcessorManager.setOutputSurfaceInfo(displaySurface, outputResolution);
}
break;
case MSG_SET_AUDIO_ATTRIBUTES:
case MSG_SET_AUX_EFFECT_INFO:
case MSG_SET_CAMERA_MOTION_LISTENER:
@ -659,6 +685,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
}
private void setOutput(@Nullable Object output) throws ExoPlaybackException {
// TODO(b/238302341) Handle output surface change in previewing.
// Handle unsupported (i.e., non-Surface) outputs by clearing the display surface.
@Nullable Surface displaySurface = output instanceof Surface ? (Surface) output : null;
@ -753,8 +780,19 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
}
displaySurface = placeholderSurface;
}
if (frameProcessorManager.isEnabled()) {
mediaFormat = frameProcessorManager.amendMediaFormatKeys(mediaFormat);
}
return MediaCodecAdapter.Configuration.createForVideoDecoding(
codecInfo, mediaFormat, format, displaySurface, crypto);
codecInfo,
mediaFormat,
format,
frameProcessorManager.isEnabled()
? frameProcessorManager.getInputSurface()
: displaySurface,
crypto);
}
@Override
@ -878,6 +916,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
return maxFrameRate == -1 ? CODEC_OPERATING_RATE_UNSET : (maxFrameRate * targetPlaybackSpeed);
}
@CallSuper
@Override
protected void onReadyToInitializeCodec(Format format) throws ExoPlaybackException {
if (!frameProcessorManager.isEnabled()) {
frameProcessorManager.maybeEnable(format);
}
}
@Override
protected void onCodecInitialized(
String name,
@ -985,6 +1031,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
decodedVideoSize =
new VideoSize(width, height, unappliedRotationDegrees, pixelWidthHeightRatio);
frameReleaseHelper.onFormatChanged(format.frameRate);
if (frameProcessorManager.isEnabled()) {
frameProcessorManager.setInputFrameInfo(width, height, pixelWidthHeightRatio);
}
}
@Override
@ -1622,6 +1672,185 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
return new MediaCodecVideoDecoderException(cause, codecInfo, displaySurface);
}
/** Manages {@link FrameProcessor} interactions. */
private static final class FrameProcessorManager {
private static final String FRAME_PROCESSOR_FACTORY_CLASS =
"com.google.android.exoplayer2.effect.GlEffectsFrameProcessor$Factory";
// TODO(b/238302341) Consider removing the reference to the containing class and make this class
// non-static.
private final MediaCodecVideoRenderer renderer;
private final ArrayDeque<Long> processedFrames;
private @MonotonicNonNull Handler handler;
@Nullable private FrameProcessor frameProcessor;
@Nullable private CopyOnWriteArrayList<Effect> videoEffects;
private boolean canEnableFrameProcessing;
/** Creates a new instance. */
public FrameProcessorManager(@UnderInitialization MediaCodecVideoRenderer renderer) {
this.renderer = renderer;
processedFrames = new ArrayDeque<>();
canEnableFrameProcessing = true;
}
/** Sets the {@linkplain Effect video effects}. */
public void setVideoEffects(List<Effect> videoEffects) {
if (this.videoEffects == null) {
this.videoEffects = new CopyOnWriteArrayList<>(videoEffects);
return;
}
this.videoEffects.clear();
this.videoEffects.addAll(videoEffects);
}
/** Returns whether frame processing is enabled. */
public boolean isEnabled() {
return frameProcessor != null;
}
/**
* Tries to enable frame processing.
*
* <p>Caller must ensure frame processing {@linkplain #isEnabled() is not enabled} before
* calling this method.
*
* @param inputFormat The {@link Format} that is input into the {@link FrameProcessor}.
* @return Whether frame processing is enabled.
* @throws ExoPlaybackException When enabling the {@link FrameProcessor} failed.
*/
@CanIgnoreReturnValue
public boolean maybeEnable(Format inputFormat) throws ExoPlaybackException {
checkState(!isEnabled());
if (!canEnableFrameProcessing) {
return false;
}
if (videoEffects == null) {
canEnableFrameProcessing = false;
return false;
}
// Playback thread handler.
handler = Util.createHandlerForCurrentLooper();
try {
frameProcessor =
((FrameProcessor.Factory)
Class.forName(FRAME_PROCESSOR_FACTORY_CLASS).getConstructor().newInstance())
.create(
renderer.context,
checkNotNull(videoEffects),
DebugViewProvider.NONE,
inputFormat.colorInfo != null
? inputFormat.colorInfo
: ColorInfo.SDR_BT709_LIMITED,
/* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED,
/* releaseFramesAutomatically= */ false,
/* executor= */ handler::post,
new FrameProcessor.Listener() {
@Override
public void onOutputSizeChanged(int width, int height) {
// TODO(b/238302341) Handle output size change.
}
@Override
public void onOutputFrameAvailable(long presentationTimeUs) {
processedFrames.add(presentationTimeUs);
}
@Override
public void onFrameProcessingError(FrameProcessingException exception) {
renderer.setPendingPlaybackException(
renderer.createRendererException(
exception,
inputFormat,
// TODO(b/238302341) Add relevant error codes for frame processing.
PlaybackException.ERROR_CODE_UNSPECIFIED));
}
@Override
public void onFrameProcessingEnded() {
throw new IllegalStateException();
}
});
} catch (Exception e) {
throw renderer.createRendererException(
e, inputFormat, PlaybackException.ERROR_CODE_UNSPECIFIED);
}
setInputFrameInfo(inputFormat.width, inputFormat.height, inputFormat.pixelWidthHeightRatio);
return true;
}
/**
* Returns the {@linkplain FrameProcessor#getInputSurface input surface} of the {@link
* FrameProcessor}.
*
* <p>Caller must ensure the {@code FrameProcessorManager} {@link #isEnabled()} before calling
* this method.
*/
public Surface getInputSurface() {
return checkNotNull(frameProcessor).getInputSurface();
}
/**
* Sets the output surface info.
*
* <p>Caller must ensure the {@code FrameProcessorManager} {@link #isEnabled()} before calling
* this method.
*
* @param outputSurface The {@link Surface} to which {@link FrameProcessor} outputs.
* @param outputResolution The {@link Size} of the output resolution.
*/
public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) {
checkNotNull(frameProcessor)
.setOutputSurfaceInfo(
new SurfaceInfo(
outputSurface, outputResolution.getWidth(), outputResolution.getHeight()));
}
/**
* Sets the input surface info.
*
* <p>Caller must ensure the {@code FrameProcessorManager} {@link #isEnabled()} before calling
* this method.
*/
public void setInputFrameInfo(int width, int height, float pixelWidthHeightRatio) {
checkNotNull(frameProcessor)
.setInputFrameInfo(
new FrameInfo(
width, height, pixelWidthHeightRatio, renderer.getOutputStreamOffsetUs()));
}
/** Sets the necessary {@link MediaFormat} keys for frame processing. */
@SuppressWarnings("InlinedApi")
public MediaFormat amendMediaFormatKeys(MediaFormat mediaFormat) {
if (Util.SDK_INT >= 29
&& renderer.context.getApplicationContext().getApplicationInfo().targetSdkVersion >= 29) {
mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0);
}
return mediaFormat;
}
/**
* Releases the resources.
*
* <p>Caller must ensure frame processing {@linkplain #isEnabled() is not enabled} before
* calling this method.
*/
public void reset() {
checkNotNull(frameProcessor).release();
frameProcessor = null;
if (handler != null) {
handler.removeCallbacksAndMessages(/* token= */ null);
}
if (videoEffects != null) {
videoEffects.clear();
}
processedFrames.clear();
canEnableFrameProcessing = true;
}
}
/**
* Returns a maximum video size to use when configuring a codec for {@code format} in a way that
* will allow possible adaptation to other compatible formats that are expected to have the same