diff --git a/library/common/src/main/java/com/google/android/exoplayer2/C.java b/library/common/src/main/java/com/google/android/exoplayer2/C.java index 6ac9821650..dd3bb3c1b0 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/C.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/C.java @@ -1103,6 +1103,12 @@ public final class C { */ public static final int COLOR_RANGE_FULL = MediaFormat.COLOR_RANGE_FULL; + /** + * Represents applying no limit to the number of input frames a {@link MediaCodec} encoder + * accepts. + */ + public static final int UNLIMITED_PENDING_FRAME_COUNT = Integer.MAX_VALUE; + /** Video projection types. */ @Documented @Retention(RetentionPolicy.SOURCE) diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/Util.java b/library/common/src/main/java/com/google/android/exoplayer2/util/Util.java index 315ffacc98..c0f0256da8 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/Util.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/Util.java @@ -16,6 +16,7 @@ package com.google.android.exoplayer2.util; import static android.content.Context.UI_MODE_SERVICE; +import static com.google.android.exoplayer2.C.UNLIMITED_PENDING_FRAME_COUNT; import static com.google.android.exoplayer2.Player.COMMAND_SEEK_BACK; import static com.google.android.exoplayer2.Player.COMMAND_SEEK_FORWARD; import static com.google.android.exoplayer2.Player.COMMAND_SEEK_IN_CURRENT_MEDIA_ITEM; @@ -50,6 +51,7 @@ import android.graphics.Point; import android.hardware.display.DisplayManager; import android.media.AudioFormat; import android.media.AudioManager; +import android.media.MediaCodec; import android.media.MediaDrm; import android.net.Uri; import android.os.Build; @@ -2670,6 +2672,46 @@ public final class Util { } } + /** + * Returns the number of maximum pending input frames that are allowed on a {@link MediaCodec} + * encoder. + */ + public static int getMaxPendingFramesCountForMediaCodecEncoders( + Context context, String codecName, boolean requestedHdrToneMapping) { + if (SDK_INT < 29 + || context.getApplicationContext().getApplicationInfo().targetSdkVersion < 29) { + // Prior to API 29, decoders may drop frames to keep their output surface from growing out of + // bounds. From API 29, if the app targets API 29 or later, the {@link + // MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame dropping even when the surface is + // full. + // Frame dropping is never desired, so a workaround is needed for older API levels. + // Allow a maximum of one frame to be pending at a time to prevent frame dropping. + // TODO(b/226330223): Investigate increasing this limit. + return 1; + } + if (Ascii.toUpperCase(codecName).startsWith("OMX.")) { + // Some OMX decoders don't correctly track their number of output buffers available, and get + // stuck if too many frames are rendered without being processed, so limit the number of + // pending frames to avoid getting stuck. This value is experimentally determined. See also + // b/213455700, b/230097284, b/229978305, and b/245491744. + // + // OMX video codecs should no longer exist from android.os.Build.DEVICE_INITIAL_SDK_INT 31+. + return 5; + } + if (requestedHdrToneMapping + && codecName.equals("c2.qti.hevc.decoder") + && MODEL.equals("SM-F936B")) { + // This decoder gets stuck if too many frames are rendered without being processed when + // tone-mapping HDR10. This value is experimentally determined. See also b/260408846. + // TODO(b/260713009): Add API version check after bug is fixed on new API versions. + return 12; + } + + // Otherwise don't limit the number of frames that can be pending at a time, to maximize + // throughput. + return UNLIMITED_PENDING_FRAME_COUNT; + } + /** * Returns string representation of a {@link C.FormatSupport} flag. * diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index 2207efb3de..ab3b2d5e33 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -21,6 +21,7 @@ import static com.google.android.exoplayer2.decoder.DecoderReuseEvaluation.DISCA import static com.google.android.exoplayer2.decoder.DecoderReuseEvaluation.REUSE_RESULT_NO; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Assertions.checkState; +import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; import static java.lang.Math.max; import static java.lang.Math.min; @@ -937,6 +938,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { if (Util.SDK_INT >= 23 && tunneling) { tunnelingOnFrameRenderedListener = new OnFrameRenderedListenerV23(checkNotNull(getCodec())); } + frameProcessorManager.onCodecInitialized(name); } @Override @@ -1137,13 +1139,20 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { && (shouldRenderFirstFrame || (isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs))); if (forceRenderOutputBuffer) { - long releaseTimeNs = System.nanoTime(); - notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); - if (Util.SDK_INT >= 21) { - renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, releaseTimeNs); + // TODO(b/238302341): Handle releasing the force rendered frames in FrameProcessor. + boolean notifyFrameMetaDataListener; + if (frameProcessorManager.isEnabled()) { + notifyFrameMetaDataListener = false; + if (!frameProcessorManager.maybeRegisterFrame()) { + // TODO(b/238302341): Handle FrameProcessor is unable to accept the force rendered buffer. + // Treat the frame as dropped for now. + return true; + } } else { - renderOutputBuffer(codec, bufferIndex, presentationTimeUs); + notifyFrameMetaDataListener = true; } + renderOutputBufferNow( + codec, format, bufferIndex, presentationTimeUs, notifyFrameMetaDataListener); updateVideoFrameProcessingOffsetCounters(earlyUs); return true; } @@ -1174,6 +1183,20 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return true; } + if (frameProcessorManager.isEnabled()) { + frameProcessorManager.releaseProcessedFrames(); + if (frameProcessorManager.maybeRegisterFrame()) { + renderOutputBufferNow( + codec, + format, + bufferIndex, + presentationTimeUs, + /* notifyFrameMetadataListener= */ false); + return true; + } + return false; + } + if (Util.SDK_INT >= 21) { // Let the underlying framework time the release. if (earlyUs < 50000) { @@ -1380,6 +1403,40 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { videoFrameProcessingOffsetCount++; } + /** + * Renders the output buffer with the specified index now. + * + * @param codec The codec that owns the output buffer. + * @param format The {@link Format} associated with the buffer. + * @param index The index of the output buffer to drop. + * @param presentationTimeUs The presentation time of the output buffer, in microseconds. + * @param notifyFrameMetadataListener Whether to notify the {@link VideoFrameMetadataListener}. + */ + private void renderOutputBufferNow( + MediaCodecAdapter codec, + Format format, + int index, + long presentationTimeUs, + boolean notifyFrameMetadataListener) { + // In previewing mode, use the presentation time as release time so that the SurfaceTexture is + // accompanied by the rendered frame's presentation time. Setting a realtime based release time + // is only relevant when rendering to a SurfaceView (that is when not using FrameProcessor) for + // better frame release. In previewing mode MediaCodec renders to FrameProcessor's input + // surface, which is not a SurfaceView. + long releaseTimeNs = + frameProcessorManager.isEnabled() + ? (presentationTimeUs + getOutputStreamOffsetUs()) * 1000 + : System.nanoTime(); + if (notifyFrameMetadataListener) { + notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format); + } + if (Util.SDK_INT >= 21) { + renderOutputBufferV21(codec, index, presentationTimeUs, releaseTimeNs); + } else { + renderOutputBuffer(codec, index, presentationTimeUs); + } + } + /** * Renders the output buffer with the specified index. This method is only called if the platform * API version of the device is less than 21. @@ -1686,12 +1743,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { private @MonotonicNonNull Handler handler; @Nullable private FrameProcessor frameProcessor; @Nullable private CopyOnWriteArrayList videoEffects; + private int frameProcessorMaxPendingFrameCount; private boolean canEnableFrameProcessing; /** Creates a new instance. */ public FrameProcessorManager(@UnderInitialization MediaCodecVideoRenderer renderer) { this.renderer = renderer; processedFrames = new ArrayDeque<>(); + frameProcessorMaxPendingFrameCount = C.LENGTH_UNSET; canEnableFrameProcessing = true; } @@ -1831,6 +1890,50 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return mediaFormat; } + /** + * Must be called when the codec is initialized. + * + *

Sets the {@code frameProcessorMaxPendingFrameCount} based on the {@code codecName}. + */ + public void onCodecInitialized(String codecName) { + frameProcessorMaxPendingFrameCount = + Util.getMaxPendingFramesCountForMediaCodecEncoders( + renderer.context, codecName, /* requestedHdrToneMapping= */ false); + } + + /** + * Tries to {@linkplain FrameProcessor#registerInputFrame register an input frame}. + * + *

Caller must ensure the {@code FrameProcessorManager} {@link #isEnabled()} before calling + * this method. + * + * @return Whether {@link MediaCodec} should render the frame to {@link FrameProcessor}. + */ + public boolean maybeRegisterFrame() { + checkStateNotNull(frameProcessor); + checkState(frameProcessorMaxPendingFrameCount != C.LENGTH_UNSET); + if (frameProcessor.getPendingInputFrameCount() < frameProcessorMaxPendingFrameCount) { + frameProcessor.registerInputFrame(); + return true; + } + return false; + } + + /** + * Releases the processed frames to the {@linkplain #setOutputSurfaceInfo output surface}. + * + *

Caller must ensure the {@code FrameProcessorManager} {@link #isEnabled()} before calling + * this method. + */ + public void releaseProcessedFrames() { + while (!processedFrames.isEmpty()) { + processedFrames.poll(); + // TODO(b/238302341): Add frame release logic. + checkNotNull(frameProcessor) + .releaseOutputFrame(FrameProcessor.RELEASE_OUTPUT_FRAME_IMMEDIATELY); + } + } + /** * Releases the resources. * diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java index 804194c260..5c0c7511e3 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java @@ -31,9 +31,6 @@ import java.nio.ByteBuffer; * buffers. */ public interface Codec { - /** Default value for the pending frame count, which represents applying no limit. */ - int UNLIMITED_PENDING_FRAME_COUNT = Integer.MAX_VALUE; - /** A factory for {@linkplain Codec decoder} instances. */ interface DecoderFactory { @@ -134,10 +131,10 @@ public interface Codec { /** * Returns the maximum number of frames that may be pending in the output {@code Codec} at a time, - * or {@link #UNLIMITED_PENDING_FRAME_COUNT} if it's not necessary to enforce a limit. + * or {@link C#UNLIMITED_PENDING_FRAME_COUNT} if it's not necessary to enforce a limit. */ default int getMaxPendingFrameCount() { - return UNLIMITED_PENDING_FRAME_COUNT; + return C.UNLIMITED_PENDING_FRAME_COUNT; } /** diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodec.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodec.java index 86585eee02..4eb1f87548 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodec.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodec.java @@ -20,7 +20,6 @@ import static com.google.android.exoplayer2.util.Assertions.checkArgument; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Assertions.checkState; import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; -import static com.google.android.exoplayer2.util.Util.MODEL; import static com.google.android.exoplayer2.util.Util.SDK_INT; import android.content.Context; @@ -40,8 +39,8 @@ import com.google.android.exoplayer2.util.Log; import com.google.android.exoplayer2.util.MediaFormatUtil; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.TraceUtil; +import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.video.ColorInfo; -import com.google.common.base.Ascii; import com.google.common.collect.ImmutableList; import java.io.IOException; import java.nio.ByteBuffer; @@ -136,7 +135,8 @@ public final class DefaultCodec implements Codec { this.mediaCodec = mediaCodec; this.inputSurface = inputSurface; maxPendingFrameCount = - getMaxPendingFrameCountInternal(context, mediaCodecName, requestedHdrToneMapping); + Util.getMaxPendingFramesCountForMediaCodecEncoders( + context, mediaCodecName, requestedHdrToneMapping); } @Override @@ -464,42 +464,6 @@ public final class DefaultCodec implements Codec { TraceUtil.endSection(); } - private static int getMaxPendingFrameCountInternal( - Context context, String codecName, boolean requestedHdrToneMapping) { - if (SDK_INT < 29 - || context.getApplicationContext().getApplicationInfo().targetSdkVersion < 29) { - // Prior to API 29, decoders may drop frames to keep their output surface from growing out of - // bounds. From API 29, if the app targets API 29 or later, the {@link - // MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame dropping even when the surface is - // full. - // Frame dropping is never desired, so a workaround is needed for older API levels. - // Allow a maximum of one frame to be pending at a time to prevent frame dropping. - // TODO(b/226330223): Investigate increasing this limit. - return 1; - } - if (Ascii.toUpperCase(codecName).startsWith("OMX.")) { - // Some OMX decoders don't correctly track their number of output buffers available, and get - // stuck if too many frames are rendered without being processed, so limit the number of - // pending frames to avoid getting stuck. This value is experimentally determined. See also - // b/213455700, b/230097284, b/229978305, and b/245491744. - // - // OMX video codecs should no longer exist from android.os.Build.DEVICE_INITIAL_SDK_INT 31+. - return 5; - } - if (requestedHdrToneMapping - && codecName.equals("c2.qti.hevc.decoder") - && MODEL.equals("SM-F936B")) { - // This decoder gets stuck if too many frames are rendered without being processed when - // tone-mapping HDR10. This value is experimentally determined. See also b/260408846. - // TODO(b/260713009): Add API version check after bug is fixed on new API versions. - return 12; - } - - // Otherwise don't limit the number of frames that can be pending at a time, to maximize - // throughput. - return UNLIMITED_PENDING_FRAME_COUNT; - } - @RequiresApi(29) private static final class Api29 { @DoNotInline diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java index fde57290a7..8bb7f2f68b 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java @@ -360,7 +360,7 @@ import org.checkerframework.dataflow.qual.Pure; return true; } - if (maxPendingFrameCount != Codec.UNLIMITED_PENDING_FRAME_COUNT + if (maxPendingFrameCount != C.UNLIMITED_PENDING_FRAME_COUNT && frameProcessor.getPendingInputFrameCount() == maxPendingFrameCount) { return false; }