From 0f3e63909cf392ec0f46bef8d93d5b26e159ba11 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Thu, 28 Apr 2022 12:59:52 +0100 Subject: [PATCH] Transformer: Restrict frame buffer size for all omx decoders API >= 29. PiperOrigin-RevId: 445119411 --- .../androidx/media3/transformer/Codec.java | 10 ++++++ .../media3/transformer/DefaultCodec.java | 25 ++++++++++++++ .../VideoTranscodingSamplePipeline.java | 34 ++----------------- 3 files changed, 37 insertions(+), 32 deletions(-) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java index 9e9f3b3baa..122470b111 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java @@ -35,6 +35,8 @@ import java.util.List; */ @UnstableApi public interface Codec { + /** Default value for the pending frame count, which represents applying no limit. */ + int UNLIMITED_PENDING_FRAME_COUNT = Integer.MAX_VALUE; /** A factory for {@linkplain Codec decoder} instances. */ interface DecoderFactory { @@ -139,6 +141,14 @@ public interface Codec { */ Surface getInputSurface(); + /** + * Returns the maximum number of frames that may be pending in the output {@code Codec} at a time, + * or {@link #UNLIMITED_PENDING_FRAME_COUNT} if it's not necessary to enforce a limit. + */ + default int getMaxPendingFrameCount() { + return UNLIMITED_PENDING_FRAME_COUNT; + } + /** * Dequeues a writable input buffer, if available. * diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodec.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodec.java index 3280df2da4..c03697d8f8 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodec.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodec.java @@ -30,7 +30,9 @@ import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.TraceUtil; import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; +import com.google.common.base.Ascii; import com.google.common.collect.ImmutableList; import java.io.IOException; import java.nio.ByteBuffer; @@ -120,6 +122,29 @@ public final class DefaultCodec implements Codec { return checkStateNotNull(inputSurface); } + @Override + public int getMaxPendingFrameCount() { + if (Util.SDK_INT < 29) { + // Prior to API 29, decoders may drop frames to keep their output surface from growing out of + // bounds. From API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame + // dropping even when the surface is full. Frame dropping is never desired, so allow a maximum + // of one frame to be pending at a time. + // TODO(b/226330223): Investigate increasing this limit. + return 1; + } + if (Ascii.toUpperCase(mediaCodec.getCodecInfo().getCanonicalName()).startsWith("OMX.")) { + // Some OMX decoders don't correctly track their number of output buffers available, and get + // stuck if too many frames are rendered without being processed, so limit the number of + // pending frames to avoid getting stuck. This value is experimentally determined. See also + // b/213455700, b/230097284, and b/229978305,. + // TODO(b/230097284): Add a maximum API check after we know which APIs will never use OMX. + return 10; + } + // Otherwise don't limit the number of frames that can be pending at a time, to maximize + // throughput. + return UNLIMITED_PENDING_FRAME_COUNT; + } + @Override @EnsuresNonNullIf(expression = "#1.data", result = true) public boolean maybeDequeueInputBuffer(DecoderInputBuffer inputBuffer) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 0587fe76f2..2304c7e74d 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -34,9 +34,6 @@ import org.checkerframework.dataflow.qual.Pure; * Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them. */ /* package */ final class VideoTranscodingSamplePipeline implements SamplePipeline { - - private static final int FRAME_COUNT_UNLIMITED = -1; - private final int outputRotationDegrees; private final DecoderInputBuffer decoderInputBuffer; private final Codec decoder; @@ -135,7 +132,7 @@ import org.checkerframework.dataflow.qual.Pure; inputFormat, frameProcessorChain.getInputSurface(), transformationRequest.enableRequestSdrToneMapping); - maxPendingFrameCount = getMaxPendingFrameCount(); + maxPendingFrameCount = decoder.getMaxPendingFrameCount(); } @Override @@ -254,7 +251,7 @@ import org.checkerframework.dataflow.qual.Pure; return false; } - if (maxPendingFrameCount != FRAME_COUNT_UNLIMITED + if (maxPendingFrameCount != Codec.UNLIMITED_PENDING_FRAME_COUNT && frameProcessorChain.getPendingFrameCount() == maxPendingFrameCount) { return false; } @@ -263,31 +260,4 @@ import org.checkerframework.dataflow.qual.Pure; decoder.releaseOutputBuffer(/* render= */ true); return true; } - - /** - * Returns the maximum number of frames that may be pending in the output {@link - * FrameProcessorChain} at a time, or {@link #FRAME_COUNT_UNLIMITED} if it's not necessary to - * enforce a limit. - */ - private static int getMaxPendingFrameCount() { - if (Util.SDK_INT < 29) { - // Prior to API 29, decoders may drop frames to keep their output surface from growing out of - // bounds, while from API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame - // dropping even when the surface is full. We never want frame dropping so allow a maximum of - // one frame to be pending at a time. - // TODO(b/226330223): Investigate increasing this limit. - return 1; - } - if (Util.SDK_INT < 33 - && ("OnePlus".equals(Util.MANUFACTURER) || "samsung".equals(Util.MANUFACTURER))) { - // Some OMX decoders don't correctly track their number of output buffers available, and get - // stuck if too many frames are rendered without being processed, so we limit the number of - // pending frames to avoid getting stuck. This value is experimentally determined. See also - // b/213455700. - return 10; - } - // Otherwise don't limit the number of frames that can be pending at a time, to maximize - // throughput. - return FRAME_COUNT_UNLIMITED; - } }