From 117456c1371ebf8d1ac7b97b12fdba8c3cb9fd6e Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Thu, 24 Mar 2022 19:19:22 +0000 Subject: [PATCH] Improve throughput on devices requiring workaround On some devices, decoding gets stuck when the number of frames pending at the `SurfaceTexture` is too high. We added a workaround that only allows one frame to be pending at a time. That fixed the issue, however, based on on-device testing it seems that it's safe to queue more than one frame. Add a method that returns a safe estimate of the number of frames that can be pending at a time, and use this to limit the number of frames that can be released from the decoder but not processed by the frame processor chain. PiperOrigin-RevId: 437057075 --- .../transformer/FrameProcessorChain.java | 12 +-- .../media3/transformer/SamplePipeline.java | 4 +- .../VideoTranscodingSamplePipeline.java | 85 +++++++++---------- 3 files changed, 49 insertions(+), 52 deletions(-) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java index 7972e5f37a..b80e32a36e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java @@ -55,7 +55,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; *

Input becomes available on its {@link #getInputSurface() input surface} asynchronously and is * processed on a background thread as it becomes available. All input frames should be {@link * #registerInputFrame() registered} before they are rendered to the input surface. {@link - * #hasPendingFrames()} can be used to check whether there are frames that have not been fully + * #getPendingFrameCount()} can be used to check whether there are frames that have not been fully * processed yet. Output is written to its {@link #configure(Surface, int, int, SurfaceView) output * surface}. */ @@ -298,16 +298,16 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } /** - * Returns whether there are input frames that have been {@link #registerInputFrame() registered} - * but not completely processed yet. + * Returns the number of input frames that have been {@link #registerInputFrame() registered} but + * not completely processed yet. */ - public boolean hasPendingFrames() { - return pendingFrameCount.get() > 0; + public int getPendingFrameCount() { + return pendingFrameCount.get(); } /** Returns whether all frames have been processed. */ public boolean isEnded() { - return inputStreamEnded && !hasPendingFrames(); + return inputStreamEnded && getPendingFrameCount() == 0; } /** Informs the {@code FrameProcessorChain} that no further input frames should be accepted. */ diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java index b2aa387111..63ec1e8c75 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java @@ -40,8 +40,8 @@ import androidx.media3.decoder.DecoderInputBuffer; void queueInputBuffer() throws TransformationException; /** - * Processes the input data and returns whether more data can be processed by calling this method - * again. + * Processes the input data and returns whether it may be possible to process more data by calling + * this method again. */ boolean processData() throws TransformationException; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index c5c35b092f..4fa5ae8520 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -17,14 +17,11 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Util.SDK_INT; import android.content.Context; import android.media.MediaCodec; -import android.media.MediaFormat; import android.util.Size; import androidx.annotation.Nullable; -import androidx.annotation.RequiresApi; import androidx.media3.common.Format; import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; @@ -37,9 +34,12 @@ import org.checkerframework.dataflow.qual.Pure; */ /* package */ final class VideoTranscodingSamplePipeline implements SamplePipeline { + private static final int FRAME_COUNT_UNLIMITED = -1; + private final int outputRotationDegrees; private final DecoderInputBuffer decoderInputBuffer; private final Codec decoder; + private final int maxPendingFrameCount; private final FrameProcessorChain frameProcessorChain; @@ -121,6 +121,7 @@ import org.checkerframework.dataflow.qual.Pure; decoder = decoderFactory.createForVideoDecoding(inputFormat, frameProcessorChain.getInputSurface()); + maxPendingFrameCount = getMaxPendingFrameCount(); } @Override @@ -148,51 +149,15 @@ import org.checkerframework.dataflow.qual.Pure; return false; } - boolean canProcessMoreDataImmediately = false; - if (SDK_INT >= 29 - && !(("samsung".equals(Util.MANUFACTURER) || "OnePlus".equals(Util.MANUFACTURER)) - && SDK_INT < 31)) { - // TODO(b/213455700): Fix Samsung and OnePlus devices filling the decoder in processDataV29(). - processDataV29(); - } else { - canProcessMoreDataImmediately = processDataDefault(); + boolean processedData = false; + while (maybeProcessDecoderOutput()) { + processedData = true; } if (decoder.isEnded()) { frameProcessorChain.signalEndOfInputStream(); } - return canProcessMoreDataImmediately; - } - - /** - * Processes input data from API 29. - * - *

In this method the decoder could decode multiple frames in one invocation; as compared to - * {@link #processDataDefault()}, in which one frame is decoded in each invocation. Consequently, - * if {@link FrameProcessorChain} processes frames slower than the decoder, decoded frames are - * queued up in the decoder's output surface. - * - *

Prior to API 29, decoders may drop frames to keep their output surface from growing out of - * bound; while after API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame - * dropping even when the surface is full. As dropping random frames is not acceptable in {@code - * Transformer}, using this method requires API level 29 or higher. - */ - @RequiresApi(29) - private void processDataV29() throws TransformationException { - while (maybeProcessDecoderOutput()) {} - } - - /** - * Processes at most one input frame and returns whether a frame was processed. - * - *

Only renders decoder output to the {@link FrameProcessorChain}'s input surface if the {@link - * FrameProcessorChain} has finished processing the previous frame. - */ - private boolean processDataDefault() throws TransformationException { - // TODO(b/214975934): Check whether this can be converted to a while-loop like processDataV29. - if (frameProcessorChain.hasPendingFrames()) { - return false; - } - return maybeProcessDecoderOutput(); + // If the decoder produced output, signal that it may be possible to process data again. + return processedData; } @Override @@ -275,8 +240,40 @@ import org.checkerframework.dataflow.qual.Pure; return false; } + if (maxPendingFrameCount != FRAME_COUNT_UNLIMITED + && frameProcessorChain.getPendingFrameCount() == maxPendingFrameCount) { + return false; + } + frameProcessorChain.registerInputFrame(); decoder.releaseOutputBuffer(/* render= */ true); return true; } + + /** + * Returns the maximum number of frames that may be pending in the output {@link + * FrameProcessorChain} at a time, or {@link #FRAME_COUNT_UNLIMITED} if it's not necessary to + * enforce a limit. + */ + private static int getMaxPendingFrameCount() { + if (Util.SDK_INT < 29) { + // Prior to API 29, decoders may drop frames to keep their output surface from growing out of + // bounds, while from API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame + // dropping even when the surface is full. We never want frame dropping so allow a maximum of + // one frame to be pending at a time. + // TODO(b/226330223): Investigate increasing this limit. + return 1; + } + if (Util.SDK_INT < 31 + && ("OnePlus".equals(Util.MANUFACTURER) || "samsung".equals(Util.MANUFACTURER))) { + // Some OMX decoders don't correctly track their number of output buffers available, and get + // stuck if too many frames are rendered without being processed, so we limit the number of + // pending frames to avoid getting stuck. This value is experimentally determined. See also + // b/213455700. + return 10; + } + // Otherwise don't limit the number of frames that can be pending at a time, to maximize + // throughput. + return FRAME_COUNT_UNLIMITED; + } }