loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-05-07 23:50:44 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Make Codec release frames to frame processor.

Browse Source 
PiperOrigin-RevId: 495406734
This commit is contained in:
claincly 2022-12-14 21:47:25 +00:00 committed by Ian Baker
parent b836212b39
commit 678b29c10d
6 changed files with 162 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
library/common/src/main/java/com/google/android/exoplayer2/C.java
View File

@ -1103,6 +1103,12 @@ public final class C {
*/
public static final int COLOR_RANGE_FULL = MediaFormat.COLOR_RANGE_FULL;
/**
* Represents applying no limit to the number of input frames a {@link MediaCodec} encoder
* accepts.
*/
public static final int UNLIMITED_PENDING_FRAME_COUNT = Integer.MAX_VALUE;
/** Video projection types. */
@Documented
@Retention(RetentionPolicy.SOURCE)

42
library/common/src/main/java/com/google/android/exoplayer2/util/Util.java
View File

@ -16,6 +16,7 @@
package com.google.android.exoplayer2.util;
import static android.content.Context.UI_MODE_SERVICE;
import static com.google.android.exoplayer2.C.UNLIMITED_PENDING_FRAME_COUNT;
import static com.google.android.exoplayer2.Player.COMMAND_SEEK_BACK;
import static com.google.android.exoplayer2.Player.COMMAND_SEEK_FORWARD;
import static com.google.android.exoplayer2.Player.COMMAND_SEEK_IN_CURRENT_MEDIA_ITEM;
@ -50,6 +51,7 @@ import android.graphics.Point;
import android.hardware.display.DisplayManager;
import android.media.AudioFormat;
import android.media.AudioManager;
import android.media.MediaCodec;
import android.media.MediaDrm;
import android.net.Uri;
import android.os.Build;
@ -2670,6 +2672,46 @@ public final class Util {
}
}
/**
* Returns the number of maximum pending input frames that are allowed on a {@link MediaCodec}
* encoder.
*/
public static int getMaxPendingFramesCountForMediaCodecEncoders(
Context context, String codecName, boolean requestedHdrToneMapping) {
if (SDK_INT < 29
|| context.getApplicationContext().getApplicationInfo().targetSdkVersion < 29) {
// Prior to API 29, decoders may drop frames to keep their output surface from growing out of
// bounds. From API 29, if the app targets API 29 or later, the {@link
// MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame dropping even when the surface is
// full.
// Frame dropping is never desired, so a workaround is needed for older API levels.
// Allow a maximum of one frame to be pending at a time to prevent frame dropping.
// TODO(b/226330223): Investigate increasing this limit.
return 1;
}
if (Ascii.toUpperCase(codecName).startsWith("OMX.")) {
// Some OMX decoders don't correctly track their number of output buffers available, and get
// stuck if too many frames are rendered without being processed, so limit the number of
// pending frames to avoid getting stuck. This value is experimentally determined. See also
// b/213455700, b/230097284, b/229978305, and b/245491744.
//
// OMX video codecs should no longer exist from android.os.Build.DEVICE_INITIAL_SDK_INT 31+.
return 5;
}
if (requestedHdrToneMapping
&& codecName.equals("c2.qti.hevc.decoder")
&& MODEL.equals("SM-F936B")) {
// This decoder gets stuck if too many frames are rendered without being processed when
// tone-mapping HDR10. This value is experimentally determined. See also b/260408846.
// TODO(b/260713009): Add API version check after bug is fixed on new API versions.
return 12;
}
// Otherwise don't limit the number of frames that can be pending at a time, to maximize
// throughput.
return UNLIMITED_PENDING_FRAME_COUNT;
}
/**
* Returns string representation of a {@link C.FormatSupport} flag.
*

113
library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java
View File

@ -21,6 +21,7 @@ import static com.google.android.exoplayer2.decoder.DecoderReuseEvaluation.DISCA
import static com.google.android.exoplayer2.decoder.DecoderReuseEvaluation.REUSE_RESULT_NO;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import static java.lang.Math.max;
import static java.lang.Math.min;
@ -937,6 +938,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
if (Util.SDK_INT >= 23 && tunneling) {
tunnelingOnFrameRenderedListener = new OnFrameRenderedListenerV23(checkNotNull(getCodec()));
}
frameProcessorManager.onCodecInitialized(name);
}
@Override
@ -1137,13 +1139,20 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
&& (shouldRenderFirstFrame
|| (isStarted && shouldForceRenderOutputBuffer(earlyUs, elapsedSinceLastRenderUs)));
if (forceRenderOutputBuffer) {
long releaseTimeNs = System.nanoTime();
notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format);
if (Util.SDK_INT >= 21) {
renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, releaseTimeNs);
// TODO(b/238302341): Handle releasing the force rendered frames in FrameProcessor.
boolean notifyFrameMetaDataListener;
if (frameProcessorManager.isEnabled()) {
notifyFrameMetaDataListener = false;
if (!frameProcessorManager.maybeRegisterFrame()) {
// TODO(b/238302341): Handle FrameProcessor is unable to accept the force rendered buffer.
// Treat the frame as dropped for now.
return true;
}
} else {
renderOutputBuffer(codec, bufferIndex, presentationTimeUs);
notifyFrameMetaDataListener = true;
}
renderOutputBufferNow(
codec, format, bufferIndex, presentationTimeUs, notifyFrameMetaDataListener);
updateVideoFrameProcessingOffsetCounters(earlyUs);
return true;
}
@ -1174,6 +1183,20 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
return true;
}
if (frameProcessorManager.isEnabled()) {
frameProcessorManager.releaseProcessedFrames();
if (frameProcessorManager.maybeRegisterFrame()) {
renderOutputBufferNow(
codec,
format,
bufferIndex,
presentationTimeUs,
/* notifyFrameMetadataListener= */ false);
return true;
}
return false;
}
if (Util.SDK_INT >= 21) {
// Let the underlying framework time the release.
if (earlyUs < 50000) {
@ -1380,6 +1403,40 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
videoFrameProcessingOffsetCount++;
}
/**
* Renders the output buffer with the specified index now.
*
* @param codec The codec that owns the output buffer.
* @param format The {@link Format} associated with the buffer.
* @param index The index of the output buffer to drop.
* @param presentationTimeUs The presentation time of the output buffer, in microseconds.
* @param notifyFrameMetadataListener Whether to notify the {@link VideoFrameMetadataListener}.
*/
private void renderOutputBufferNow(
MediaCodecAdapter codec,
Format format,
int index,
long presentationTimeUs,
boolean notifyFrameMetadataListener) {
// In previewing mode, use the presentation time as release time so that the SurfaceTexture is
// accompanied by the rendered frame's presentation time. Setting a realtime based release time
// is only relevant when rendering to a SurfaceView (that is when not using FrameProcessor) for
// better frame release. In previewing mode MediaCodec renders to FrameProcessor's input
// surface, which is not a SurfaceView.
long releaseTimeNs =
frameProcessorManager.isEnabled()
? (presentationTimeUs + getOutputStreamOffsetUs()) * 1000
: System.nanoTime();
if (notifyFrameMetadataListener) {
notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format);
}
if (Util.SDK_INT >= 21) {
renderOutputBufferV21(codec, index, presentationTimeUs, releaseTimeNs);
} else {
renderOutputBuffer(codec, index, presentationTimeUs);
}
}
/**
* Renders the output buffer with the specified index. This method is only called if the platform
* API version of the device is less than 21.
@ -1686,12 +1743,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
private @MonotonicNonNull Handler handler;
@Nullable private FrameProcessor frameProcessor;
@Nullable private CopyOnWriteArrayList<Effect> videoEffects;
private int frameProcessorMaxPendingFrameCount;
private boolean canEnableFrameProcessing;
/** Creates a new instance. */
public FrameProcessorManager(@UnderInitialization MediaCodecVideoRenderer renderer) {
this.renderer = renderer;
processedFrames = new ArrayDeque<>();
frameProcessorMaxPendingFrameCount = C.LENGTH_UNSET;
canEnableFrameProcessing = true;
}
@ -1831,6 +1890,50 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
return mediaFormat;
}
/**
* Must be called when the codec is initialized.
*
* <p>Sets the {@code frameProcessorMaxPendingFrameCount} based on the {@code codecName}.
*/
public void onCodecInitialized(String codecName) {
frameProcessorMaxPendingFrameCount =
Util.getMaxPendingFramesCountForMediaCodecEncoders(
renderer.context, codecName, /* requestedHdrToneMapping= */ false);
}
/**
* Tries to {@linkplain FrameProcessor#registerInputFrame register an input frame}.
*
* <p>Caller must ensure the {@code FrameProcessorManager} {@link #isEnabled()} before calling
* this method.
*
* @return Whether {@link MediaCodec} should render the frame to {@link FrameProcessor}.
*/
public boolean maybeRegisterFrame() {
checkStateNotNull(frameProcessor);
checkState(frameProcessorMaxPendingFrameCount != C.LENGTH_UNSET);
if (frameProcessor.getPendingInputFrameCount() < frameProcessorMaxPendingFrameCount) {
frameProcessor.registerInputFrame();
return true;
}
return false;
}
/**
* Releases the processed frames to the {@linkplain #setOutputSurfaceInfo output surface}.
*
* <p>Caller must ensure the {@code FrameProcessorManager} {@link #isEnabled()} before calling
* this method.
*/
public void releaseProcessedFrames() {
while (!processedFrames.isEmpty()) {
processedFrames.poll();
// TODO(b/238302341): Add frame release logic.
checkNotNull(frameProcessor)
.releaseOutputFrame(FrameProcessor.RELEASE_OUTPUT_FRAME_IMMEDIATELY);
}
}
/**
* Releases the resources.
*

7
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Codec.java
View File

@ -31,9 +31,6 @@ import java.nio.ByteBuffer;
* buffers.
*/
public interface Codec {
/** Default value for the pending frame count, which represents applying no limit. */
int UNLIMITED_PENDING_FRAME_COUNT = Integer.MAX_VALUE;
/** A factory for {@linkplain Codec decoder} instances. */
interface DecoderFactory {
@ -134,10 +131,10 @@ public interface Codec {
/**
* Returns the maximum number of frames that may be pending in the output {@code Codec} at a time,
* or {@link #UNLIMITED_PENDING_FRAME_COUNT} if it's not necessary to enforce a limit.
* or {@link C#UNLIMITED_PENDING_FRAME_COUNT} if it's not necessary to enforce a limit.
*/
default int getMaxPendingFrameCount() {
return UNLIMITED_PENDING_FRAME_COUNT;
return C.UNLIMITED_PENDING_FRAME_COUNT;
}
/**

42
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultCodec.java
View File

@ -20,7 +20,6 @@ import static com.google.android.exoplayer2.util.Assertions.checkArgument;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;
import static com.google.android.exoplayer2.util.Util.MODEL;
import static com.google.android.exoplayer2.util.Util.SDK_INT;
import android.content.Context;
@ -40,8 +39,8 @@ import com.google.android.exoplayer2.util.Log;
import com.google.android.exoplayer2.util.MediaFormatUtil;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.TraceUtil;
import com.google.android.exoplayer2.util.Util;
import com.google.android.exoplayer2.video.ColorInfo;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.nio.ByteBuffer;
@ -136,7 +135,8 @@ public final class DefaultCodec implements Codec {
this.mediaCodec = mediaCodec;
this.inputSurface = inputSurface;
maxPendingFrameCount =
getMaxPendingFrameCountInternal(context, mediaCodecName, requestedHdrToneMapping);
Util.getMaxPendingFramesCountForMediaCodecEncoders(
context, mediaCodecName, requestedHdrToneMapping);
}
@Override
@ -464,42 +464,6 @@ public final class DefaultCodec implements Codec {
TraceUtil.endSection();
}
private static int getMaxPendingFrameCountInternal(
Context context, String codecName, boolean requestedHdrToneMapping) {
if (SDK_INT < 29
|| context.getApplicationContext().getApplicationInfo().targetSdkVersion < 29) {
// Prior to API 29, decoders may drop frames to keep their output surface from growing out of
// bounds. From API 29, if the app targets API 29 or later, the {@link
// MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame dropping even when the surface is
// full.
// Frame dropping is never desired, so a workaround is needed for older API levels.
// Allow a maximum of one frame to be pending at a time to prevent frame dropping.
// TODO(b/226330223): Investigate increasing this limit.
return 1;
}
if (Ascii.toUpperCase(codecName).startsWith("OMX.")) {
// Some OMX decoders don't correctly track their number of output buffers available, and get
// stuck if too many frames are rendered without being processed, so limit the number of
// pending frames to avoid getting stuck. This value is experimentally determined. See also
// b/213455700, b/230097284, b/229978305, and b/245491744.
//
// OMX video codecs should no longer exist from android.os.Build.DEVICE_INITIAL_SDK_INT 31+.
return 5;
}
if (requestedHdrToneMapping
&& codecName.equals("c2.qti.hevc.decoder")
&& MODEL.equals("SM-F936B")) {
// This decoder gets stuck if too many frames are rendered without being processed when
// tone-mapping HDR10. This value is experimentally determined. See also b/260408846.
// TODO(b/260713009): Add API version check after bug is fixed on new API versions.
return 12;
}
// Otherwise don't limit the number of frames that can be pending at a time, to maximize
// throughput.
return UNLIMITED_PENDING_FRAME_COUNT;
}
@RequiresApi(29)
private static final class Api29 {
@DoNotInline

2
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java
View File

@ -360,7 +360,7 @@ import org.checkerframework.dataflow.qual.Pure;
return true;
}
if (maxPendingFrameCount != Codec.UNLIMITED_PENDING_FRAME_COUNT
if (maxPendingFrameCount != C.UNLIMITED_PENDING_FRAME_COUNT
&& frameProcessor.getPendingInputFrameCount() == maxPendingFrameCount) {
return false;
}