From f574ec952ab622c6640c192c744299944e3902c0 Mon Sep 17 00:00:00 2001 From: kimvde Date: Thu, 26 Aug 2021 13:13:22 +0100 Subject: [PATCH] Make TransformerTranscodingVideoRenderer support API < 23 PiperOrigin-RevId: 393100075 --- .../AsynchronousMediaCodecAdapter.java | 19 ++++++++- .../mediacodec/MediaCodecAdapter.java | 30 ++++++++++++-- .../SynchronousMediaCodecAdapter.java | 41 ++++++++++++------- .../transformer/MediaCodecAdapterWrapper.java | 19 +++++---- .../TransformerTranscodingVideoRenderer.java | 16 +++----- .../testutil/CapturingRenderersFactory.java | 6 +++ 6 files changed, 93 insertions(+), 38 deletions(-) diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapter.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapter.java index dffdef4280..9cc9382238 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapter.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/AsynchronousMediaCodecAdapter.java @@ -118,7 +118,8 @@ import java.nio.ByteBuffer; configuration.mediaFormat, configuration.surface, configuration.crypto, - configuration.flags); + configuration.flags, + configuration.createInputSurface); return codecAdapter; } catch (Exception e) { if (codecAdapter != null) { @@ -146,6 +147,7 @@ import java.nio.ByteBuffer; private final boolean synchronizeCodecInteractionsWithQueueing; private boolean codecReleased; @State private int state; + @Nullable private Surface inputSurface; private AsynchronousMediaCodecAdapter( MediaCodec codec, @@ -166,11 +168,15 @@ import java.nio.ByteBuffer; @Nullable MediaFormat mediaFormat, @Nullable Surface surface, @Nullable MediaCrypto crypto, - int flags) { + int flags, + boolean createInputSurface) { asynchronousMediaCodecCallback.initialize(codec); TraceUtil.beginSection("configureCodec"); codec.configure(mediaFormat, surface, crypto, flags); TraceUtil.endSection(); + if (createInputSurface) { + inputSurface = codec.createInputSurface(); + } bufferEnqueuer.start(); TraceUtil.beginSection("startCodec"); codec.start(); @@ -226,6 +232,12 @@ import java.nio.ByteBuffer; return codec.getInputBuffer(index); } + @Override + @Nullable + public Surface getInputSurface() { + return inputSurface; + } + @Override @Nullable public ByteBuffer getOutputBuffer(int index) { @@ -253,6 +265,9 @@ import java.nio.ByteBuffer; } state = STATE_SHUT_DOWN; } finally { + if (inputSurface != null) { + inputSurface.release(); + } if (!codecReleased) { codec.release(); codecReleased = true; diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecAdapter.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecAdapter.java index 183adfc417..c620e0e4a2 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecAdapter.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecAdapter.java @@ -45,10 +45,7 @@ public interface MediaCodecAdapter { public final MediaFormat mediaFormat; /** The {@link Format} for which the codec is being configured. */ public final Format format; - /** - * For video decoding, the output where the object will render the decoded frames; for video - * encoding, this is the input surface from which the decoded frames are retrieved. - */ + /** For video decoding, the output where the object will render the decoded frames. */ @Nullable public final Surface surface; /** For DRM protected playbacks, a {@link MediaCrypto} to use for decryption. */ @Nullable public final MediaCrypto crypto; @@ -58,6 +55,11 @@ public interface MediaCodecAdapter { * @see MediaCodec#configure */ public final int flags; + /** + * Whether to request a {@link Surface} and use it as to the input to an encoder. This can only + * be set to {@code true} on API 18+. + */ + public final boolean createInputSurface; public Configuration( MediaCodecInfo codecInfo, @@ -66,12 +68,24 @@ public interface MediaCodecAdapter { @Nullable Surface surface, @Nullable MediaCrypto crypto, int flags) { + this(codecInfo, mediaFormat, format, surface, crypto, flags, /* createInputSurface= */ false); + } + + public Configuration( + MediaCodecInfo codecInfo, + MediaFormat mediaFormat, + Format format, + @Nullable Surface surface, + @Nullable MediaCrypto crypto, + int flags, + boolean createInputSurface) { this.codecInfo = codecInfo; this.mediaFormat = mediaFormat; this.format = format; this.surface = surface; this.crypto = crypto; this.flags = flags; + this.createInputSurface = createInputSurface; } } @@ -129,6 +143,14 @@ public interface MediaCodecAdapter { @Nullable ByteBuffer getInputBuffer(int index); + /** + * Returns the input {@link Surface}, or null if the input is not a surface. + * + * @see MediaCodec#createInputSurface() + */ + @Nullable + Surface getInputSurface(); + /** * Returns a read-only ByteBuffer for a dequeued output buffer index. * diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/SynchronousMediaCodecAdapter.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/SynchronousMediaCodecAdapter.java index d74e41b2cf..4150592a39 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/SynchronousMediaCodecAdapter.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/SynchronousMediaCodecAdapter.java @@ -21,7 +21,6 @@ import static com.google.android.exoplayer2.util.Util.castNonNull; import android.media.MediaCodec; import android.media.MediaFormat; -import android.os.Build; import android.os.Bundle; import android.os.Handler; import android.view.Surface; @@ -47,31 +46,34 @@ public class SynchronousMediaCodecAdapter implements MediaCodecAdapter { @RequiresApi(16) public MediaCodecAdapter createAdapter(Configuration configuration) throws IOException { @Nullable MediaCodec codec = null; - boolean isEncoder = configuration.flags == MediaCodec.CONFIGURE_FLAG_ENCODE; - @Nullable Surface decoderOutputSurface = isEncoder ? null : configuration.surface; + @Nullable Surface inputSurface = null; try { codec = createCodec(configuration); TraceUtil.beginSection("configureCodec"); codec.configure( configuration.mediaFormat, - decoderOutputSurface, + configuration.surface, configuration.crypto, configuration.flags); TraceUtil.endSection(); - if (isEncoder && configuration.surface != null) { - if (Build.VERSION.SDK_INT >= 23) { - Api23.setCodecInputSurface(codec, configuration.surface); + + if (configuration.createInputSurface) { + if (Util.SDK_INT >= 18) { + inputSurface = Api18.createCodecInputSurface(codec); } else { throw new IllegalStateException( - "Encoding from a surface is only supported on API 23 and up"); + "Encoding from a surface is only supported on API 18 and up."); } } TraceUtil.beginSection("startCodec"); codec.start(); TraceUtil.endSection(); - return new SynchronousMediaCodecAdapter(codec); + return new SynchronousMediaCodecAdapter(codec, inputSurface); } catch (IOException | RuntimeException e) { + if (inputSurface != null) { + inputSurface.release(); + } if (codec != null) { codec.release(); } @@ -91,11 +93,13 @@ public class SynchronousMediaCodecAdapter implements MediaCodecAdapter { } private final MediaCodec codec; + @Nullable private final Surface inputSurface; @Nullable private ByteBuffer[] inputByteBuffers; @Nullable private ByteBuffer[] outputByteBuffers; - private SynchronousMediaCodecAdapter(MediaCodec mediaCodec) { + private SynchronousMediaCodecAdapter(MediaCodec mediaCodec, @Nullable Surface inputSurface) { this.codec = mediaCodec; + this.inputSurface = inputSurface; if (Util.SDK_INT < 21) { inputByteBuffers = codec.getInputBuffers(); outputByteBuffers = codec.getOutputBuffers(); @@ -140,6 +144,12 @@ public class SynchronousMediaCodecAdapter implements MediaCodecAdapter { } } + @Override + @Nullable + public Surface getInputSurface() { + return inputSurface; + } + @Override @Nullable public ByteBuffer getOutputBuffer(int index) { @@ -183,6 +193,9 @@ public class SynchronousMediaCodecAdapter implements MediaCodecAdapter { public void release() { inputByteBuffers = null; outputByteBuffers = null; + if (inputSurface != null) { + inputSurface.release(); + } codec.release(); } @@ -213,11 +226,11 @@ public class SynchronousMediaCodecAdapter implements MediaCodecAdapter { codec.setVideoScalingMode(scalingMode); } - @RequiresApi(23) - private static final class Api23 { + @RequiresApi(18) + private static final class Api18 { @DoNotInline - public static void setCodecInputSurface(MediaCodec codec, Surface surface) { - codec.setInputSurface(surface); + public static Surface createCodecInputSurface(MediaCodec codec) { + return codec.createInputSurface(); } } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java index 6e30f3f27e..30a2f94ff2 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java @@ -48,6 +48,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * through {@link MediaCodecAdapter}. This is done by simplifying the calls needed to queue and * dequeue buffers, removing the need to track buffer indices and codec events. */ +@RequiresApi(18) /* package */ final class MediaCodecAdapterWrapper { // MediaCodec decoders always output 16 bit PCM, unless configured to output PCM float. @@ -142,7 +143,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @return A configured and started decoder wrapper. * @throws IOException If the underlying codec cannot be created. */ - @RequiresApi(23) public static MediaCodecAdapterWrapper createForVideoDecoding(Format format, Surface surface) throws IOException { @Nullable MediaCodecAdapter adapter = null; @@ -163,7 +163,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; surface, /* crypto= */ null, /* flags= */ 0)); - adapter.setOutputSurface(surface); return new MediaCodecAdapterWrapper(adapter); } catch (Exception e) { if (adapter != null) { @@ -217,13 +216,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * * @param format The {@link Format} (of the output data) used to determine the underlying {@link * MediaCodec} and its configuration values. - * @param surface The {@link Surface} from which the encoder obtains the frame input. * @return A configured and started encoder wrapper. * @throws IOException If the underlying codec cannot be created. */ - @RequiresApi(18) - public static MediaCodecAdapterWrapper createForVideoEncoding(Format format, Surface surface) - throws IOException { + public static MediaCodecAdapterWrapper createForVideoEncoding(Format format) throws IOException { @Nullable MediaCodecAdapter adapter = null; try { MediaFormat mediaFormat = @@ -242,9 +238,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; createPlaceholderMediaCodecInfo(), mediaFormat, format, - surface, + /* surface= */ null, /* crypto= */ null, - MediaCodec.CONFIGURE_FLAG_ENCODE)); + MediaCodec.CONFIGURE_FLAG_ENCODE, + /* createInputSurface= */ true)); return new MediaCodecAdapterWrapper(adapter); } catch (Exception e) { if (adapter != null) { @@ -261,6 +258,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputBufferIndex = C.INDEX_UNSET; } + /** Returns the input {@link Surface}, or null if the input is not a surface. */ + @Nullable + public Surface getInputSurface() { + return codec.getInputSurface(); + } + /** * Dequeues a writable input buffer, if available. * diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerTranscodingVideoRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerTranscodingVideoRenderer.java index a7adae8535..064142186d 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerTranscodingVideoRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerTranscodingVideoRenderer.java @@ -19,7 +19,6 @@ package com.google.android.exoplayer2.transformer; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import android.media.MediaCodec; -import android.view.Surface; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import com.google.android.exoplayer2.C; @@ -32,7 +31,7 @@ import com.google.android.exoplayer2.source.SampleStream; import java.io.IOException; import java.nio.ByteBuffer; -@RequiresApi(23) +@RequiresApi(18) /* package */ final class TransformerTranscodingVideoRenderer extends TransformerBaseRenderer { private static final String TAG = "TransformerTranscodingVideoRenderer"; @@ -41,8 +40,6 @@ import java.nio.ByteBuffer; /** The format the encoder is configured to output, may differ from the actual output format. */ private final Format encoderConfigurationOutputFormat; - private final Surface surface; - @Nullable private MediaCodecAdapterWrapper decoder; @Nullable private MediaCodecAdapterWrapper encoder; /** Whether encoder's actual output format is obtained. */ @@ -58,7 +55,6 @@ import java.nio.ByteBuffer; super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformation); decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); - surface = MediaCodec.createPersistentInputSurface(); this.encoderConfigurationOutputFormat = encoderConfigurationOutputFormat; } @@ -93,7 +89,6 @@ import java.nio.ByteBuffer; protected void onReset() { decoderInputBuffer.clear(); decoderInputBuffer.data = null; - surface.release(); if (decoder != null) { decoder.release(); decoder = null; @@ -121,8 +116,11 @@ import java.nio.ByteBuffer; } Format inputFormat = checkNotNull(formatHolder.format); + MediaCodecAdapterWrapper encoder = checkNotNull(this.encoder); try { - decoder = MediaCodecAdapterWrapper.createForVideoDecoding(inputFormat, surface); + decoder = + MediaCodecAdapterWrapper.createForVideoDecoding( + inputFormat, checkNotNull(encoder.getInputSurface())); } catch (IOException e) { throw createRendererException( e, formatHolder.format, PlaybackException.ERROR_CODE_DECODER_INIT_FAILED); @@ -136,9 +134,7 @@ import java.nio.ByteBuffer; } try { - encoder = - MediaCodecAdapterWrapper.createForVideoEncoding( - encoderConfigurationOutputFormat, surface); + encoder = MediaCodecAdapterWrapper.createForVideoEncoding(encoderConfigurationOutputFormat); } catch (IOException e) { throw createRendererException( // TODO(claincly): should be "ENCODER_INIT_FAILED" diff --git a/testutils/src/main/java/com/google/android/exoplayer2/testutil/CapturingRenderersFactory.java b/testutils/src/main/java/com/google/android/exoplayer2/testutil/CapturingRenderersFactory.java index 86b4d508c4..0a2f5832ac 100644 --- a/testutils/src/main/java/com/google/android/exoplayer2/testutil/CapturingRenderersFactory.java +++ b/testutils/src/main/java/com/google/android/exoplayer2/testutil/CapturingRenderersFactory.java @@ -193,6 +193,12 @@ public class CapturingRenderersFactory implements RenderersFactory, Dumper.Dumpa return inputBuffer; } + @Nullable + @Override + public Surface getInputSurface() { + return delegate.getInputSurface(); + } + @Nullable @Override public ByteBuffer getOutputBuffer(int index) {