From 649fe702f2149ee761947ea921136d2252477e6c Mon Sep 17 00:00:00 2001 From: hschlueter Date: Tue, 26 Oct 2021 14:20:24 +0100 Subject: [PATCH] Deduce encoder video format from decoder format. When no encoder video MIME type is specified, the `TransformerTranscodingVideoRenderer` now uses the video MIME type of the input for the encoder format. The input format is now read in a new method `ensureInputFormatRead` which is called before the other configuration methods. This removes the logic for reading the input format from `ensureDecoderConfigured`, because it is now needed for both encoder and decoder configuration but the encoder needs to be configured before GL and GL needs to be configured before the decoder, so the decoder can't read the format. The width and height are now inferred from the input and the frame rate and bit rate are still hard-coded but set by the `MediaCodecAdapterWrapper` instead of `TranscodingTransformer`. PiperOrigin-RevId: 405631263 --- .../transformer/MediaCodecAdapterWrapper.java | 13 ++-- .../transformer/TranscodingTransformer.java | 18 ++--- .../transformer/Transformation.java | 5 +- .../exoplayer2/transformer/Transformer.java | 3 +- .../transformer/TransformerAudioRenderer.java | 14 ++-- .../TransformerTranscodingVideoRenderer.java | 67 +++++++++++-------- 6 files changed, 65 insertions(+), 55 deletions(-) diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java index 4afa84dfd3..be05034d70 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java @@ -19,7 +19,6 @@ package com.google.android.exoplayer2.transformer; import static com.google.android.exoplayer2.util.Assertions.checkArgument; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Assertions.checkState; -import static java.lang.Math.ceil; import android.media.MediaCodec; import android.media.MediaCodec.BufferInfo; @@ -202,9 +201,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * MediaCodecAdapter} video encoder. * * @param format The {@link Format} (of the output data) used to determine the underlying {@link - * MediaCodec} and its configuration values. {@link Format#width}, {@link Format#height}, - * {@link Format#frameRate} and {@link Format#averageBitrate} must be set to those of the - * desired output video format. + * MediaCodec} and its configuration values. {@link Format#sampleMimeType}, {@link + * Format#width} and {@link Format#height} must be set to those of the desired output video + * format. * @param additionalEncoderConfig A map of {@link MediaFormat}'s integer settings, where the keys * are from {@code MediaFormat.KEY_*} constants. Its values will override those in {@code * format}. @@ -215,8 +214,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Format format, Map additionalEncoderConfig) throws IOException { checkArgument(format.width != Format.NO_VALUE); checkArgument(format.height != Format.NO_VALUE); - checkArgument(format.frameRate != Format.NO_VALUE); - checkArgument(format.averageBitrate != Format.NO_VALUE); @Nullable MediaCodecAdapter adapter = null; try { @@ -224,9 +221,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; MediaFormat.createVideoFormat( checkNotNull(format.sampleMimeType), format.width, format.height); mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, CodecCapabilities.COLOR_FormatSurface); - mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, (int) ceil(format.frameRate)); + mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30); mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1); - mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.averageBitrate); + mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, 413_000); for (Map.Entry encoderSetting : additionalEncoderConfig.entrySet()) { mediaFormat.setInteger(encoderSetting.getKey(), encoderSetting.getValue()); diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TranscodingTransformer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TranscodingTransformer.java index 61feaf5dd0..8e622c2a9a 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TranscodingTransformer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TranscodingTransformer.java @@ -38,7 +38,6 @@ import androidx.annotation.VisibleForTesting; import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.DefaultLoadControl; import com.google.android.exoplayer2.ExoPlayer; -import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.MediaItem; import com.google.android.exoplayer2.PlaybackException; import com.google.android.exoplayer2.Player; @@ -339,7 +338,12 @@ public final class TranscodingTransformer { } Transformation transformation = new Transformation( - removeAudio, removeVideo, flattenForSlowMotion, outputMimeType, audioMimeType); + removeAudio, + removeVideo, + flattenForSlowMotion, + outputMimeType, + audioMimeType, + /* videoMimeType= */ null); return new TranscodingTransformer( context, mediaSourceFactory, muxerFactory, transformation, listener, looper, clock); } @@ -639,17 +643,9 @@ public final class TranscodingTransformer { index++; } if (!transformation.removeVideo) { - Format encoderOutputFormat = - new Format.Builder() - .setSampleMimeType(MimeTypes.VIDEO_H264) - .setWidth(480) - .setHeight(360) - .setAverageBitrate(413_000) - .setFrameRate(30) - .build(); renderers[index] = new TransformerTranscodingVideoRenderer( - context, muxerWrapper, mediaClock, transformation, encoderOutputFormat); + context, muxerWrapper, mediaClock, transformation); index++; } return renderers; diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformation.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformation.java index a224fe3a93..e273c1fde5 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformation.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformation.java @@ -26,17 +26,20 @@ import androidx.annotation.Nullable; public final boolean flattenForSlowMotion; public final String outputMimeType; @Nullable public final String audioMimeType; + @Nullable public final String videoMimeType; public Transformation( boolean removeAudio, boolean removeVideo, boolean flattenForSlowMotion, String outputMimeType, - @Nullable String audioMimeType) { + @Nullable String audioMimeType, + @Nullable String videoMimeType) { this.removeAudio = removeAudio; this.removeVideo = removeVideo; this.flattenForSlowMotion = flattenForSlowMotion; this.outputMimeType = outputMimeType; this.audioMimeType = audioMimeType; + this.videoMimeType = videoMimeType; } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java index 34d10137b8..1a79061f66 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java @@ -304,7 +304,8 @@ public final class Transformer { removeVideo, flattenForSlowMotion, outputMimeType, - /* audioMimeType= */ null); + /* audioMimeType= */ null, + /* videoMimeType= */ null); return new Transformer( context, mediaSourceFactory, muxerFactory, transformation, listener, looper, clock); } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java index 64dc53d58c..9a8da0ecf6 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java @@ -51,7 +51,7 @@ import java.nio.ByteBuffer; @Nullable private MediaCodecAdapterWrapper decoder; @Nullable private MediaCodecAdapterWrapper encoder; @Nullable private SpeedProvider speedProvider; - @Nullable private Format inputFormat; + @Nullable private Format decoderInputFormat; @Nullable private AudioFormat encoderInputAudioFormat; private ByteBuffer sonicOutputBuffer; @@ -100,7 +100,7 @@ import java.nio.ByteBuffer; encoder = null; } speedProvider = null; - inputFormat = null; + decoderInputFormat = null; encoderInputAudioFormat = null; sonicOutputBuffer = AudioProcessor.EMPTY_BUFFER; nextEncoderInputBufferTimeUs = 0; @@ -352,7 +352,7 @@ import java.nio.ByteBuffer; } String audioMimeType = transformation.audioMimeType == null - ? checkNotNull(inputFormat).sampleMimeType + ? checkNotNull(decoderInputFormat).sampleMimeType : transformation.audioMimeType; try { encoder = @@ -385,14 +385,14 @@ import java.nio.ByteBuffer; if (result != C.RESULT_FORMAT_READ) { return false; } - inputFormat = checkNotNull(formatHolder.format); + decoderInputFormat = checkNotNull(formatHolder.format); try { - decoder = MediaCodecAdapterWrapper.createForAudioDecoding(inputFormat); + decoder = MediaCodecAdapterWrapper.createForAudioDecoding(decoderInputFormat); } catch (IOException e) { // TODO (internal b/184262323): Assign an adequate error code. throw createRendererException(e, PlaybackException.ERROR_CODE_UNSPECIFIED); } - speedProvider = new SegmentSpeedProvider(inputFormat); + speedProvider = new SegmentSpeedProvider(decoderInputFormat); currentSpeed = speedProvider.getSpeed(0); return true; } @@ -418,7 +418,7 @@ import java.nio.ByteBuffer; cause, TAG, getIndex(), - inputFormat, + decoderInputFormat, /* rendererFormatSupport= */ C.FORMAT_HANDLED, /* isRecoverable= */ false, errorCode); diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerTranscodingVideoRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerTranscodingVideoRenderer.java index 79d28a208e..8c1e11df61 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerTranscodingVideoRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerTranscodingVideoRenderer.java @@ -42,6 +42,7 @@ import com.google.android.exoplayer2.util.GlUtil; import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.nio.ByteBuffer; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @RequiresApi(18) /* package */ final class TransformerTranscodingVideoRenderer extends TransformerBaseRenderer { @@ -53,12 +54,12 @@ import java.nio.ByteBuffer; private static final String TAG = "TransformerTranscodingVideoRenderer"; private final Context context; - /** The format the encoder is configured to output, may differ from the actual output format. */ - private final Format encoderConfigurationOutputFormat; private final DecoderInputBuffer decoderInputBuffer; private final float[] decoderTextureTransformMatrix; + private @MonotonicNonNull Format decoderInputFormat; + @Nullable private EGLDisplay eglDisplay; @Nullable private EGLContext eglContext; @Nullable private EGLSurface eglSurface; @@ -81,11 +82,9 @@ import java.nio.ByteBuffer; Context context, MuxerWrapper muxerWrapper, TransformerMediaClock mediaClock, - Transformation transformation, - Format encoderConfigurationOutputFormat) { + Transformation transformation) { super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformation); this.context = context; - this.encoderConfigurationOutputFormat = encoderConfigurationOutputFormat; decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); decoderTextureTransformMatrix = new float[16]; decoderTextureId = GlUtil.TEXTURE_ID_UNSET; @@ -97,15 +96,13 @@ import java.nio.ByteBuffer; } @Override - protected void onStarted() throws ExoPlaybackException { - super.onStarted(); + public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { + if (!isRendererStarted || isEnded() || !ensureInputFormatRead()) { + return; + } ensureEncoderConfigured(); ensureOpenGlConfigured(); - } - - @Override - public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { - if (!isRendererStarted || isEnded() || !ensureDecoderConfigured()) { + if (!ensureDecoderConfigured()) { return; } @@ -153,6 +150,22 @@ import java.nio.ByteBuffer; muxerWrapperTrackEnded = false; } + private boolean ensureInputFormatRead() { + if (decoderInputFormat != null) { + return true; + } + FormatHolder formatHolder = getFormatHolder(); + @SampleStream.ReadDataResult + int result = + readSource( + formatHolder, decoderInputBuffer, /* readFlags= */ SampleStream.FLAG_REQUIRE_FORMAT); + if (result != C.RESULT_FORMAT_READ) { + return false; + } + decoderInputFormat = checkNotNull(formatHolder.format); + return true; + } + private void ensureEncoderConfigured() throws ExoPlaybackException { if (encoder != null) { return; @@ -161,7 +174,15 @@ import java.nio.ByteBuffer; try { encoder = MediaCodecAdapterWrapper.createForVideoEncoding( - encoderConfigurationOutputFormat, ImmutableMap.of()); + new Format.Builder() + .setWidth(checkNotNull(decoderInputFormat).width) + .setHeight(decoderInputFormat.height) + .setSampleMimeType( + transformation.videoMimeType != null + ? transformation.videoMimeType + : decoderInputFormat.sampleMimeType) + .build(), + ImmutableMap.of()); } catch (IOException e) { throw createRendererException( // TODO(claincly): should be "ENCODER_INIT_FAILED" @@ -190,8 +211,8 @@ import java.nio.ByteBuffer; eglDisplay, eglContext, eglSurface, - encoderConfigurationOutputFormat.width, - encoderConfigurationOutputFormat.height); + checkNotNull(decoderInputFormat).width, + decoderInputFormat.height); decoderTextureId = GlUtil.createExternalTexture(); String vertexShaderCode; String fragmentShaderCode; @@ -248,26 +269,18 @@ import java.nio.ByteBuffer; return true; } - FormatHolder formatHolder = getFormatHolder(); - @SampleStream.ReadDataResult - int result = - readSource( - formatHolder, decoderInputBuffer, /* readFlags= */ SampleStream.FLAG_REQUIRE_FORMAT); - if (result != C.RESULT_FORMAT_READ) { - return false; - } - - Format inputFormat = checkNotNull(formatHolder.format); checkState(decoderTextureId != GlUtil.TEXTURE_ID_UNSET); decoderSurfaceTexture = new SurfaceTexture(decoderTextureId); decoderSurfaceTexture.setOnFrameAvailableListener( surfaceTexture -> isDecoderSurfacePopulated = true); decoderSurface = new Surface(decoderSurfaceTexture); try { - decoder = MediaCodecAdapterWrapper.createForVideoDecoding(inputFormat, decoderSurface); + decoder = + MediaCodecAdapterWrapper.createForVideoDecoding( + checkNotNull(decoderInputFormat), decoderSurface); } catch (IOException e) { throw createRendererException( - e, formatHolder.format, PlaybackException.ERROR_CODE_DECODER_INIT_FAILED); + e, decoderInputFormat, PlaybackException.ERROR_CODE_DECODER_INIT_FAILED); } return true; }