From 081700f72b2a43e1e0d01a0130cf283c7d59b9bc Mon Sep 17 00:00:00 2001 From: hschlueter Date: Wed, 19 Jan 2022 18:32:59 +0000 Subject: [PATCH] Communicate sample MIME type changes to FallbackListener. We may fall back to a different sample MIME type because a) the sample MIME type inferred from the input is not supported by the muxer or b) no encoders are available for the the requested sample MIME type. PiperOrigin-RevId: 422849036 --- .../transformer/AudioSamplePipeline.java | 53 ++++++++++++------- .../PassthroughSamplePipeline.java | 6 ++- .../exoplayer2/transformer/Transformer.java | 12 ++++- .../transformer/TransformerAudioRenderer.java | 10 ++-- .../transformer/TransformerBaseRenderer.java | 6 ++- .../transformer/TransformerVideoRenderer.java | 7 ++- .../transformer/VideoSamplePipeline.java | 42 ++++++++++----- .../transformer/TransformerTest.java | 1 + 8 files changed, 98 insertions(+), 39 deletions(-) diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java index 505677d046..f529a949e7 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java @@ -29,7 +29,9 @@ import com.google.android.exoplayer2.audio.AudioProcessor; import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat; import com.google.android.exoplayer2.audio.SonicAudioProcessor; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import com.google.android.exoplayer2.util.Util; import java.nio.ByteBuffer; +import org.checkerframework.dataflow.qual.Pure; /** * Pipeline to decode audio samples, apply transformations on the raw samples, and re-encode them. @@ -39,13 +41,12 @@ import java.nio.ByteBuffer; private static final String TAG = "AudioSamplePipeline"; private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024; - private final TransformationRequest transformationRequest; - private final Codec decoder; private final DecoderInputBuffer decoderInputBuffer; private final SonicAudioProcessor sonicAudioProcessor; private final SpeedProvider speedProvider; + private final boolean flattenForSlowMotion; private final Codec encoder; private final AudioFormat encoderInputAudioFormat; @@ -63,9 +64,9 @@ import java.nio.ByteBuffer; Format inputFormat, TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, - Codec.DecoderFactory decoderFactory) + Codec.DecoderFactory decoderFactory, + FallbackListener fallbackListener) throws TransformationException { - this.transformationRequest = transformationRequest; decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); encoderInputBuffer = @@ -75,6 +76,7 @@ import java.nio.ByteBuffer; this.decoder = decoderFactory.createForAudioDecoding(inputFormat); + this.flattenForSlowMotion = transformationRequest.flattenForSlowMotion; sonicAudioProcessor = new SonicAudioProcessor(); sonicOutputBuffer = AudioProcessor.EMPTY_BUFFER; speedProvider = new SegmentSpeedProvider(inputFormat); @@ -86,7 +88,7 @@ import java.nio.ByteBuffer; // The decoder uses ENCODING_PCM_16BIT by default. // https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers C.ENCODING_PCM_16BIT); - if (transformationRequest.flattenForSlowMotion) { + if (flattenForSlowMotion) { try { encoderInputAudioFormat = sonicAudioProcessor.configure(encoderInputAudioFormat); } catch (AudioProcessor.UnhandledAudioFormatException impossible) { @@ -96,19 +98,23 @@ import java.nio.ByteBuffer; sonicAudioProcessor.setPitch(currentSpeed); sonicAudioProcessor.flush(); } - - encoder = - encoderFactory.createForAudioEncoding( - new Format.Builder() - .setSampleMimeType( - transformationRequest.audioMimeType == null - ? inputFormat.sampleMimeType - : transformationRequest.audioMimeType) - .setSampleRate(encoderInputAudioFormat.sampleRate) - .setChannelCount(encoderInputAudioFormat.channelCount) - .setAverageBitrate(DEFAULT_ENCODER_BITRATE) - .build()); this.encoderInputAudioFormat = encoderInputAudioFormat; + + Format requestedOutputFormat = + new Format.Builder() + .setSampleMimeType( + transformationRequest.audioMimeType == null + ? inputFormat.sampleMimeType + : transformationRequest.audioMimeType) + .setSampleRate(encoderInputAudioFormat.sampleRate) + .setChannelCount(encoderInputAudioFormat.channelCount) + .setAverageBitrate(DEFAULT_ENCODER_BITRATE) + .build(); + encoder = encoderFactory.createForAudioEncoding(requestedOutputFormat); + + fallbackListener.onTransformationRequestFinalized( + createFallbackRequest( + transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat())); } @Override @@ -293,7 +299,7 @@ import java.nio.ByteBuffer; } private boolean isSpeedChanging(BufferInfo bufferInfo) { - if (!transformationRequest.flattenForSlowMotion) { + if (!flattenForSlowMotion) { return false; } float newSpeed = speedProvider.getSpeed(bufferInfo.presentationTimeUs); @@ -325,4 +331,15 @@ import java.nio.ByteBuffer; } nextEncoderInputBufferTimeUs += bufferDurationUs; } + + @Pure + private static TransformationRequest createFallbackRequest( + TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) { + // TODO(b/210591626): Also update bitrate and other params once encoder configuration and + // fallback are implemented. + if (Util.areEqual(requestedFormat.sampleMimeType, actualFormat.sampleMimeType)) { + return transformationRequest; + } + return transformationRequest.buildUpon().setAudioMimeType(actualFormat.sampleMimeType).build(); + } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java index 8b540ee105..042819b0fb 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java @@ -28,10 +28,14 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer; private boolean hasPendingBuffer; - public PassthroughSamplePipeline(Format format) { + public PassthroughSamplePipeline( + Format format, + TransformationRequest transformationRequest, + FallbackListener fallbackListener) { this.format = format; buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); hasPendingBuffer = false; + fallbackListener.onTransformationRequestFinalized(transformationRequest); } @Override diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java index bd9eac24ac..f312d84643 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java @@ -703,6 +703,7 @@ public final class Transformer { transformationRequest, encoderFactory, decoderFactory, + new FallbackListener(mediaItem, listeners, transformationRequest), debugViewProvider)) .setMediaSourceFactory(mediaSourceFactory) .setTrackSelector(trackSelector) @@ -805,6 +806,7 @@ public final class Transformer { private final TransformationRequest transformationRequest; private final Codec.EncoderFactory encoderFactory; private final Codec.DecoderFactory decoderFactory; + private final FallbackListener fallbackListener; private final Transformer.DebugViewProvider debugViewProvider; public TransformerRenderersFactory( @@ -815,6 +817,7 @@ public final class Transformer { TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, + FallbackListener fallbackListener, Transformer.DebugViewProvider debugViewProvider) { this.context = context; this.muxerWrapper = muxerWrapper; @@ -823,6 +826,7 @@ public final class Transformer { this.transformationRequest = transformationRequest; this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; + this.fallbackListener = fallbackListener; this.debugViewProvider = debugViewProvider; mediaClock = new TransformerMediaClock(); } @@ -840,7 +844,12 @@ public final class Transformer { if (!removeAudio) { renderers[index] = new TransformerAudioRenderer( - muxerWrapper, mediaClock, transformationRequest, encoderFactory, decoderFactory); + muxerWrapper, + mediaClock, + transformationRequest, + encoderFactory, + decoderFactory, + fallbackListener); index++; } if (!removeVideo) { @@ -852,6 +861,7 @@ public final class Transformer { transformationRequest, encoderFactory, decoderFactory, + fallbackListener, debugViewProvider); index++; } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java index 7925a0ec45..ee04edcac2 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java @@ -41,8 +41,9 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult; TransformerMediaClock mediaClock, TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, - Codec.DecoderFactory decoderFactory) { - super(C.TRACK_TYPE_AUDIO, muxerWrapper, mediaClock, transformationRequest); + Codec.DecoderFactory decoderFactory, + FallbackListener fallbackListener) { + super(C.TRACK_TYPE_AUDIO, muxerWrapper, mediaClock, transformationRequest, fallbackListener); this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; decoderInputBuffer = @@ -78,11 +79,12 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult; TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); } if (shouldPassthrough(inputFormat)) { - samplePipeline = new PassthroughSamplePipeline(inputFormat); + samplePipeline = + new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); } else { samplePipeline = new AudioSamplePipeline( - inputFormat, transformationRequest, encoderFactory, decoderFactory); + inputFormat, transformationRequest, encoderFactory, decoderFactory, fallbackListener); } return true; } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java index eecb0cae62..80244bc2a7 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java @@ -39,6 +39,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; protected final MuxerWrapper muxerWrapper; protected final TransformerMediaClock mediaClock; protected final TransformationRequest transformationRequest; + protected final FallbackListener fallbackListener; protected boolean isRendererStarted; protected boolean muxerWrapperTrackAdded; @@ -50,11 +51,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; int trackType, MuxerWrapper muxerWrapper, TransformerMediaClock mediaClock, - TransformationRequest transformationRequest) { + TransformationRequest transformationRequest, + FallbackListener fallbackListener) { super(trackType); this.muxerWrapper = muxerWrapper; this.mediaClock = mediaClock; this.transformationRequest = transformationRequest; + this.fallbackListener = fallbackListener; } /** @@ -112,6 +115,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @Override protected final void onEnabled(boolean joining, boolean mayRenderStartOfStream) { muxerWrapper.registerTrack(); + fallbackListener.registerTrack(); mediaClock.updateTimeForTrackType(getTrackType(), 0L); } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java index 18d3cb996f..11579d5649 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java @@ -48,8 +48,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, + FallbackListener fallbackListener, Transformer.DebugViewProvider debugViewProvider) { - super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest); + super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener); this.context = context; this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; @@ -87,7 +88,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); } if (shouldPassthrough(inputFormat)) { - samplePipeline = new PassthroughSamplePipeline(inputFormat); + samplePipeline = + new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); } else { samplePipeline = new VideoSamplePipeline( @@ -96,6 +98,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; transformationRequest, encoderFactory, decoderFactory, + fallbackListener, debugViewProvider); } if (transformationRequest.flattenForSlowMotion) { diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java index 0158e70a60..8795f333bb 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java @@ -28,7 +28,9 @@ import androidx.annotation.RequiresApi; import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import com.google.android.exoplayer2.util.Util; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.dataflow.qual.Pure; /** * Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them. @@ -54,6 +56,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, + FallbackListener fallbackListener, Transformer.DebugViewProvider debugViewProvider) throws TransformationException { decoderInputBuffer = @@ -63,7 +66,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // Scale width and height to desired transformationRequest.outputHeight, preserving aspect // ratio. - // TODO(internal b/209781577): Think about which edge length should be set for portrait videos. + // TODO(b/209781577): Think about which edge length should be set for portrait videos. float inputFormatAspectRatio = (float) inputFormat.width / inputFormat.height; int outputWidth = inputFormat.width; int outputHeight = inputFormat.height; @@ -102,17 +105,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // postRotate in a later vertex shader. transformationMatrix.postRotate(outputRotationDegrees); - encoder = - encoderFactory.createForVideoEncoding( - new Format.Builder() - .setWidth(outputWidth) - .setHeight(outputHeight) - .setRotationDegrees(0) - .setSampleMimeType( - transformationRequest.videoMimeType != null - ? transformationRequest.videoMimeType - : inputFormat.sampleMimeType) - .build()); + Format requestedOutputFormat = + new Format.Builder() + .setWidth(outputWidth) + .setHeight(outputHeight) + .setRotationDegrees(0) + .setSampleMimeType( + transformationRequest.videoMimeType != null + ? transformationRequest.videoMimeType + : inputFormat.sampleMimeType) + .build(); + encoder = encoderFactory.createForVideoEncoding(requestedOutputFormat); + fallbackListener.onTransformationRequestFinalized( + createFallbackRequest( + transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat())); + if (inputFormat.height != outputHeight || inputFormat.width != outputWidth || !transformationMatrix.isIdentity()) { @@ -261,4 +268,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; decoder.release(); encoder.release(); } + + @Pure + private static TransformationRequest createFallbackRequest( + TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) { + // TODO(b/210591626): Also update resolution, bitrate etc. once encoder configuration and + // fallback are implemented. + if (Util.areEqual(requestedFormat.sampleMimeType, actualFormat.sampleMimeType)) { + return transformationRequest; + } + return transformationRequest.buildUpon().setVideoMimeType(actualFormat.sampleMimeType).build(); + } } diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerTest.java index 3dc75ca189..80ccd14fb2 100644 --- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerTest.java +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerTest.java @@ -65,6 +65,7 @@ import org.robolectric.shadows.ShadowMediaCodec; /** Unit test for {@link Transformer}. */ @RunWith(AndroidJUnit4.class) public final class TransformerTest { + // TODO(b/214973843): Disable fallback for all tests that aren't specifically testing fallback. private static final String URI_PREFIX = "asset:///media/"; private static final String FILE_VIDEO_ONLY = "mp4/sample_18byte_nclx_colr.mp4";