diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java index 505677d046..f529a949e7 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java @@ -29,7 +29,9 @@ import com.google.android.exoplayer2.audio.AudioProcessor; import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat; import com.google.android.exoplayer2.audio.SonicAudioProcessor; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import com.google.android.exoplayer2.util.Util; import java.nio.ByteBuffer; +import org.checkerframework.dataflow.qual.Pure; /** * Pipeline to decode audio samples, apply transformations on the raw samples, and re-encode them. @@ -39,13 +41,12 @@ import java.nio.ByteBuffer; private static final String TAG = "AudioSamplePipeline"; private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024; - private final TransformationRequest transformationRequest; - private final Codec decoder; private final DecoderInputBuffer decoderInputBuffer; private final SonicAudioProcessor sonicAudioProcessor; private final SpeedProvider speedProvider; + private final boolean flattenForSlowMotion; private final Codec encoder; private final AudioFormat encoderInputAudioFormat; @@ -63,9 +64,9 @@ import java.nio.ByteBuffer; Format inputFormat, TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, - Codec.DecoderFactory decoderFactory) + Codec.DecoderFactory decoderFactory, + FallbackListener fallbackListener) throws TransformationException { - this.transformationRequest = transformationRequest; decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); encoderInputBuffer = @@ -75,6 +76,7 @@ import java.nio.ByteBuffer; this.decoder = decoderFactory.createForAudioDecoding(inputFormat); + this.flattenForSlowMotion = transformationRequest.flattenForSlowMotion; sonicAudioProcessor = new SonicAudioProcessor(); sonicOutputBuffer = AudioProcessor.EMPTY_BUFFER; speedProvider = new SegmentSpeedProvider(inputFormat); @@ -86,7 +88,7 @@ import java.nio.ByteBuffer; // The decoder uses ENCODING_PCM_16BIT by default. // https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers C.ENCODING_PCM_16BIT); - if (transformationRequest.flattenForSlowMotion) { + if (flattenForSlowMotion) { try { encoderInputAudioFormat = sonicAudioProcessor.configure(encoderInputAudioFormat); } catch (AudioProcessor.UnhandledAudioFormatException impossible) { @@ -96,19 +98,23 @@ import java.nio.ByteBuffer; sonicAudioProcessor.setPitch(currentSpeed); sonicAudioProcessor.flush(); } - - encoder = - encoderFactory.createForAudioEncoding( - new Format.Builder() - .setSampleMimeType( - transformationRequest.audioMimeType == null - ? inputFormat.sampleMimeType - : transformationRequest.audioMimeType) - .setSampleRate(encoderInputAudioFormat.sampleRate) - .setChannelCount(encoderInputAudioFormat.channelCount) - .setAverageBitrate(DEFAULT_ENCODER_BITRATE) - .build()); this.encoderInputAudioFormat = encoderInputAudioFormat; + + Format requestedOutputFormat = + new Format.Builder() + .setSampleMimeType( + transformationRequest.audioMimeType == null + ? inputFormat.sampleMimeType + : transformationRequest.audioMimeType) + .setSampleRate(encoderInputAudioFormat.sampleRate) + .setChannelCount(encoderInputAudioFormat.channelCount) + .setAverageBitrate(DEFAULT_ENCODER_BITRATE) + .build(); + encoder = encoderFactory.createForAudioEncoding(requestedOutputFormat); + + fallbackListener.onTransformationRequestFinalized( + createFallbackRequest( + transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat())); } @Override @@ -293,7 +299,7 @@ import java.nio.ByteBuffer; } private boolean isSpeedChanging(BufferInfo bufferInfo) { - if (!transformationRequest.flattenForSlowMotion) { + if (!flattenForSlowMotion) { return false; } float newSpeed = speedProvider.getSpeed(bufferInfo.presentationTimeUs); @@ -325,4 +331,15 @@ import java.nio.ByteBuffer; } nextEncoderInputBufferTimeUs += bufferDurationUs; } + + @Pure + private static TransformationRequest createFallbackRequest( + TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) { + // TODO(b/210591626): Also update bitrate and other params once encoder configuration and + // fallback are implemented. + if (Util.areEqual(requestedFormat.sampleMimeType, actualFormat.sampleMimeType)) { + return transformationRequest; + } + return transformationRequest.buildUpon().setAudioMimeType(actualFormat.sampleMimeType).build(); + } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java index 8b540ee105..042819b0fb 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java @@ -28,10 +28,14 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer; private boolean hasPendingBuffer; - public PassthroughSamplePipeline(Format format) { + public PassthroughSamplePipeline( + Format format, + TransformationRequest transformationRequest, + FallbackListener fallbackListener) { this.format = format; buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); hasPendingBuffer = false; + fallbackListener.onTransformationRequestFinalized(transformationRequest); } @Override diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java index bd9eac24ac..f312d84643 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java @@ -703,6 +703,7 @@ public final class Transformer { transformationRequest, encoderFactory, decoderFactory, + new FallbackListener(mediaItem, listeners, transformationRequest), debugViewProvider)) .setMediaSourceFactory(mediaSourceFactory) .setTrackSelector(trackSelector) @@ -805,6 +806,7 @@ public final class Transformer { private final TransformationRequest transformationRequest; private final Codec.EncoderFactory encoderFactory; private final Codec.DecoderFactory decoderFactory; + private final FallbackListener fallbackListener; private final Transformer.DebugViewProvider debugViewProvider; public TransformerRenderersFactory( @@ -815,6 +817,7 @@ public final class Transformer { TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, + FallbackListener fallbackListener, Transformer.DebugViewProvider debugViewProvider) { this.context = context; this.muxerWrapper = muxerWrapper; @@ -823,6 +826,7 @@ public final class Transformer { this.transformationRequest = transformationRequest; this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; + this.fallbackListener = fallbackListener; this.debugViewProvider = debugViewProvider; mediaClock = new TransformerMediaClock(); } @@ -840,7 +844,12 @@ public final class Transformer { if (!removeAudio) { renderers[index] = new TransformerAudioRenderer( - muxerWrapper, mediaClock, transformationRequest, encoderFactory, decoderFactory); + muxerWrapper, + mediaClock, + transformationRequest, + encoderFactory, + decoderFactory, + fallbackListener); index++; } if (!removeVideo) { @@ -852,6 +861,7 @@ public final class Transformer { transformationRequest, encoderFactory, decoderFactory, + fallbackListener, debugViewProvider); index++; } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java index 7925a0ec45..ee04edcac2 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java @@ -41,8 +41,9 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult; TransformerMediaClock mediaClock, TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, - Codec.DecoderFactory decoderFactory) { - super(C.TRACK_TYPE_AUDIO, muxerWrapper, mediaClock, transformationRequest); + Codec.DecoderFactory decoderFactory, + FallbackListener fallbackListener) { + super(C.TRACK_TYPE_AUDIO, muxerWrapper, mediaClock, transformationRequest, fallbackListener); this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; decoderInputBuffer = @@ -78,11 +79,12 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult; TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); } if (shouldPassthrough(inputFormat)) { - samplePipeline = new PassthroughSamplePipeline(inputFormat); + samplePipeline = + new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); } else { samplePipeline = new AudioSamplePipeline( - inputFormat, transformationRequest, encoderFactory, decoderFactory); + inputFormat, transformationRequest, encoderFactory, decoderFactory, fallbackListener); } return true; } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java index eecb0cae62..80244bc2a7 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java @@ -39,6 +39,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; protected final MuxerWrapper muxerWrapper; protected final TransformerMediaClock mediaClock; protected final TransformationRequest transformationRequest; + protected final FallbackListener fallbackListener; protected boolean isRendererStarted; protected boolean muxerWrapperTrackAdded; @@ -50,11 +51,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; int trackType, MuxerWrapper muxerWrapper, TransformerMediaClock mediaClock, - TransformationRequest transformationRequest) { + TransformationRequest transformationRequest, + FallbackListener fallbackListener) { super(trackType); this.muxerWrapper = muxerWrapper; this.mediaClock = mediaClock; this.transformationRequest = transformationRequest; + this.fallbackListener = fallbackListener; } /** @@ -112,6 +115,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @Override protected final void onEnabled(boolean joining, boolean mayRenderStartOfStream) { muxerWrapper.registerTrack(); + fallbackListener.registerTrack(); mediaClock.updateTimeForTrackType(getTrackType(), 0L); } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java index 18d3cb996f..11579d5649 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java @@ -48,8 +48,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, + FallbackListener fallbackListener, Transformer.DebugViewProvider debugViewProvider) { - super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest); + super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener); this.context = context; this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; @@ -87,7 +88,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); } if (shouldPassthrough(inputFormat)) { - samplePipeline = new PassthroughSamplePipeline(inputFormat); + samplePipeline = + new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); } else { samplePipeline = new VideoSamplePipeline( @@ -96,6 +98,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; transformationRequest, encoderFactory, decoderFactory, + fallbackListener, debugViewProvider); } if (transformationRequest.flattenForSlowMotion) { diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java index 0158e70a60..8795f333bb 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java @@ -28,7 +28,9 @@ import androidx.annotation.RequiresApi; import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import com.google.android.exoplayer2.util.Util; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.dataflow.qual.Pure; /** * Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them. @@ -54,6 +56,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, + FallbackListener fallbackListener, Transformer.DebugViewProvider debugViewProvider) throws TransformationException { decoderInputBuffer = @@ -63,7 +66,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // Scale width and height to desired transformationRequest.outputHeight, preserving aspect // ratio. - // TODO(internal b/209781577): Think about which edge length should be set for portrait videos. + // TODO(b/209781577): Think about which edge length should be set for portrait videos. float inputFormatAspectRatio = (float) inputFormat.width / inputFormat.height; int outputWidth = inputFormat.width; int outputHeight = inputFormat.height; @@ -102,17 +105,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // postRotate in a later vertex shader. transformationMatrix.postRotate(outputRotationDegrees); - encoder = - encoderFactory.createForVideoEncoding( - new Format.Builder() - .setWidth(outputWidth) - .setHeight(outputHeight) - .setRotationDegrees(0) - .setSampleMimeType( - transformationRequest.videoMimeType != null - ? transformationRequest.videoMimeType - : inputFormat.sampleMimeType) - .build()); + Format requestedOutputFormat = + new Format.Builder() + .setWidth(outputWidth) + .setHeight(outputHeight) + .setRotationDegrees(0) + .setSampleMimeType( + transformationRequest.videoMimeType != null + ? transformationRequest.videoMimeType + : inputFormat.sampleMimeType) + .build(); + encoder = encoderFactory.createForVideoEncoding(requestedOutputFormat); + fallbackListener.onTransformationRequestFinalized( + createFallbackRequest( + transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat())); + if (inputFormat.height != outputHeight || inputFormat.width != outputWidth || !transformationMatrix.isIdentity()) { @@ -261,4 +268,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; decoder.release(); encoder.release(); } + + @Pure + private static TransformationRequest createFallbackRequest( + TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) { + // TODO(b/210591626): Also update resolution, bitrate etc. once encoder configuration and + // fallback are implemented. + if (Util.areEqual(requestedFormat.sampleMimeType, actualFormat.sampleMimeType)) { + return transformationRequest; + } + return transformationRequest.buildUpon().setVideoMimeType(actualFormat.sampleMimeType).build(); + } } diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerTest.java index 3dc75ca189..80ccd14fb2 100644 --- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerTest.java +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerTest.java @@ -65,6 +65,7 @@ import org.robolectric.shadows.ShadowMediaCodec; /** Unit test for {@link Transformer}. */ @RunWith(AndroidJUnit4.class) public final class TransformerTest { + // TODO(b/214973843): Disable fallback for all tests that aren't specifically testing fallback. private static final String URI_PREFIX = "asset:///media/"; private static final String FILE_VIDEO_ONLY = "mp4/sample_18byte_nclx_colr.mp4";