From d277deb335b69020bf643dfcc215810cb52a5878 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Fri, 21 Jan 2022 10:24:36 +0000 Subject: [PATCH] Transcode to a muxer-supported sample MIME type. If the output sample MIME type is inferred from the input but is not supported by the muxer, we fallback to transcoding to a supported sample MIME type. The audio and video renderers need to make sure not to select the PassthroughSamplePipeline for this case. Which sample MIME type to choose is decided by the EncoderFactory. PiperOrigin-RevId: 423272812 --- .../mp4/sample_ac3.mp4.fallback.dump | 61 ++++++ .../transformer/AudioSamplePipeline.java | 6 +- .../androidx/media3/transformer/Codec.java | 27 ++- .../transformer/DefaultCodecFactory.java | 33 ++- .../transformer/TransformerAudioRenderer.java | 21 +- .../transformer/TransformerVideoRenderer.java | 17 +- .../transformer/VideoSamplePipeline.java | 6 +- .../media3/transformer/TransformerTest.java | 191 ++++++++---------- 8 files changed, 210 insertions(+), 152 deletions(-) create mode 100644 libraries/test_data/src/test/assets/transformerdumps/mp4/sample_ac3.mp4.fallback.dump diff --git a/libraries/test_data/src/test/assets/transformerdumps/mp4/sample_ac3.mp4.fallback.dump b/libraries/test_data/src/test/assets/transformerdumps/mp4/sample_ac3.mp4.fallback.dump new file mode 100644 index 0000000000..d97f474a1d --- /dev/null +++ b/libraries/test_data/src/test/assets/transformerdumps/mp4/sample_ac3.mp4.fallback.dump @@ -0,0 +1,61 @@ +containerMimeType = video/mp4 +format 0: + sampleMimeType = audio/mp4a-latm + channelCount = 6 + sampleRate = 48000 + pcmEncoding = 2 +sample: + trackIndex = 0 + dataHashCode = 1896404418 + size = 1536 + isKeyFrame = true + presentationTimeUs = 0 +sample: + trackIndex = 0 + dataHashCode = -2134951116 + size = 1536 + isKeyFrame = true + presentationTimeUs = 2667 +sample: + trackIndex = 0 + dataHashCode = 97556101 + size = 1536 + isKeyFrame = true + presentationTimeUs = 5334 +sample: + trackIndex = 0 + dataHashCode = -1448980924 + size = 1536 + isKeyFrame = true + presentationTimeUs = 8000 +sample: + trackIndex = 0 + dataHashCode = 1871012467 + size = 1536 + isKeyFrame = true + presentationTimeUs = 10667 +sample: + trackIndex = 0 + dataHashCode = -1317831364 + size = 1536 + isKeyFrame = true + presentationTimeUs = 13334 +sample: + trackIndex = 0 + dataHashCode = -1728189539 + size = 1536 + isKeyFrame = true + presentationTimeUs = 16000 +sample: + trackIndex = 0 + dataHashCode = -1715881661 + size = 1536 + isKeyFrame = true + presentationTimeUs = 18667 +sample: + trackIndex = 0 + dataHashCode = -1428554542 + size = 1536 + isKeyFrame = true + presentationTimeUs = 21334 +released = true diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java index b3533b8703..8a857b8d0b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java @@ -31,6 +31,7 @@ import androidx.media3.exoplayer.audio.AudioProcessor; import androidx.media3.exoplayer.audio.AudioProcessor.AudioFormat; import androidx.media3.exoplayer.audio.SonicAudioProcessor; import java.nio.ByteBuffer; +import java.util.List; import org.checkerframework.dataflow.qual.Pure; /** @@ -63,8 +64,9 @@ import org.checkerframework.dataflow.qual.Pure; public AudioSamplePipeline( Format inputFormat, TransformationRequest transformationRequest, - Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, + Codec.EncoderFactory encoderFactory, + List allowedOutputMimeTypes, FallbackListener fallbackListener) throws TransformationException { decoderInputBuffer = @@ -110,7 +112,7 @@ import org.checkerframework.dataflow.qual.Pure; .setChannelCount(encoderInputAudioFormat.channelCount) .setAverageBitrate(DEFAULT_ENCODER_BITRATE) .build(); - encoder = encoderFactory.createForAudioEncoding(requestedOutputFormat); + encoder = encoderFactory.createForAudioEncoding(requestedOutputFormat, allowedOutputMimeTypes); fallbackListener.onTransformationRequestFinalized( createFallbackRequest( diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java index 817e9c866c..7be6175134 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java @@ -31,6 +31,7 @@ import androidx.media3.common.util.UnstableApi; import androidx.media3.decoder.DecoderInputBuffer; import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; +import java.util.List; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -56,7 +57,7 @@ public final class Codec { * @param format The {@link Format} (of the input data) used to determine the underlying {@link * MediaCodec} and its configuration values. * @return A configured and started decoder wrapper. - * @throws TransformationException If the underlying codec cannot be created. + * @throws TransformationException If no suitable codec can be created. */ Codec createForAudioDecoding(Format format) throws TransformationException; @@ -67,7 +68,7 @@ public final class Codec { * MediaCodec} and its configuration values. * @param outputSurface The {@link Surface} to which the decoder output is rendered. * @return A configured and started decoder wrapper. - * @throws TransformationException If the underlying codec cannot be created. + * @throws TransformationException If no suitable codec can be created. */ Codec createForVideoDecoding(Format format, Surface outputSurface) throws TransformationException; @@ -82,25 +83,39 @@ public final class Codec { /** * Returns a {@link Codec} for audio encoding. * + *

This method must validate that the {@link Codec} is configured to produce one of the + * {@code allowedMimeTypes}. The {@link Format#sampleMimeType sample MIME type} given in {@code + * format} is not necessarily allowed. + * * @param format The {@link Format} (of the output data) used to determine the underlying {@link * MediaCodec} and its configuration values. + * @param allowedMimeTypes The non-empty list of allowed output sample {@link MimeTypes MIME + * types}. * @return A configured and started encoder wrapper. - * @throws TransformationException If the underlying codec cannot be created. + * @throws TransformationException If no suitable codec can be created. */ - Codec createForAudioEncoding(Format format) throws TransformationException; + Codec createForAudioEncoding(Format format, List allowedMimeTypes) + throws TransformationException; /** * Returns a {@link Codec} for video encoding. * + *

This method must validate that the {@link Codec} is configured to produce one of the + * {@code allowedMimeTypes}. The {@link Format#sampleMimeType sample MIME type} given in {@code + * format} is not necessarily allowed. + * * @param format The {@link Format} (of the output data) used to determine the underlying {@link * MediaCodec} and its configuration values. {@link Format#sampleMimeType}, {@link * Format#width} and {@link Format#height} must be set to those of the desired output video * format. {@link Format#rotationDegrees} should be 0. The video should always be in * landscape orientation. + * @param allowedMimeTypes The non-empty list of allowed output sample {@link MimeTypes MIME + * types}. * @return A configured and started encoder wrapper. - * @throws TransformationException If the underlying codec cannot be created. + * @throws TransformationException If no suitable codec can be created. */ - Codec createForVideoEncoding(Format format) throws TransformationException; + Codec createForVideoEncoding(Format format, List allowedMimeTypes) + throws TransformationException; } // MediaCodec decoders always output 16 bit PCM, unless configured to output PCM float. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodecFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodecFactory.java index ef0443a8b0..0c7c1040a6 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodecFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodecFactory.java @@ -34,13 +34,15 @@ import androidx.media3.common.util.MediaFormatUtil; import androidx.media3.common.util.TraceUtil; import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import java.io.IOException; +import java.util.List; import org.checkerframework.checker.nullness.qual.RequiresNonNull; /** A default {@link Codec.DecoderFactory} and {@link Codec.EncoderFactory}. */ /* package */ final class DefaultCodecFactory implements Codec.DecoderFactory, Codec.EncoderFactory { + // TODO(b/214973843): Add option to disable fallback. - // TODO(b/210591626) Fall back adaptively to H265 if possible. + // TODO(b/210591626): Fall back adaptively to H265 if possible. private static final String DEFAULT_FALLBACK_MIME_TYPE = MimeTypes.VIDEO_H264; private static final int DEFAULT_COLOR_FORMAT = CodecCapabilities.COLOR_FormatSurface; private static final int DEFAULT_FRAME_RATE = 60; @@ -85,7 +87,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } @Override - public Codec createForAudioEncoding(Format format) throws TransformationException { + public Codec createForAudioEncoding(Format format, List allowedMimeTypes) + throws TransformationException { + checkArgument(!allowedMimeTypes.isEmpty()); + if (!allowedMimeTypes.contains(format.sampleMimeType)) { + // TODO(b/210591626): Pick fallback MIME type using same strategy as for encoder + // capabilities limitations. + format = format.buildUpon().setSampleMimeType(allowedMimeTypes.get(0)).build(); + } MediaFormat mediaFormat = MediaFormat.createAudioFormat( checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount); @@ -100,7 +109,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } @Override - public Codec createForVideoEncoding(Format format) throws TransformationException { + public Codec createForVideoEncoding(Format format, List allowedMimeTypes) + throws TransformationException { checkArgument(format.width != Format.NO_VALUE); checkArgument(format.height != Format.NO_VALUE); // According to interface Javadoc, format.rotationDegrees should be 0. The video should always @@ -108,7 +118,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; checkArgument(format.height <= format.width); checkArgument(format.rotationDegrees == 0); checkNotNull(format.sampleMimeType); - format = getVideoEncoderSupportedFormat(format); + + checkArgument(!allowedMimeTypes.isEmpty()); + + format = getVideoEncoderSupportedFormat(format, allowedMimeTypes); MediaFormat mediaFormat = MediaFormat.createVideoFormat( @@ -191,14 +204,18 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } @RequiresNonNull("#1.sampleMimeType") - private static Format getVideoEncoderSupportedFormat(Format requestedFormat) - throws TransformationException { + private static Format getVideoEncoderSupportedFormat( + Format requestedFormat, List allowedMimeTypes) throws TransformationException { String mimeType = requestedFormat.sampleMimeType; Format.Builder formatBuilder = requestedFormat.buildUpon(); // TODO(b/210591626) Implement encoder filtering. - if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) { - mimeType = DEFAULT_FALLBACK_MIME_TYPE; + if (!allowedMimeTypes.contains(mimeType) + || EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) { + mimeType = + allowedMimeTypes.contains(DEFAULT_FALLBACK_MIME_TYPE) + ? DEFAULT_FALLBACK_MIME_TYPE + : allowedMimeTypes.get(0); if (EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) { throw createTransformationException( new IllegalArgumentException( diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java index 2ac47e6f30..ca4a4dd393 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java @@ -68,23 +68,18 @@ import androidx.media3.extractor.metadata.mp4.SlowMotionData; return false; } Format inputFormat = checkNotNull(formatHolder.format); - String sampleMimeType = checkNotNull(inputFormat.sampleMimeType); - if (transformationRequest.audioMimeType == null - && !muxerWrapper.supportsSampleMimeType(sampleMimeType)) { - throw TransformationException.createForMuxer( - new IllegalArgumentException( - "The output sample MIME inferred from the input format is not supported by the muxer." - + " Sample MIME type: " - + sampleMimeType), - TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); - } if (shouldPassthrough(inputFormat)) { samplePipeline = new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); } else { samplePipeline = new AudioSamplePipeline( - inputFormat, transformationRequest, encoderFactory, decoderFactory, fallbackListener); + inputFormat, + transformationRequest, + decoderFactory, + encoderFactory, + muxerWrapper.getSupportedSampleMimeTypes(getTrackType()), + fallbackListener); } return true; } @@ -94,6 +89,10 @@ import androidx.media3.extractor.metadata.mp4.SlowMotionData; && !transformationRequest.audioMimeType.equals(inputFormat.sampleMimeType)) { return false; } + if (transformationRequest.audioMimeType == null + && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) { + return false; + } if (transformationRequest.flattenForSlowMotion && isSlowMotion(inputFormat)) { return false; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java index ae8fef409a..1cae7cc6f1 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java @@ -77,16 +77,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return false; } Format inputFormat = checkNotNull(formatHolder.format); - String sampleMimeType = checkNotNull(inputFormat.sampleMimeType); - if (transformationRequest.videoMimeType == null - && !muxerWrapper.supportsSampleMimeType(sampleMimeType)) { - throw TransformationException.createForMuxer( - new IllegalArgumentException( - "The output sample MIME inferred from the input format is not supported by the muxer." - + " Sample MIME type: " - + sampleMimeType), - TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); - } if (shouldPassthrough(inputFormat)) { samplePipeline = new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); @@ -96,8 +86,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; context, inputFormat, transformationRequest, - encoderFactory, decoderFactory, + encoderFactory, + muxerWrapper.getSupportedSampleMimeTypes(getTrackType()), fallbackListener, debugViewProvider); } @@ -112,6 +103,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; && !transformationRequest.videoMimeType.equals(inputFormat.sampleMimeType)) { return false; } + if (transformationRequest.videoMimeType == null + && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) { + return false; + } if (transformationRequest.outputHeight != C.LENGTH_UNSET && transformationRequest.outputHeight != inputFormat.height) { return false; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java index e3e7568d56..32e4025712 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java @@ -29,6 +29,7 @@ import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; +import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.dataflow.qual.Pure; @@ -54,8 +55,9 @@ import org.checkerframework.dataflow.qual.Pure; Context context, Format inputFormat, TransformationRequest transformationRequest, - Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, + Codec.EncoderFactory encoderFactory, + List allowedOutputMimeTypes, FallbackListener fallbackListener, Transformer.DebugViewProvider debugViewProvider) throws TransformationException { @@ -115,7 +117,7 @@ import org.checkerframework.dataflow.qual.Pure; ? transformationRequest.videoMimeType : inputFormat.sampleMimeType) .build(); - encoder = encoderFactory.createForVideoEncoding(requestedOutputFormat); + encoder = encoderFactory.createForVideoEncoding(requestedOutputFormat, allowedOutputMimeTypes); fallbackListener.onTransformationRequestFinalized( createFallbackRequest( transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat())); diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerTest.java index 3df816852b..cec7ff3d29 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerTest.java @@ -23,6 +23,7 @@ import static androidx.media3.transformer.Transformer.PROGRESS_STATE_WAITING_FOR import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -75,7 +76,6 @@ public final class TransformerTest { private static final String FILE_AUDIO_UNSUPPORTED_BY_DECODER = "amr/sample_wb.amr"; private static final String FILE_AUDIO_UNSUPPORTED_BY_ENCODER = "amr/sample_nb.amr"; private static final String FILE_AUDIO_UNSUPPORTED_BY_MUXER = "mp4/sample_ac3.mp4"; - private static final String FILE_VIDEO_UNSUPPORTED = "vp9/bear-vp9.webm"; private static final String FILE_UNKNOWN_DURATION = "mp4/sample_fragmented.mp4"; public static final String DUMP_FILE_OUTPUT_DIRECTORY = "transformerdumps"; public static final String DUMP_FILE_EXTENSION = "dump"; @@ -103,11 +103,7 @@ public final class TransformerTest { @Test public void startTransformation_videoOnlyPassthrough_completesSuccessfully() throws Exception { - Transformer transformer = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) - .build(); + Transformer transformer = createTransformerBuilder().build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_VIDEO_ONLY); transformer.startTransformation(mediaItem, outputPath); @@ -118,11 +114,7 @@ public final class TransformerTest { @Test public void startTransformation_audioOnlyPassthrough_completesSuccessfully() throws Exception { - Transformer transformer = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) - .build(); + Transformer transformer = createTransformerBuilder().build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_AUDIO_UNSUPPORTED_BY_ENCODER); @@ -136,9 +128,7 @@ public final class TransformerTest { @Test public void startTransformation_audioOnlyTranscoding_completesSuccessfully() throws Exception { Transformer transformer = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) + createTransformerBuilder() .setTransformationRequest( new TransformationRequest.Builder() .setAudioMimeType(MimeTypes.AUDIO_AAC) // supported by encoder and muxer @@ -155,11 +145,7 @@ public final class TransformerTest { @Test public void startTransformation_audioAndVideo_completesSuccessfully() throws Exception { - Transformer transformer = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) - .build(); + Transformer transformer = createTransformerBuilder().build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_AUDIO_VIDEO); transformer.startTransformation(mediaItem, outputPath); @@ -171,9 +157,7 @@ public final class TransformerTest { @Test public void startTransformation_withSubtitles_completesSuccessfully() throws Exception { Transformer transformer = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) + createTransformerBuilder() .setTransformationRequest( new TransformationRequest.Builder().setAudioMimeType(MimeTypes.AUDIO_AAC).build()) .build(); @@ -188,11 +172,7 @@ public final class TransformerTest { @Test public void startTransformation_successiveTransformations_completesSuccessfully() throws Exception { - Transformer transformer = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) - .build(); + Transformer transformer = createTransformerBuilder().build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_AUDIO_VIDEO); // Transform first media item. @@ -209,7 +189,7 @@ public final class TransformerTest { @Test public void startTransformation_concurrentTransformations_throwsError() throws Exception { - Transformer transformer = new Transformer.Builder(context).setClock(clock).build(); + Transformer transformer = createTransformerBuilder().build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_VIDEO_ONLY); transformer.startTransformation(mediaItem, outputPath); @@ -220,12 +200,7 @@ public final class TransformerTest { @Test public void startTransformation_removeAudio_completesSuccessfully() throws Exception { - Transformer transformer = - new Transformer.Builder(context) - .setRemoveAudio(true) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) - .build(); + Transformer transformer = createTransformerBuilder().setRemoveAudio(true).build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_AUDIO_VIDEO); transformer.startTransformation(mediaItem, outputPath); @@ -237,12 +212,7 @@ public final class TransformerTest { @Test public void startTransformation_removeVideo_completesSuccessfully() throws Exception { - Transformer transformer = - new Transformer.Builder(context) - .setRemoveVideo(true) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) - .build(); + Transformer transformer = createTransformerBuilder().setRemoveVideo(true).build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_AUDIO_VIDEO); transformer.startTransformation(mediaItem, outputPath); @@ -258,9 +228,7 @@ public final class TransformerTest { Transformer.Listener mockListener2 = mock(Transformer.Listener.class); Transformer.Listener mockListener3 = mock(Transformer.Listener.class); Transformer transformer = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) + createTransformerBuilder() .addListener(mockListener1) .addListener(mockListener2) .addListener(mockListener3) @@ -281,14 +249,14 @@ public final class TransformerTest { Transformer.Listener mockListener2 = mock(Transformer.Listener.class); Transformer.Listener mockListener3 = mock(Transformer.Listener.class); Transformer transformer = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) + createTransformerBuilder() .addListener(mockListener1) .addListener(mockListener2) .addListener(mockListener3) + .setTransformationRequest( // Request transcoding so that decoder is used. + new TransformationRequest.Builder().setAudioMimeType(MimeTypes.AUDIO_AAC).build()) .build(); - MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_AUDIO_UNSUPPORTED_BY_MUXER); + MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_AUDIO_UNSUPPORTED_BY_DECODER); transformer.startTransformation(mediaItem, outputPath); TransformationException exception = TransformerTestRunner.runUntilError(transformer); @@ -298,6 +266,34 @@ public final class TransformerTest { verify(mockListener3, times(1)).onTransformationError(mediaItem, exception); } + @Test + public void startTransformation_withMultipleListeners_callsEachOnFallback() throws Exception { + Transformer.Listener mockListener1 = mock(Transformer.Listener.class); + Transformer.Listener mockListener2 = mock(Transformer.Listener.class); + Transformer.Listener mockListener3 = mock(Transformer.Listener.class); + TransformationRequest originalTransformationRequest = + new TransformationRequest.Builder().build(); + TransformationRequest fallbackTransformationRequest = + new TransformationRequest.Builder().setAudioMimeType(MimeTypes.AUDIO_AAC).build(); + Transformer transformer = + createTransformerBuilder() + .addListener(mockListener1) + .addListener(mockListener2) + .addListener(mockListener3) + .build(); + MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_AUDIO_UNSUPPORTED_BY_MUXER); + + transformer.startTransformation(mediaItem, outputPath); + TransformerTestRunner.runUntilCompleted(transformer); + + verify(mockListener1, times(1)) + .onFallbackApplied(mediaItem, originalTransformationRequest, fallbackTransformationRequest); + verify(mockListener2, times(1)) + .onFallbackApplied(mediaItem, originalTransformationRequest, fallbackTransformationRequest); + verify(mockListener3, times(1)) + .onFallbackApplied(mediaItem, originalTransformationRequest, fallbackTransformationRequest); + } + @Test public void startTransformation_afterBuildUponWithListenerRemoved_onlyCallsRemainingListeners() throws Exception { @@ -305,9 +301,7 @@ public final class TransformerTest { Transformer.Listener mockListener2 = mock(Transformer.Listener.class); Transformer.Listener mockListener3 = mock(Transformer.Listener.class); Transformer transformer1 = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) + createTransformerBuilder() .addListener(mockListener1) .addListener(mockListener2) .addListener(mockListener3) @@ -319,16 +313,14 @@ public final class TransformerTest { TransformerTestRunner.runUntilCompleted(transformer2); verify(mockListener1, times(1)).onTransformationCompleted(mediaItem); - verify(mockListener2, times(0)).onTransformationCompleted(mediaItem); + verify(mockListener2, never()).onTransformationCompleted(mediaItem); verify(mockListener3, times(1)).onTransformationCompleted(mediaItem); } @Test public void startTransformation_flattenForSlowMotion_completesSuccessfully() throws Exception { Transformer transformer = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) + createTransformerBuilder() .setTransformationRequest( new TransformationRequest.Builder().setFlattenForSlowMotion(true).build()) .build(); @@ -344,9 +336,7 @@ public final class TransformerTest { public void startTransformation_withAudioEncoderFormatUnsupported_completesWithError() throws Exception { Transformer transformer = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) + createTransformerBuilder() .setTransformationRequest( new TransformationRequest.Builder() .setAudioMimeType( @@ -367,9 +357,7 @@ public final class TransformerTest { public void startTransformation_withAudioDecoderFormatUnsupported_completesWithError() throws Exception { Transformer transformer = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) + createTransformerBuilder() .setTransformationRequest( new TransformationRequest.Builder() .setAudioMimeType(MimeTypes.AUDIO_AAC) // supported by encoder and muxer @@ -389,9 +377,7 @@ public final class TransformerTest { public void startTransformation_withVideoEncoderFormatUnsupported_completesWithError() throws Exception { Transformer transformer = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) + createTransformerBuilder() .setTransformationRequest( new TransformationRequest.Builder() .setVideoMimeType(MimeTypes.VIDEO_H263) // unsupported encoder MIME type @@ -409,7 +395,7 @@ public final class TransformerTest { @Test public void startTransformation_withIoError_completesWithError() throws Exception { - Transformer transformer = new Transformer.Builder(context).setClock(clock).build(); + Transformer transformer = createTransformerBuilder().build(); MediaItem mediaItem = MediaItem.fromUri("asset:///non-existing-path.mp4"); transformer.startTransformation(mediaItem, outputPath); @@ -420,50 +406,32 @@ public final class TransformerTest { } @Test - public void startTransformation_withAudioMuxerFormatUnsupported_completesWithError() + public void startTransformation_withAudioMuxerFormatFallback_completesSuccessfully() throws Exception { - Transformer transformer = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) - .build(); + Transformer.Listener mockListener = mock(Transformer.Listener.class); + TransformationRequest originalTransformationRequest = + new TransformationRequest.Builder().build(); + TransformationRequest fallbackTransformationRequest = + new TransformationRequest.Builder().setAudioMimeType(MimeTypes.AUDIO_AAC).build(); + Transformer transformer = createTransformerBuilder().addListener(mockListener).build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_AUDIO_UNSUPPORTED_BY_MUXER); transformer.startTransformation(mediaItem, outputPath); - TransformationException exception = TransformerTestRunner.runUntilError(transformer); + TransformerTestRunner.runUntilCompleted(transformer); - assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException.class); - assertThat(exception).hasCauseThat().hasMessageThat().contains("audio"); - assertThat(exception.errorCode) - .isEqualTo(TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); + DumpFileAsserts.assertOutput( + context, testMuxer, getDumpFileName(FILE_AUDIO_UNSUPPORTED_BY_MUXER + ".fallback")); + verify(mockListener, times(1)) + .onFallbackApplied(mediaItem, originalTransformationRequest, fallbackTransformationRequest); } - @Test - public void startTransformation_withVideoMuxerFormatUnsupported_completesWithError() - throws Exception { - Transformer transformer = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) - .build(); - MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_VIDEO_UNSUPPORTED); - - transformer.startTransformation(mediaItem, outputPath); - TransformationException exception = TransformerTestRunner.runUntilError(transformer); - - assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException.class); - assertThat(exception).hasCauseThat().hasMessageThat().contains("video"); - assertThat(exception.errorCode) - .isEqualTo(TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); - } + // TODO(b/214012830): Add a test to check that the correct exception is thrown when the muxer + // doesn't support the output sample MIME type inferred from the input once it is possible to + // disable fallback. @Test public void startTransformation_afterCancellation_completesSuccessfully() throws Exception { - Transformer transformer = - new Transformer.Builder(context) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) - .build(); + Transformer transformer = createTransformerBuilder().build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_AUDIO_VIDEO); transformer.startTransformation(mediaItem, outputPath); @@ -482,12 +450,7 @@ public final class TransformerTest { HandlerThread anotherThread = new HandlerThread("AnotherThread"); anotherThread.start(); Looper looper = anotherThread.getLooper(); - Transformer transformer = - new Transformer.Builder(context) - .setLooper(looper) - .setClock(clock) - .setMuxerFactory(new TestMuxerFactory()) - .build(); + Transformer transformer = createTransformerBuilder().setLooper(looper).build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_AUDIO_VIDEO); AtomicReference exception = new AtomicReference<>(); CountDownLatch countDownLatch = new CountDownLatch(1); @@ -512,7 +475,7 @@ public final class TransformerTest { @Test public void startTransformation_fromWrongThread_throwsError() throws Exception { - Transformer transformer = new Transformer.Builder(context).setClock(clock).build(); + Transformer transformer = createTransformerBuilder().build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_AUDIO_VIDEO); HandlerThread anotherThread = new HandlerThread("AnotherThread"); AtomicReference illegalStateException = new AtomicReference<>(); @@ -539,7 +502,7 @@ public final class TransformerTest { @Test public void getProgress_knownDuration_returnsConsistentStates() throws Exception { - Transformer transformer = new Transformer.Builder(context).setClock(clock).build(); + Transformer transformer = createTransformerBuilder().build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_VIDEO_ONLY); AtomicInteger previousProgressState = new AtomicInteger(PROGRESS_STATE_WAITING_FOR_AVAILABILITY); @@ -585,7 +548,7 @@ public final class TransformerTest { @Test public void getProgress_knownDuration_givesIncreasingPercentages() throws Exception { - Transformer transformer = new Transformer.Builder(context).setClock(clock).build(); + Transformer transformer = createTransformerBuilder().build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_VIDEO_ONLY); List progresses = new ArrayList<>(); Handler progressHandler = @@ -620,7 +583,7 @@ public final class TransformerTest { @Test public void getProgress_noCurrentTransformation_returnsNoTransformation() throws Exception { - Transformer transformer = new Transformer.Builder(context).setClock(clock).build(); + Transformer transformer = createTransformerBuilder().build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_VIDEO_ONLY); @Transformer.ProgressState int stateBeforeTransform = transformer.getProgress(progressHolder); @@ -634,7 +597,7 @@ public final class TransformerTest { @Test public void getProgress_unknownDuration_returnsConsistentStates() throws Exception { - Transformer transformer = new Transformer.Builder(context).setClock(clock).build(); + Transformer transformer = createTransformerBuilder().build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_UNKNOWN_DURATION); AtomicInteger previousProgressState = new AtomicInteger(PROGRESS_STATE_WAITING_FOR_AVAILABILITY); @@ -677,7 +640,7 @@ public final class TransformerTest { @Test public void getProgress_fromWrongThread_throwsError() throws Exception { - Transformer transformer = new Transformer.Builder(context).setClock(clock).build(); + Transformer transformer = createTransformerBuilder().build(); HandlerThread anotherThread = new HandlerThread("AnotherThread"); AtomicReference illegalStateException = new AtomicReference<>(); CountDownLatch countDownLatch = new CountDownLatch(1); @@ -701,7 +664,7 @@ public final class TransformerTest { @Test public void cancel_afterCompletion_doesNotThrow() throws Exception { - Transformer transformer = new Transformer.Builder(context).setClock(clock).build(); + Transformer transformer = createTransformerBuilder().build(); MediaItem mediaItem = MediaItem.fromUri(URI_PREFIX + FILE_VIDEO_ONLY); transformer.startTransformation(mediaItem, outputPath); @@ -711,7 +674,7 @@ public final class TransformerTest { @Test public void cancel_fromWrongThread_throwsError() throws Exception { - Transformer transformer = new Transformer.Builder(context).setClock(clock).build(); + Transformer transformer = createTransformerBuilder().build(); HandlerThread anotherThread = new HandlerThread("AnotherThread"); AtomicReference illegalStateException = new AtomicReference<>(); CountDownLatch countDownLatch = new CountDownLatch(1); @@ -733,6 +696,10 @@ public final class TransformerTest { assertThat(illegalStateException.get()).isNotNull(); } + private Transformer.Builder createTransformerBuilder() { + return new Transformer.Builder(context).setClock(clock).setMuxerFactory(new TestMuxerFactory()); + } + private static void createEncodersAndDecoders() { ShadowMediaCodec.CodecConfig codecConfig = new ShadowMediaCodec.CodecConfig(