diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index ac322c90a9..e8ab80a5d7 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -36,7 +36,6 @@ import com.google.common.collect.ImmutableList; import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.util.List; import org.json.JSONException; import org.json.JSONObject; @@ -472,9 +471,8 @@ public final class AndroidTestUtil { } @Override - public Codec createForAudioEncoding(Format format, List allowedMimeTypes) - throws TransformationException { - return encoderFactory.createForAudioEncoding(format, allowedMimeTypes); + public Codec createForAudioEncoding(Format format) throws TransformationException { + return encoderFactory.createForAudioEncoding(format); } @Override diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java index 070e3cb744..e79123a3c3 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java @@ -35,7 +35,6 @@ import com.google.common.base.Ascii; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.File; import java.io.IOException; -import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeoutException; @@ -446,9 +445,8 @@ public class TransformerAndroidTestRunner { } @Override - public Codec createForAudioEncoding(Format format, List allowedMimeTypes) - throws TransformationException { - Codec audioEncoder = encoderFactory.createForAudioEncoding(format, allowedMimeTypes); + public Codec createForAudioEncoding(Format format) throws TransformationException { + Codec audioEncoder = encoderFactory.createForAudioEncoding(format); audioEncoderName = audioEncoder.getName(); return audioEncoder; } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index b468ab8f93..5b330255b3 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -26,7 +26,6 @@ import androidx.media3.common.Format; import androidx.media3.common.MediaItem; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; -import java.util.List; import org.junit.Test; import org.junit.runner.RunWith; @@ -140,9 +139,8 @@ public class TransformerEndToEndTest { } @Override - public Codec createForAudioEncoding(Format format, List allowedMimeTypes) - throws TransformationException { - return encoderFactory.createForAudioEncoding(format, allowedMimeTypes); + public Codec createForAudioEncoding(Format format) throws TransformationException { + return encoderFactory.createForAudioEncoding(format); } @Override diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java index dc34bfe275..4e2d00bf4a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java @@ -31,6 +31,7 @@ import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; +import java.util.List; import org.checkerframework.dataflow.qual.Pure; /** @@ -103,20 +104,33 @@ import org.checkerframework.dataflow.qual.Pure; audioProcessingPipeline.flush(); + String requestedMimeType = + transformationRequest.audioMimeType != null + ? transformationRequest.audioMimeType + : checkNotNull(inputFormat.sampleMimeType); Format requestedOutputFormat = new Format.Builder() - .setSampleMimeType( - transformationRequest.audioMimeType == null - ? inputFormat.sampleMimeType - : transformationRequest.audioMimeType) + .setSampleMimeType(requestedMimeType) .setSampleRate(encoderInputAudioFormat.sampleRate) .setChannelCount(encoderInputAudioFormat.channelCount) .setAverageBitrate(DEFAULT_ENCODER_BITRATE) .build(); + + ImmutableList muxerSupportedMimeTypes = + muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO); + + // TODO(b/259570024): investigate overhauling fallback. + @Nullable + String supportedMimeType = + selectEncoderAndMuxerSupportedMimeType(requestedMimeType, muxerSupportedMimeTypes); + if (supportedMimeType == null) { + throw createNoSupportedMimeTypeException(requestedOutputFormat); + } + encoder = encoderFactory.createForAudioEncoding( - requestedOutputFormat, muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO)); - + requestedOutputFormat.buildUpon().setSampleMimeType(supportedMimeType).build()); + checkState(supportedMimeType.equals(encoder.getConfigurationFormat().sampleMimeType)); fallbackListener.onTransformationRequestFinalized( createFallbackTransformationRequest( transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat())); @@ -284,6 +298,23 @@ import org.checkerframework.dataflow.qual.Pure; encoder.queueInputBuffer(encoderInputBuffer); } + @Nullable + private static String selectEncoderAndMuxerSupportedMimeType( + String requestedMimeType, List muxerSupportedMimeTypes) { + if (!EncoderUtil.getSupportedEncoders(requestedMimeType).isEmpty()) { + return requestedMimeType; + } else { + // No encoder supports the requested MIME type. + for (int i = 0; i < muxerSupportedMimeTypes.size(); i++) { + String mimeType = muxerSupportedMimeTypes.get(i); + if (!EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) { + return mimeType; + } + } + } + return null; + } + @Pure private static TransformationRequest createFallbackTransformationRequest( TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java index a81dbb3246..1fe8488f64 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java @@ -58,6 +58,17 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; : null; } + protected static TransformationException createNoSupportedMimeTypeException( + Format requestedEncoderFormat) { + return TransformationException.createForCodec( + new IllegalArgumentException("No MIME type is supported by both encoder and muxer."), + MimeTypes.isVideo(requestedEncoderFormat.sampleMimeType), + /* isDecoder= */ false, + requestedEncoderFormat, + /* mediaCodecName= */ null, + TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); + } + @Nullable @Override public DecoderInputBuffer dequeueInputBuffer() throws TransformationException { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java index fdb0b3cd67..ef818e2f69 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java @@ -21,11 +21,9 @@ import android.view.Surface; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; -import androidx.media3.common.MimeTypes; import androidx.media3.common.util.UnstableApi; import androidx.media3.decoder.DecoderInputBuffer; import java.nio.ByteBuffer; -import java.util.List; /** * Provides a layer of abstraction for interacting with decoders and encoders. @@ -72,19 +70,18 @@ public interface Codec { /** * Returns a {@link Codec} for audio encoding. * - *

This method must validate that the {@link Codec} is configured to produce one of the - * {@code allowedMimeTypes}. The {@linkplain Format#sampleMimeType sample MIME type} given in - * {@code format} is not necessarily allowed. + *

The caller should ensure the {@linkplain Format#sampleMimeType MIME type} is supported on + * the device before calling this method. * * @param format The {@link Format} (of the output data) used to determine the underlying - * encoder and its configuration values. - * @param allowedMimeTypes The non-empty list of allowed output sample {@linkplain MimeTypes - * MIME types}. - * @return A {@link Codec} for audio encoding. + * encoder and its configuration values. {@link Format#sampleMimeType}, {@link + * Format#sampleRate}, {@link Format#channelCount} and {@link Format#bitrate} are set to + * those of the desired output video format. + * @return A {@link Codec} for encoding audio to the requested {@link Format#sampleMimeType MIME + * type}. * @throws TransformationException If no suitable {@link Codec} can be created. */ - Codec createForAudioEncoding(Format format, List allowedMimeTypes) - throws TransformationException; + Codec createForAudioEncoding(Format format) throws TransformationException; /** * Returns a {@link Codec} for video encoding. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java index a563ef853b..12158d5491 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java @@ -174,23 +174,12 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { } @Override - public Codec createForAudioEncoding(Format format, List allowedMimeTypes) - throws TransformationException { + public DefaultCodec createForAudioEncoding(Format format) throws TransformationException { // TODO(b/210591626) Add encoder selection for audio. - checkArgument(!allowedMimeTypes.isEmpty()); checkNotNull(format.sampleMimeType); - if (!allowedMimeTypes.contains(format.sampleMimeType)) { - if (enableFallback) { - // TODO(b/210591626): Pick fallback MIME type using same strategy as for encoder - // capabilities limitations. - format = format.buildUpon().setSampleMimeType(allowedMimeTypes.get(0)).build(); - } else { - throw createTransformationException(format); - } - } MediaFormat mediaFormat = MediaFormat.createAudioFormat( - checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount); + format.sampleMimeType, format.sampleRate, format.channelCount); mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.bitrate); @Nullable diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java index 44117f39ed..169a4cb3ac 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java @@ -446,9 +446,7 @@ public final class EncoderUtil { } String[] supportedMimeTypes = mediaCodecInfo.getSupportedTypes(); for (String mimeType : supportedMimeTypes) { - if (MimeTypes.isVideo(mimeType)) { - encoderInfosBuilder.put(Ascii.toLowerCase(mimeType), mediaCodecInfo); - } + encoderInfosBuilder.put(Ascii.toLowerCase(mimeType), mediaCodecInfo); } } return encoderInfosBuilder.build(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index e7d2187532..21f742f91f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -461,13 +461,7 @@ import org.checkerframework.dataflow.qual.Pure; selectEncoderAndMuxerSupportedMimeType( requestedOutputMimeType, muxerSupportedMimeTypes, requestedEncoderFormat.colorInfo); if (supportedMimeType == null) { - throw TransformationException.createForCodec( - new IllegalArgumentException("No MIME type is supported by both encoder and muxer."), - /* isVideo= */ true, - /* isDecoder= */ false, - requestedEncoderFormat, - /* mediaCodecName= */ null, - TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); + throw createNoSupportedMimeTypeException(requestedEncoderFormat); } encoder = diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java index b144512f1c..0e9c9d5d90 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -453,18 +453,25 @@ public final class TransformerEndToEndTest { } @Test - public void startTransformation_withAudioMuxerFormatUnsupported_completesWithError() + public void startTransformation_withAudioMuxerFormatUnsupported_completesSuccessfully() throws Exception { - Transformer transformer = createTransformerBuilder(/* enableFallback= */ false).build(); + // Test succeeds because MIME type fallback is mandatory. + Transformer.Listener mockListener = mock(Transformer.Listener.class); + TransformationRequest originalTransformationRequest = + new TransformationRequest.Builder().build(); + TransformationRequest fallbackTransformationRequest = + new TransformationRequest.Builder().setAudioMimeType(MimeTypes.AUDIO_AAC).build(); + Transformer transformer = + createTransformerBuilder(/* enableFallback= */ false).addListener(mockListener).build(); MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_UNSUPPORTED_BY_MUXER); transformer.startTransformation(mediaItem, outputPath); - TransformationException exception = TransformerTestRunner.runUntilError(transformer); + TransformerTestRunner.runUntilCompleted(transformer); - assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException.class); - assertThat(exception).hasMessageThat().contains("audio"); - assertThat(exception.errorCode) - .isEqualTo(TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); + DumpFileAsserts.assertOutput( + context, testMuxer, getDumpFileName(FILE_AUDIO_UNSUPPORTED_BY_MUXER + ".fallback")); + verify(mockListener) + .onFallbackApplied(mediaItem, originalTransformationRequest, fallbackTransformationRequest); } @Test diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java index 8bdab50f29..551e3e91cb 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java @@ -32,7 +32,6 @@ import androidx.media3.common.util.Clock; import androidx.media3.common.util.ListenerSet; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; -import java.util.List; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -165,7 +164,7 @@ public final class VideoEncoderWrapperTest { } @Override - public Codec createForAudioEncoding(Format format, List allowedMimeTypes) { + public Codec createForAudioEncoding(Format format) { throw new UnsupportedOperationException(); }