loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move audio MIME type fallback away to ATSP

Browse Source 
PiperOrigin-RevId: 491660842
This commit is contained in:
claincly 2022-11-29 17:35:55 +00:00 committed by Rohit Singh
parent d12afe0596
commit 4033013ff8
11 changed files with 81 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View File

@ -36,7 +36,6 @@ import com.google.common.collect.ImmutableList;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.List;
import org.json.JSONException;
import org.json.JSONObject;
@ -472,9 +471,8 @@ public final class AndroidTestUtil {
}
@Override
public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
throws TransformationException {
return encoderFactory.createForAudioEncoding(format, allowedMimeTypes);
public Codec createForAudioEncoding(Format format) throws TransformationException {
return encoderFactory.createForAudioEncoding(format);
}
@Override

6
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java
View File

@ -35,7 +35,6 @@ import com.google.common.base.Ascii;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeoutException;
@ -446,9 +445,8 @@ public class TransformerAndroidTestRunner {
}
@Override
public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
throws TransformationException {
Codec audioEncoder = encoderFactory.createForAudioEncoding(format, allowedMimeTypes);
public Codec createForAudioEncoding(Format format) throws TransformationException {
Codec audioEncoder = encoderFactory.createForAudioEncoding(format);
audioEncoderName = audioEncoder.getName();
return audioEncoder;
}

6
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -26,7 +26,6 @@ import androidx.media3.common.Format;
import androidx.media3.common.MediaItem;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -140,9 +139,8 @@ public class TransformerEndToEndTest {
}
@Override
public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
throws TransformationException {
return encoderFactory.createForAudioEncoding(format, allowedMimeTypes);
public Codec createForAudioEncoding(Format format) throws TransformationException {
return encoderFactory.createForAudioEncoding(format);
}
@Override

43
libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java
View File

@ -31,6 +31,7 @@ import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer;
import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer;
import java.util.List;
import org.checkerframework.dataflow.qual.Pure;
/**
@ -103,20 +104,33 @@ import org.checkerframework.dataflow.qual.Pure;
audioProcessingPipeline.flush();
String requestedMimeType =
transformationRequest.audioMimeType != null
? transformationRequest.audioMimeType
: checkNotNull(inputFormat.sampleMimeType);
Format requestedOutputFormat =
new Format.Builder()
.setSampleMimeType(
transformationRequest.audioMimeType == null
? inputFormat.sampleMimeType
: transformationRequest.audioMimeType)
.setSampleMimeType(requestedMimeType)
.setSampleRate(encoderInputAudioFormat.sampleRate)
.setChannelCount(encoderInputAudioFormat.channelCount)
.setAverageBitrate(DEFAULT_ENCODER_BITRATE)
.build();
ImmutableList<String> muxerSupportedMimeTypes =
muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO);
// TODO(b/259570024): investigate overhauling fallback.
@Nullable
String supportedMimeType =
selectEncoderAndMuxerSupportedMimeType(requestedMimeType, muxerSupportedMimeTypes);
if (supportedMimeType == null) {
throw createNoSupportedMimeTypeException(requestedOutputFormat);
}
encoder =
encoderFactory.createForAudioEncoding(
requestedOutputFormat, muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO));
requestedOutputFormat.buildUpon().setSampleMimeType(supportedMimeType).build());
checkState(supportedMimeType.equals(encoder.getConfigurationFormat().sampleMimeType));
fallbackListener.onTransformationRequestFinalized(
createFallbackTransformationRequest(
transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat()));
@ -284,6 +298,23 @@ import org.checkerframework.dataflow.qual.Pure;
encoder.queueInputBuffer(encoderInputBuffer);
}
@Nullable
private static String selectEncoderAndMuxerSupportedMimeType(
String requestedMimeType, List<String> muxerSupportedMimeTypes) {
if (!EncoderUtil.getSupportedEncoders(requestedMimeType).isEmpty()) {
return requestedMimeType;
} else {
// No encoder supports the requested MIME type.
for (int i = 0; i < muxerSupportedMimeTypes.size(); i++) {
String mimeType = muxerSupportedMimeTypes.get(i);
if (!EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) {
return mimeType;
}
}
}
return null;
}
@Pure
private static TransformationRequest createFallbackTransformationRequest(
TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) {

11
libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java
View File

@ -58,6 +58,17 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
: null;
}
protected static TransformationException createNoSupportedMimeTypeException(
Format requestedEncoderFormat) {
return TransformationException.createForCodec(
new IllegalArgumentException("No MIME type is supported by both encoder and muxer."),
MimeTypes.isVideo(requestedEncoderFormat.sampleMimeType),
/* isDecoder= */ false,
requestedEncoderFormat,
/* mediaCodecName= */ null,
TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
}
@Nullable
@Override
public DecoderInputBuffer dequeueInputBuffer() throws TransformationException {

19
libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java
View File

@ -21,11 +21,9 @@ import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.decoder.DecoderInputBuffer;
import java.nio.ByteBuffer;
import java.util.List;
/**
* Provides a layer of abstraction for interacting with decoders and encoders.
@ -72,19 +70,18 @@ public interface Codec {
/**
* Returns a {@link Codec} for audio encoding.
*
* <p>This method must validate that the {@link Codec} is configured to produce one of the
* {@code allowedMimeTypes}. The {@linkplain Format#sampleMimeType sample MIME type} given in
* {@code format} is not necessarily allowed.
* <p>The caller should ensure the {@linkplain Format#sampleMimeType MIME type} is supported on
* the device before calling this method.
*
* @param format The {@link Format} (of the output data) used to determine the underlying
* encoder and its configuration values.
* @param allowedMimeTypes The non-empty list of allowed output sample {@linkplain MimeTypes
* MIME types}.
* @return A {@link Codec} for audio encoding.
* encoder and its configuration values. {@link Format#sampleMimeType}, {@link
* Format#sampleRate}, {@link Format#channelCount} and {@link Format#bitrate} are set to
* those of the desired output video format.
* @return A {@link Codec} for encoding audio to the requested {@link Format#sampleMimeType MIME
* type}.
* @throws TransformationException If no suitable {@link Codec} can be created.
*/
Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
throws TransformationException;
Codec createForAudioEncoding(Format format) throws TransformationException;
/**
* Returns a {@link Codec} for video encoding.

15
libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java
View File

@ -174,23 +174,12 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
}
@Override
public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
throws TransformationException {
public DefaultCodec createForAudioEncoding(Format format) throws TransformationException {
// TODO(b/210591626) Add encoder selection for audio.
checkArgument(!allowedMimeTypes.isEmpty());
checkNotNull(format.sampleMimeType);
if (!allowedMimeTypes.contains(format.sampleMimeType)) {
if (enableFallback) {
// TODO(b/210591626): Pick fallback MIME type using same strategy as for encoder
// capabilities limitations.
format = format.buildUpon().setSampleMimeType(allowedMimeTypes.get(0)).build();
} else {
throw createTransformationException(format);
}
}
MediaFormat mediaFormat =
MediaFormat.createAudioFormat(
checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount);
format.sampleMimeType, format.sampleRate, format.channelCount);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.bitrate);
@Nullable

4
libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java
View File

@ -446,9 +446,7 @@ public final class EncoderUtil {
}
String[] supportedMimeTypes = mediaCodecInfo.getSupportedTypes();
for (String mimeType : supportedMimeTypes) {
if (MimeTypes.isVideo(mimeType)) {
encoderInfosBuilder.put(Ascii.toLowerCase(mimeType), mediaCodecInfo);
}
encoderInfosBuilder.put(Ascii.toLowerCase(mimeType), mediaCodecInfo);
}
}
return encoderInfosBuilder.build();

8
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -461,13 +461,7 @@ import org.checkerframework.dataflow.qual.Pure;
selectEncoderAndMuxerSupportedMimeType(
requestedOutputMimeType, muxerSupportedMimeTypes, requestedEncoderFormat.colorInfo);
if (supportedMimeType == null) {
throw TransformationException.createForCodec(
new IllegalArgumentException("No MIME type is supported by both encoder and muxer."),
/* isVideo= */ true,
/* isDecoder= */ false,
requestedEncoderFormat,
/* mediaCodecName= */ null,
TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
throw createNoSupportedMimeTypeException(requestedEncoderFormat);
}
encoder =

21
libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -453,18 +453,25 @@ public final class TransformerEndToEndTest {
}
@Test
public void startTransformation_withAudioMuxerFormatUnsupported_completesWithError()
public void startTransformation_withAudioMuxerFormatUnsupported_completesSuccessfully()
throws Exception {
Transformer transformer = createTransformerBuilder(/* enableFallback= */ false).build();
// Test succeeds because MIME type fallback is mandatory.
Transformer.Listener mockListener = mock(Transformer.Listener.class);
TransformationRequest originalTransformationRequest =
new TransformationRequest.Builder().build();
TransformationRequest fallbackTransformationRequest =
new TransformationRequest.Builder().setAudioMimeType(MimeTypes.AUDIO_AAC).build();
Transformer transformer =
createTransformerBuilder(/* enableFallback= */ false).addListener(mockListener).build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_UNSUPPORTED_BY_MUXER);
transformer.startTransformation(mediaItem, outputPath);
TransformationException exception = TransformerTestRunner.runUntilError(transformer);
TransformerTestRunner.runUntilCompleted(transformer);
assertThat(exception).hasCauseThat().isInstanceOf(IllegalArgumentException.class);
assertThat(exception).hasMessageThat().contains("audio");
assertThat(exception.errorCode)
.isEqualTo(TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
DumpFileAsserts.assertOutput(
context, testMuxer, getDumpFileName(FILE_AUDIO_UNSUPPORTED_BY_MUXER + ".fallback"));
verify(mockListener)
.onFallbackApplied(mediaItem, originalTransformationRequest, fallbackTransformationRequest);
}
@Test

3
libraries/transformer/src/test/java/androidx/media3/transformer/VideoEncoderWrapperTest.java
View File

@ -32,7 +32,6 @@ import androidx.media3.common.util.Clock;
import androidx.media3.common.util.ListenerSet;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.util.List;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -165,7 +164,7 @@ public final class VideoEncoderWrapperTest {
}
@Override
public Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes) {
public Codec createForAudioEncoding(Format format) {
throw new UnsupportedOperationException();
}