Ensure audio components check incoming data is valid.

Default PCM encoding is only set for decoders outputting raw.

Tests migrated to abide by tighter restrictions.

PiperOrigin-RevId: 558129452
This commit is contained in:
samrobinson 2023-08-18 14:31:03 +01:00 committed by Julia Bibik
parent 15650c6bf3
commit 2db6f0aee7
8 changed files with 1353 additions and 389 deletions

View File

@ -9,12 +9,6 @@ format 0:
initializationData: initializationData:
data = length 30, hash F6F3D010 data = length 30, hash F6F3D010
data = length 10, hash 7A0D0F2B data = length 10, hash 7A0D0F2B
format 1:
averageBitrate = 131072
sampleMimeType = audio/mp4a-latm
channelCount = 1
sampleRate = 44100
pcmEncoding = 2
sample: sample:
trackIndex = 0 trackIndex = 0
dataHashCode = -252482306 dataHashCode = -252482306
@ -195,178 +189,4 @@ sample:
size = 486 size = 486
isKeyFrame = false isKeyFrame = false
presentationTimeUs = 933000 presentationTimeUs = 933000
sample:
trackIndex = 1
dataHashCode = 555688582
size = 416
isKeyFrame = true
presentationTimeUs = 0
sample:
trackIndex = 1
dataHashCode = 2000837254
size = 418
isKeyFrame = true
presentationTimeUs = 4716
sample:
trackIndex = 1
dataHashCode = -1593942879
size = 418
isKeyFrame = true
presentationTimeUs = 9455
sample:
trackIndex = 1
dataHashCode = 587837542
size = 418
isKeyFrame = true
presentationTimeUs = 14195
sample:
trackIndex = 1
dataHashCode = -1836423877
size = 418
isKeyFrame = true
presentationTimeUs = 18934
sample:
trackIndex = 1
dataHashCode = 874705099
size = 418
isKeyFrame = true
presentationTimeUs = 23673
sample:
trackIndex = 1
dataHashCode = -269206181
size = 418
isKeyFrame = true
presentationTimeUs = 28412
sample:
trackIndex = 1
dataHashCode = -58682425
size = 418
isKeyFrame = true
presentationTimeUs = 33151
sample:
trackIndex = 1
dataHashCode = -859796970
size = 418
isKeyFrame = true
presentationTimeUs = 37891
sample:
trackIndex = 1
dataHashCode = 711911523
size = 418
isKeyFrame = true
presentationTimeUs = 42630
sample:
trackIndex = 1
dataHashCode = -694513071
size = 418
isKeyFrame = true
presentationTimeUs = 47369
sample:
trackIndex = 1
dataHashCode = -1124371059
size = 418
isKeyFrame = true
presentationTimeUs = 52108
sample:
trackIndex = 1
dataHashCode = 297166745
size = 418
isKeyFrame = true
presentationTimeUs = 56848
sample:
trackIndex = 1
dataHashCode = -937110638
size = 418
isKeyFrame = true
presentationTimeUs = 61587
sample:
trackIndex = 1
dataHashCode = -1050158990
size = 418
isKeyFrame = true
presentationTimeUs = 66326
sample:
trackIndex = 1
dataHashCode = 1109510229
size = 418
isKeyFrame = true
presentationTimeUs = 71065
sample:
trackIndex = 1
dataHashCode = 1297086772
size = 418
isKeyFrame = true
presentationTimeUs = 75804
sample:
trackIndex = 1
dataHashCode = -1739939803
size = 418
isKeyFrame = true
presentationTimeUs = 80544
sample:
trackIndex = 1
dataHashCode = -1149727930
size = 418
isKeyFrame = true
presentationTimeUs = 85283
sample:
trackIndex = 1
dataHashCode = -1627652713
size = 418
isKeyFrame = true
presentationTimeUs = 90022
sample:
trackIndex = 1
dataHashCode = -551926260
size = 418
isKeyFrame = true
presentationTimeUs = 94761
sample:
trackIndex = 1
dataHashCode = 45987178
size = 418
isKeyFrame = true
presentationTimeUs = 99501
sample:
trackIndex = 1
dataHashCode = -903675808
size = 418
isKeyFrame = true
presentationTimeUs = 104240
sample:
trackIndex = 1
dataHashCode = -755916991
size = 418
isKeyFrame = true
presentationTimeUs = 108979
sample:
trackIndex = 1
dataHashCode = -1355207303
size = 418
isKeyFrame = true
presentationTimeUs = 113718
sample:
trackIndex = 1
dataHashCode = -975703389
size = 418
isKeyFrame = true
presentationTimeUs = 118458
sample:
trackIndex = 1
dataHashCode = 1933194670
size = 418
isKeyFrame = true
presentationTimeUs = 123197
sample:
trackIndex = 1
dataHashCode = -565778989
size = 418
isKeyFrame = true
presentationTimeUs = 127936
sample:
trackIndex = 1
dataHashCode = 1454083383
size = 418
isKeyFrame = true
presentationTimeUs = 132675
released = true released = true

View File

@ -11,14 +11,6 @@ format 0:
data = length 33, hash D3FB879D data = length 33, hash D3FB879D
data = length 10, hash 7A0D0F2B data = length 10, hash 7A0D0F2B
container metadata = entries=[mdta: key=com.android.version, value=10, mdta: key=com.android.video.temporal_layers_count, value=4, mdta: key=com.android.capture.fps, value=240.0, SlowMotion: segments=[Segment: startTimeMs=88, endTimeMs=879, speedDivisor=2, Segment: startTimeMs=1255, endTimeMs=1970, speedDivisor=8], smta: captureFrameRate=240.0, svcTemporalLayerCount=4, Mp4Timestamp: creation time=3686904890, modification time=3686904890, timescale=1000] container metadata = entries=[mdta: key=com.android.version, value=10, mdta: key=com.android.video.temporal_layers_count, value=4, mdta: key=com.android.capture.fps, value=240.0, SlowMotion: segments=[Segment: startTimeMs=88, endTimeMs=879, speedDivisor=2, Segment: startTimeMs=1255, endTimeMs=1970, speedDivisor=8], smta: captureFrameRate=240.0, svcTemporalLayerCount=4, Mp4Timestamp: creation time=3686904890, modification time=3686904890, timescale=1000]
format 1:
averageBitrate = 131072
sampleMimeType = audio/mp4a-latm
channelCount = 2
sampleRate = 12000
pcmEncoding = 2
metadata = entries=[mdta: key=com.android.version, value=10, mdta: key=com.android.video.temporal_layers_count, value=4, SlowMotion: segments=[Segment: startTimeMs=88, endTimeMs=879, speedDivisor=2, Segment: startTimeMs=1255, endTimeMs=1970, speedDivisor=8], smta: captureFrameRate=240.0, svcTemporalLayerCount=4, Mp4Timestamp: creation time=3686904890, modification time=3686904890, timescale=1000]
container metadata = entries=[mdta: key=com.android.version, value=10, mdta: key=com.android.video.temporal_layers_count, value=4, SlowMotion: segments=[Segment: startTimeMs=88, endTimeMs=879, speedDivisor=2, Segment: startTimeMs=1255, endTimeMs=1970, speedDivisor=8], smta: captureFrameRate=240.0, svcTemporalLayerCount=4, Mp4Timestamp: creation time=3686904890, modification time=3686904890, timescale=1000]
sample: sample:
trackIndex = 0 trackIndex = 0
dataHashCode = 1949079733 dataHashCode = 1949079733
@ -193,160 +185,4 @@ sample:
size = 138 size = 138
isKeyFrame = false isKeyFrame = false
presentationTimeUs = 834083 presentationTimeUs = 834083
sample:
trackIndex = 1
dataHashCode = -212376212
size = 20
isKeyFrame = true
presentationTimeUs = 0
sample:
trackIndex = 1
dataHashCode = -1948569090
size = 72
isKeyFrame = true
presentationTimeUs = 416
sample:
trackIndex = 1
dataHashCode = -1316750072
size = 84
isKeyFrame = true
presentationTimeUs = 1916
sample:
trackIndex = 1
dataHashCode = 1016428949
size = 88
isKeyFrame = true
presentationTimeUs = 3666
sample:
trackIndex = 1
dataHashCode = -1127325245
size = 96
isKeyFrame = true
presentationTimeUs = 5500
sample:
trackIndex = 1
dataHashCode = 1148147726
size = 92
isKeyFrame = true
presentationTimeUs = 7500
sample:
trackIndex = 1
dataHashCode = -2125685540
size = 76
isKeyFrame = true
presentationTimeUs = 9416
sample:
trackIndex = 1
dataHashCode = 473329679
size = 24
isKeyFrame = true
presentationTimeUs = 11000
sample:
trackIndex = 1
dataHashCode = 240990900
size = 176
isKeyFrame = true
presentationTimeUs = 11500
sample:
trackIndex = 1
dataHashCode = 777637182
size = 196
isKeyFrame = true
presentationTimeUs = 15166
sample:
trackIndex = 1
dataHashCode = 1872106264
size = 180
isKeyFrame = true
presentationTimeUs = 19250
sample:
trackIndex = 1
dataHashCode = -1520711499
size = 140
isKeyFrame = true
presentationTimeUs = 23000
sample:
trackIndex = 1
dataHashCode = 1580199067
size = 232
isKeyFrame = true
presentationTimeUs = 25916
sample:
trackIndex = 1
dataHashCode = 475464086
size = 184
isKeyFrame = true
presentationTimeUs = 30750
sample:
trackIndex = 1
dataHashCode = -211754132
size = 172
isKeyFrame = true
presentationTimeUs = 34583
sample:
trackIndex = 1
dataHashCode = 1236547164
size = 172
isKeyFrame = true
presentationTimeUs = 38166
sample:
trackIndex = 1
dataHashCode = -2064216186
size = 188
isKeyFrame = true
presentationTimeUs = 41750
sample:
trackIndex = 1
dataHashCode = -682950885
size = 260
isKeyFrame = true
presentationTimeUs = 45666
sample:
trackIndex = 1
dataHashCode = 1301206627
size = 236
isKeyFrame = true
presentationTimeUs = 51083
sample:
trackIndex = 1
dataHashCode = 256580525
size = 236
isKeyFrame = true
presentationTimeUs = 56000
sample:
trackIndex = 1
dataHashCode = -1086601304
size = 236
isKeyFrame = true
presentationTimeUs = 60916
sample:
trackIndex = 1
dataHashCode = -2046131588
size = 224
isKeyFrame = true
presentationTimeUs = 65833
sample:
trackIndex = 1
dataHashCode = 1550955865
size = 224
isKeyFrame = true
presentationTimeUs = 70500
sample:
trackIndex = 1
dataHashCode = -274800552
size = 220
isKeyFrame = true
presentationTimeUs = 75166
sample:
trackIndex = 1
dataHashCode = 382420909
size = 224
isKeyFrame = true
presentationTimeUs = 79750
sample:
trackIndex = 1
dataHashCode = -1431575865
size = 232
isKeyFrame = true
presentationTimeUs = 84416
released = true released = true

View File

@ -0,0 +1,73 @@
format 0:
averageBitrate = 131072
sampleMimeType = audio/mp4a-latm
channelCount = 1
sampleRate = 48000
pcmEncoding = 2
sample:
trackIndex = 0
dataHashCode = -206878443
size = 9598
isKeyFrame = true
presentationTimeUs = 0
sample:
trackIndex = 0
dataHashCode = 1925380271
size = 9600
isKeyFrame = true
presentationTimeUs = 99979
sample:
trackIndex = 0
dataHashCode = -1463691457
size = 9600
isKeyFrame = true
presentationTimeUs = 199979
sample:
trackIndex = 0
dataHashCode = -1949646832
size = 9600
isKeyFrame = true
presentationTimeUs = 299979
sample:
trackIndex = 0
dataHashCode = 448289446
size = 9600
isKeyFrame = true
presentationTimeUs = 399979
sample:
trackIndex = 0
dataHashCode = 774
size = 2
isKeyFrame = true
presentationTimeUs = 499979
sample:
trackIndex = 0
dataHashCode = 389300669
size = 19198
isKeyFrame = true
presentationTimeUs = 500000
sample:
trackIndex = 0
dataHashCode = 1861606067
size = 9600
isKeyFrame = true
presentationTimeUs = 699979
sample:
trackIndex = 0
dataHashCode = -1038569478
size = 9600
isKeyFrame = true
presentationTimeUs = 799979
sample:
trackIndex = 0
dataHashCode = 646441790
size = 9600
isKeyFrame = true
presentationTimeUs = 899979
sample:
trackIndex = 0
dataHashCode = 340
size = 2
isKeyFrame = true
presentationTimeUs = 999979
released = true

View File

@ -27,6 +27,7 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREAS
import static androidx.media3.transformer.AndroidTestUtil.MP4_REMOTE_8K24_FORMAT; import static androidx.media3.transformer.AndroidTestUtil.MP4_REMOTE_8K24_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.MP4_REMOTE_8K24_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.MP4_REMOTE_8K24_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped; import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped;
import static com.google.common.truth.Truth.assertThat;
import android.content.Context; import android.content.Context;
import android.net.Uri; import android.net.Uri;
@ -39,6 +40,7 @@ import androidx.media3.transformer.AndroidTestUtil.ForceEncodeEncoderFactory;
import androidx.media3.transformer.DefaultEncoderFactory; import androidx.media3.transformer.DefaultEncoderFactory;
import androidx.media3.transformer.EditedMediaItem; import androidx.media3.transformer.EditedMediaItem;
import androidx.media3.transformer.Effects; import androidx.media3.transformer.Effects;
import androidx.media3.transformer.ExportTestResult;
import androidx.media3.transformer.Transformer; import androidx.media3.transformer.Transformer;
import androidx.media3.transformer.TransformerAndroidTestRunner; import androidx.media3.transformer.TransformerAndroidTestRunner;
import androidx.media3.transformer.VideoEncoderSettings; import androidx.media3.transformer.VideoEncoderSettings;
@ -237,9 +239,13 @@ public class ExportTest {
new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_SEF_URI_STRING))) new EditedMediaItem.Builder(MediaItem.fromUri(Uri.parse(MP4_ASSET_SEF_URI_STRING)))
.setFlattenForSlowMotion(true) .setFlattenForSlowMotion(true)
.build(); .build();
new TransformerAndroidTestRunner.Builder(context, transformer) ExportTestResult result =
.build() new TransformerAndroidTestRunner.Builder(context, transformer)
.run(testId, editedMediaItem); .build()
.run(testId, editedMediaItem);
assertThat(result.exportResult.durationMs).isGreaterThan(800);
assertThat(result.exportResult.durationMs).isLessThan(950);
} }
@Test @Test

View File

@ -68,7 +68,9 @@ import java.util.concurrent.atomic.AtomicReference;
public AudioGraphInput(EditedMediaItem item, Format inputFormat) public AudioGraphInput(EditedMediaItem item, Format inputFormat)
throws UnhandledAudioFormatException { throws UnhandledAudioFormatException {
checkArgument(inputFormat.pcmEncoding != Format.NO_VALUE); AudioFormat inputAudioFormat = new AudioFormat(inputFormat);
checkArgument(isInputAudioFormatValid(inputAudioFormat), /* errorMessage= */ inputAudioFormat);
availableInputBuffers = new ConcurrentLinkedDeque<>(); availableInputBuffers = new ConcurrentLinkedDeque<>();
ByteBuffer emptyBuffer = ByteBuffer.allocateDirect(0).order(ByteOrder.nativeOrder()); ByteBuffer emptyBuffer = ByteBuffer.allocateDirect(0).order(ByteOrder.nativeOrder());
for (int i = 0; i < MAX_INPUT_BUFFER_COUNT; i++) { for (int i = 0; i < MAX_INPUT_BUFFER_COUNT; i++) {
@ -78,7 +80,6 @@ import java.util.concurrent.atomic.AtomicReference;
} }
pendingInputBuffers = new ConcurrentLinkedDeque<>(); pendingInputBuffers = new ConcurrentLinkedDeque<>();
pendingMediaItemChange = new AtomicReference<>(); pendingMediaItemChange = new AtomicReference<>();
AudioFormat inputAudioFormat = new AudioFormat(inputFormat);
silentAudioGenerator = new SilentAudioGenerator(inputAudioFormat); silentAudioGenerator = new SilentAudioGenerator(inputAudioFormat);
audioProcessingPipeline = audioProcessingPipeline =
configureProcessing( configureProcessing(
@ -127,7 +128,8 @@ import java.util.concurrent.atomic.AtomicReference;
"Could not generate silent audio because duration is unknown."); "Could not generate silent audio because duration is unknown.");
} else { } else {
checkState(MimeTypes.isAudio(trackFormat.sampleMimeType)); checkState(MimeTypes.isAudio(trackFormat.sampleMimeType));
checkState(trackFormat.pcmEncoding != Format.NO_VALUE); AudioFormat trackAudioFormat = new AudioFormat(trackFormat);
checkState(isInputAudioFormatValid(trackAudioFormat), /* errorMessage= */ trackAudioFormat);
} }
pendingMediaItemChange.set( pendingMediaItemChange.set(
new MediaItemChange(editedMediaItem, durationUs, trackFormat, isLast)); new MediaItemChange(editedMediaItem, durationUs, trackFormat, isLast));
@ -365,6 +367,22 @@ import java.util.concurrent.atomic.AtomicReference;
return audioProcessingPipeline; return audioProcessingPipeline;
} }
private static boolean isInputAudioFormatValid(AudioFormat format) {
if (format.encoding == Format.NO_VALUE) {
return false;
}
if (format.sampleRate == Format.NO_VALUE) {
return false;
}
if (format.channelCount == Format.NO_VALUE) {
return false;
}
if (format.bytesPerFrame == Format.NO_VALUE) {
return false;
}
return true;
}
private static final class MediaItemChange { private static final class MediaItemChange {
public final EditedMediaItem editedMediaItem; public final EditedMediaItem editedMediaItem;
public final long durationUs; public final long durationUs;

View File

@ -45,6 +45,7 @@ import androidx.media3.decoder.DecoderInputBuffer;
import androidx.media3.effect.DebugTraceUtil; import androidx.media3.effect.DebugTraceUtil;
import java.io.IOException; import java.io.IOException;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.Objects;
import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.checkerframework.checker.initialization.qual.UnknownInitialization;
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@ -72,7 +73,6 @@ public final class DefaultCodec implements Codec {
private @MonotonicNonNull Format outputFormat; private @MonotonicNonNull Format outputFormat;
@Nullable private ByteBuffer outputBuffer; @Nullable private ByteBuffer outputBuffer;
private int inputBufferIndex; private int inputBufferIndex;
private int outputBufferIndex; private int outputBufferIndex;
private boolean inputStreamEnded; private boolean inputStreamEnded;
@ -413,16 +413,12 @@ public final class DefaultCodec implements Codec {
private static Format convertToFormat( private static Format convertToFormat(
MediaFormat mediaFormat, boolean isDecoder, @Nullable Metadata metadata) { MediaFormat mediaFormat, boolean isDecoder, @Nullable Metadata metadata) {
Format.Builder formatBuilder = Format format = MediaFormatUtil.createFormatFromMediaFormat(mediaFormat);
MediaFormatUtil.createFormatFromMediaFormat(mediaFormat).buildUpon().setMetadata(metadata); Format.Builder formatBuilder = format.buildUpon().setMetadata(metadata);
if (isDecoder) {
// TODO(b/178685617): Restrict this to only set the PCM encoding for audio/raw once we have
// a way to simulate more realistic codec input/output formats in tests.
// With Robolectric, codecs do not actually encode/decode. The format of buffers is passed if (isDecoder
// through. However downstream components need to know the PCM encoding of the data being && format.pcmEncoding == Format.NO_VALUE
// output, so if a decoder is not outputting raw audio, we need to set the PCM && Objects.equals(format.sampleMimeType, MimeTypes.AUDIO_RAW)) {
// encoding to the default.
formatBuilder.setPcmEncoding(DEFAULT_PCM_ENCODING); formatBuilder.setPcmEncoding(DEFAULT_PCM_ENCODING);
} }
return formatBuilder.build(); return formatBuilder.build();

View File

@ -20,7 +20,6 @@ import static androidx.media3.test.utils.robolectric.RobolectricUtil.runLooperUn
import static androidx.media3.transformer.AssetLoader.SUPPORTED_OUTPUT_TYPE_DECODED; import static androidx.media3.transformer.AssetLoader.SUPPORTED_OUTPUT_TYPE_DECODED;
import static androidx.media3.transformer.AssetLoader.SUPPORTED_OUTPUT_TYPE_ENCODED; import static androidx.media3.transformer.AssetLoader.SUPPORTED_OUTPUT_TYPE_ENCODED;
import static androidx.media3.transformer.TestUtil.ASSET_URI_PREFIX; import static androidx.media3.transformer.TestUtil.ASSET_URI_PREFIX;
import static androidx.media3.transformer.TestUtil.FILE_AUDIO_AC3_UNSUPPORTED_BY_MUXER;
import static androidx.media3.transformer.TestUtil.FILE_AUDIO_AMR_NB; import static androidx.media3.transformer.TestUtil.FILE_AUDIO_AMR_NB;
import static androidx.media3.transformer.TestUtil.FILE_AUDIO_AMR_WB; import static androidx.media3.transformer.TestUtil.FILE_AUDIO_AMR_WB;
import static androidx.media3.transformer.TestUtil.FILE_AUDIO_RAW; import static androidx.media3.transformer.TestUtil.FILE_AUDIO_RAW;
@ -123,8 +122,8 @@ public final class MediaItemExportTest {
muxerFactory = new CapturingMuxer.Factory(); muxerFactory = new CapturingMuxer.Factory();
progressHolder = new ProgressHolder(); progressHolder = new ProgressHolder();
compositionArgumentCaptor = ArgumentCaptor.forClass(Composition.class); compositionArgumentCaptor = ArgumentCaptor.forClass(Composition.class);
addAudioDecoders(MimeTypes.AUDIO_RAW, MimeTypes.AUDIO_AAC, MimeTypes.AUDIO_AC3); addAudioDecoders(MimeTypes.AUDIO_RAW);
addAudioEncoders(MimeTypes.AUDIO_AAC, MimeTypes.AUDIO_AC3); addAudioEncoders(MimeTypes.AUDIO_AAC);
} }
@After @After
@ -176,7 +175,7 @@ public final class MediaItemExportTest {
} }
@Test @Test
public void start_audioAndVideo_completesSuccessfully() throws Exception { public void start_audioAndVideoPassthrough_completesSuccessfully() throws Exception {
Transformer transformer = Transformer transformer =
createTransformerBuilder(muxerFactory, /* enableFallback= */ false).build(); createTransformerBuilder(muxerFactory, /* enableFallback= */ false).build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO); MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO);
@ -189,7 +188,7 @@ public final class MediaItemExportTest {
} }
@Test @Test
public void start_audioAndVideo_withClippingStartAtKeyFrame_completesSuccessfully() public void start_audioAndVideoPassthrough_withClippingStartAtKeyFrame_completesSuccessfully()
throws Exception { throws Exception {
Transformer transformer = Transformer transformer =
createTransformerBuilder(muxerFactory, /* enableFallback= */ false).build(); createTransformerBuilder(muxerFactory, /* enableFallback= */ false).build();
@ -214,18 +213,21 @@ public final class MediaItemExportTest {
} }
@Test @Test
public void start_withSubtitles_completesSuccessfully() throws Exception { public void start_withSubtitlesVideoOnly_completesSuccessfully() throws Exception {
Transformer transformer = Transformer transformer =
createTransformerBuilder(muxerFactory, /* enableFallback= */ false) createTransformerBuilder(muxerFactory, /* enableFallback= */ false)
.setAudioMimeType(MimeTypes.AUDIO_AAC) .setAudioMimeType(MimeTypes.AUDIO_AAC)
.build(); .build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_WITH_SUBTITLES); EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_WITH_SUBTITLES))
.setRemoveAudio(true)
.build();
transformer.start(mediaItem, outputPath); transformer.start(editedMediaItem, outputPath);
TransformerTestRunner.runLooper(transformer); TransformerTestRunner.runLooper(transformer);
DumpFileAsserts.assertOutput( DumpFileAsserts.assertOutput(
context, muxerFactory.getCreatedMuxer(), getDumpFileName(FILE_WITH_SUBTITLES)); context, muxerFactory.getCreatedMuxer(), getDumpFileName(FILE_WITH_SUBTITLES) + ".noaudio");
} }
@Test @Test
@ -429,7 +431,7 @@ public final class MediaItemExportTest {
sonicAudioProcessor.setOutputSampleRateHz(48000); sonicAudioProcessor.setOutputSampleRateHz(48000);
Transformer transformer = Transformer transformer =
createTransformerBuilder(muxerFactory, /* enableFallback= */ false).build(); createTransformerBuilder(muxerFactory, /* enableFallback= */ false).build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO); MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW);
EditedMediaItem editedMediaItem = EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem) new EditedMediaItem.Builder(mediaItem)
@ -440,7 +442,7 @@ public final class MediaItemExportTest {
TransformerTestRunner.runLooper(transformer); TransformerTestRunner.runLooper(transformer);
DumpFileAsserts.assertOutput( DumpFileAsserts.assertOutput(
context, muxerFactory.getCreatedMuxer(), getDumpFileName(FILE_AUDIO_VIDEO + ".48000hz")); context, muxerFactory.getCreatedMuxer(), getDumpFileName(FILE_AUDIO_RAW + ".48000hz"));
} }
@Test @Test
@ -449,7 +451,7 @@ public final class MediaItemExportTest {
sonicAudioProcessor.setOutputSampleRateHz(48000); sonicAudioProcessor.setOutputSampleRateHz(48000);
Transformer transformer = Transformer transformer =
createTransformerBuilder(muxerFactory, /* enableFallback= */ false).build(); createTransformerBuilder(muxerFactory, /* enableFallback= */ false).build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO); MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW);
EditedMediaItem editedMediaItem = EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem) new EditedMediaItem.Builder(mediaItem)
.setEffects(createAudioEffects(sonicAudioProcessor)) .setEffects(createAudioEffects(sonicAudioProcessor))
@ -463,9 +465,8 @@ public final class MediaItemExportTest {
transformer.start(composition, outputPath); transformer.start(composition, outputPath);
TransformerTestRunner.runLooper(transformer); TransformerTestRunner.runLooper(transformer);
DumpFileAsserts.assertOutput( DumpFileAsserts.assertOutput(
context, muxerFactory.getCreatedMuxer(), getDumpFileName(FILE_AUDIO_VIDEO + ".48000hz")); context, muxerFactory.getCreatedMuxer(), getDumpFileName(FILE_AUDIO_RAW + ".48000hz"));
} }
@Test @Test
@ -530,9 +531,9 @@ public final class MediaItemExportTest {
.addListener(mockListener2) .addListener(mockListener2)
.addListener(mockListener3) .addListener(mockListener3)
.build(); .build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_AC3_UNSUPPORTED_BY_MUXER);
transformer.start(mediaItem, outputPath); // No RAW encoder/muxer support, so fallback.
transformer.start(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW), outputPath);
TransformerTestRunner.runLooper(transformer); TransformerTestRunner.runLooper(transformer);
verify(mockListener1) verify(mockListener1)
@ -643,9 +644,9 @@ public final class MediaItemExportTest {
} }
}) })
.build(); .build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_AC3_UNSUPPORTED_BY_MUXER);
transformer.start(mediaItem, outputPath); // No RAW encoder/muxer support, so fallback.
transformer.start(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW), outputPath);
TransformerTestRunner.runLooper(transformer); TransformerTestRunner.runLooper(transformer);
assertThat(deprecatedFallbackCalled.get()).isTrue(); assertThat(deprecatedFallbackCalled.get()).isTrue();
@ -675,19 +676,22 @@ public final class MediaItemExportTest {
} }
@Test @Test
public void start_flattenForSlowMotion_completesSuccessfully() throws Exception { public void start_flattenForSlowMotionVideoOnly_completesSuccessfully() throws Exception {
Transformer transformer = Transformer transformer =
createTransformerBuilder(muxerFactory, /* enableFallback= */ false).build(); createTransformerBuilder(muxerFactory, /* enableFallback= */ false).build();
EditedMediaItem editedMediaItem = EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_WITH_SEF_SLOW_MOTION)) new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_WITH_SEF_SLOW_MOTION))
.setFlattenForSlowMotion(true) .setFlattenForSlowMotion(true)
.setRemoveAudio(true)
.build(); .build();
transformer.start(editedMediaItem, outputPath); transformer.start(editedMediaItem, outputPath);
TransformerTestRunner.runLooper(transformer); TransformerTestRunner.runLooper(transformer);
DumpFileAsserts.assertOutput( DumpFileAsserts.assertOutput(
context, muxerFactory.getCreatedMuxer(), getDumpFileName(FILE_WITH_SEF_SLOW_MOTION)); context,
muxerFactory.getCreatedMuxer(),
getDumpFileName(FILE_WITH_SEF_SLOW_MOTION) + ".noaudio");
} }
@Test @Test
@ -762,25 +766,29 @@ public final class MediaItemExportTest {
public void public void
start_withAudioFormatUnsupportedByMuxer_ignoresDisabledFallbackAndCompletesSuccessfully() start_withAudioFormatUnsupportedByMuxer_ignoresDisabledFallbackAndCompletesSuccessfully()
throws Exception { throws Exception {
// Test succeeds because MIME type fallback is mandatory. removeEncodersAndDecoders();
addAudioDecoders(MimeTypes.AUDIO_RAW);
// RAW supported by encoder, unsupported by muxer.
// AAC supported by encoder and muxer.
addAudioEncoders(MimeTypes.AUDIO_RAW, MimeTypes.AUDIO_AAC);
Transformer.Listener mockListener = mock(Transformer.Listener.class); Transformer.Listener mockListener = mock(Transformer.Listener.class);
TransformationRequest originalTransformationRequest = TransformationRequest originalTransformationRequest =
new TransformationRequest.Builder().build(); new TransformationRequest.Builder().build();
TransformationRequest fallbackTransformationRequest = TransformationRequest fallbackTransformationRequest =
new TransformationRequest.Builder().setAudioMimeType(MimeTypes.AUDIO_AAC).build(); new TransformationRequest.Builder().setAudioMimeType(MimeTypes.AUDIO_AAC).build();
// MIME type fallback is mandatory.
Transformer transformer = Transformer transformer =
createTransformerBuilder(muxerFactory, /* enableFallback= */ false) createTransformerBuilder(muxerFactory, /* enableFallback= */ false)
.addListener(mockListener) .addListener(mockListener)
.build(); .build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_AC3_UNSUPPORTED_BY_MUXER); MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW);
transformer.start(mediaItem, outputPath); transformer.start(mediaItem, outputPath);
TransformerTestRunner.runLooper(transformer); TransformerTestRunner.runLooper(transformer);
DumpFileAsserts.assertOutput( DumpFileAsserts.assertOutput(
context, context, muxerFactory.getCreatedMuxer(), getDumpFileName(FILE_AUDIO_RAW + ".aac"));
muxerFactory.getCreatedMuxer(),
getDumpFileName(FILE_AUDIO_AC3_UNSUPPORTED_BY_MUXER + ".fallback"));
verify(mockListener) verify(mockListener)
.onFallbackApplied( .onFallbackApplied(
any(Composition.class), any(Composition.class),
@ -791,6 +799,12 @@ public final class MediaItemExportTest {
@Test @Test
public void start_withAudioFormatUnsupportedByMuxer_fallsBackAndCompletesSuccessfully() public void start_withAudioFormatUnsupportedByMuxer_fallsBackAndCompletesSuccessfully()
throws Exception { throws Exception {
removeEncodersAndDecoders();
addAudioDecoders(MimeTypes.AUDIO_RAW);
// RAW supported by encoder, unsupported by muxer.
// AAC supported by encoder and muxer.
addAudioEncoders(MimeTypes.AUDIO_RAW, MimeTypes.AUDIO_AAC);
Transformer.Listener mockListener = mock(Transformer.Listener.class); Transformer.Listener mockListener = mock(Transformer.Listener.class);
TransformationRequest originalTransformationRequest = TransformationRequest originalTransformationRequest =
new TransformationRequest.Builder().build(); new TransformationRequest.Builder().build();
@ -800,15 +814,13 @@ public final class MediaItemExportTest {
createTransformerBuilder(muxerFactory, /* enableFallback= */ true) createTransformerBuilder(muxerFactory, /* enableFallback= */ true)
.addListener(mockListener) .addListener(mockListener)
.build(); .build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_AC3_UNSUPPORTED_BY_MUXER); MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW);
transformer.start(mediaItem, outputPath); transformer.start(mediaItem, outputPath);
TransformerTestRunner.runLooper(transformer); TransformerTestRunner.runLooper(transformer);
DumpFileAsserts.assertOutput( DumpFileAsserts.assertOutput(
context, context, muxerFactory.getCreatedMuxer(), getDumpFileName(FILE_AUDIO_RAW + ".aac"));
muxerFactory.getCreatedMuxer(),
getDumpFileName(FILE_AUDIO_AC3_UNSUPPORTED_BY_MUXER + ".fallback"));
verify(mockListener) verify(mockListener)
.onFallbackApplied( .onFallbackApplied(
any(Composition.class), any(Composition.class),