From 1865e381087252e91ea1efb1e8fd11851a63ff66 Mon Sep 17 00:00:00 2001 From: kimvde Date: Thu, 9 Mar 2023 18:43:39 +0000 Subject: [PATCH] Add test for audio and video from different sources PiperOrigin-RevId: 515379858 --- .../TransformerAndroidTestRunner.java | 74 +++++++++++---- .../transformer/TransformerEndToEndTest.java | 48 ++++++++++ .../ExoAssetLoaderBaseRenderer.java | 50 +++++----- .../media3/transformer/Transformer.java | 1 - .../transformer/TransformerInternal.java | 2 +- .../transformer/CompositionExportTest.java | 91 +++++++++++++++++++ 6 files changed, 221 insertions(+), 45 deletions(-) create mode 100644 libraries/transformer/src/test/java/androidx/media3/transformer/CompositionExportTest.java diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java index 48cfedfc97..baccb5dd18 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java @@ -15,6 +15,7 @@ */ package androidx.media3.transformer; +import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static java.util.concurrent.TimeUnit.SECONDS; @@ -31,6 +32,7 @@ import androidx.media3.common.util.SystemClock; import androidx.media3.common.util.Util; import androidx.test.platform.app.InstrumentationRegistry; import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.File; import java.io.IOException; @@ -174,20 +176,20 @@ public class TransformerAndroidTestRunner { } /** - * Exports the {@link EditedMediaItem}, saving a summary of the export to the application cache. + * Exports the {@link Composition}, saving a summary of the export to the application cache. * * @param testId A unique identifier for the transformer test run. - * @param editedMediaItem The {@link EditedMediaItem} to export. + * @param composition The {@link Composition} to export. * @return The {@link ExportTestResult}. * @throws Exception The cause of the export not completing. */ - public ExportTestResult run(String testId, EditedMediaItem editedMediaItem) throws Exception { + public ExportTestResult run(String testId, Composition composition) throws Exception { JSONObject resultJson = new JSONObject(); if (inputValues != null) { resultJson.put("inputValues", JSONObject.wrap(inputValues)); } try { - ExportTestResult exportTestResult = runInternal(testId, editedMediaItem); + ExportTestResult exportTestResult = runInternal(testId, composition); resultJson.put("exportResult", exportTestResult.asJsonObject()); if (exportTestResult.exportResult.exportException != null) { throw exportTestResult.exportResult.exportException; @@ -209,6 +211,21 @@ public class TransformerAndroidTestRunner { } } + /** + * Exports the {@link EditedMediaItem}, saving a summary of the export to the application cache. + * + * @param testId A unique identifier for the transformer test run. + * @param editedMediaItem The {@link EditedMediaItem} to export. + * @return The {@link ExportTestResult}. + * @throws Exception The cause of the export not completing. + */ + public ExportTestResult run(String testId, EditedMediaItem editedMediaItem) throws Exception { + EditedMediaItemSequence sequence = + new EditedMediaItemSequence(ImmutableList.of(editedMediaItem)); + Composition composition = new Composition.Builder(ImmutableList.of(sequence)).build(); + return run(testId, composition); + } + /** * Exports the {@link MediaItem}, saving a summary of the export to the application cache. * @@ -223,33 +240,49 @@ public class TransformerAndroidTestRunner { } /** - * Exports the {@link EditedMediaItem}. + * Exports the {@link Composition}. * * @param testId An identifier for the test. - * @param editedMediaItem The {@link EditedMediaItem} to export. + * @param composition The {@link Composition} to export. * @return The {@link ExportTestResult}. - * @throws IllegalStateException See {@link Transformer#start(EditedMediaItem, String)}. + * @throws IllegalStateException See {@link Transformer#start(Composition, String)}. * @throws InterruptedException If the thread is interrupted whilst waiting for transformer to * complete. * @throws IOException If an error occurs opening the output file for writing. * @throws TimeoutException If the export has not completed after {@linkplain * Builder#setTimeoutSeconds(int) the given timeout}. */ - private ExportTestResult runInternal(String testId, EditedMediaItem editedMediaItem) + private ExportTestResult runInternal(String testId, Composition composition) throws InterruptedException, IOException, TimeoutException { - MediaItem mediaItem = editedMediaItem.mediaItem; - if (!mediaItem.clippingConfiguration.equals(MediaItem.ClippingConfiguration.UNSET) - && requestCalculateSsim) { - throw new UnsupportedOperationException( + if (requestCalculateSsim) { + checkArgument( + composition.sequences.size() == 1 + && composition.sequences.get(0).editedMediaItems.size() == 1, + "SSIM is only relevant for single MediaItem compositions"); + checkArgument( + composition + .sequences + .get(0) + .editedMediaItems + .get(0) + .mediaItem + .clippingConfiguration + .equals(MediaItem.ClippingConfiguration.UNSET), "SSIM calculation is not supported for clipped inputs."); } - - Uri mediaItemUri = checkNotNull(mediaItem.localConfiguration).uri; - String scheme = checkNotNull(mediaItemUri.getScheme()); - if ((scheme.equals("http") || scheme.equals("https")) && !hasNetworkConnection(context)) { - throw new UnsupportedOperationException( - "Input network file requested on device with no network connection. Input file name: " - + mediaItemUri); + if (!hasNetworkConnection(context)) { + for (EditedMediaItemSequence sequence : composition.sequences) { + for (EditedMediaItem editedMediaItem : sequence.editedMediaItems) { + Uri mediaItemUri = checkNotNull(editedMediaItem.mediaItem.localConfiguration).uri; + String scheme = checkNotNull(mediaItemUri.getScheme()); + if ((scheme.equals("http") || scheme.equals("https"))) { + throw new IllegalArgumentException( + "Input network file requested on device with no network connection. Input file" + + " name: " + + mediaItemUri); + } + } + } } AtomicReference<@NullableType FallbackDetails> fallbackDetailsReference = @@ -307,7 +340,7 @@ public class TransformerAndroidTestRunner { .runOnMainSync( () -> { try { - testTransformer.start(editedMediaItem, outputVideoFile.getAbsolutePath()); + testTransformer.start(composition, outputVideoFile.getAbsolutePath()); // Catch all exceptions to report. Exceptions thrown here and not caught will NOT // propagate. } catch (Exception e) { @@ -359,6 +392,7 @@ public class TransformerAndroidTestRunner { return testResultBuilder.build(); } try { + MediaItem mediaItem = composition.sequences.get(0).editedMediaItems.get(0).mediaItem; double ssim = SsimHelper.calculate( context, diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 0835836c27..c332f24892 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -27,7 +27,10 @@ import androidx.media3.common.C; import androidx.media3.common.Effect; import androidx.media3.common.Format; import androidx.media3.common.MediaItem; +import androidx.media3.common.audio.AudioProcessor; +import androidx.media3.common.audio.SonicAudioProcessor; import androidx.media3.effect.Presentation; +import androidx.media3.effect.RgbFilter; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; @@ -185,6 +188,51 @@ public class TransformerEndToEndTest { assertThat(exception).hasMessageThat().contains("video"); } + @Test + public void start_audioVideoTranscodedFromDifferentSequences_producesExpectedResult() + throws Exception { + Transformer transformer = new Transformer.Builder(context).build(); + String testId = "start_audioVideoTranscodedFromDifferentSequences_producesExpectedResult"; + ImmutableList audioProcessors = ImmutableList.of(new SonicAudioProcessor()); + ImmutableList videoEffects = ImmutableList.of(RgbFilter.createGrayscaleFilter()); + MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(mediaItem) + .setEffects(new Effects(audioProcessors, videoEffects)) + .build(); + ExportTestResult expectedResult = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, editedMediaItem); + EditedMediaItem audioEditedMediaItem = + new EditedMediaItem.Builder(mediaItem) + .setEffects(new Effects(audioProcessors, /* videoEffects= */ ImmutableList.of())) + .setRemoveVideo(true) + .build(); + EditedMediaItemSequence audioSequence = + new EditedMediaItemSequence(ImmutableList.of(audioEditedMediaItem)); + EditedMediaItem videoEditedMediaItem = + new EditedMediaItem.Builder(mediaItem) + .setEffects(new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects)) + .setRemoveAudio(true) + .build(); + EditedMediaItemSequence videoSequence = + new EditedMediaItemSequence(ImmutableList.of(videoEditedMediaItem)); + Composition composition = + new Composition.Builder(ImmutableList.of(audioSequence, videoSequence)).build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, composition); + + assertThat(result.exportResult.channelCount) + .isEqualTo(expectedResult.exportResult.channelCount); + assertThat(result.exportResult.videoFrameCount) + .isEqualTo(expectedResult.exportResult.videoFrameCount); + assertThat(result.exportResult.durationMs).isEqualTo(expectedResult.exportResult.durationMs); + } + private static final class VideoUnsupportedEncoderFactory implements Codec.EncoderFactory { private final Codec.EncoderFactory encoderFactory; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderBaseRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderBaseRenderer.java index e9d569e2fe..7cfa53456a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderBaseRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderBaseRenderer.java @@ -17,7 +17,6 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.decoder.DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED; import static androidx.media3.exoplayer.source.SampleStream.FLAG_REQUIRE_FORMAT; import static androidx.media3.transformer.AssetLoader.SUPPORTED_OUTPUT_TYPE_DECODED; @@ -54,6 +53,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private boolean isRunning; private long streamStartPositionUs; + private boolean shouldInitDecoder; public ExoAssetLoaderBaseRenderer( @C.TrackType int trackType, @@ -97,7 +97,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @Override public void render(long positionUs, long elapsedRealtimeUs) { try { - if (!isRunning || isEnded() || !hasReadInputFormat()) { + if (!isRunning || isEnded() || !readInputFormatAndInitDecoderIfNeeded()) { return; } @@ -195,35 +195,39 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * {@linkplain Codec decoder}. */ @EnsuresNonNullIf(expression = "inputFormat", result = true) - private boolean hasReadInputFormat() throws ExportException { - if (inputFormat != null) { + private boolean readInputFormatAndInitDecoderIfNeeded() throws ExportException { + if (inputFormat != null && !shouldInitDecoder) { return true; } - FormatHolder formatHolder = getFormatHolder(); - @ReadDataResult - int result = readSource(formatHolder, decoderInputBuffer, /* readFlags= */ FLAG_REQUIRE_FORMAT); - if (result != C.RESULT_FORMAT_READ) { - return false; + if (inputFormat == null) { + FormatHolder formatHolder = getFormatHolder(); + @ReadDataResult + int result = + readSource(formatHolder, decoderInputBuffer, /* readFlags= */ FLAG_REQUIRE_FORMAT); + if (result != C.RESULT_FORMAT_READ) { + return false; + } + inputFormat = overrideFormat(checkNotNull(formatHolder.format)); + onInputFormatRead(inputFormat); + shouldInitDecoder = + assetLoaderListener.onTrackAdded( + inputFormat, + SUPPORTED_OUTPUT_TYPE_DECODED | SUPPORTED_OUTPUT_TYPE_ENCODED, + streamStartPositionUs, + streamOffsetUs); } - inputFormat = overrideFormat(checkNotNull(formatHolder.format)); - onInputFormatRead(inputFormat); - boolean decodeOutput = - assetLoaderListener.onTrackAdded( - inputFormat, - SUPPORTED_OUTPUT_TYPE_DECODED | SUPPORTED_OUTPUT_TYPE_ENCODED, - streamStartPositionUs, - streamOffsetUs); - if (decodeOutput) { - if (getProcessedTrackType(inputFormat.sampleMimeType) == C.TRACK_TYPE_AUDIO) { - initDecoder(inputFormat); - } else { + if (shouldInitDecoder) { + if (getProcessedTrackType(inputFormat.sampleMimeType) == C.TRACK_TYPE_VIDEO) { // TODO(b/237674316): Move surface creation out of video sampleConsumer. Init decoder and // get decoder output Format before init sampleConsumer. - checkState(ensureSampleConsumerInitialized()); - initDecoder(inputFormat); + if (!ensureSampleConsumerInitialized()) { + return false; + } } + initDecoder(inputFormat); + shouldInitDecoder = false; } return true; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 9fc1527926..f08b44cca7 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -702,7 +702,6 @@ public final class Transformer { * @throws IllegalStateException If an export is already in progress. */ public void start(Composition composition, String path) { - checkArgument(composition.sequences.size() == 1); checkArgument(composition.effects.audioProcessors.isEmpty()); // Only supports Presentation in video effects. ImmutableList videoEffects = composition.effects.videoEffects; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index 50ef5ccff5..b1a415bf6c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -465,7 +465,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } else { outputHasVideo.set(true); } - if (trackCountsToReport.get() == 0 && tracksToAdd.decrementAndGet() == 0) { + if (tracksToAdd.decrementAndGet() == 0 && trackCountsToReport.get() == 0) { int outputTrackCount = (outputHasAudio.get() ? 1 : 0) + (outputHasVideo.get() ? 1 : 0); muxerWrapper.setTrackCount(outputTrackCount); fallbackListener.setTrackCount(outputTrackCount); diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionExportTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionExportTest.java new file mode 100644 index 0000000000..a960ac59a8 --- /dev/null +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionExportTest.java @@ -0,0 +1,91 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.transformer.TestUtil.ASSET_URI_PREFIX; +import static androidx.media3.transformer.TestUtil.FILE_AUDIO_VIDEO; +import static androidx.media3.transformer.TestUtil.createTransformerBuilder; +import static com.google.common.truth.Truth.assertThat; + +import android.content.Context; +import androidx.media3.common.MediaItem; +import androidx.media3.common.util.Util; +import androidx.media3.transformer.TestUtil.TestMuxerFactory.TestMuxerHolder; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import java.nio.file.Files; +import java.nio.file.Paths; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * End-to-end test for exporting a {@link Composition} containing multiple {@link + * EditedMediaItemSequence} instances with {@link Transformer}. + */ +@RunWith(AndroidJUnit4.class) +public class CompositionExportTest { + + private String outputPath; + private TestMuxerHolder testMuxerHolder; + + @Before + public void setUp() throws Exception { + Context context = ApplicationProvider.getApplicationContext(); + outputPath = Util.createTempFile(context, "TransformerTest").getPath(); + testMuxerHolder = new TestUtil.TestMuxerFactory.TestMuxerHolder(); + } + + @After + public void tearDown() throws Exception { + Files.delete(Paths.get(outputPath)); + } + + @Test + public void start_audioVideoTransmuxedFromDifferentSequences_producesExpectedResult() + throws Exception { + Transformer transformer = + createTransformerBuilder(testMuxerHolder, /* enableFallback= */ false).build(); + MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO); + transformer.start(mediaItem, outputPath); + ExportResult expectedExportResult = TransformerTestRunner.runLooper(transformer); + EditedMediaItem audioEditedMediaItem = + new EditedMediaItem.Builder(mediaItem).setRemoveVideo(true).build(); + EditedMediaItemSequence audioSequence = + new EditedMediaItemSequence(ImmutableList.of(audioEditedMediaItem)); + EditedMediaItem videoEditedMediaItem = + new EditedMediaItem.Builder(mediaItem).setRemoveAudio(true).build(); + EditedMediaItemSequence videoSequence = + new EditedMediaItemSequence(ImmutableList.of(videoEditedMediaItem)); + Composition composition = + new Composition.Builder(ImmutableList.of(audioSequence, videoSequence)) + .setTransmuxAudio(true) + .setTransmuxVideo(true) + .build(); + + transformer.start(composition, outputPath); + ExportResult exportResult = TransformerTestRunner.runLooper(transformer); + + // We can't compare the muxer output against a dump file because the asset loaders in each + // sequence load samples from their own thread, independently of each other, which makes the + // output non-deterministic. + assertThat(exportResult.channelCount).isEqualTo(expectedExportResult.channelCount); + assertThat(exportResult.videoFrameCount).isEqualTo(expectedExportResult.videoFrameCount); + assertThat(exportResult.durationMs).isEqualTo(expectedExportResult.durationMs); + } +}