From b1185657301361f64ca2c70568f67b18942cd08c Mon Sep 17 00:00:00 2001 From: kimvde Date: Fri, 24 Mar 2023 09:38:07 +0000 Subject: [PATCH] Add an API entry point for looping a sequence Also - Add unit tests - Fix bug discovered by unit tests PiperOrigin-RevId: 519092249 --- .../media3/transformer/AndroidTestUtil.java | 3 + .../transformer/TransformerEndToEndTest.java | 99 ++++++++++++++++++- .../transformer/EditedMediaItemSequence.java | 25 ++++- .../ExoAssetLoaderAudioRenderer.java | 15 ++- .../ExoAssetLoaderBaseRenderer.java | 6 +- .../transformer/CompositionExportTest.java | 62 ++++++++++++ .../androidx/media3/transformer/TestUtil.java | 1 + 7 files changed, 199 insertions(+), 12 deletions(-) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index 91c184cf1f..02725ad4cc 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -478,6 +478,9 @@ public final class AndroidTestUtil { .setFrameRate(23.163f) .setCodecs("hvc1.1.6.L183.B0") .build(); + + public static final String MP3_ASSET_URI_STRING = "asset:///media/mp3/test.mp3"; + /** * Log in logcat and in an analysis file that this test was skipped. * diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 6bd5fa6abf..ee1acaa32b 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -15,6 +15,7 @@ */ package androidx.media3.transformer; +import static androidx.media3.transformer.AndroidTestUtil.MP3_ASSET_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.PNG_ASSET_URI_STRING; @@ -227,10 +228,9 @@ public class TransformerEndToEndTest { } @Test - public void start_audioVideoTranscodedFromDifferentSequences_producesExpectedResult() - throws Exception { + public void audioVideoTranscodedFromDifferentSequences_producesExpectedResult() throws Exception { Transformer transformer = new Transformer.Builder(context).build(); - String testId = "start_audioVideoTranscodedFromDifferentSequences_producesExpectedResult"; + String testId = "audioVideoTranscodedFromDifferentSequences_producesExpectedResult"; ImmutableList audioProcessors = ImmutableList.of(new SonicAudioProcessor()); ImmutableList videoEffects = ImmutableList.of(RgbFilter.createGrayscaleFilter()); MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)); @@ -271,6 +271,99 @@ public class TransformerEndToEndTest { assertThat(result.exportResult.durationMs).isEqualTo(expectedResult.exportResult.durationMs); } + @Test + public void loopingTranscodedAudio_producesExpectedResult() throws Exception { + Transformer transformer = new Transformer.Builder(context).build(); + String testId = "loopingTranscodedAudio_producesExpectedResult"; + EditedMediaItem audioEditedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(MP3_ASSET_URI_STRING)).build(); + EditedMediaItemSequence audioSequence = + new EditedMediaItemSequence( + ImmutableList.of(audioEditedMediaItem, audioEditedMediaItem), /* isLooping= */ true); + EditedMediaItem videoEditedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET_URI_STRING)) + .setRemoveAudio(true) + .build(); + EditedMediaItemSequence videoSequence = + new EditedMediaItemSequence( + ImmutableList.of(videoEditedMediaItem, videoEditedMediaItem, videoEditedMediaItem)); + Composition composition = + new Composition.Builder(ImmutableList.of(audioSequence, videoSequence)) + .setTransmuxVideo(true) + .build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, composition); + + assertThat(result.exportResult.processedInputs).hasSize(6); + assertThat(result.exportResult.channelCount).isEqualTo(1); + assertThat(result.exportResult.videoFrameCount).isEqualTo(90); + assertThat(result.exportResult.durationMs).isEqualTo(3015); + } + + @Test + public void loopingTranscodedVideo_producesExpectedResult() throws Exception { + Transformer transformer = new Transformer.Builder(context).build(); + String testId = "loopingTranscodedVideo_producesExpectedResult"; + EditedMediaItem audioEditedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(MP3_ASSET_URI_STRING)).build(); + EditedMediaItemSequence audioSequence = + new EditedMediaItemSequence( + ImmutableList.of(audioEditedMediaItem, audioEditedMediaItem, audioEditedMediaItem)); + EditedMediaItem videoEditedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET_URI_STRING)) + .setRemoveAudio(true) + .build(); + EditedMediaItemSequence videoSequence = + new EditedMediaItemSequence( + ImmutableList.of(videoEditedMediaItem, videoEditedMediaItem), /* isLooping= */ true); + Composition composition = + new Composition.Builder(ImmutableList.of(audioSequence, videoSequence)).build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, composition); + + assertThat(result.exportResult.processedInputs).hasSize(7); + assertThat(result.exportResult.channelCount).isEqualTo(1); + assertThat(result.exportResult.videoFrameCount).isEqualTo(92); + assertThat(result.exportResult.durationMs).isEqualTo(3105); + } + + @Test + public void loopingImage_producesExpectedResult() throws Exception { + Transformer transformer = new Transformer.Builder(context).build(); + String testId = "loopingImage_producesExpectedResult"; + EditedMediaItem audioEditedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(MP3_ASSET_URI_STRING)).build(); + EditedMediaItemSequence audioSequence = + new EditedMediaItemSequence( + ImmutableList.of(audioEditedMediaItem, audioEditedMediaItem, audioEditedMediaItem)); + EditedMediaItem imageEditedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(PNG_ASSET_URI_STRING)) + .setDurationUs(1_000_000) + .setFrameRate(30) + .build(); + EditedMediaItemSequence imageSequence = + new EditedMediaItemSequence( + ImmutableList.of(imageEditedMediaItem, imageEditedMediaItem), /* isLooping= */ true); + Composition composition = + new Composition.Builder(ImmutableList.of(audioSequence, imageSequence)).build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, composition); + + assertThat(result.exportResult.processedInputs).hasSize(7); + assertThat(result.exportResult.channelCount).isEqualTo(1); + assertThat(result.exportResult.videoFrameCount).isEqualTo(94); + assertThat(result.exportResult.durationMs).isEqualTo(3100); + } + private static final class VideoUnsupportedEncoderFactory implements Codec.EncoderFactory { private final Codec.EncoderFactory encoderFactory; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/EditedMediaItemSequence.java b/libraries/transformer/src/main/java/androidx/media3/transformer/EditedMediaItemSequence.java index 9715cba951..354db579c3 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/EditedMediaItemSequence.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/EditedMediaItemSequence.java @@ -35,8 +35,17 @@ public final class EditedMediaItemSequence { *

This list must not be empty. */ public final ImmutableList editedMediaItems; - - /* package */ final boolean isLooping; + /** + * Whether this sequence is looping. + * + *

This value indicates whether to loop over the {@link EditedMediaItem} instances in this + * sequence until all the non-looping sequences in the {@link Composition} have ended. + * + *

A looping sequence ends at the same time as the longest non-looping sequence. This means + * that the last exported {@link EditedMediaItem} from a looping sequence can be only partially + * exported. + */ + public final boolean isLooping; /** * Creates an instance. @@ -44,8 +53,18 @@ public final class EditedMediaItemSequence { * @param editedMediaItems The {@link #editedMediaItems}. */ public EditedMediaItemSequence(List editedMediaItems) { + this(editedMediaItems, /* isLooping= */ false); + } + + /** + * Creates an instance. + * + * @param editedMediaItems The {@link #editedMediaItems}. + * @param isLooping Whether the sequence {@linkplain #isLooping is looping}. + */ + public EditedMediaItemSequence(List editedMediaItems, boolean isLooping) { checkArgument(!editedMediaItems.isEmpty()); this.editedMediaItems = ImmutableList.copyOf(editedMediaItems); - isLooping = false; + this.isLooping = isLooping; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderAudioRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderAudioRenderer.java index 7de36e799a..844e073388 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderAudioRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderAudioRenderer.java @@ -31,6 +31,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private final Codec.DecoderFactory decoderFactory; + private boolean hasPendingConsumerInput; + public ExoAssetLoaderAudioRenderer( Codec.DecoderFactory decoderFactory, TransformerMediaClock mediaClock, @@ -63,10 +65,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return false; } - ByteBuffer sampleConsumerInputData = checkNotNull(sampleConsumerInputBuffer.data); - if (sampleConsumerInputData.position() == 0) { + if (!hasPendingConsumerInput) { if (decoder.isEnded()) { - sampleConsumerInputData.limit(0); + checkNotNull(sampleConsumerInputBuffer.data).limit(0); sampleConsumerInputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM); isEnded = sampleConsumer.queueInputBuffer(); return false; @@ -83,8 +84,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; sampleConsumerInputBuffer.timeUs = bufferInfo.presentationTimeUs; sampleConsumerInputBuffer.setFlags(bufferInfo.flags); decoder.releaseOutputBuffer(/* render= */ false); + hasPendingConsumerInput = true; } - return sampleConsumer.queueInputBuffer(); + if (!sampleConsumer.queueInputBuffer()) { + return false; + } + + hasPendingConsumerInput = false; + return true; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderBaseRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderBaseRenderer.java index 8aca0ad7f2..381fe9ab3e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderBaseRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderBaseRenderer.java @@ -54,6 +54,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private boolean isRunning; private long streamStartPositionUs; private boolean shouldInitDecoder; + private boolean hasPendingConsumerInput; public ExoAssetLoaderBaseRenderer( @C.TrackType int trackType, @@ -309,14 +310,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return false; } - if (checkNotNull(sampleConsumerInputBuffer.data).position() == 0 - && !sampleConsumerInputBuffer.isEndOfStream()) { + if (!hasPendingConsumerInput) { if (!readInput(sampleConsumerInputBuffer)) { return false; } if (shouldDropInputBuffer(sampleConsumerInputBuffer)) { return true; } + hasPendingConsumerInput = true; } boolean isInputEnded = sampleConsumerInputBuffer.isEndOfStream(); @@ -324,6 +325,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return false; } + hasPendingConsumerInput = false; isEnded = isInputEnded; return !isEnded; } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionExportTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionExportTest.java index a960ac59a8..2d81c93326 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionExportTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionExportTest.java @@ -16,7 +16,9 @@ package androidx.media3.transformer; import static androidx.media3.transformer.TestUtil.ASSET_URI_PREFIX; +import static androidx.media3.transformer.TestUtil.FILE_AUDIO_ONLY; import static androidx.media3.transformer.TestUtil.FILE_AUDIO_VIDEO; +import static androidx.media3.transformer.TestUtil.FILE_VIDEO_ONLY; import static androidx.media3.transformer.TestUtil.createTransformerBuilder; import static com.google.common.truth.Truth.assertThat; @@ -88,4 +90,64 @@ public class CompositionExportTest { assertThat(exportResult.videoFrameCount).isEqualTo(expectedExportResult.videoFrameCount); assertThat(exportResult.durationMs).isEqualTo(expectedExportResult.durationMs); } + + @Test + public void start_loopingTransmuxedAudio_producesExpectedResult() throws Exception { + Transformer transformer = + createTransformerBuilder(testMuxerHolder, /* enableFallback= */ false).build(); + EditedMediaItem audioEditedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_ONLY)).build(); + EditedMediaItemSequence audioSequence = + new EditedMediaItemSequence( + ImmutableList.of(audioEditedMediaItem, audioEditedMediaItem), /* isLooping= */ true); + EditedMediaItem videoEditedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_VIDEO_ONLY)).build(); + EditedMediaItemSequence videoSequence = + new EditedMediaItemSequence( + ImmutableList.of(videoEditedMediaItem, videoEditedMediaItem, videoEditedMediaItem)); + Composition composition = + new Composition.Builder(ImmutableList.of(audioSequence, videoSequence)) + .setTransmuxAudio(true) + .setTransmuxVideo(true) + .build(); + + transformer.start(composition, outputPath); + ExportResult exportResult = TransformerTestRunner.runLooper(transformer); + + assertThat(exportResult.processedInputs).hasSize(6); + assertThat(exportResult.channelCount).isEqualTo(1); + assertThat(exportResult.videoFrameCount).isEqualTo(90); + assertThat(exportResult.durationMs).isEqualTo(2977); + assertThat(exportResult.fileSizeBytes).isEqualTo(293660); + } + + @Test + public void start_loopingTransmuxedVideo_producesExpectedResult() throws Exception { + Transformer transformer = + createTransformerBuilder(testMuxerHolder, /* enableFallback= */ false).build(); + EditedMediaItem audioEditedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_ONLY)).build(); + EditedMediaItemSequence audioSequence = + new EditedMediaItemSequence( + ImmutableList.of(audioEditedMediaItem, audioEditedMediaItem, audioEditedMediaItem)); + EditedMediaItem videoEditedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_VIDEO_ONLY)).build(); + EditedMediaItemSequence videoSequence = + new EditedMediaItemSequence( + ImmutableList.of(videoEditedMediaItem, videoEditedMediaItem), /* isLooping= */ true); + Composition composition = + new Composition.Builder(ImmutableList.of(audioSequence, videoSequence)) + .setTransmuxAudio(true) + .setTransmuxVideo(true) + .build(); + + transformer.start(composition, outputPath); + ExportResult exportResult = TransformerTestRunner.runLooper(transformer); + + assertThat(exportResult.processedInputs).hasSize(7); + assertThat(exportResult.channelCount).isEqualTo(1); + assertThat(exportResult.videoFrameCount).isEqualTo(93); + assertThat(exportResult.durationMs).isEqualTo(3108); + assertThat(exportResult.fileSizeBytes).isEqualTo(337308); + } } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/TestUtil.java b/libraries/transformer/src/test/java/androidx/media3/transformer/TestUtil.java index aa79533cd0..5abe75acb1 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/TestUtil.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/TestUtil.java @@ -145,6 +145,7 @@ public final class TestUtil { public static final String ASSET_URI_PREFIX = "asset:///media/"; public static final String FILE_VIDEO_ONLY = "mp4/sample_18byte_nclx_colr.mp4"; + public static final String FILE_AUDIO_ONLY = "mp3/test.mp3"; public static final String FILE_AUDIO_VIDEO = "mp4/sample.mp4"; public static final String FILE_AUDIO_VIDEO_INCREASING_TIMESTAMPS_15S = "mp4/sample_with_increasing_timestamps_320w_240h.mp4";