From bdbcdf1660a727f192c713ad81ed88fe75fce265 Mon Sep 17 00:00:00 2001 From: sheenachhabra Date: Tue, 25 Feb 2025 04:30:24 -0800 Subject: [PATCH] Implement blank frame insertion when video track goes missing If an sequence has both audio and video tracks running but then the next media item has only audio, then insert blank frames to continue video track. PiperOrigin-RevId: 730828269 --- .../transformer/TransformerVideoGapsTest.java | 130 ++++++++++++++++++ .../transformer/SequenceAssetLoader.java | 59 +++++--- 2 files changed, 171 insertions(+), 18 deletions(-) create mode 100644 libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerVideoGapsTest.java diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerVideoGapsTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerVideoGapsTest.java new file mode 100644 index 0000000000..eb59bb0005 --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerVideoGapsTest.java @@ -0,0 +1,130 @@ +/* + * Copyright 2025 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET; +import static androidx.media3.transformer.AndroidTestUtil.getVideoTrackOutput; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import android.content.Context; +import androidx.media3.common.MediaItem; +import androidx.media3.extractor.mp4.Mp4Extractor; +import androidx.media3.extractor.text.DefaultSubtitleParserFactory; +import androidx.media3.test.utils.FakeExtractorOutput; +import androidx.media3.test.utils.FakeTrackOutput; +import androidx.media3.test.utils.TestUtil; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +/** + * End-to-end instrumentation tests for {@link Transformer} when the {@link EditedMediaItemSequence} + * has video gaps. + */ +@RunWith(AndroidJUnit4.class) +public class TransformerVideoGapsTest { + private static final EditedMediaItem AUDIO_VIDEO_MEDIA_ITEM = + new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET.uri)).build(); + private static final EditedMediaItem AUDIO_ONLY_MEDIA_ITEM = + AUDIO_VIDEO_MEDIA_ITEM.buildUpon().setRemoveVideo(true).build(); + + private final Context context = ApplicationProvider.getApplicationContext(); + @Rule public final TestName testName = new TestName(); + + private String testId; + + @Before + public void setUpTestId() { + testId = testName.getMethodName(); + } + + // TODO: b/391111085 - Change test when gaps at the start of the sequence are supported. + @Test + public void export_withThreeMediaItemsAndFirstMediaItemHavingNoVideo_throws() { + Transformer transformer = new Transformer.Builder(context).build(); + Composition composition = + new Composition.Builder( + new EditedMediaItemSequence.Builder( + AUDIO_ONLY_MEDIA_ITEM, AUDIO_VIDEO_MEDIA_ITEM, AUDIO_VIDEO_MEDIA_ITEM) + .build()) + .build(); + + TransformerAndroidTestRunner transformerAndroidTestRunner = + new TransformerAndroidTestRunner.Builder(context, transformer).build(); + + assertThrows( + ExportException.class, () -> transformerAndroidTestRunner.run(testId, composition)); + } + + @Test + public void + export_withThreeMediaItemsAndSecondMediaItemHavingNoVideo_insertsBlankFrameForSecondMediaItem() + throws Exception { + Transformer transformer = new Transformer.Builder(context).build(); + Composition composition = + new Composition.Builder( + new EditedMediaItemSequence.Builder( + AUDIO_VIDEO_MEDIA_ITEM, AUDIO_ONLY_MEDIA_ITEM, AUDIO_VIDEO_MEDIA_ITEM) + .build()) + .build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, composition); + + FakeExtractorOutput fakeExtractorOutput = + TestUtil.extractAllSamplesFromFilePath( + new Mp4Extractor(new DefaultSubtitleParserFactory()), result.filePath); + FakeTrackOutput videoTrackOutput = getVideoTrackOutput(fakeExtractorOutput); + // The gap is for 1024ms with 30 fps. + int expectedBlankFrames = 31; + assertThat(videoTrackOutput.getSampleCount()) + .isEqualTo(2 * MP4_ASSET.videoFrameCount + expectedBlankFrames); + } + + @Test + public void + export_withThreeMediaItemsAndLastMediaItemHavingNoVideo_insertsBlankFrameForLastMediaItem() + throws Exception { + Transformer transformer = new Transformer.Builder(context).build(); + Composition composition = + new Composition.Builder( + new EditedMediaItemSequence.Builder( + AUDIO_VIDEO_MEDIA_ITEM, AUDIO_VIDEO_MEDIA_ITEM, AUDIO_ONLY_MEDIA_ITEM) + .build()) + .build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, composition); + + FakeExtractorOutput fakeExtractorOutput = + TestUtil.extractAllSamplesFromFilePath( + new Mp4Extractor(new DefaultSubtitleParserFactory()), result.filePath); + FakeTrackOutput videoTrackOutput = getVideoTrackOutput(fakeExtractorOutput); + // The gap is for 1024ms with 30 fps. + int expectedBlankFrames = 31; + assertThat(videoTrackOutput.getSampleCount()) + .isEqualTo(2 * MP4_ASSET.videoFrameCount + expectedBlankFrames); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceAssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceAssetLoader.java index d161109c0a..eff4402352 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceAssetLoader.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceAssetLoader.java @@ -27,15 +27,18 @@ import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED import static androidx.media3.transformer.TransformerUtil.getProcessedTrackType; import android.graphics.Bitmap; +import android.graphics.Color; import android.os.Looper; import android.view.Surface; import androidx.annotation.Nullable; import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; import androidx.media3.common.Format; import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; import androidx.media3.common.OnInputFrameProcessedListener; import androidx.media3.common.util.Clock; +import androidx.media3.common.util.ConstantRateTimestampIterator; import androidx.media3.common.util.HandlerWrapper; import androidx.media3.common.util.TimestampIterator; import androidx.media3.common.util.Util; @@ -62,6 +65,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; .setChannelCount(2) .build(); + private static final Format BLANK_IMAGE_BITMAP_FORMAT = + new Format.Builder() + .setWidth(1) + .setHeight(1) + .setSampleMimeType(MimeTypes.IMAGE_RAW) + .setColorInfo(ColorInfo.SRGB_BT709_FULL) + .build(); + + private static final float BLANK_IMAGE_FRAME_RATE = 30.0f; + + private static final int RETRY_DELAY_MS = 10; + private final List editedMediaItems; private final boolean isLooping; private final boolean forceAudioTrack; @@ -314,14 +329,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; new SampleConsumerWrapper(wrappedAudioSampleConsumer, C.TRACK_TYPE_AUDIO)); } } else { - // TODO: b/270533049 - Remove the check below when implementing blank video frames generation. - boolean videoTrackDisappeared = - reportedTrackCount.get() == 1 - && trackType == C.TRACK_TYPE_AUDIO - && sampleConsumersByTrackType.size() == 2; - checkState( - !videoTrackDisappeared, - "Inputs with no video track are not supported when the output contains a video track"); sampleConsumer = checkStateNotNull( sampleConsumersByTrackType.get(trackType), @@ -335,17 +342,36 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } onMediaItemChanged(trackType, format); if (reportedTrackCount.get() == 1 && sampleConsumersByTrackType.size() == 2) { - for (Map.Entry entry : - sampleConsumersByTrackType.entrySet()) { - int outputTrackType = entry.getKey(); - if (trackType != outputTrackType) { - onMediaItemChanged(outputTrackType, /* outputFormat= */ null); - } + // One track is missing from the current media item. + if (trackType == C.TRACK_TYPE_AUDIO) { + // Fill video gap with blank frames. + onMediaItemChanged(C.TRACK_TYPE_VIDEO, /* outputFormat= */ BLANK_IMAGE_BITMAP_FORMAT); + nonEndedTrackCount.incrementAndGet(); + Bitmap bitmap = + Bitmap.createBitmap( + new int[] {Color.BLACK}, /* width= */ 1, /* height= */ 1, Bitmap.Config.ARGB_8888); + handler.post(() -> insertBlankFrames(bitmap)); + } else { + // Generate audio silence in the AudioGraph by signalling null format. + onMediaItemChanged(C.TRACK_TYPE_AUDIO, /* outputFormat= */ null); } } return sampleConsumer; } + private void insertBlankFrames(Bitmap bitmap) { + SampleConsumerWrapper videoSampleConsumer = + checkNotNull(sampleConsumersByTrackType.get(C.TRACK_TYPE_VIDEO)); + if (videoSampleConsumer.queueInputBitmap( + bitmap, + new ConstantRateTimestampIterator(currentAssetDurationUs, BLANK_IMAGE_FRAME_RATE)) + != SampleConsumer.INPUT_RESULT_SUCCESS) { + handler.postDelayed(() -> insertBlankFrames(bitmap), RETRY_DELAY_MS); + } else { + videoSampleConsumer.signalEndOfVideoInput(); + } + } + private void onMediaItemChanged(int trackType, @Nullable Format outputFormat) { @Nullable OnMediaItemChangedListener onMediaItemChangedListener = @@ -646,8 +672,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; */ private final class GapSignalingAssetLoader implements AssetLoader { - private static final int OUTPUT_FORMAT_RETRY_DELAY_MS = 10; - private final long durationUs; private final Format trackFormat; private final Format decodedFormat; @@ -700,8 +724,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputtedFormat = true; sampleConsumerWrapper.onGapSignalled(); } else { - handler.postDelayed( - this::outputFormatToSequenceAssetLoader, OUTPUT_FORMAT_RETRY_DELAY_MS); + handler.postDelayed(this::outputFormatToSequenceAssetLoader, RETRY_DELAY_MS); } } catch (ExportException e) {