From c5c8e988e8ff246c7ea94c6e4f14cd9e53318d21 Mon Sep 17 00:00:00 2001 From: tofunmi Date: Thu, 7 Dec 2023 10:36:47 -0800 Subject: [PATCH] Abandon trim optimization when transcoding effects are set PiperOrigin-RevId: 588839072 --- .../transformer/TransformerEndToEndTest.java | 34 +++++++++++++ .../media3/transformer/Mp4MetadataInfo.java | 24 ++++++++-- .../media3/transformer/Transformer.java | 48 +++++++++++++------ .../transformer/TransformerInternal.java | 19 +++++--- .../media3/transformer/TransformerUtil.java | 4 -- .../transformer/Mp4MetadataInfoTest.java | 32 ++++++++++++- 6 files changed, 132 insertions(+), 29 deletions(-) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 9ca40982f0..1786ab9b43 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -28,9 +28,11 @@ import static androidx.media3.transformer.AndroidTestUtil.PNG_ASSET_URI_STRING; import static androidx.media3.transformer.AndroidTestUtil.createOpenGlObjects; import static androidx.media3.transformer.AndroidTestUtil.generateTextureFromBitmap; import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped; +import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED; import static androidx.media3.transformer.ExportResult.OPTIMIZATION_SUCCEEDED; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; +import static org.junit.Assume.assumeFalse; import static org.junit.Assume.assumeTrue; import android.content.Context; @@ -478,6 +480,38 @@ public class TransformerEndToEndTest { assertThat(result.exportResult.durationMs).isAtMost(2000); } + @Test + public void videoEditing_trimOptimizationEnabled_fallbackToNormalExport() throws Exception { + String testId = "videoEditing_trimOptimizationEnabled_fallbackToNormalExport"; + Transformer transformer = + new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build(); + // The trim optimization is only guaranteed to work on emulator for this file. + assumeTrue(isRunningOnEmulator()); + // MediaCodec returns a segmentation fault fails at this SDK level on emulators. + assumeFalse(Util.SDK_INT == 26); + MediaItem mediaItem = + new MediaItem.Builder() + .setUri("asset:///media/mp4/crow_emulator_transformer_output.mp4") + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setStartPositionMs(500) + .setEndPositionMs(2500) + .build()) + .build(); + ImmutableList videoEffects = ImmutableList.of(Presentation.createForHeight(480)); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(mediaItem) + .setEffects(new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects)) + .build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, editedMediaItem); + + assertThat(result.exportResult.optimizationResult).isEqualTo(OPTIMIZATION_ABANDONED); + } + @Test public void videoEncoderFormatUnsupported_completesWithError() throws Exception { String testId = "videoEncoderFormatUnsupported_completesWithError"; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Mp4MetadataInfo.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Mp4MetadataInfo.java index b0c1de5119..a3e06bdebe 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Mp4MetadataInfo.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Mp4MetadataInfo.java @@ -64,15 +64,20 @@ import org.checkerframework.checker.nullness.qual.Nullable; /** The video {@link Format} or {@code null} if there is no video track. */ public final @Nullable Format videoFormat; + /** The audio {@link Format} or {@code null} if there is no audio track. */ + public final @Nullable Format audioFormat; + private Mp4MetadataInfo( long durationUs, long lastSyncSampleTimestampUs, long firstSyncSampleTimestampUsAfterTimeUs, - @Nullable Format videoFormat) { + @Nullable Format videoFormat, + @Nullable Format audioFormat) { this.durationUs = durationUs; this.lastSyncSampleTimestampUs = lastSyncSampleTimestampUs; this.firstSyncSampleTimestampUsAfterTimeUs = firstSyncSampleTimestampUsAfterTimeUs; this.videoFormat = videoFormat; + this.audioFormat = audioFormat; } /** @@ -129,11 +134,11 @@ import org.checkerframework.checker.nullness.qual.Nullable; throw new IllegalStateException("The MP4 file is invalid"); } } + long durationUs = mp4Extractor.getDurationUs(); long lastSyncSampleTimestampUs = C.TIME_UNSET; long firstSyncSampleTimestampUsAfterTimeUs = C.TIME_UNSET; @Nullable Format videoFormat = null; - if (extractorOutput.videoTrackId != C.INDEX_UNSET) { ExtractorOutputImpl.TrackOutputImpl videoTrackOutput = checkNotNull(extractorOutput.trackTypeToTrackOutput.get(C.TRACK_TYPE_VIDEO)); @@ -155,11 +160,20 @@ import org.checkerframework.checker.nullness.qual.Nullable; } } } + + @Nullable Format audioFormat = null; + if (extractorOutput.audioTrackId != C.INDEX_UNSET) { + ExtractorOutputImpl.TrackOutputImpl audioTrackOutput = + checkNotNull(extractorOutput.trackTypeToTrackOutput.get(C.TRACK_TYPE_AUDIO)); + audioFormat = checkNotNull(audioTrackOutput.format); + } + return new Mp4MetadataInfo( durationUs, lastSyncSampleTimestampUs, firstSyncSampleTimestampUsAfterTimeUs, - videoFormat); + videoFormat, + audioFormat); } finally { DataSourceUtil.closeQuietly(dataSource); mp4Extractor.release(); @@ -168,12 +182,14 @@ import org.checkerframework.checker.nullness.qual.Nullable; private static final class ExtractorOutputImpl implements ExtractorOutput { public int videoTrackId; + public int audioTrackId; public boolean seekMapInitialized; final Map trackTypeToTrackOutput; public ExtractorOutputImpl() { videoTrackId = C.INDEX_UNSET; + audioTrackId = C.INDEX_UNSET; trackTypeToTrackOutput = new HashMap<>(); } @@ -181,6 +197,8 @@ import org.checkerframework.checker.nullness.qual.Nullable; public TrackOutput track(int id, @C.TrackType int type) { if (type == C.TRACK_TYPE_VIDEO) { videoTrackId = id; + } else if (type == C.TRACK_TYPE_AUDIO) { + audioTrackId = id; } @Nullable TrackOutputImpl trackOutput = trackTypeToTrackOutput.get(type); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 5e2d265440..2e5711d262 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -22,6 +22,8 @@ import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.transformer.Composition.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR; import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED; import static androidx.media3.transformer.ExportResult.OPTIMIZATION_FAILED_EXTRACTION_FAILED; +import static androidx.media3.transformer.TransformerUtil.shouldTranscodeAudio; +import static androidx.media3.transformer.TransformerUtil.shouldTranscodeVideo; import static androidx.media3.transformer.TransmuxTranscodeHelper.buildNewCompositionWithClipTimes; import static java.lang.annotation.ElementType.TYPE_USE; @@ -301,14 +303,13 @@ public final class Transformer { *
  • Progress updates will be unavailable. * * - *

    {@link ExportResult#optimizationResult} will indicate whether the optimization was - * applied. - * *

    This process relies on the given {@linkplain #setEncoderFactory EncoderFactory} providing * the right encoder level and profiles when transcoding, so that the transcoded and transmuxed * segments of the file can be stitched together. If the file segments can't be stitched - * together, the {@linkplain #start(Composition, String) export operation} will throw an - * exception. + * together, Transformer throw away any progress and proceed with unoptimized export instead. + * + *

    The {@link ExportResult#optimizationResult} will indicate whether the optimization was + * applied. * * @param enabled Whether to enable trim optimization. * @return This builder. @@ -1257,7 +1258,32 @@ public final class Transformer { processFullInput(); return; } - + remuxingMuxerWrapper = + new MuxerWrapper( + checkNotNull(outputFilePath), + muxerFactory, + componentListener, + MuxerWrapper.MUXER_MODE_MUX_PARTIAL); + if (shouldTranscodeVideo( + checkNotNull(mp4MetadataInfo.videoFormat), + composition, + /* sequenceIndex= */ 0, + transformationRequest, + encoderFactory, + remuxingMuxerWrapper) + || (mp4MetadataInfo.audioFormat != null + && shouldTranscodeAudio( + mp4MetadataInfo.audioFormat, + composition, + /* sequenceIndex= */ 0, + transformationRequest, + encoderFactory, + remuxingMuxerWrapper))) { + remuxingMuxerWrapper = null; + exportResultBuilder.setOptimizationResult(OPTIMIZATION_ABANDONED); + processFullInput(); + return; + } Transformer.this.mp4MetadataInfo = mp4MetadataInfo; Composition trancodeComposition = buildNewCompositionWithClipTimes( @@ -1267,17 +1293,9 @@ public final class Transformer { mp4MetadataInfo.durationUs, /* startsAtKeyFrame= */ false); - // TODO: b/304476154 - Check for cases where we shouldTranscode anyway and proceed with - // normal export instead. - remuxingMuxerWrapper = - new MuxerWrapper( - checkNotNull(outputFilePath), - muxerFactory, - componentListener, - MuxerWrapper.MUXER_MODE_MUX_PARTIAL); startInternal( trancodeComposition, - remuxingMuxerWrapper, + checkNotNull(remuxingMuxerWrapper), componentListener, /* initialTimestampOffsetUs= */ 0); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index 946ed61862..3f62e9a938 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -45,6 +45,7 @@ import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Format; +import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.Clock; @@ -692,12 +693,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } else if (trackType == C.TRACK_TYPE_VIDEO) { shouldTranscode = shouldTranscodeVideo( - inputFormat, - composition, - sequenceIndex, - transformationRequest, - encoderFactory, - muxerWrapper); + inputFormat, + composition, + sequenceIndex, + transformationRequest, + encoderFactory, + muxerWrapper) + || clippingRequiresTranscode(firstEditedMediaItem.mediaItem); } checkState(!shouldTranscode || assetLoaderCanOutputDecoded); @@ -706,6 +708,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } + private static boolean clippingRequiresTranscode(MediaItem mediaItem) { + return mediaItem.clippingConfiguration.startPositionMs > 0 + && !mediaItem.clippingConfiguration.startsAtKeyFrame; + } + /** Tracks the inputs and outputs of {@link AssetLoader AssetLoaders}. */ private static final class AssetLoaderInputTracker { private final List sequencesMetadata; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java index db065f5960..1c7fb37d3e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java @@ -125,10 +125,6 @@ import com.google.common.collect.ImmutableList; } EditedMediaItem firstEditedMediaItem = composition.sequences.get(sequenceIndex).editedMediaItems.get(0); - if (firstEditedMediaItem.mediaItem.clippingConfiguration.startPositionMs > 0 - && !firstEditedMediaItem.mediaItem.clippingConfiguration.startsAtKeyFrame) { - return true; - } if (encoderFactory.videoNeedsEncoding()) { return true; } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/Mp4MetadataInfoTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/Mp4MetadataInfoTest.java index f1dc18fcf2..330f664c88 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/Mp4MetadataInfoTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/Mp4MetadataInfoTest.java @@ -15,6 +15,8 @@ */ package androidx.media3.transformer; +import static androidx.media3.common.MimeTypes.AUDIO_AAC; +import static androidx.media3.common.MimeTypes.VIDEO_H264; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; @@ -125,7 +127,7 @@ public class Mp4MetadataInfoTest { } @Test - public void videoFormat_outputsFormatObjectWithCorrectInitializationData() throws IOException { + public void videoFormat_outputsFormatObjectWithCorrectRelevantFormatData() throws IOException { String mp4FilePath = "asset:///media/mp4/sample.mp4"; Mp4MetadataInfo mp4MetadataInfo = Mp4MetadataInfo.create(context, mp4FilePath); byte[] expectedCsd0 = { @@ -139,6 +141,11 @@ public class Mp4MetadataInfoTest { assertThat(actualFormat).isNotNull(); assertThat(actualFormat.initializationData.get(0)).isEqualTo(expectedCsd0); assertThat(actualFormat.initializationData.get(1)).isEqualTo(expectedCsd1); + assertThat(actualFormat.sampleMimeType).isEqualTo(VIDEO_H264); + assertThat(actualFormat.width).isEqualTo(1080); + assertThat(actualFormat.height).isEqualTo(720); + assertThat(actualFormat.rotationDegrees).isEqualTo(0); + assertThat(actualFormat.pixelWidthHeightRatio).isEqualTo(1); } @Test @@ -148,4 +155,27 @@ public class Mp4MetadataInfoTest { assertThat(mp4MetadataInfo.videoFormat).isNull(); } + + @Test + public void audioFormat_outputsFormatObjectWithCorrectRelevantFormatData() throws IOException { + String mp4FilePath = "asset:///media/mp4/sample.mp4"; + Mp4MetadataInfo mp4MetadataInfo = Mp4MetadataInfo.create(context, mp4FilePath); + byte[] expectedCsd0 = {18, 8}; + + Format actualFormat = mp4MetadataInfo.audioFormat; + + assertThat(actualFormat).isNotNull(); + assertThat(actualFormat.sampleMimeType).isEqualTo(AUDIO_AAC); + assertThat(actualFormat.channelCount).isEqualTo(1); + assertThat(actualFormat.sampleRate).isEqualTo(44100); + assertThat(actualFormat.initializationData.get(0)).isEqualTo(expectedCsd0); + } + + @Test + public void audioFormat_videoOnlyMp4File_outputsNull() throws IOException { + String mp4FilePath = "asset:///media/mp4/sample_18byte_nclx_colr.mp4"; + Mp4MetadataInfo mp4MetadataInfo = Mp4MetadataInfo.create(context, mp4FilePath); + + assertThat(mp4MetadataInfo.audioFormat).isNull(); + } }