Abandon trim optimization when transcoding effects are set

PiperOrigin-RevId: 588839072

libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java

@@ -28,9 +28,11 @@ import static androidx.media3.transformer.AndroidTestUtil.PNG_ASSET_URI_STRING;
 import static androidx.media3.transformer.AndroidTestUtil.createOpenGlObjects;
 import static androidx.media3.transformer.AndroidTestUtil.generateTextureFromBitmap;
 import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped;
+import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED;
 import static androidx.media3.transformer.ExportResult.OPTIMIZATION_SUCCEEDED;
 import static com.google.common.truth.Truth.assertThat;
 import static org.junit.Assert.assertThrows;
+import static org.junit.Assume.assumeFalse;
 import static org.junit.Assume.assumeTrue;
 
 import android.content.Context;
@@ -478,6 +480,38 @@ public class TransformerEndToEndTest {
     assertThat(result.exportResult.durationMs).isAtMost(2000);
   }
 
+  @Test
+  public void videoEditing_trimOptimizationEnabled_fallbackToNormalExport() throws Exception {
+    String testId = "videoEditing_trimOptimizationEnabled_fallbackToNormalExport";
+    Transformer transformer =
+        new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build();
+    // The trim optimization is only guaranteed to work on emulator for this file.
+    assumeTrue(isRunningOnEmulator());
+    // MediaCodec returns a segmentation fault fails at this SDK level on emulators.
+    assumeFalse(Util.SDK_INT == 26);
+    MediaItem mediaItem =
+        new MediaItem.Builder()
+            .setUri("asset:///media/mp4/crow_emulator_transformer_output.mp4")
+            .setClippingConfiguration(
+                new MediaItem.ClippingConfiguration.Builder()
+                    .setStartPositionMs(500)
+                    .setEndPositionMs(2500)
+                    .build())
+            .build();
+    ImmutableList<Effect> videoEffects = ImmutableList.of(Presentation.createForHeight(480));
+    EditedMediaItem editedMediaItem =
+        new EditedMediaItem.Builder(mediaItem)
+            .setEffects(new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects))
+            .build();
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, transformer)
+            .build()
+            .run(testId, editedMediaItem);
+
+    assertThat(result.exportResult.optimizationResult).isEqualTo(OPTIMIZATION_ABANDONED);
+  }
+
   @Test
   public void videoEncoderFormatUnsupported_completesWithError() throws Exception {
     String testId = "videoEncoderFormatUnsupported_completesWithError";

libraries/transformer/src/main/java/androidx/media3/transformer/Mp4MetadataInfo.java

@@ -64,15 +64,20 @@ import org.checkerframework.checker.nullness.qual.Nullable;
   /** The video {@link Format} or {@code null} if there is no video track. */
   public final @Nullable Format videoFormat;
 
+  /** The audio {@link Format} or {@code null} if there is no audio track. */
+  public final @Nullable Format audioFormat;
+
   private Mp4MetadataInfo(
       long durationUs,
       long lastSyncSampleTimestampUs,
       long firstSyncSampleTimestampUsAfterTimeUs,
-      @Nullable Format videoFormat) {
+      @Nullable Format videoFormat,
+      @Nullable Format audioFormat) {
     this.durationUs = durationUs;
     this.lastSyncSampleTimestampUs = lastSyncSampleTimestampUs;
     this.firstSyncSampleTimestampUsAfterTimeUs = firstSyncSampleTimestampUsAfterTimeUs;
     this.videoFormat = videoFormat;
+    this.audioFormat = audioFormat;
   }
 
   /**
@@ -129,11 +134,11 @@ import org.checkerframework.checker.nullness.qual.Nullable;
           throw new IllegalStateException("The MP4 file is invalid");
         }
       }
+
       long durationUs = mp4Extractor.getDurationUs();
       long lastSyncSampleTimestampUs = C.TIME_UNSET;
       long firstSyncSampleTimestampUsAfterTimeUs = C.TIME_UNSET;
       @Nullable Format videoFormat = null;
-
       if (extractorOutput.videoTrackId != C.INDEX_UNSET) {
         ExtractorOutputImpl.TrackOutputImpl videoTrackOutput =
             checkNotNull(extractorOutput.trackTypeToTrackOutput.get(C.TRACK_TYPE_VIDEO));
@@ -155,11 +160,20 @@ import org.checkerframework.checker.nullness.qual.Nullable;
           }
         }
       }
+
+      @Nullable Format audioFormat = null;
+      if (extractorOutput.audioTrackId != C.INDEX_UNSET) {
+        ExtractorOutputImpl.TrackOutputImpl audioTrackOutput =
+            checkNotNull(extractorOutput.trackTypeToTrackOutput.get(C.TRACK_TYPE_AUDIO));
+        audioFormat = checkNotNull(audioTrackOutput.format);
+      }
+
       return new Mp4MetadataInfo(
           durationUs,
           lastSyncSampleTimestampUs,
           firstSyncSampleTimestampUsAfterTimeUs,
-          videoFormat);
+          videoFormat,
+          audioFormat);
     } finally {
       DataSourceUtil.closeQuietly(dataSource);
       mp4Extractor.release();
@@ -168,12 +182,14 @@ import org.checkerframework.checker.nullness.qual.Nullable;
 
   private static final class ExtractorOutputImpl implements ExtractorOutput {
     public int videoTrackId;
+    public int audioTrackId;
     public boolean seekMapInitialized;
 
     final Map<Integer, TrackOutputImpl> trackTypeToTrackOutput;
 
     public ExtractorOutputImpl() {
       videoTrackId = C.INDEX_UNSET;
+      audioTrackId = C.INDEX_UNSET;
       trackTypeToTrackOutput = new HashMap<>();
     }
 
@@ -181,6 +197,8 @@ import org.checkerframework.checker.nullness.qual.Nullable;
     public TrackOutput track(int id, @C.TrackType int type) {
       if (type == C.TRACK_TYPE_VIDEO) {
         videoTrackId = id;
+      } else if (type == C.TRACK_TYPE_AUDIO) {
+        audioTrackId = id;
       }
 
       @Nullable TrackOutputImpl trackOutput = trackTypeToTrackOutput.get(type);

libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java

@@ -22,6 +22,8 @@ import static androidx.media3.common.util.Assertions.checkState;
 import static androidx.media3.transformer.Composition.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR;
 import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED;
 import static androidx.media3.transformer.ExportResult.OPTIMIZATION_FAILED_EXTRACTION_FAILED;
+import static androidx.media3.transformer.TransformerUtil.shouldTranscodeAudio;
+import static androidx.media3.transformer.TransformerUtil.shouldTranscodeVideo;
 import static androidx.media3.transformer.TransmuxTranscodeHelper.buildNewCompositionWithClipTimes;
 import static java.lang.annotation.ElementType.TYPE_USE;
 
@@ -301,14 +303,13 @@ public final class Transformer {
      *   <li>Progress updates will be unavailable.
      * </ul>
      *
-     * <p>{@link ExportResult#optimizationResult} will indicate whether the optimization was
-     * applied.
-     *
      * <p>This process relies on the given {@linkplain #setEncoderFactory EncoderFactory} providing
      * the right encoder level and profiles when transcoding, so that the transcoded and transmuxed
      * segments of the file can be stitched together. If the file segments can't be stitched
-     * together, the {@linkplain #start(Composition, String) export operation} will throw an
+     * together, Transformer throw away any progress and proceed with unoptimized export instead.
-     * exception.
+     *
+     * <p>The {@link ExportResult#optimizationResult} will indicate whether the optimization was
+     * applied.
      *
      * @param enabled Whether to enable trim optimization.
      * @return This builder.
@@ -1257,7 +1258,32 @@ public final class Transformer {
               processFullInput();
               return;
             }
-
+            remuxingMuxerWrapper =
+                new MuxerWrapper(
+                    checkNotNull(outputFilePath),
+                    muxerFactory,
+                    componentListener,
+                    MuxerWrapper.MUXER_MODE_MUX_PARTIAL);
+            if (shouldTranscodeVideo(
+                    checkNotNull(mp4MetadataInfo.videoFormat),
+                    composition,
+                    /* sequenceIndex= */ 0,
+                    transformationRequest,
+                    encoderFactory,
+                    remuxingMuxerWrapper)
+                || (mp4MetadataInfo.audioFormat != null
+                    && shouldTranscodeAudio(
+                        mp4MetadataInfo.audioFormat,
+                        composition,
+                        /* sequenceIndex= */ 0,
+                        transformationRequest,
+                        encoderFactory,
+                        remuxingMuxerWrapper))) {
+              remuxingMuxerWrapper = null;
+              exportResultBuilder.setOptimizationResult(OPTIMIZATION_ABANDONED);
+              processFullInput();
+              return;
+            }
             Transformer.this.mp4MetadataInfo = mp4MetadataInfo;
             Composition trancodeComposition =
                 buildNewCompositionWithClipTimes(
@@ -1267,17 +1293,9 @@ public final class Transformer {
                     mp4MetadataInfo.durationUs,
                     /* startsAtKeyFrame= */ false);
 
-            // TODO: b/304476154 - Check for cases where we shouldTranscode anyway and proceed with
-            //  normal export instead.
-            remuxingMuxerWrapper =
-                new MuxerWrapper(
-                    checkNotNull(outputFilePath),
-                    muxerFactory,
-                    componentListener,
-                    MuxerWrapper.MUXER_MODE_MUX_PARTIAL);
             startInternal(
                 trancodeComposition,
-                remuxingMuxerWrapper,
+                checkNotNull(remuxingMuxerWrapper),
                 componentListener,
                 /* initialTimestampOffsetUs= */ 0);
           }

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java

@@ -45,6 +45,7 @@ import androidx.annotation.Nullable;
 import androidx.media3.common.C;
 import androidx.media3.common.DebugViewProvider;
 import androidx.media3.common.Format;
+import androidx.media3.common.MediaItem;
 import androidx.media3.common.MimeTypes;
 import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.util.Clock;
@@ -692,12 +693,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       } else if (trackType == C.TRACK_TYPE_VIDEO) {
         shouldTranscode =
             shouldTranscodeVideo(
-                inputFormat,
+                    inputFormat,
-                composition,
+                    composition,
-                sequenceIndex,
+                    sequenceIndex,
-                transformationRequest,
+                    transformationRequest,
-                encoderFactory,
+                    encoderFactory,
-                muxerWrapper);
+                    muxerWrapper)
+                || clippingRequiresTranscode(firstEditedMediaItem.mediaItem);
       }
 
       checkState(!shouldTranscode || assetLoaderCanOutputDecoded);
@@ -706,6 +708,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     }
   }
 
+  private static boolean clippingRequiresTranscode(MediaItem mediaItem) {
+    return mediaItem.clippingConfiguration.startPositionMs > 0
+        && !mediaItem.clippingConfiguration.startsAtKeyFrame;
+  }
+
   /** Tracks the inputs and outputs of {@link AssetLoader AssetLoaders}. */
   private static final class AssetLoaderInputTracker {
     private final List<SequenceMetadata> sequencesMetadata;

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java

@@ -125,10 +125,6 @@ import com.google.common.collect.ImmutableList;
     }
     EditedMediaItem firstEditedMediaItem =
         composition.sequences.get(sequenceIndex).editedMediaItems.get(0);
-    if (firstEditedMediaItem.mediaItem.clippingConfiguration.startPositionMs > 0
-        && !firstEditedMediaItem.mediaItem.clippingConfiguration.startsAtKeyFrame) {
-      return true;
-    }
     if (encoderFactory.videoNeedsEncoding()) {
       return true;
     }

libraries/transformer/src/test/java/androidx/media3/transformer/Mp4MetadataInfoTest.java

@@ -15,6 +15,8 @@
  */
 package androidx.media3.transformer;
 
+import static androidx.media3.common.MimeTypes.AUDIO_AAC;
+import static androidx.media3.common.MimeTypes.VIDEO_H264;
 import static com.google.common.truth.Truth.assertThat;
 import static org.junit.Assert.assertThrows;
 
@@ -125,7 +127,7 @@ public class Mp4MetadataInfoTest {
   }
 
   @Test
-  public void videoFormat_outputsFormatObjectWithCorrectInitializationData() throws IOException {
+  public void videoFormat_outputsFormatObjectWithCorrectRelevantFormatData() throws IOException {
     String mp4FilePath = "asset:///media/mp4/sample.mp4";
     Mp4MetadataInfo mp4MetadataInfo = Mp4MetadataInfo.create(context, mp4FilePath);
     byte[] expectedCsd0 = {
@@ -139,6 +141,11 @@ public class Mp4MetadataInfoTest {
     assertThat(actualFormat).isNotNull();
     assertThat(actualFormat.initializationData.get(0)).isEqualTo(expectedCsd0);
     assertThat(actualFormat.initializationData.get(1)).isEqualTo(expectedCsd1);
+    assertThat(actualFormat.sampleMimeType).isEqualTo(VIDEO_H264);
+    assertThat(actualFormat.width).isEqualTo(1080);
+    assertThat(actualFormat.height).isEqualTo(720);
+    assertThat(actualFormat.rotationDegrees).isEqualTo(0);
+    assertThat(actualFormat.pixelWidthHeightRatio).isEqualTo(1);
   }
 
   @Test
@@ -148,4 +155,27 @@ public class Mp4MetadataInfoTest {
 
     assertThat(mp4MetadataInfo.videoFormat).isNull();
   }
+
+  @Test
+  public void audioFormat_outputsFormatObjectWithCorrectRelevantFormatData() throws IOException {
+    String mp4FilePath = "asset:///media/mp4/sample.mp4";
+    Mp4MetadataInfo mp4MetadataInfo = Mp4MetadataInfo.create(context, mp4FilePath);
+    byte[] expectedCsd0 = {18, 8};
+
+    Format actualFormat = mp4MetadataInfo.audioFormat;
+
+    assertThat(actualFormat).isNotNull();
+    assertThat(actualFormat.sampleMimeType).isEqualTo(AUDIO_AAC);
+    assertThat(actualFormat.channelCount).isEqualTo(1);
+    assertThat(actualFormat.sampleRate).isEqualTo(44100);
+    assertThat(actualFormat.initializationData.get(0)).isEqualTo(expectedCsd0);
+  }
+
+  @Test
+  public void audioFormat_videoOnlyMp4File_outputsNull() throws IOException {
+    String mp4FilePath = "asset:///media/mp4/sample_18byte_nclx_colr.mp4";
+    Mp4MetadataInfo mp4MetadataInfo = Mp4MetadataInfo.create(context, mp4FilePath);
+
+    assertThat(mp4MetadataInfo.audioFormat).isNull();
+  }
 }