From c7b4ec4d6514b18bec2e8d1fd29957b52eba4fa2 Mon Sep 17 00:00:00 2001
From: claincly <claincly@google.com>
Date: Fri, 24 Feb 2023 16:56:42 +0000
Subject: [PATCH] Allow video format change.

Uses the first mediaItem's format as the output format.

If there is `Presentation` supplied in the `Composition.effects`, add it as the
last effect of the first EditedMediaItem.

PiperOrigin-RevId: 512082659
---
 .../effect/FinalShaderProgramWrapper.java     |  4 +-
 .../media3/transformer/Transformer.java       | 16 +++++--
 .../transformer/TransformerInternal.java      | 15 ++++++-
 .../transformer/VideoSamplePipeline.java      | 43 +++++++++++--------
 4 files changed, 55 insertions(+), 23 deletions(-)

diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
index a02b9016b3..d714d0d953 100644
--- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
+++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
@@ -321,6 +321,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
   private synchronized boolean ensureConfigured(int inputWidth, int inputHeight)
       throws VideoFrameProcessingException, GlUtil.GlException {
 
+    boolean inputSizeChanged = false;
     if (this.inputWidth != inputWidth
         || this.inputHeight != inputHeight
         || this.outputSizeBeforeSurfaceTransformation == null) {
@@ -337,6 +338,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
                     outputSizeBeforeSurfaceTransformation.getWidth(),
                     outputSizeBeforeSurfaceTransformation.getHeight()));
       }
+      inputSizeChanged = true;
     }
 
     if (outputSurfaceInfo == null) {
@@ -372,7 +374,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       this.debugSurfaceView = debugSurfaceView;
     }
 
-    if (defaultShaderProgram != null && outputSizeOrRotationChanged) {
+    if (defaultShaderProgram != null && (outputSizeOrRotationChanged || inputSizeChanged)) {
       defaultShaderProgram.release();
       defaultShaderProgram = null;
       outputSizeOrRotationChanged = false;
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
index eb0a8fa377..b7e8baee75 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
@@ -39,6 +39,7 @@ import androidx.media3.common.util.ListenerSet;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
 import androidx.media3.effect.DefaultVideoFrameProcessor;
+import androidx.media3.effect.Presentation;
 import androidx.media3.exoplayer.source.DefaultMediaSourceFactory;
 import com.google.common.collect.ImmutableList;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
@@ -653,8 +654,12 @@ public final class Transformer {
    *
    * <ul>
    *   <li>The {@link Composition} must contain exactly one {@link Composition#sequences
-   *       EditedMediaItemSequence} and its {@link Composition#effects Effects} must be {@linkplain
-   *       Effects#EMPTY empty}.
+   *       EditedMediaItemSequence}. Its {@link Composition#effects Effects} must
+   *       <ul>
+   *         <li>Contain no {@linkplain Effects#audioProcessors audio effects}.
+   *         <li>Either contain no {@linkplain Effects#videoEffects video effects}, or exactly one
+   *             {@link Presentation}.
+   *       </ul>
    *   <li>The {@link EditedMediaItem} instances in the {@link EditedMediaItemSequence} must:
    *       <ul>
    *         <li>have identical tracks of the same format (after {@linkplain
@@ -690,7 +695,12 @@ public final class Transformer {
    */
   public void start(Composition composition, String path) {
     checkArgument(composition.sequences.size() == 1);
-    checkArgument(composition.effects == Effects.EMPTY);
+    checkArgument(composition.effects.audioProcessors.isEmpty());
+    // Only supports Presentation in video effects.
+    ImmutableList<Effect> videoEffects = composition.effects.videoEffects;
+    checkArgument(
+        videoEffects.isEmpty()
+            || (videoEffects.size() == 1 && videoEffects.get(0) instanceof Presentation));
     verifyApplicationThread();
     checkState(transformerInternal == null, "There is already an export in progress.");
 
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
index 4ee5dd01f2..4e2d727be4 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
@@ -41,6 +41,7 @@ import androidx.media3.common.util.Clock;
 import androidx.media3.common.util.ConditionVariable;
 import androidx.media3.common.util.HandlerWrapper;
 import androidx.media3.effect.GlEffect;
+import androidx.media3.effect.Presentation;
 import androidx.media3.extractor.metadata.mp4.SlowMotionData;
 import com.google.common.collect.ImmutableList;
 import java.lang.annotation.Documented;
@@ -131,9 +132,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     internalHandlerThread.start();
     Looper internalLooper = internalHandlerThread.getLooper();
     EditedMediaItemSequence sequence = composition.sequences.get(0);
+    ImmutableList<Effect> compositionVideoEffects = composition.effects.videoEffects;
+    @Nullable
+    Presentation presentation =
+        compositionVideoEffects.isEmpty() ? null : (Presentation) compositionVideoEffects.get(0);
     ComponentListener componentListener =
         new ComponentListener(
-            sequence, composition.transmuxAudio, composition.transmuxVideo, fallbackListener);
+            sequence,
+            presentation,
+            composition.transmuxAudio,
+            composition.transmuxVideo,
+            fallbackListener);
     compositeAssetLoader =
         new CompositeAssetLoader(
             sequence,
@@ -313,6 +322,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
     // The first EditedMediaItem in the sequence determines which SamplePipeline to use.
     private final EditedMediaItem firstEditedMediaItem;
+    @Nullable private final Presentation compositionPresentation;
     private final int mediaItemCount;
     private final boolean transmuxAudio;
     private final boolean transmuxVideo;
@@ -323,10 +333,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
     public ComponentListener(
         EditedMediaItemSequence sequence,
+        @Nullable Presentation compositionPresentation,
         boolean transmuxAudio,
         boolean transmuxVideo,
         FallbackListener fallbackListener) {
       firstEditedMediaItem = sequence.editedMediaItems.get(0);
+      this.compositionPresentation = compositionPresentation;
       mediaItemCount = sequence.editedMediaItems.size();
       this.transmuxAudio = transmuxAudio;
       this.transmuxVideo = transmuxVideo;
@@ -458,6 +470,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
               streamOffsetUs,
               transformationRequest,
               firstEditedMediaItem.effects.videoEffects,
+              compositionPresentation,
               firstEditedMediaItem.effects.videoFrameProcessorFactory,
               encoderFactory,
               muxerWrapper,
diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java
index ce0f38f51d..c6c08cff98 100644
--- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java
+++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java
@@ -41,11 +41,14 @@ import androidx.media3.common.VideoFrameProcessingException;
 import androidx.media3.common.VideoFrameProcessor;
 import androidx.media3.common.util.Consumer;
 import androidx.media3.common.util.Log;
+import androidx.media3.common.util.Size;
 import androidx.media3.common.util.Util;
 import androidx.media3.decoder.DecoderInputBuffer;
+import androidx.media3.effect.Presentation;
 import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.MoreExecutors;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicLong;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@@ -60,7 +63,6 @@ import org.checkerframework.dataflow.qual.Pure;
   private final AtomicLong mediaItemOffsetUs;
   private final VideoFrameProcessor videoFrameProcessor;
   private final ColorInfo videoFrameProcessorInputColor;
-  private final FrameInfo firstFrameInfo;
   private final EncoderWrapper encoderWrapper;
   private final DecoderInputBuffer encoderOutputBuffer;
 
@@ -77,6 +79,7 @@ import org.checkerframework.dataflow.qual.Pure;
       long streamOffsetUs,
       TransformationRequest transformationRequest,
       ImmutableList<Effect> effects,
+      @Nullable Presentation presentation,
       VideoFrameProcessor.Factory videoFrameProcessorFactory,
       Codec.EncoderFactory encoderFactory,
       MuxerWrapper muxerWrapper,
@@ -84,6 +87,7 @@ import org.checkerframework.dataflow.qual.Pure;
       FallbackListener fallbackListener,
       DebugViewProvider debugViewProvider)
       throws ExportException {
+    // TODO(b/262693177) Add tests for input format change.
     super(firstInputFormat, streamStartPositionUs, muxerWrapper);
 
     mediaItemOffsetUs = new AtomicLong();
@@ -119,11 +123,15 @@ import org.checkerframework.dataflow.qual.Pure;
                 .setColorTransfer(C.COLOR_TRANSFER_GAMMA_2_2)
                 .build()
             : encoderInputColor;
+    List<Effect> effectsWithPresentation = new ArrayList<>(effects);
+    if (presentation != null) {
+      effectsWithPresentation.add(presentation);
+    }
     try {
       videoFrameProcessor =
           videoFrameProcessorFactory.create(
               context,
-              effects,
+              effectsWithPresentation,
               debugViewProvider,
               videoFrameProcessorInputColor,
               videoFrameProcessorOutputColor,
@@ -171,20 +179,6 @@ import org.checkerframework.dataflow.qual.Pure;
       throw ExportException.createForVideoFrameProcessingException(
           e, ExportException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED);
     }
-    // The decoder rotates encoded frames for display by firstInputFormat.rotationDegrees.
-    int decodedWidth =
-        (firstInputFormat.rotationDegrees % 180 == 0)
-            ? firstInputFormat.width
-            : firstInputFormat.height;
-    int decodedHeight =
-        (firstInputFormat.rotationDegrees % 180 == 0)
-            ? firstInputFormat.height
-            : firstInputFormat.width;
-    firstFrameInfo =
-        new FrameInfo.Builder(decodedWidth, decodedHeight)
-            .setPixelWidthHeightRatio(firstInputFormat.pixelWidthHeightRatio)
-            .setStreamOffsetUs(streamOffsetUs)
-            .build();
   }
 
   @Override
@@ -193,8 +187,14 @@ import org.checkerframework.dataflow.qual.Pure;
       long durationUs,
       @Nullable Format trackFormat,
       boolean isLast) {
-    videoFrameProcessor.setInputFrameInfo(
-        new FrameInfo.Builder(firstFrameInfo).setOffsetToAddUs(mediaItemOffsetUs.get()).build());
+    if (trackFormat != null) {
+      Size decodedSize = getDecodedSize(trackFormat);
+      videoFrameProcessor.setInputFrameInfo(
+          new FrameInfo.Builder(decodedSize.getWidth(), decodedSize.getHeight())
+              .setPixelWidthHeightRatio(trackFormat.pixelWidthHeightRatio)
+              .setOffsetToAddUs(mediaItemOffsetUs.get())
+              .build());
+    }
     mediaItemOffsetUs.addAndGet(durationUs);
   }
 
@@ -314,6 +314,13 @@ import org.checkerframework.dataflow.qual.Pure;
     return supportedRequestBuilder.build();
   }
 
+  private static Size getDecodedSize(Format format) {
+    // The decoder rotates encoded frames for display by firstInputFormat.rotationDegrees.
+    int decodedWidth = (format.rotationDegrees % 180 == 0) ? format.width : format.height;
+    int decodedHeight = (format.rotationDegrees % 180 == 0) ? format.height : format.width;
+    return new Size(decodedWidth, decodedHeight);
+  }
+
   /**
    * Wraps an {@linkplain Codec encoder} and provides its input {@link Surface}.
    *