Communicate sample MIME type changes to FallbackListener.

We may fall back to a different sample MIME type because
a) the sample MIME type inferred from the input is not supported
by the muxer or b) no encoders are available for the the requested
sample MIME type.

PiperOrigin-RevId: 422849036

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java

@@ -29,7 +29,9 @@ import com.google.android.exoplayer2.audio.AudioProcessor;
 import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
 import com.google.android.exoplayer2.audio.SonicAudioProcessor;
 import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
+import com.google.android.exoplayer2.util.Util;
 import java.nio.ByteBuffer;
+import org.checkerframework.dataflow.qual.Pure;
 
 /**
  * Pipeline to decode audio samples, apply transformations on the raw samples, and re-encode them.
@@ -39,13 +41,12 @@ import java.nio.ByteBuffer;
   private static final String TAG = "AudioSamplePipeline";
   private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024;
 
-  private final TransformationRequest transformationRequest;
-
   private final Codec decoder;
   private final DecoderInputBuffer decoderInputBuffer;
 
   private final SonicAudioProcessor sonicAudioProcessor;
   private final SpeedProvider speedProvider;
+  private final boolean flattenForSlowMotion;
 
   private final Codec encoder;
   private final AudioFormat encoderInputAudioFormat;
@@ -63,9 +64,9 @@ import java.nio.ByteBuffer;
       Format inputFormat,
       TransformationRequest transformationRequest,
       Codec.EncoderFactory encoderFactory,
-      Codec.DecoderFactory decoderFactory)
+      Codec.DecoderFactory decoderFactory,
+      FallbackListener fallbackListener)
       throws TransformationException {
-    this.transformationRequest = transformationRequest;
     decoderInputBuffer =
         new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
     encoderInputBuffer =
@@ -75,6 +76,7 @@ import java.nio.ByteBuffer;
 
     this.decoder = decoderFactory.createForAudioDecoding(inputFormat);
 
+    this.flattenForSlowMotion = transformationRequest.flattenForSlowMotion;
     sonicAudioProcessor = new SonicAudioProcessor();
     sonicOutputBuffer = AudioProcessor.EMPTY_BUFFER;
     speedProvider = new SegmentSpeedProvider(inputFormat);
@@ -86,7 +88,7 @@ import java.nio.ByteBuffer;
             // The decoder uses ENCODING_PCM_16BIT by default.
             // https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers
             C.ENCODING_PCM_16BIT);
-    if (transformationRequest.flattenForSlowMotion) {
+    if (flattenForSlowMotion) {
       try {
         encoderInputAudioFormat = sonicAudioProcessor.configure(encoderInputAudioFormat);
       } catch (AudioProcessor.UnhandledAudioFormatException impossible) {
@@ -96,19 +98,23 @@ import java.nio.ByteBuffer;
       sonicAudioProcessor.setPitch(currentSpeed);
       sonicAudioProcessor.flush();
     }
-
-    encoder =
-        encoderFactory.createForAudioEncoding(
-            new Format.Builder()
-                .setSampleMimeType(
-                    transformationRequest.audioMimeType == null
-                        ? inputFormat.sampleMimeType
-                        : transformationRequest.audioMimeType)
-                .setSampleRate(encoderInputAudioFormat.sampleRate)
-                .setChannelCount(encoderInputAudioFormat.channelCount)
-                .setAverageBitrate(DEFAULT_ENCODER_BITRATE)
-                .build());
     this.encoderInputAudioFormat = encoderInputAudioFormat;
+
+    Format requestedOutputFormat =
+        new Format.Builder()
+            .setSampleMimeType(
+                transformationRequest.audioMimeType == null
+                    ? inputFormat.sampleMimeType
+                    : transformationRequest.audioMimeType)
+            .setSampleRate(encoderInputAudioFormat.sampleRate)
+            .setChannelCount(encoderInputAudioFormat.channelCount)
+            .setAverageBitrate(DEFAULT_ENCODER_BITRATE)
+            .build();
+    encoder = encoderFactory.createForAudioEncoding(requestedOutputFormat);
+
+    fallbackListener.onTransformationRequestFinalized(
+        createFallbackRequest(
+            transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat()));
   }
 
   @Override
@@ -293,7 +299,7 @@ import java.nio.ByteBuffer;
   }
 
   private boolean isSpeedChanging(BufferInfo bufferInfo) {
-    if (!transformationRequest.flattenForSlowMotion) {
+    if (!flattenForSlowMotion) {
       return false;
     }
     float newSpeed = speedProvider.getSpeed(bufferInfo.presentationTimeUs);
@@ -325,4 +331,15 @@ import java.nio.ByteBuffer;
     }
     nextEncoderInputBufferTimeUs += bufferDurationUs;
   }
+
+  @Pure
+  private static TransformationRequest createFallbackRequest(
+      TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) {
+    // TODO(b/210591626): Also update bitrate and other params once encoder configuration and
+    // fallback are implemented.
+    if (Util.areEqual(requestedFormat.sampleMimeType, actualFormat.sampleMimeType)) {
+      return transformationRequest;
+    }
+    return transformationRequest.buildUpon().setAudioMimeType(actualFormat.sampleMimeType).build();
+  }
 }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java

@@ -28,10 +28,14 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
 
   private boolean hasPendingBuffer;
 
-  public PassthroughSamplePipeline(Format format) {
+  public PassthroughSamplePipeline(
+      Format format,
+      TransformationRequest transformationRequest,
+      FallbackListener fallbackListener) {
     this.format = format;
     buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT);
     hasPendingBuffer = false;
+    fallbackListener.onTransformationRequestFinalized(transformationRequest);
   }
 
   @Override

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java

@@ -703,6 +703,7 @@ public final class Transformer {
                     transformationRequest,
                     encoderFactory,
                     decoderFactory,
+                    new FallbackListener(mediaItem, listeners, transformationRequest),
                     debugViewProvider))
             .setMediaSourceFactory(mediaSourceFactory)
             .setTrackSelector(trackSelector)
@@ -805,6 +806,7 @@ public final class Transformer {
     private final TransformationRequest transformationRequest;
     private final Codec.EncoderFactory encoderFactory;
     private final Codec.DecoderFactory decoderFactory;
+    private final FallbackListener fallbackListener;
     private final Transformer.DebugViewProvider debugViewProvider;
 
     public TransformerRenderersFactory(
@@ -815,6 +817,7 @@ public final class Transformer {
         TransformationRequest transformationRequest,
         Codec.EncoderFactory encoderFactory,
         Codec.DecoderFactory decoderFactory,
+        FallbackListener fallbackListener,
         Transformer.DebugViewProvider debugViewProvider) {
       this.context = context;
       this.muxerWrapper = muxerWrapper;
@@ -823,6 +826,7 @@ public final class Transformer {
       this.transformationRequest = transformationRequest;
       this.encoderFactory = encoderFactory;
       this.decoderFactory = decoderFactory;
+      this.fallbackListener = fallbackListener;
       this.debugViewProvider = debugViewProvider;
       mediaClock = new TransformerMediaClock();
     }
@@ -840,7 +844,12 @@ public final class Transformer {
       if (!removeAudio) {
         renderers[index] =
             new TransformerAudioRenderer(
-                muxerWrapper, mediaClock, transformationRequest, encoderFactory, decoderFactory);
+                muxerWrapper,
+                mediaClock,
+                transformationRequest,
+                encoderFactory,
+                decoderFactory,
+                fallbackListener);
         index++;
       }
       if (!removeVideo) {
@@ -852,6 +861,7 @@ public final class Transformer {
                 transformationRequest,
                 encoderFactory,
                 decoderFactory,
+                fallbackListener,
                 debugViewProvider);
         index++;
       }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java

@@ -41,8 +41,9 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult;
       TransformerMediaClock mediaClock,
       TransformationRequest transformationRequest,
       Codec.EncoderFactory encoderFactory,
-      Codec.DecoderFactory decoderFactory) {
+      Codec.DecoderFactory decoderFactory,
-    super(C.TRACK_TYPE_AUDIO, muxerWrapper, mediaClock, transformationRequest);
+      FallbackListener fallbackListener) {
+    super(C.TRACK_TYPE_AUDIO, muxerWrapper, mediaClock, transformationRequest, fallbackListener);
     this.encoderFactory = encoderFactory;
     this.decoderFactory = decoderFactory;
     decoderInputBuffer =
@@ -78,11 +79,12 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult;
           TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
     }
     if (shouldPassthrough(inputFormat)) {
-      samplePipeline = new PassthroughSamplePipeline(inputFormat);
+      samplePipeline =
+          new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener);
     } else {
       samplePipeline =
           new AudioSamplePipeline(
-              inputFormat, transformationRequest, encoderFactory, decoderFactory);
+              inputFormat, transformationRequest, encoderFactory, decoderFactory, fallbackListener);
     }
     return true;
   }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java

@@ -39,6 +39,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   protected final MuxerWrapper muxerWrapper;
   protected final TransformerMediaClock mediaClock;
   protected final TransformationRequest transformationRequest;
+  protected final FallbackListener fallbackListener;
 
   protected boolean isRendererStarted;
   protected boolean muxerWrapperTrackAdded;
@@ -50,11 +51,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
       int trackType,
       MuxerWrapper muxerWrapper,
       TransformerMediaClock mediaClock,
-      TransformationRequest transformationRequest) {
+      TransformationRequest transformationRequest,
+      FallbackListener fallbackListener) {
     super(trackType);
     this.muxerWrapper = muxerWrapper;
     this.mediaClock = mediaClock;
     this.transformationRequest = transformationRequest;
+    this.fallbackListener = fallbackListener;
   }
 
   /**
@@ -112,6 +115,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   @Override
   protected final void onEnabled(boolean joining, boolean mayRenderStartOfStream) {
     muxerWrapper.registerTrack();
+    fallbackListener.registerTrack();
     mediaClock.updateTimeForTrackType(getTrackType(), 0L);
   }
 

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java

@@ -48,8 +48,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
       TransformationRequest transformationRequest,
       Codec.EncoderFactory encoderFactory,
       Codec.DecoderFactory decoderFactory,
+      FallbackListener fallbackListener,
       Transformer.DebugViewProvider debugViewProvider) {
-    super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest);
+    super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener);
     this.context = context;
     this.encoderFactory = encoderFactory;
     this.decoderFactory = decoderFactory;
@@ -87,7 +88,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
           TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
     }
     if (shouldPassthrough(inputFormat)) {
-      samplePipeline = new PassthroughSamplePipeline(inputFormat);
+      samplePipeline =
+          new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener);
     } else {
       samplePipeline =
           new VideoSamplePipeline(
@@ -96,6 +98,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
               transformationRequest,
               encoderFactory,
               decoderFactory,
+              fallbackListener,
               debugViewProvider);
     }
     if (transformationRequest.flattenForSlowMotion) {

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java

@@ -28,7 +28,9 @@ import androidx.annotation.RequiresApi;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
+import com.google.android.exoplayer2.util.Util;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
+import org.checkerframework.dataflow.qual.Pure;
 
 /**
  * Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them.
@@ -54,6 +56,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
       TransformationRequest transformationRequest,
       Codec.EncoderFactory encoderFactory,
       Codec.DecoderFactory decoderFactory,
+      FallbackListener fallbackListener,
       Transformer.DebugViewProvider debugViewProvider)
       throws TransformationException {
     decoderInputBuffer =
@@ -63,7 +66,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
     // Scale width and height to desired transformationRequest.outputHeight, preserving aspect
     // ratio.
-    // TODO(internal b/209781577): Think about which edge length should be set for portrait videos.
+    // TODO(b/209781577): Think about which edge length should be set for portrait videos.
     float inputFormatAspectRatio = (float) inputFormat.width / inputFormat.height;
     int outputWidth = inputFormat.width;
     int outputHeight = inputFormat.height;
@@ -102,17 +105,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     // postRotate in a later vertex shader.
     transformationMatrix.postRotate(outputRotationDegrees);
 
-    encoder =
+    Format requestedOutputFormat =
-        encoderFactory.createForVideoEncoding(
+        new Format.Builder()
-            new Format.Builder()
+            .setWidth(outputWidth)
-                .setWidth(outputWidth)
+            .setHeight(outputHeight)
-                .setHeight(outputHeight)
+            .setRotationDegrees(0)
-                .setRotationDegrees(0)
+            .setSampleMimeType(
-                .setSampleMimeType(
+                transformationRequest.videoMimeType != null
-                    transformationRequest.videoMimeType != null
+                    ? transformationRequest.videoMimeType
-                        ? transformationRequest.videoMimeType
+                    : inputFormat.sampleMimeType)
-                        : inputFormat.sampleMimeType)
+            .build();
-                .build());
+    encoder = encoderFactory.createForVideoEncoding(requestedOutputFormat);
+    fallbackListener.onTransformationRequestFinalized(
+        createFallbackRequest(
+            transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat()));
+
     if (inputFormat.height != outputHeight
         || inputFormat.width != outputWidth
         || !transformationMatrix.isIdentity()) {
@@ -261,4 +268,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     decoder.release();
     encoder.release();
   }
+
+  @Pure
+  private static TransformationRequest createFallbackRequest(
+      TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) {
+    // TODO(b/210591626): Also update resolution, bitrate etc. once encoder configuration and
+    // fallback are implemented.
+    if (Util.areEqual(requestedFormat.sampleMimeType, actualFormat.sampleMimeType)) {
+      return transformationRequest;
+    }
+    return transformationRequest.buildUpon().setVideoMimeType(actualFormat.sampleMimeType).build();
+  }
 }

library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerTest.java

@@ -65,6 +65,7 @@ import org.robolectric.shadows.ShadowMediaCodec;
 /** Unit test for {@link Transformer}. */
 @RunWith(AndroidJUnit4.class)
 public final class TransformerTest {
+  // TODO(b/214973843): Disable fallback for all tests that aren't specifically testing fallback.
 
   private static final String URI_PREFIX = "asset:///media/";
   private static final String FILE_VIDEO_ONLY = "mp4/sample_18byte_nclx_colr.mp4";