Improve format propagation in transformer

- Store output format in `MediaCodecAdapterWrapper` when we get a format from
  the codec, instead of creating it on demand.
- Make format building code not audio-specific.
- Remove `MediaCodecAdapterWrapper.getConfigFormat` and instead keep track of
  the input/output formats in the renderer. This will mean that the code still
  works if an audio processor changes the audio format in future.
- Make exceptions thrown during audio rendering use the same (input) renderer
  format.
- Misc other minor cleanup.

#minor-release

PiperOrigin-RevId: 354556619

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java

@@ -29,11 +29,12 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
 import com.google.android.exoplayer2.mediacodec.MediaCodecAdapter;
 import com.google.android.exoplayer2.mediacodec.MediaFormatUtil;
 import com.google.android.exoplayer2.mediacodec.SynchronousMediaCodecAdapter;
+import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.common.collect.ImmutableList;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
-import org.checkerframework.checker.nullness.qual.RequiresNonNull;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 /**
  * A wrapper around {@link MediaCodecAdapter}.
@@ -44,17 +45,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
  */
 /* package */ final class MediaCodecAdapterWrapper {
 
+  // MediaCodec decoders always output 16 bit PCM, unless configured to output PCM float.
+  // https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers.
+  private static final int MEDIA_CODEC_PCM_ENCODING = C.ENCODING_PCM_16BIT;
+
   private final BufferInfo outputBufferInfo;
   private final MediaCodecAdapter codec;
-  private final Format format;
 
+  private @MonotonicNonNull Format outputFormat;
   @Nullable private ByteBuffer outputBuffer;
 
   private int inputBufferIndex;
   private int outputBufferIndex;
   private boolean inputStreamEnded;
   private boolean outputStreamEnded;
-  private boolean hasOutputFormat;
 
   /**
    * Returns a {@link MediaCodecAdapterWrapper} for a configured and started {@link
@@ -65,12 +69,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
    * @return A configured and started decoder wrapper.
    * @throws IOException If the underlying codec cannot be created.
    */
-  @RequiresNonNull("#1.sampleMimeType")
   public static MediaCodecAdapterWrapper createForAudioDecoding(Format format) throws IOException {
     @Nullable MediaCodec decoder = null;
     @Nullable MediaCodecAdapter adapter = null;
     try {
-      decoder = MediaCodec.createDecoderByType(format.sampleMimeType);
+      decoder = MediaCodec.createDecoderByType(checkNotNull(format.sampleMimeType));
       MediaFormat mediaFormat =
           MediaFormat.createAudioFormat(
               format.sampleMimeType, format.sampleRate, format.channelCount);
@@ -78,7 +81,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
       adapter = new SynchronousMediaCodecAdapter.Factory().createAdapter(decoder);
       adapter.configure(mediaFormat, /* surface= */ null, /* crypto= */ null, /* flags= */ 0);
       adapter.start();
-      return new MediaCodecAdapterWrapper(adapter, format);
+      return new MediaCodecAdapterWrapper(adapter);
     } catch (Exception e) {
       if (adapter != null) {
         adapter.release();
@@ -98,12 +101,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
    * @return A configured and started encoder wrapper.
    * @throws IOException If the underlying codec cannot be created.
    */
-  @RequiresNonNull("#1.sampleMimeType")
   public static MediaCodecAdapterWrapper createForAudioEncoding(Format format) throws IOException {
     @Nullable MediaCodec encoder = null;
     @Nullable MediaCodecAdapter adapter = null;
     try {
-      encoder = MediaCodec.createEncoderByType(format.sampleMimeType);
+      encoder = MediaCodec.createEncoderByType(checkNotNull(format.sampleMimeType));
       MediaFormat mediaFormat =
           MediaFormat.createAudioFormat(
               format.sampleMimeType, format.sampleRate, format.channelCount);
@@ -115,7 +117,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
           /* crypto= */ null,
           /* flags= */ MediaCodec.CONFIGURE_FLAG_ENCODE);
       adapter.start();
-      return new MediaCodecAdapterWrapper(adapter, format);
+      return new MediaCodecAdapterWrapper(adapter);
     } catch (Exception e) {
       if (adapter != null) {
         adapter.release();
@@ -126,9 +128,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     }
   }
 
-  private MediaCodecAdapterWrapper(MediaCodecAdapter codec, Format format) {
+  private MediaCodecAdapterWrapper(MediaCodecAdapter codec) {
     this.codec = codec;
-    this.format = format;
     outputBufferInfo = new BufferInfo();
     inputBufferIndex = C.INDEX_UNSET;
     outputBufferIndex = C.INDEX_UNSET;
@@ -202,8 +203,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
 
     outputBufferIndex = codec.dequeueOutputBufferIndex(outputBufferInfo);
     if (outputBufferIndex < 0) {
-      if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED && !hasOutputFormat) {
+      if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
-        hasOutputFormat = true;
+        outputFormat = getFormat(codec.getOutputFormat());
       }
       return false;
     }
@@ -228,41 +229,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     return true;
   }
 
-  /**
+  /** Returns the current output format, if available. */
-   * Returns a {@link Format} based on the {@link MediaCodecAdapter#getOutputFormat() mediaFormat},
-   * if available.
-   */
   @Nullable
   public Format getOutputFormat() {
-    @Nullable MediaFormat mediaFormat = hasOutputFormat ? codec.getOutputFormat() : null;
+    return outputFormat;
-    if (mediaFormat == null) {
-      return null;
-    }
-
-    ImmutableList.Builder<byte[]> csdBuffers = new ImmutableList.Builder<>();
-    int csdIndex = 0;
-    while (true) {
-      @Nullable ByteBuffer csdByteBuffer = mediaFormat.getByteBuffer("csd-" + csdIndex);
-      if (csdByteBuffer == null) {
-        break;
-      }
-      byte[] csdBufferData = new byte[csdByteBuffer.remaining()];
-      csdByteBuffer.get(csdBufferData);
-      csdBuffers.add(csdBufferData);
-      csdIndex++;
-    }
-
-    return new Format.Builder()
-        .setSampleMimeType(mediaFormat.getString(MediaFormat.KEY_MIME))
-        .setChannelCount(mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT))
-        .setSampleRate(mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE))
-        .setInitializationData(csdBuffers.build())
-        .build();
-  }
-
-  /** Returns the {@link Format} used to create and configure the underlying {@link MediaCodec}. */
-  public Format getConfigFormat() {
-    return format;
   }
 
   /** Returns the current output {@link ByteBuffer}, if available. */
@@ -299,4 +269,37 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     outputBuffer = null;
     codec.release();
   }
+
+  private static Format getFormat(MediaFormat mediaFormat) {
+    ImmutableList.Builder<byte[]> csdBuffers = new ImmutableList.Builder<>();
+    int csdIndex = 0;
+    while (true) {
+      @Nullable ByteBuffer csdByteBuffer = mediaFormat.getByteBuffer("csd-" + csdIndex);
+      if (csdByteBuffer == null) {
+        break;
+      }
+      byte[] csdBufferData = new byte[csdByteBuffer.remaining()];
+      csdByteBuffer.get(csdBufferData);
+      csdBuffers.add(csdBufferData);
+      csdIndex++;
+    }
+    String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
+    Format.Builder formatBuilder =
+        new Format.Builder()
+            .setSampleMimeType(mediaFormat.getString(MediaFormat.KEY_MIME))
+            .setInitializationData(csdBuffers.build());
+    if (MimeTypes.isVideo(mimeType)) {
+      formatBuilder
+          .setWidth(mediaFormat.getInteger(MediaFormat.KEY_WIDTH))
+          .setHeight(mediaFormat.getInteger(MediaFormat.KEY_HEIGHT));
+    } else if (MimeTypes.isAudio(mimeType)) {
+      // TODO(internal b/178685617): Only set the PCM encoding for audio/raw, once we have a way to
+      // simulate more realistic codec input/output formats in tests.
+      formatBuilder
+          .setChannelCount(mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT))
+          .setSampleRate(mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE))
+          .setPcmEncoding(MEDIA_CODEC_PCM_ENCODING);
+    }
+    return formatBuilder.build();
+  }
 }

library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java

@@ -32,7 +32,6 @@ import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
 import com.google.android.exoplayer2.audio.SonicAudioProcessor;
 import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
 import com.google.android.exoplayer2.source.SampleStream;
-import com.google.android.exoplayer2.util.Util;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 
@@ -40,9 +39,6 @@ import java.nio.ByteBuffer;
 /* package */ final class TransformerAudioRenderer extends TransformerBaseRenderer {
 
   private static final String TAG = "TransformerAudioRenderer";
-  // MediaCodec decoders always output 16 bit PCM, unless configured to output PCM float.
-  // https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers.
-  private static final int MEDIA_CODEC_PCM_ENCODING = C.ENCODING_PCM_16BIT;
   private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024;
   private static final float SPEED_UNSET = -1f;
 
@@ -53,6 +49,8 @@ import java.nio.ByteBuffer;
   @Nullable private MediaCodecAdapterWrapper decoder;
   @Nullable private MediaCodecAdapterWrapper encoder;
   @Nullable private SpeedProvider speedProvider;
+  @Nullable private Format inputFormat;
+  @Nullable private AudioFormat encoderInputAudioFormat;
 
   private ByteBuffer sonicOutputBuffer;
   private long nextEncoderInputBufferTimeUs;
@@ -100,6 +98,8 @@ import java.nio.ByteBuffer;
       encoder = null;
     }
     speedProvider = null;
+    inputFormat = null;
+    encoderInputAudioFormat = null;
     sonicOutputBuffer = AudioProcessor.EMPTY_BUFFER;
     nextEncoderInputBufferTimeUs = 0;
     currentSpeed = SPEED_UNSET;
@@ -307,6 +307,7 @@ import java.nio.ByteBuffer;
    * returns whether it may be possible to write more data.
    */
   private boolean feedEncoder(ByteBuffer inputBuffer) {
+    AudioFormat encoderInputAudioFormat = checkNotNull(this.encoderInputAudioFormat);
     MediaCodecAdapterWrapper encoder = checkNotNull(this.encoder);
     ByteBuffer encoderInputBufferData = checkNotNull(encoderInputBuffer.data);
     int bufferLimit = inputBuffer.limit();
@@ -317,9 +318,8 @@ import java.nio.ByteBuffer;
     nextEncoderInputBufferTimeUs +=
         getBufferDurationUs(
             /* bytesWritten= */ encoderInputBufferData.position(),
-            /* bytesPerFrame= */ Util.getPcmFrameSize(
+            encoderInputAudioFormat.bytesPerFrame,
-                MEDIA_CODEC_PCM_ENCODING, encoder.getConfigFormat().channelCount),
+            encoderInputAudioFormat.sampleRate);
-            encoder.getConfigFormat().sampleRate);
 
     encoderInputBuffer.setFlags(0);
     encoderInputBuffer.flip();
@@ -342,30 +342,35 @@ import java.nio.ByteBuffer;
    * yet.
    */
   private void setupEncoderAndMaybeSonic() throws ExoPlaybackException {
-    MediaCodecAdapterWrapper decoder = checkNotNull(this.decoder);
-
     if (encoder != null) {
       return;
     }
-
+    // TODO(b/161127201): Use the decoder output format once the decoder is fed before setting up
-    Format decoderFormat = decoder.getConfigFormat();
+    // the encoder.
+    AudioFormat outputAudioFormat =
+        new AudioFormat(
+            checkNotNull(inputFormat).sampleRate, inputFormat.channelCount, C.ENCODING_PCM_16BIT);
     if (transformation.flattenForSlowMotion) {
       try {
-        configureSonic(decoderFormat);
+        outputAudioFormat = sonicAudioProcessor.configure(outputAudioFormat);
+        flushSonicAndSetSpeed(currentSpeed);
       } catch (AudioProcessor.UnhandledAudioFormatException e) {
-        throw ExoPlaybackException.createForRenderer(
+        throw createRendererException(e);
-            e, TAG, getIndex(), /* rendererFormat= */ null, C.FORMAT_HANDLED);
       }
     }
-    Format encoderFormat =
-        decoderFormat.buildUpon().setAverageBitrate(DEFAULT_ENCODER_BITRATE).build();
-    checkNotNull(encoderFormat.sampleMimeType);
     try {
-      encoder = MediaCodecAdapterWrapper.createForAudioEncoding(encoderFormat);
+      encoder =
+          MediaCodecAdapterWrapper.createForAudioEncoding(
+              new Format.Builder()
+                  .setSampleMimeType(checkNotNull(inputFormat).sampleMimeType)
+                  .setSampleRate(outputAudioFormat.sampleRate)
+                  .setChannelCount(outputAudioFormat.channelCount)
+                  .setAverageBitrate(DEFAULT_ENCODER_BITRATE)
+                  .build());
     } catch (IOException e) {
-      throw ExoPlaybackException.createForRenderer(
+      throw createRendererException(e);
-          e, TAG, getIndex(), encoderFormat, /* rendererFormatSupport= */ C.FORMAT_HANDLED);
     }
+    encoderInputAudioFormat = outputAudioFormat;
   }
 
   /**
@@ -383,15 +388,13 @@ import java.nio.ByteBuffer;
     if (result != C.RESULT_FORMAT_READ) {
       return false;
     }
-    Format decoderFormat = checkNotNull(formatHolder.format);
+    inputFormat = checkNotNull(formatHolder.format);
-    checkNotNull(decoderFormat.sampleMimeType);
     try {
-      decoder = MediaCodecAdapterWrapper.createForAudioDecoding(decoderFormat);
+      decoder = MediaCodecAdapterWrapper.createForAudioDecoding(inputFormat);
     } catch (IOException e) {
-      throw ExoPlaybackException.createForRenderer(
+      throw createRendererException(e);
-          e, TAG, getIndex(), decoderFormat, /* rendererFormatSupport= */ C.FORMAT_HANDLED);
     }
-    speedProvider = new SegmentSpeedProvider(decoderFormat);
+    speedProvider = new SegmentSpeedProvider(inputFormat);
     currentSpeed = speedProvider.getSpeed(0);
     return true;
   }
@@ -406,18 +409,17 @@ import java.nio.ByteBuffer;
     return speedChanging;
   }
 
-  private void configureSonic(Format format) throws AudioProcessor.UnhandledAudioFormatException {
-    sonicAudioProcessor.configure(
-        new AudioFormat(format.sampleRate, format.channelCount, MEDIA_CODEC_PCM_ENCODING));
-    flushSonicAndSetSpeed(currentSpeed);
-  }
-
   private void flushSonicAndSetSpeed(float speed) {
     sonicAudioProcessor.setSpeed(speed);
     sonicAudioProcessor.setPitch(speed);
     sonicAudioProcessor.flush();
   }
 
+  private ExoPlaybackException createRendererException(Throwable cause) {
+    return ExoPlaybackException.createForRenderer(
+        cause, TAG, getIndex(), inputFormat, /* rendererFormatSupport= */ C.FORMAT_HANDLED);
+  }
+
   private static long getBufferDurationUs(long bytesWritten, int bytesPerFrame, int sampleRate) {
     long framesWritten = bytesWritten / bytesPerFrame;
     return framesWritten * C.MICROS_PER_SECOND / sampleRate;

testdata/src/test/assets/transformerdumps/amr/sample_nb.amr.dump

@@ -3,6 +3,7 @@ format 0:
   sampleMimeType = audio/3gpp
   channelCount = 1
   sampleRate = 8000
+  pcmEncoding = 2
 sample:
   trackIndex = 0
   dataHashCode = 924517484

testdata/src/test/assets/transformerdumps/mp4/sample.mp4.dump

@@ -3,6 +3,7 @@ format 0:
   sampleMimeType = audio/mp4a-latm
   channelCount = 1
   sampleRate = 44100
+  pcmEncoding = 2
 format 1:
   id = 1
   sampleMimeType = video/avc

testdata/src/test/assets/transformerdumps/mp4/sample.mp4.novideo.dump

@@ -3,6 +3,7 @@ format 0:
   sampleMimeType = audio/mp4a-latm
   channelCount = 1
   sampleRate = 44100
+  pcmEncoding = 2
 sample:
   trackIndex = 0
   dataHashCode = 1205768497

testdata/src/test/assets/transformerdumps/mp4/sample_sef_slow_motion.mp4.dump

@@ -3,6 +3,7 @@ format 0:
   sampleMimeType = audio/mp4a-latm
   channelCount = 2
   sampleRate = 12000
+  pcmEncoding = 2
 format 1:
   id = 2
   sampleMimeType = video/avc