Improve Format propagation within the MediaCodecRenderer.

Handles pixel aspect ratio changes in playlists where video
resolution does not change.

Issue:#6646
PiperOrigin-RevId: 307817028

RELEASENOTES.md

@@ -38,6 +38,8 @@
     *   Rename `MediaCodecRenderer.onOutputFormatChanged` to
         `MediaCodecRenderer.onOutputMediaFormatChanged`, further clarifying the
         distinction between `Format` and `MediaFormat`.
+    *   Improve `Format` propagation within the media codec renderer
+        ([#6646](https://github.com/google/ExoPlayer/issues/6646)).
     *   Move player message-related constants from `C` to `Renderer`, to avoid
         having the constants class depend on player/renderer classes.
     *   Split out `common` and `extractor` submodules.

library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java

@@ -368,7 +368,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   private final long[] pendingOutputStreamSwitchTimesUs;
 
   @Nullable private Format inputFormat;
-  private Format outputFormat;
+  @Nullable private Format outputFormat;
   @Nullable private DrmSession codecDrmSession;
   @Nullable private DrmSession sourceDrmSession;
   @Nullable private MediaCrypto mediaCrypto;
@@ -420,6 +420,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
   protected DecoderCounters decoderCounters;
   private long outputStreamOffsetUs;
   private int pendingOutputStreamOffsetCount;
+  private boolean receivedOutputMediaFormatChange;
 
   /**
    * @param trackType The track type that the renderer handles. One of the {@code C.TRACK_TYPE_*}
@@ -635,13 +636,18 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
    * method if they are taking over responsibility for output format propagation (e.g., when using
    * video tunneling).
    */
-  @Nullable
-  protected final Format updateOutputFormatForTime(long presentationTimeUs) {
-    Format format = formatQueue.pollFloor(presentationTimeUs);
+  protected final void updateOutputFormatForTime(long presentationTimeUs) {
+    @Nullable Format format = formatQueue.pollFloor(presentationTimeUs);
     if (format != null) {
       outputFormat = format;
+      onOutputFormatChanged(outputFormat);
+    } else if (receivedOutputMediaFormatChange && outputFormat != null) {
+      // No Format change with the MediaFormat change, so we need to update based on the existing
+      // Format.
+      configureOutput(outputFormat);
     }
-    return format;
+
+    receivedOutputMediaFormatChange = false;
   }
 
   @Nullable
@@ -1445,6 +1451,28 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
     // Do nothing.
   }
 
+  /**
+   * Called when the output {@link Format} changes.
+   *
+   * <p>The default implementation is a no-op.
+   *
+   * @param outputFormat The new output {@link Format}.
+   */
+  protected void onOutputFormatChanged(Format outputFormat) {
+    // Do nothing.
+  }
+
+  /**
+   * Configures the renderer output based on a {@link Format}.
+   *
+   * <p>The default implementation is a no-op.
+   *
+   * @param outputFormat The format to configure the output with.
+   */
+  protected void configureOutput(Format outputFormat) {
+    // Do nothing.
+  }
+
   /**
    * Handles supplemental data associated with an input buffer.
    *
@@ -1650,6 +1678,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
       if (outputIndex < 0) {
         if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED /* (-2) */) {
           processOutputMediaFormat();
+          receivedOutputMediaFormatChange = true;
           return true;
         } else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED /* (-3) */) {
           processOutputBuffersChanged();

library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java

@@ -128,9 +128,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   private long totalVideoFrameProcessingOffsetUs;
   private int videoFrameProcessingOffsetCount;
 
-  private int pendingRotationDegrees;
-  private float pendingPixelWidthHeightRatio;
   @Nullable private MediaFormat currentMediaFormat;
+  private int mediaFormatWidth;
+  private int mediaFormatHeight;
   private int currentWidth;
   private int currentHeight;
   private int currentUnappliedRotationDegrees;
@@ -235,8 +235,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     currentWidth = Format.NO_VALUE;
     currentHeight = Format.NO_VALUE;
     currentPixelWidthHeightRatio = Format.NO_VALUE;
-    pendingPixelWidthHeightRatio = Format.NO_VALUE;
     scalingMode = VIDEO_SCALING_MODE_DEFAULT;
+    mediaFormatWidth = Format.NO_VALUE;
+    mediaFormatHeight = Format.NO_VALUE;
     clearReportedVideoSize();
   }
 
@@ -603,10 +604,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
   @Override
   protected void onInputFormatChanged(FormatHolder formatHolder) throws ExoPlaybackException {
     super.onInputFormatChanged(formatHolder);
-    Format newFormat = formatHolder.format;
-    eventDispatcher.inputFormatChanged(newFormat);
-    pendingPixelWidthHeightRatio = newFormat.pixelWidthHeightRatio;
-    pendingRotationDegrees = newFormat.rotationDegrees;
+    eventDispatcher.inputFormatChanged(formatHolder.format);
   }
 
   /**
@@ -637,22 +635,55 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
             && outputMediaFormat.containsKey(KEY_CROP_LEFT)
             && outputMediaFormat.containsKey(KEY_CROP_BOTTOM)
             && outputMediaFormat.containsKey(KEY_CROP_TOP);
-    int mediaFormatWidth =
+    mediaFormatWidth =
         hasCrop
             ? outputMediaFormat.getInteger(KEY_CROP_RIGHT)
                 - outputMediaFormat.getInteger(KEY_CROP_LEFT)
                 + 1
             : outputMediaFormat.getInteger(MediaFormat.KEY_WIDTH);
-    int mediaFormatHeight =
+    mediaFormatHeight =
         hasCrop
             ? outputMediaFormat.getInteger(KEY_CROP_BOTTOM)
                 - outputMediaFormat.getInteger(KEY_CROP_TOP)
                 + 1
             : outputMediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
-    processOutputFormat(codec, mediaFormatWidth, mediaFormatHeight);
+
+    // Must be applied each time the output MediaFormat changes.
+    codec.setVideoScalingMode(scalingMode);
     maybeNotifyVideoFrameProcessingOffset();
   }
 
+  @Override
+  protected void onOutputFormatChanged(Format outputFormat) {
+    configureOutput(outputFormat);
+  }
+
+  @Override
+  protected void configureOutput(Format outputFormat) {
+    if (tunneling) {
+      currentWidth = outputFormat.width;
+      currentHeight = outputFormat.height;
+    } else {
+      currentWidth = mediaFormatWidth;
+      currentHeight = mediaFormatHeight;
+    }
+    currentPixelWidthHeightRatio = outputFormat.pixelWidthHeightRatio;
+    if (Util.SDK_INT >= 21) {
+      // On API level 21 and above the decoder applies the rotation when rendering to the surface.
+      // Hence currentUnappliedRotation should always be 0. For 90 and 270 degree rotations, we need
+      // to flip the width, height and pixel aspect ratio to reflect the rotation that was applied.
+      if (outputFormat.rotationDegrees == 90 || outputFormat.rotationDegrees == 270) {
+        int rotatedHeight = currentWidth;
+        currentWidth = currentHeight;
+        currentHeight = rotatedHeight;
+        currentPixelWidthHeightRatio = 1 / currentPixelWidthHeightRatio;
+      }
+    } else {
+      // On API level 20 and below the decoder does not apply the rotation.
+      currentUnappliedRotationDegrees = outputFormat.rotationDegrees;
+    }
+  }
+
   @Override
   @TargetApi(29) // codecHandlesHdr10PlusOutOfBandMetadata is false if Util.SDK_INT < 29
   protected void handleInputBufferSupplementalData(DecoderInputBuffer buffer)
@@ -814,28 +845,6 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
     return false;
   }
 
-  private void processOutputFormat(MediaCodec codec, int width, int height) {
-    currentWidth = width;
-    currentHeight = height;
-    currentPixelWidthHeightRatio = pendingPixelWidthHeightRatio;
-    if (Util.SDK_INT >= 21) {
-      // On API level 21 and above the decoder applies the rotation when rendering to the surface.
-      // Hence currentUnappliedRotation should always be 0. For 90 and 270 degree rotations, we need
-      // to flip the width, height and pixel aspect ratio to reflect the rotation that was applied.
-      if (pendingRotationDegrees == 90 || pendingRotationDegrees == 270) {
-        int rotatedHeight = currentWidth;
-        currentWidth = currentHeight;
-        currentHeight = rotatedHeight;
-        currentPixelWidthHeightRatio = 1 / currentPixelWidthHeightRatio;
-      }
-    } else {
-      // On API level 20 and below the decoder does not apply the rotation.
-      currentUnappliedRotationDegrees = pendingRotationDegrees;
-    }
-    // Must be applied each time the output MediaFormat changes.
-    codec.setVideoScalingMode(scalingMode);
-  }
-
   private void notifyFrameMetadataListener(
       long presentationTimeUs, long releaseTimeNs, Format format, MediaFormat mediaFormat) {
     if (frameMetadataListener != null) {
@@ -846,10 +855,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
 
   /** Called when a buffer was processed in tunneling mode. */
   protected void onProcessedTunneledBuffer(long presentationTimeUs) {
-    @Nullable Format format = updateOutputFormatForTime(presentationTimeUs);
-    if (format != null) {
-      processOutputFormat(getCodec(), format.width, format.height);
-    }
+    updateOutputFormatForTime(presentationTimeUs);
     maybeNotifyVideoSizeChanged();
     decoderCounters.renderedOutputBufferCount++;
     maybeNotifyRenderedFirstFrame();