Simplify extension video renderers

- This change also adds support for VideoFrameMetadataListener in the
  AV1 renderer
- This is a preliminary step prior to adding FfmpegVideoDecoder

Issue: #2159
PiperOrigin-RevId: 301702460

extensions/av1/src/main/java/com/google/android/exoplayer2/ext/av1/Libgav1VideoRenderer.java

@@ -21,37 +21,17 @@ import android.os.Handler;
 import android.view.Surface;
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.C;
-import com.google.android.exoplayer2.ExoPlaybackException;
-import com.google.android.exoplayer2.ExoPlayer;
 import com.google.android.exoplayer2.Format;
-import com.google.android.exoplayer2.PlayerMessage.Target;
 import com.google.android.exoplayer2.RendererCapabilities;
-import com.google.android.exoplayer2.decoder.DecoderException;
-import com.google.android.exoplayer2.decoder.SimpleDecoder;
 import com.google.android.exoplayer2.drm.ExoMediaCrypto;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.TraceUtil;
 import com.google.android.exoplayer2.util.Util;
 import com.google.android.exoplayer2.video.DecoderVideoRenderer;
-import com.google.android.exoplayer2.video.VideoDecoderInputBuffer;
 import com.google.android.exoplayer2.video.VideoDecoderOutputBuffer;
-import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer;
 import com.google.android.exoplayer2.video.VideoRendererEventListener;
 
-/**
+/** Decodes and renders video using libgav1 decoder. */
- * Decodes and renders video using libgav1 decoder.
- *
- * <p>This renderer accepts the following messages sent via {@link ExoPlayer#createMessage(Target)}
- * on the playback thread:
- *
- * <ul>
- *   <li>Message with type {@link #MSG_SET_SURFACE} to set the output surface. The message payload
- *       should be the target {@link Surface}, or null.
- *   <li>Message with type {@link #MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER} to set the output
- *       buffer renderer. The message payload should be the target {@link
- *       VideoDecoderOutputBufferRenderer}, or null.
- * </ul>
- */
 public class Libgav1VideoRenderer extends DecoderVideoRenderer {
 
   private static final int DEFAULT_NUM_OF_INPUT_BUFFERS = 4;
@@ -73,7 +53,7 @@ public class Libgav1VideoRenderer extends DecoderVideoRenderer {
   @Nullable private Gav1Decoder decoder;
 
   /**
-   * Creates a Libgav1VideoRenderer.
+   * Creates a new instance.
    *
    * @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer
    *     can attempt to seamlessly join an ongoing playback.
@@ -99,7 +79,7 @@ public class Libgav1VideoRenderer extends DecoderVideoRenderer {
   }
 
   /**
-   * Creates a Libgav1VideoRenderer.
+   * Creates a new instance.
    *
    * @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer
    *     can attempt to seamlessly join an ongoing playback.
@@ -140,9 +120,8 @@ public class Libgav1VideoRenderer extends DecoderVideoRenderer {
   }
 
   @Override
-  protected SimpleDecoder<
+  protected Gav1Decoder createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto)
-          VideoDecoderInputBuffer, ? extends VideoDecoderOutputBuffer, ? extends DecoderException>
+      throws Gav1DecoderException {
-      createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto) throws DecoderException {
     TraceUtil.beginSection("createGav1Decoder");
     int initialInputBufferSize =
         format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE;
@@ -170,17 +149,4 @@ public class Libgav1VideoRenderer extends DecoderVideoRenderer {
       decoder.setOutputMode(outputMode);
     }
   }
-
-  // PlayerMessage.Target implementation.
-
-  @Override
-  public void handleMessage(int messageType, @Nullable Object message) throws ExoPlaybackException {
-    if (messageType == MSG_SET_SURFACE) {
-      setOutputSurface((Surface) message);
-    } else if (messageType == MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER) {
-      setOutputBufferRenderer((VideoDecoderOutputBufferRenderer) message);
-    } else {
-      super.handleMessage(messageType, message);
-    }
-  }
 }

extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/LibvpxVideoRenderer.java

@@ -21,37 +21,16 @@ import android.os.Handler;
 import android.view.Surface;
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.C;
-import com.google.android.exoplayer2.ExoPlaybackException;
-import com.google.android.exoplayer2.ExoPlayer;
 import com.google.android.exoplayer2.Format;
-import com.google.android.exoplayer2.PlayerMessage.Target;
 import com.google.android.exoplayer2.RendererCapabilities;
-import com.google.android.exoplayer2.decoder.DecoderException;
-import com.google.android.exoplayer2.decoder.SimpleDecoder;
 import com.google.android.exoplayer2.drm.ExoMediaCrypto;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.TraceUtil;
 import com.google.android.exoplayer2.video.DecoderVideoRenderer;
-import com.google.android.exoplayer2.video.VideoDecoderInputBuffer;
 import com.google.android.exoplayer2.video.VideoDecoderOutputBuffer;
-import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer;
-import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
 import com.google.android.exoplayer2.video.VideoRendererEventListener;
 
-/**
+/** Decodes and renders video using the native VP9 decoder. */
- * Decodes and renders video using the native VP9 decoder.
- *
- * <p>This renderer accepts the following messages sent via {@link ExoPlayer#createMessage(Target)}
- * on the playback thread:
- *
- * <ul>
- *   <li>Message with type {@link #MSG_SET_SURFACE} to set the output surface. The message payload
- *       should be the target {@link Surface}, or null.
- *   <li>Message with type {@link #MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER} to set the output
- *       buffer renderer. The message payload should be the target {@link
- *       VideoDecoderOutputBufferRenderer}, or null.
- * </ul>
- */
 public class LibvpxVideoRenderer extends DecoderVideoRenderer {
 
   /** The number of input buffers. */
@@ -70,9 +49,10 @@ public class LibvpxVideoRenderer extends DecoderVideoRenderer {
   private final int threads;
 
   @Nullable private VpxDecoder decoder;
-  @Nullable private VideoFrameMetadataListener frameMetadataListener;
 
   /**
+   * Creates a new instance.
+   *
    * @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer
    *     can attempt to seamlessly join an ongoing playback.
    */
@@ -81,6 +61,8 @@ public class LibvpxVideoRenderer extends DecoderVideoRenderer {
   }
 
   /**
+   * Creates a new instance.
+   *
    * @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer
    *     can attempt to seamlessly join an ongoing playback.
    * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be
@@ -105,6 +87,8 @@ public class LibvpxVideoRenderer extends DecoderVideoRenderer {
   }
 
   /**
+   * Creates a new instance.
+   *
    * @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer
    *     can attempt to seamlessly join an ongoing playback.
    * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be
@@ -147,9 +131,8 @@ public class LibvpxVideoRenderer extends DecoderVideoRenderer {
   }
 
   @Override
-  protected SimpleDecoder<
+  protected VpxDecoder createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto)
-          VideoDecoderInputBuffer, ? extends VideoDecoderOutputBuffer, ? extends DecoderException>
+      throws VpxDecoderException {
-      createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto) throws DecoderException {
     TraceUtil.beginSection("createVpxDecoder");
     int initialInputBufferSize =
         format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE;
@@ -161,17 +144,6 @@ public class LibvpxVideoRenderer extends DecoderVideoRenderer {
     return decoder;
   }
 
-  @Override
-  protected void renderOutputBuffer(
-      VideoDecoderOutputBuffer outputBuffer, long presentationTimeUs, Format outputFormat)
-      throws DecoderException {
-    if (frameMetadataListener != null) {
-      frameMetadataListener.onVideoFrameAboutToBeRendered(
-          presentationTimeUs, System.nanoTime(), outputFormat, /* mediaFormat= */ null);
-    }
-    super.renderOutputBuffer(outputBuffer, presentationTimeUs, outputFormat);
-  }
-
   @Override
   protected void renderOutputBufferToSurface(VideoDecoderOutputBuffer outputBuffer, Surface surface)
       throws VpxDecoderException {
@@ -189,19 +161,4 @@ public class LibvpxVideoRenderer extends DecoderVideoRenderer {
       decoder.setOutputMode(outputMode);
     }
   }
-
-  // PlayerMessage.Target implementation.
-
-  @Override
-  public void handleMessage(int messageType, @Nullable Object message) throws ExoPlaybackException {
-    if (messageType == MSG_SET_SURFACE) {
-      setOutputSurface((Surface) message);
-    } else if (messageType == MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER) {
-      setOutputBufferRenderer((VideoDecoderOutputBufferRenderer) message);
-    } else if (messageType == MSG_SET_VIDEO_FRAME_METADATA_LISTENER) {
-      frameMetadataListener = (VideoFrameMetadataListener) message;
-    } else {
-      super.handleMessage(messageType, message);
-    }
-  }
 }

library/core/src/main/java/com/google/android/exoplayer2/video/DecoderVideoRenderer.java

@@ -24,8 +24,10 @@ import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.BaseRenderer;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.ExoPlaybackException;
+import com.google.android.exoplayer2.ExoPlayer;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.FormatHolder;
+import com.google.android.exoplayer2.PlayerMessage.Target;
 import com.google.android.exoplayer2.decoder.Decoder;
 import com.google.android.exoplayer2.decoder.DecoderCounters;
 import com.google.android.exoplayer2.decoder.DecoderException;
@@ -42,7 +44,23 @@ import java.lang.annotation.Documented;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 
-/** Decodes and renders video using a {@link Decoder}. */
+/**
+ * Decodes and renders video using a {@link Decoder}.
+ *
+ * <p>This renderer accepts the following messages sent via {@link ExoPlayer#createMessage(Target)}
+ * on the playback thread:
+ *
+ * <ul>
+ *   <li>Message with type {@link #MSG_SET_SURFACE} to set the output surface. The message payload
+ *       should be the target {@link Surface}, or null.
+ *   <li>Message with type {@link #MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER} to set the output
+ *       buffer renderer. The message payload should be the target {@link
+ *       VideoDecoderOutputBufferRenderer}, or null.
+ *   <li>Message with type {@link #MSG_SET_VIDEO_FRAME_METADATA_LISTENER} to set a listener for
+ *       metadata associated with frames being rendered. The message payload should be the {@link
+ *       VideoFrameMetadataListener}, or null.
+ * </ul>
+ */
 public abstract class DecoderVideoRenderer extends BaseRenderer {
 
   /** Decoder reinitialization states. */
@@ -84,6 +102,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
   private VideoDecoderOutputBuffer outputBuffer;
   @Nullable private Surface surface;
   @Nullable private VideoDecoderOutputBufferRenderer outputBufferRenderer;
+  @Nullable private VideoFrameMetadataListener frameMetadataListener;
   @C.VideoOutputMode private int outputMode;
 
   @Nullable private DrmSession<ExoMediaCrypto> decoderDrmSession;
@@ -214,6 +233,21 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
     }
   }
 
+  // PlayerMessage.Target implementation.
+
+  @Override
+  public void handleMessage(int messageType, @Nullable Object message) throws ExoPlaybackException {
+    if (messageType == MSG_SET_SURFACE) {
+      setOutputSurface((Surface) message);
+    } else if (messageType == MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER) {
+      setOutputBufferRenderer((VideoDecoderOutputBufferRenderer) message);
+    } else if (messageType == MSG_SET_VIDEO_FRAME_METADATA_LISTENER) {
+      frameMetadataListener = (VideoFrameMetadataListener) message;
+    } else {
+      super.handleMessage(messageType, message);
+    }
+  }
+
   // Protected methods.
 
   @Override
@@ -507,6 +541,10 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
   protected void renderOutputBuffer(
       VideoDecoderOutputBuffer outputBuffer, long presentationTimeUs, Format outputFormat)
       throws DecoderException {
+    if (frameMetadataListener != null) {
+      frameMetadataListener.onVideoFrameAboutToBeRendered(
+          presentationTimeUs, System.nanoTime(), outputFormat, /* mediaFormat= */ null);
+    }
     lastRenderTimeUs = C.msToUs(SystemClock.elapsedRealtime() * 1000);
     int bufferMode = outputBuffer.mode;
     boolean renderSurface = bufferMode == C.VIDEO_OUTPUT_MODE_SURFACE_YUV && surface != null;

library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java

@@ -73,6 +73,9 @@ import java.util.List;
  *       payload should be one of the integer scaling modes in {@link VideoScalingMode}. Note that
  *       the scaling mode only applies if the {@link Surface} targeted by this renderer is owned by
  *       a {@link android.view.SurfaceView}.
+ *   <li>Message with type {@link #MSG_SET_VIDEO_FRAME_METADATA_LISTENER} to set a listener for
+ *       metadata associated with frames being rendered. The message payload should be the {@link
+ *       VideoFrameMetadataListener}, or null.
  * </ul>
  */
 public class MediaCodecVideoRenderer extends MediaCodecRenderer {

library/core/src/main/java/com/google/android/exoplayer2/video/VideoFrameMetadataListener.java

@@ -19,14 +19,14 @@ import android.media.MediaFormat;
 import androidx.annotation.Nullable;
 import com.google.android.exoplayer2.Format;
 
-/** A listener for metadata corresponding to video frame being rendered. */
+/** A listener for metadata corresponding to video frames being rendered. */
 public interface VideoFrameMetadataListener {
   /**
-   * Called when the video frame about to be rendered. This method is called on the playback thread.
+   * Called on the playback thread when a video frame is about to be rendered.
    *
-   * @param presentationTimeUs The presentation time of the output buffer, in microseconds.
+   * @param presentationTimeUs The presentation time of the frame, in microseconds.
    * @param releaseTimeNs The wallclock time at which the frame should be displayed, in nanoseconds.
-   *     If the platform API version of the device is less than 21, then this is the best effort.
+   *     If the platform API version of the device is less than 21, then this is a best effort.
    * @param format The format associated with the frame.
    * @param mediaFormat The framework media format associated with the frame, or {@code null} if not
    *     known or not applicable (e.g., because the frame was not output by a {@link