From 4cd8d64446724e5301b2b6471a05fa2fa3f56b5e Mon Sep 17 00:00:00 2001 From: tonihei Date: Fri, 14 Jun 2024 05:31:21 -0700 Subject: [PATCH] Image support in PlayerView Images are rendered into an ImageView (on top of the video shutter). The image view is set to the images emitted by ExoPlayer's ImageOutput and cleared when there is no longer a selected image track. In order to keep the existing behavior of video tracks to only clear the old output once the new first frame is rendered (avoiding short periods of black between playlist items), we have to reorder this code slightly to make it work for video and images. Both are treated in the same way. If both are enabled, video takes precedence. As the UI module only depends on the common module, we can't direcly add the ImageOutput to ExoPlayer. This is done via reflection if the provided Player is an ExoPlayer. #cherrypick PiperOrigin-RevId: 643320666 --- RELEASENOTES.md | 2 + libraries/ui/proguard-rules.txt | 9 +- .../java/androidx/media3/ui/PlayerView.java | 273 ++++++++++++++++-- .../src/main/res/layout/exo_player_view.xml | 9 +- libraries/ui/src/main/res/values/attrs.xml | 5 + libraries/ui/src/main/res/values/ids.xml | 1 + 6 files changed, 269 insertions(+), 30 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 36be4f6665..739fe7419a 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -49,6 +49,8 @@ error code and message are set but the state of the plaftorm session remains different to `STATE_ERROR`. * UI: + * Add image display support to `PlayerView` when connected to an + `ExoPlayer` ([#1144](https://github.com/androidx/media/issues/1144)). * Add customisation of various icons in `PlayerControlView` through xml attributes to allow different drawables per `PlayerView` instance, rather than global overrides diff --git a/libraries/ui/proguard-rules.txt b/libraries/ui/proguard-rules.txt index e60edc4649..c6f7553c44 100644 --- a/libraries/ui/proguard-rules.txt +++ b/libraries/ui/proguard-rules.txt @@ -1,6 +1,6 @@ # Proguard rules specific to the UI module. -# Constructor method accessed via reflection in PlayerView +# Constructor method and classes accessed via reflection in PlayerView -dontnote androidx.media3.exoplayer.video.spherical.SphericalGLSurfaceView -keepclassmembers class androidx.media3.exoplayer.video.spherical.SphericalGLSurfaceView { (android.content.Context); @@ -9,6 +9,13 @@ -keepclassmembers class androidx.media3.exoplayer.video.VideoDecoderGLSurfaceView { (android.content.Context); } +-keepnames class androidx.media3.exoplayer.ExoPlayer {} +-keepclassmembers class androidx.media3.exoplayer.ExoPlayer { + void setImageOutput(androidx.media3.exoplayer.image.ImageOutput); +} +-keepclasseswithmembers class androidx.media3.exoplayer.image.ImageOutput { + void onImageAvailable(long, android.graphics.Bitmap); +} # Constructor method accessed via reflection in TrackSelectionDialogBuilder -dontnote androidx.appcompat.app.AlertDialog.Builder diff --git a/libraries/ui/src/main/java/androidx/media3/ui/PlayerView.java b/libraries/ui/src/main/java/androidx/media3/ui/PlayerView.java index 872e8850ad..78808438e6 100644 --- a/libraries/ui/src/main/java/androidx/media3/ui/PlayerView.java +++ b/libraries/ui/src/main/java/androidx/media3/ui/PlayerView.java @@ -36,6 +36,7 @@ import android.graphics.RectF; import android.graphics.drawable.BitmapDrawable; import android.graphics.drawable.Drawable; import android.opengl.GLSurfaceView; +import android.os.Handler; import android.os.Looper; import android.util.AttributeSet; import android.view.KeyEvent; @@ -77,10 +78,12 @@ import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; import java.util.ArrayList; import java.util.List; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; /** * A high level view for {@link Player} media playbacks. It displays video, subtitles and album art @@ -106,6 +109,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; *
  • Corresponding method: {@link #setDefaultArtwork(Drawable)} *
  • Default: {@code null} * + *
  • {@code image_display_mode} - The {@link ImageDisplayMode mode} in which images are + * displayed. + *
      + *
    • Corresponding method: {@link #setImageDisplayMode(int)} + *
    • Default: {@code #IMAGE_DISPLAY_MODE_FIT} + *
    *
  • {@code use_controller} - Whether the playback controls can be shown. *
      *
    • Corresponding method: {@link #setUseController(boolean)} @@ -227,6 +236,26 @@ public class PlayerView extends FrameLayout implements AdViewProvider { */ @UnstableApi public static final int ARTWORK_DISPLAY_MODE_FILL = 2; + /** + * Determines the image display mode. {@link #IMAGE_DISPLAY_MODE_FIT} or {@link + * #IMAGE_DISPLAY_MODE_FILL}. + */ + @UnstableApi + @Documented + @Retention(RetentionPolicy.SOURCE) + @Target(TYPE_USE) + @IntDef({IMAGE_DISPLAY_MODE_FIT, IMAGE_DISPLAY_MODE_FILL}) + public @interface ImageDisplayMode {} + + /** The image is fit into the player view and centered creating a letterbox style. */ + @UnstableApi public static final int IMAGE_DISPLAY_MODE_FIT = 0; + + /** + * The image covers the entire space of the player view. If the aspect ratio of the image is + * different than the player view some areas of the image are cropped. + */ + @UnstableApi public static final int IMAGE_DISPLAY_MODE_FILL = 1; + /** * Determines when the buffering view is shown. One of {@link #SHOW_BUFFERING_NEVER}, {@link * #SHOW_BUFFERING_WHEN_PLAYING} or {@link #SHOW_BUFFERING_ALWAYS}. @@ -264,6 +293,7 @@ public class PlayerView extends FrameLayout implements AdViewProvider { @Nullable private final View shutterView; @Nullable private final View surfaceView; private final boolean surfaceViewIgnoresVideoAspectRatio; + @Nullable private final ImageView imageView; @Nullable private final ImageView artworkView; @Nullable private final SubtitleView subtitleView; @Nullable private final View bufferingView; @@ -271,6 +301,10 @@ public class PlayerView extends FrameLayout implements AdViewProvider { @Nullable private final PlayerControlView controller; @Nullable private final FrameLayout adOverlayFrameLayout; @Nullable private final FrameLayout overlayFrameLayout; + private final Handler mainLooperHandler; + @Nullable private final Class exoPlayerClazz; + @Nullable private final Method setImageOutputMethod; + @Nullable private final Object imageOutput; @Nullable private Player player; private boolean useController; @@ -285,6 +319,7 @@ public class PlayerView extends FrameLayout implements AdViewProvider { @Nullable private FullscreenButtonClickListener fullscreenButtonClickListener; private @ArtworkDisplayMode int artworkDisplayMode; + private @ImageDisplayMode int imageDisplayMode; @Nullable private Drawable defaultArtwork; private @ShowBuffering int showBuffering; @@ -296,7 +331,6 @@ public class PlayerView extends FrameLayout implements AdViewProvider { private boolean controllerHideDuringAds; private boolean controllerHideOnTouch; private int textureViewRotation; - private boolean isTouching; public PlayerView(Context context) { this(context, /* attrs= */ null); @@ -312,12 +346,14 @@ public class PlayerView extends FrameLayout implements AdViewProvider { super(context, attrs, defStyleAttr); componentListener = new ComponentListener(); + mainLooperHandler = new Handler(Looper.getMainLooper()); if (isInEditMode()) { contentFrame = null; shutterView = null; surfaceView = null; surfaceViewIgnoresVideoAspectRatio = false; + imageView = null; artworkView = null; subtitleView = null; bufferingView = null; @@ -325,6 +361,9 @@ public class PlayerView extends FrameLayout implements AdViewProvider { controller = null; adOverlayFrameLayout = null; overlayFrameLayout = null; + exoPlayerClazz = null; + setImageOutputMethod = null; + imageOutput = null; ImageView logo = new ImageView(context); if (Util.SDK_INT >= 23) { configureEditModeLogoV23(context, getResources(), logo); @@ -340,6 +379,7 @@ public class PlayerView extends FrameLayout implements AdViewProvider { int playerLayoutId = R.layout.exo_player_view; boolean useArtwork = true; int artworkDisplayMode = ARTWORK_DISPLAY_MODE_FIT; + int imageDisplayMode = IMAGE_DISPLAY_MODE_FIT; int defaultArtworkId = 0; boolean useController = true; int surfaceType = SURFACE_TYPE_SURFACE_VIEW; @@ -364,6 +404,7 @@ public class PlayerView extends FrameLayout implements AdViewProvider { a.getInt(R.styleable.PlayerView_artwork_display_mode, artworkDisplayMode); defaultArtworkId = a.getResourceId(R.styleable.PlayerView_default_artwork, defaultArtworkId); + imageDisplayMode = a.getInt(R.styleable.PlayerView_image_display_mode, imageDisplayMode); useController = a.getBoolean(R.styleable.PlayerView_use_controller, useController); surfaceType = a.getInt(R.styleable.PlayerView_surface_type, surfaceType); resizeMode = a.getInt(R.styleable.PlayerView_resize_mode, resizeMode); @@ -455,6 +496,38 @@ public class PlayerView extends FrameLayout implements AdViewProvider { // Overlay frame layout. overlayFrameLayout = findViewById(R.id.exo_overlay); + // Image view. + imageView = findViewById(R.id.exo_image); + this.imageDisplayMode = imageDisplayMode; + + // ExoPlayer image output classes and methods. + Class exoPlayerClazz; + Method setImageOutputMethod; + Object imageOutput; + try { + exoPlayerClazz = Class.forName("androidx.media3.exoplayer.ExoPlayer"); + Class imageOutputClazz = Class.forName("androidx.media3.exoplayer.image.ImageOutput"); + setImageOutputMethod = exoPlayerClazz.getMethod("setImageOutput", imageOutputClazz); + imageOutput = + Proxy.newProxyInstance( + imageOutputClazz.getClassLoader(), + new Class[] {imageOutputClazz}, + (proxy, method, args) -> { + if (method.getName().equals("onImageAvailable")) { + onImageAvailable((Bitmap) args[1]); + } + return null; + }); + } catch (ClassNotFoundException | NoSuchMethodException e) { + // Expected if ExoPlayer module not available. + exoPlayerClazz = null; + setImageOutputMethod = null; + imageOutput = null; + } + this.exoPlayerClazz = exoPlayerClazz; + this.setImageOutputMethod = setImageOutputMethod; + this.imageOutput = imageOutput; + // Artwork view. artworkView = findViewById(R.id.exo_artwork); boolean isArtworkEnabled = @@ -568,6 +641,7 @@ public class PlayerView extends FrameLayout implements AdViewProvider { if (this.player == player) { return; } + @Nullable Player oldPlayer = this.player; if (oldPlayer != null) { oldPlayer.removeListener(componentListener); @@ -578,6 +652,7 @@ public class PlayerView extends FrameLayout implements AdViewProvider { oldPlayer.clearVideoSurfaceView((SurfaceView) surfaceView); } } + clearImageOutput(oldPlayer); } if (subtitleView != null) { subtitleView.setCues(null); @@ -606,12 +681,34 @@ public class PlayerView extends FrameLayout implements AdViewProvider { subtitleView.setCues(player.getCurrentCues().cues); } player.addListener(componentListener); + setImageOutput(player); maybeShowController(false); } else { hideController(); } } + private void setImageOutput(Player player) { + if (exoPlayerClazz != null && exoPlayerClazz.isAssignableFrom(player.getClass())) { + try { + checkNotNull(setImageOutputMethod).invoke(player, checkNotNull(imageOutput)); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + } + + @SuppressWarnings("argument.type.incompatible") // null allowed as method parameter in invoke + private void clearImageOutput(Player player) { + if (exoPlayerClazz != null && exoPlayerClazz.isAssignableFrom(player.getClass())) { + try { + checkNotNull(setImageOutputMethod).invoke(player, new Object[] {null}); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + } + @Override public void setVisibility(int visibility) { super.setVisibility(visibility); @@ -694,6 +791,22 @@ public class PlayerView extends FrameLayout implements AdViewProvider { } } + /** Sets how images are displayed if present in the media. */ + @UnstableApi + public void setImageDisplayMode(@ImageDisplayMode int imageDisplayMode) { + Assertions.checkState(imageView != null); + if (this.imageDisplayMode != imageDisplayMode) { + this.imageDisplayMode = imageDisplayMode; + updateImageViewAspectRatio(); + } + } + + /** Returns the {@link ImageDisplayMode image display mode}. */ + @UnstableApi + public @ImageDisplayMode int getImageDisplayMode() { + return imageDisplayMode; + } + /** Returns whether the playback controls can be shown. */ public boolean getUseController() { return useController; @@ -1326,7 +1439,6 @@ public class PlayerView extends FrameLayout implements AdViewProvider { return false; } - @EnsuresNonNullIf(expression = "artworkView", result = true) private boolean useArtwork() { if (artworkDisplayMode != ARTWORK_DISPLAY_MODE_OFF) { Assertions.checkStateNotNull(artworkView); @@ -1391,32 +1503,46 @@ public class PlayerView extends FrameLayout implements AdViewProvider { private void updateForCurrentTrackSelections(boolean isNewPlayer) { @Nullable Player player = this.player; - if (player == null - || !player.isCommandAvailable(COMMAND_GET_TRACKS) - || player.getCurrentTracks().isEmpty()) { - if (!keepContentOnPlayerReset) { - hideArtwork(); - closeShutter(); - } - return; - } - if (isNewPlayer && !keepContentOnPlayerReset) { - // Hide any video from the previous player. - closeShutter(); - } - - if (player.getCurrentTracks().isTypeSelected(C.TRACK_TYPE_VIDEO)) { - // Video enabled, so artwork must be hidden. If the shutter is closed, it will be opened - // in onRenderedFirstFrame(). + // Unless configured to keep, clear output completely when tracks are empty or for a new player. + boolean hasTracks = + player != null + && player.isCommandAvailable(COMMAND_GET_TRACKS) + && !player.getCurrentTracks().isEmpty(); + if (!keepContentOnPlayerReset && (!hasTracks || isNewPlayer)) { hideArtwork(); + closeShutter(); + hideAndClearImage(); + } + if (!hasTracks) { + // Nothing else to check. return; } - // Video disabled so the shutter must be closed. - closeShutter(); + boolean hasSelectedVideoTrack = hasSelectedVideoTrack(); + boolean hasSelectedImageTrack = hasSelectedImageTrack(); + + // When video and image are disabled, close shutter and clear image. Do nothing if one or both + // are enabled to keep the current state (previous frame or video). Once ready, the first frame + // or image will update the view. + if (!hasSelectedVideoTrack && !hasSelectedImageTrack) { + closeShutter(); + hideAndClearImage(); + } + // Exception: If both were enabled (shutter open and image set) and we switch to one only, we + // can hide the other output immediately as no further callbacks will arrive. + boolean wasVideoAndImageSet = + shutterView != null && shutterView.getVisibility() == INVISIBLE && isImageSet(); + if (hasSelectedImageTrack && !hasSelectedVideoTrack && wasVideoAndImageSet) { + closeShutter(); + showImage(); + } else if (hasSelectedVideoTrack && !hasSelectedImageTrack && wasVideoAndImageSet) { + hideAndClearImage(); + } + // Display artwork if enabled and available, else hide it. - if (useArtwork()) { + boolean shouldShowArtwork = !hasSelectedVideoTrack && !hasSelectedImageTrack && useArtwork(); + if (shouldShowArtwork) { if (setArtworkFromMediaMetadata(player)) { return; } @@ -1428,9 +1554,8 @@ public class PlayerView extends FrameLayout implements AdViewProvider { hideArtwork(); } - @RequiresNonNull("artworkView") - private boolean setArtworkFromMediaMetadata(Player player) { - if (!player.isCommandAvailable(COMMAND_GET_METADATA)) { + private boolean setArtworkFromMediaMetadata(@Nullable Player player) { + if (player == null || !player.isCommandAvailable(COMMAND_GET_METADATA)) { return false; } MediaMetadata mediaMetadata = player.getMediaMetadata(); @@ -1443,9 +1568,8 @@ public class PlayerView extends FrameLayout implements AdViewProvider { return setDrawableArtwork(new BitmapDrawable(getResources(), bitmap)); } - @RequiresNonNull("artworkView") private boolean setDrawableArtwork(@Nullable Drawable drawable) { - if (drawable != null) { + if (artworkView != null && drawable != null) { int drawableWidth = drawable.getIntrinsicWidth(); int drawableHeight = drawable.getIntrinsicHeight(); if (drawableWidth > 0 && drawableHeight > 0) { @@ -1472,6 +1596,94 @@ public class PlayerView extends FrameLayout implements AdViewProvider { } } + private boolean hasSelectedImageTrack() { + @Nullable Player player = this.player; + return player != null + && imageOutput != null + && player.isCommandAvailable(COMMAND_GET_TRACKS) + && player.getCurrentTracks().isTypeSelected(C.TRACK_TYPE_IMAGE); + } + + private boolean hasSelectedVideoTrack() { + @Nullable Player player = this.player; + return player != null + && player.isCommandAvailable(COMMAND_GET_TRACKS) + && player.getCurrentTracks().isTypeSelected(C.TRACK_TYPE_VIDEO); + } + + private boolean isImageSet() { + if (imageView == null) { + return false; + } + @Nullable Drawable drawable = imageView.getDrawable(); + // The transparent placeholder has an alpha value of 0, but BitmapDrawables never have alpha 0. + return drawable != null && drawable.getAlpha() != 0; + } + + private void setImage(Drawable drawable) { + if (imageView == null) { + return; + } + imageView.setImageDrawable(drawable); + updateImageViewAspectRatio(); + } + + private void updateImageViewAspectRatio() { + if (imageView == null) { + return; + } + @Nullable Drawable drawable = imageView.getDrawable(); + if (drawable == null) { + return; + } + int drawableWidth = drawable.getIntrinsicWidth(); + int drawableHeight = drawable.getIntrinsicHeight(); + if (drawableWidth <= 0 || drawableHeight <= 0) { + return; + } + float drawableLayoutAspectRatio = (float) drawableWidth / drawableHeight; + ImageView.ScaleType scaleStyle = ImageView.ScaleType.FIT_XY; + if (imageDisplayMode == IMAGE_DISPLAY_MODE_FILL) { + drawableLayoutAspectRatio = (float) getWidth() / getHeight(); + scaleStyle = ImageView.ScaleType.CENTER_CROP; + } + if (imageView.getVisibility() == VISIBLE) { + onContentAspectRatioChanged(contentFrame, drawableLayoutAspectRatio); + } + imageView.setScaleType(scaleStyle); + } + + private void hideAndClearImage() { + hideImage(); + if (imageView != null) { + imageView.setImageResource(android.R.color.transparent); + } + } + + private void showImage() { + if (imageView != null) { + imageView.setVisibility(VISIBLE); + updateImageViewAspectRatio(); + } + } + + private void hideImage() { + if (imageView != null) { + imageView.setVisibility(INVISIBLE); + } + } + + private void onImageAvailable(Bitmap bitmap) { + mainLooperHandler.post( + () -> { + setImage(new BitmapDrawable(getResources(), bitmap)); + if (!hasSelectedVideoTrack()) { + showImage(); + closeShutter(); + } + }); + } + private void closeShutter() { if (shutterView != null) { shutterView.setVisibility(View.VISIBLE); @@ -1654,6 +1866,11 @@ public class PlayerView extends FrameLayout implements AdViewProvider { public void onRenderedFirstFrame() { if (shutterView != null) { shutterView.setVisibility(INVISIBLE); + if (hasSelectedImageTrack()) { + hideImage(); + } else { + hideAndClearImage(); + } } } diff --git a/libraries/ui/src/main/res/layout/exo_player_view.xml b/libraries/ui/src/main/res/layout/exo_player_view.xml index 3d0fe4b312..bb1ad0b8ac 100644 --- a/libraries/ui/src/main/res/layout/exo_player_view.xml +++ b/libraries/ui/src/main/res/layout/exo_player_view.xml @@ -27,10 +27,17 @@ android:layout_height="match_parent" android:background="@android:color/black"/> + + + android:scaleType="fitXY" + android:contentDescription="@null"/> + + + + @@ -115,6 +119,7 @@ + diff --git a/libraries/ui/src/main/res/values/ids.xml b/libraries/ui/src/main/res/values/ids.xml index b0c1632693..c36331e587 100644 --- a/libraries/ui/src/main/res/values/ids.xml +++ b/libraries/ui/src/main/res/values/ids.xml @@ -16,6 +16,7 @@ +