diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 33f1165450..7d9ab1c365 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -12,6 +12,22 @@ by the user of the device. Apps can opt-out of contributing to platform diagnostics for ExoPlayer with `ExoPlayer.Builder.setUsePlatformDiagnostics(false)`. + * Fix bug that tracks are reset too often when using `MergingMediaSource`, + for example when side-loading subtitles and changing the selected + subtitle mid-playback + ([#10248](https://github.com/google/ExoPlayer/issues/10248)). + * Stop detecting 5G-NSA network type on API 29 and 30. These playbacks + will assume a 4G network. + * Disallow passing `null` to + `MediaSource.Factory.setDrmSessionManagerProvider` and + `MediaSource.Factory.setLoadErrorHandlingPolicy`. Instances of + `DefaultDrmSessionManagerProvider` and `DefaultLoadErrorHandlingPolicy` + can be passed explicitly if required. + * Add `MediaItem.RequestMetadata` to represent metadata needed to play + media when the exact `LocalConfiguration` is not known. Also remove + `MediaMetadata.mediaUrl` as this is now included in `RequestMetadata`. + * Add `Player.Command.COMMAND_SET_MEDIA_ITEM` to enable players to allow + setting a single item. * Track selection: * Flatten `TrackSelectionOverrides` class into `TrackSelectionParameters`, and promote `TrackSelectionOverride` to a top level class. @@ -19,17 +35,58 @@ `Tracks.Group`. `Player.getCurrentTracksInfo` and `Player.Listener.onTracksInfoChanged` have also been renamed to `Player.getCurrentTracks` and `Player.Listener.onTracksChanged`. + * Change `DefaultTrackSelector.buildUponParameters` and + `DefaultTrackSelector.Parameters.buildUpon` to return + `DefaultTrackSelector.Parameters.Builder` instead of the deprecated + `DefaultTrackSelector.ParametersBuilder`. + * Add + `DefaultTrackSelector.Parameters.constrainAudioChannelCountToDeviceCapabilities`. + which is enabled by default. When enabled, the `DefaultTrackSelector` + will prefer audio tracks whose channel count does not exceed the device + output capabilities. On handheld devices, the `DefaultTrackSelector` + will prefer stereo/mono over multichannel audio formats, unless the + multichannel format can be + [Spatialized](https://developer.android.com/reference/android/media/Spatializer) + (Android 12L+) or is a Dolby surround sound format. In addition, on + devices that support audio spatialization, the `DefaultTrackSelector` + will monitor for changes in the + [Spatializer properties](https://developer.android.com/reference/android/media/Spatializer.OnSpatializerStateChangedListener) + and trigger a new track selection upon these. Devices with a + `television` + [UI mode](https://developer.android.com/guide/topics/resources/providing-resources#UiModeQualifier) + are excluded from these constraints and the format with the highest + channel count will be preferred. To enable this feature, the + `DefaultTrackSelector` instance must be constructed with a `Context`. * Video: - * Rename `DummySurface` to `PlaceHolderSurface`. + * Rename `DummySurface` to `PlaceholderSurface`. + * Add AV1 support to the `MediaCodecVideoRenderer.getCodecMaxInputSize`. * Audio: * Use LG AC3 audio decoder advertising non-standard MIME type. + * Change the return type of `AudioAttributes.getAudioAttributesV21()` from + `android.media.AudioAttributes` to a new `AudioAttributesV21` wrapper + class, to prevent slow ART verification on API < 21. + * Query the platform (API 29+) or assume the audio encoding channel count + for audio passthrough when the format audio channel count is unset, + which occurs with HLS chunkless preparation + ([10204](https://github.com/google/ExoPlayer/issues/10204)). * Ad playback / IMA: - * Decrease ad polling rate from every 100ms to every 200ms, to line up with - Media Rating Council (MRC) recommendations. + * Decrease ad polling rate from every 100ms to every 200ms, to line up + with Media Rating Council (MRC) recommendations. +* Text: + * Change `Player.getCurrentCues()` to return `CueGroup` instead of + `List`. + * SSA: Support `OutlineColour` style setting when `BorderStyle == 3` (i.e. + `OutlineColour` sets the background of the cue) + ([#8435](https://github.com/google/ExoPlayer/issues/8435)). + * CEA-708: Parse data into multiple service blocks and ignore blocks not + associated with the currently selected service number. + * Remove `RawCcExtractor`, which was only used to handle a Google-internal + subtitle format. * Extractors: * Matroska: Parse `DiscardPadding` for Opus tracks. - * Parse bitrates from `esds` boxes. - * MP4: Parse initialization data from AV1 tracks. + * MP4: Parse bitrates from `esds` boxes. + * Ogg: Allow duplicate Opus ID and comment headers + ([#10038](https://github.com/google/ExoPlayer/issues/10038)). * UI: * Fix delivery of events to `OnClickListener`s set on `PlayerView` and `LegacyPlayerView`, in the case that `useController=false` @@ -49,17 +106,38 @@ * Don't show forced text tracks in the `PlayerView` track selector, and keep a suitable forced text track selected if "None" is selected ([#9432](https://github.com/google/ExoPlayer/issues/9432)). +* DASH: + * Parse channel count from DTS `AudioChannelConfiguration` elements. This + re-enables audio passthrough for DTS streams + ([#10159](https://github.com/google/ExoPlayer/issues/10159)). + * Disallow passing `null` to + `DashMediaSource.Factory.setCompositeSequenceableLoaderFactory`. + Instances of `DefaultCompositeSequenceableLoaderFactory` can be passed + explicitly if required. * HLS: - * Fallback to chunkful preparation if the playlist CODECS attribute - does not contain the audio codec + * Fallback to chunkful preparation if the playlist CODECS attribute does + not contain the audio codec ([#10065](https://github.com/google/ExoPlayer/issues/10065)). + * Disallow passing `null` to + `HlsMediaSource.Factory.setCompositeSequenceableLoaderFactory`, + `HlsMediaSource.Factory.setPlaylistParserFactory`, and + `HlsMediaSource.Factory.setPlaylistTrackerFactory`. Instances of + `DefaultCompositeSequenceableLoaderFactory`, + `DefaultHlsPlaylistParserFactory`, or a reference to + `DefaultHlsPlaylistTracker.FACTORY` can be passed explicitly if + required. +* Smooth Streaming: + * Disallow passing `null` to + `SsMediaSource.Factory.setCompositeSequenceableLoaderFactory`. Instances + of `DefaultCompositeSequenceableLoaderFactory` can be passed explicitly + if required. * RTSP: * Add RTP reader for MPEG4 - ([#35](https://github.com/androidx/media/pull/35)) + ([#35](https://github.com/androidx/media/pull/35)). * Add RTP reader for HEVC ([#36](https://github.com/androidx/media/pull/36)). - * Add RTP reader for AMR. Currently only mono-channel, non-interleaved - AMR streams are supported. Compound AMR RTP payload is not supported. + * Add RTP reader for AMR. Currently only mono-channel, non-interleaved AMR + streams are supported. Compound AMR RTP payload is not supported. ([#46](https://github.com/androidx/media/pull/46)) * Add RTP reader for VP8 ([#47](https://github.com/androidx/media/pull/47)). @@ -67,13 +145,37 @@ ([#56](https://github.com/androidx/media/pull/56)). * Fix RTSP basic authorization header. ([#9544](https://github.com/google/ExoPlayer/issues/9544)). + * Stop checking mandatory SDP fields as ExoPlayer doesn't need them + ([#10049](https://github.com/google/ExoPlayer/issues/10049)). * Throw checked exception when parsing RTSP timing ([#10165](https://github.com/google/ExoPlayer/issues/10165)). + * Add RTP reader for VP9 + ([#47](https://github.com/androidx/media/pull/64)). + * Add RTP reader for OPUS + ([#53](https://github.com/androidx/media/pull/53)). * Session: * Fix NPE in MediaControllerImplLegacy - ([#59](https://github.com/androidx/media/pull/59)) + ([#59](https://github.com/androidx/media/pull/59)). + * Update session position info on timeline + change([#51](https://github.com/androidx/media/issues/51)). + * Fix NPE in MediaControllerImplBase after releasing controller + ([#74](https://github.com/androidx/media/issues/74)). + * Rename `MediaSession.MediaSessionCallback` to `MediaSession.Callback`, + `MediaLibrarySession.MediaLibrarySessionCallback` to + `MediaLibrarySession.Callback` and + `MediaSession.Builder.setSessionCallback` to `setCallback`. + * Replace `MediaSession.MediaItemFiler` with + `MediaSession.Callback.onAddMediaItems` to allow asynchronous resolution + of requests. + * Forward legacy `MediaController` calls to play media to + `MediaSession.Callback.onAddMediaItems` instead of `onSetMediaUri`. * Data sources: - * Rename `DummyDataSource` to `PlaceHolderDataSource`. + * Rename `DummyDataSource` to `PlaceholderDataSource`. + * Workaround OkHttp interrupt handling. +* FFmpeg extension: + * Update CMake version to `3.21.0+` to avoid a CMake bug causing + AndroidStudio's gradle sync to fail + ([#9933](https://github.com/google/ExoPlayer/issues/9933)). * Remove deprecated symbols: * Remove `Player.Listener.onTracksChanged`. Use `Player.Listener.onTracksInfoChanged` instead. @@ -87,10 +189,10 @@ `DEFAULT_TRACK_SELECTOR_PARAMETERS` constants. Use `getDefaultTrackSelectorParameters(Context)` instead when possible, and `DEFAULT_TRACK_SELECTOR_PARAMETERS_WITHOUT_CONTEXT` otherwise. - * FFmpeg extension: - * Update CMake version to `3.21.0+` to avoid a CMake bug causing - AndroidStudio's gradle sync to fail - ([#9933](https://github.com/google/ExoPlayer/issues/9933)). + * Remove constructor `DefaultTrackSelector(ExoTrackSelection.Factory)`. + Use `DefaultTrackSelector(Context, ExoTrackSelection.Factory)` instead. + * Remove `Transformer.Builder.setContext`. The `Context` should be passed + to the `Transformer.Builder` constructor instead. ### 1.0.0-alpha03 (2022-03-14) diff --git a/build.gradle b/build.gradle index aafc0e790c..0c15bce9e5 100644 --- a/build.gradle +++ b/build.gradle @@ -17,7 +17,7 @@ buildscript { mavenCentral() } dependencies { - classpath 'com.android.tools.build:gradle:7.0.3' + classpath 'com.android.tools.build:gradle:7.2.1' classpath 'com.google.android.gms:strict-version-matcher-plugin:1.2.2' classpath 'org.jetbrains.kotlin:kotlin-gradle-plugin:1.5.21' } diff --git a/common_library_config.gradle b/common_library_config.gradle index 5ac9e337f6..9d14a1f601 100644 --- a/common_library_config.gradle +++ b/common_library_config.gradle @@ -29,5 +29,10 @@ android { targetCompatibility JavaVersion.VERSION_1_8 } - testOptions.unitTests.includeAndroidResources = true + testOptions { + unitTests.all { + jvmArgs "-Xmx2g" + } + unitTests.includeAndroidResources true + } } diff --git a/constants.gradle b/constants.gradle index 5545b5f4ab..0d49ac63ab 100644 --- a/constants.gradle +++ b/constants.gradle @@ -26,7 +26,7 @@ project.ext { // https://cs.android.com/android/platform/superproject/+/master:external/guava/METADATA guavaVersion = '31.0.1-android' mockitoVersion = '3.12.4' - robolectricVersion = '4.8-alpha-1' + robolectricVersion = '4.8.1' // Keep this in sync with Google's internal Checker Framework version. checkerframeworkVersion = '3.13.0' checkerframeworkCompatVersion = '2.5.5' diff --git a/demos/cast/src/main/java/androidx/media3/demo/cast/MainActivity.java b/demos/cast/src/main/java/androidx/media3/demo/cast/MainActivity.java index d6aec8c4a1..aa9bd9f08e 100644 --- a/demos/cast/src/main/java/androidx/media3/demo/cast/MainActivity.java +++ b/demos/cast/src/main/java/androidx/media3/demo/cast/MainActivity.java @@ -230,8 +230,8 @@ public class MainActivity extends AppCompatActivity @Override public boolean onMove( RecyclerView list, RecyclerView.ViewHolder origin, RecyclerView.ViewHolder target) { - int fromPosition = origin.getAdapterPosition(); - int toPosition = target.getAdapterPosition(); + int fromPosition = origin.getBindingAdapterPosition(); + int toPosition = target.getBindingAdapterPosition(); if (draggingFromPosition == C.INDEX_UNSET) { // A drag has started, but changes to the media queue will be reflected in clearView(). draggingFromPosition = fromPosition; @@ -243,7 +243,7 @@ public class MainActivity extends AppCompatActivity @Override public void onSwiped(RecyclerView.ViewHolder viewHolder, int direction) { - int position = viewHolder.getAdapterPosition(); + int position = viewHolder.getBindingAdapterPosition(); QueueItemViewHolder queueItemHolder = (QueueItemViewHolder) viewHolder; if (playerManager.removeItem(queueItemHolder.item)) { mediaQueueListAdapter.notifyItemRemoved(position); @@ -282,7 +282,7 @@ public class MainActivity extends AppCompatActivity @Override public void onClick(View v) { - playerManager.selectQueueItem(getAdapterPosition()); + playerManager.selectQueueItem(getBindingAdapterPosition()); } } diff --git a/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java b/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java index a9329b5343..793e1fd493 100644 --- a/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java +++ b/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java @@ -29,6 +29,7 @@ import android.opengl.GLUtils; import androidx.media3.common.C; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.Log; import java.io.IOException; import java.util.Locale; import javax.microedition.khronos.opengles.GL10; @@ -41,6 +42,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* package */ final class BitmapOverlayVideoProcessor implements VideoProcessingGLSurfaceView.VideoProcessor { + private static final String TAG = "BitmapOverlayVP"; private static final int OVERLAY_WIDTH = 512; private static final int OVERLAY_HEIGHT = 256; @@ -85,11 +87,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* fragmentShaderFilePath= */ "bitmap_overlay_video_processor_fragment.glsl"); } catch (IOException e) { throw new IllegalStateException(e); + } catch (GlUtil.GlException e) { + Log.e(TAG, "Failed to initialize the shader program", e); + return; } program.setBufferAttribute( - "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); program.setBufferAttribute( - "aTexCoords", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); + "aTexCoords", + GlUtil.getTextureCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); GLES20.glGenTextures(1, textures, 0); GLES20.glBindTexture(GL10.GL_TEXTURE_2D, textures[0]); GLES20.glTexParameterf(GL10.GL_TEXTURE_2D, GL10.GL_TEXTURE_MIN_FILTER, GL10.GL_NEAREST); @@ -115,7 +124,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; GLES20.glBindTexture(GL10.GL_TEXTURE_2D, textures[0]); GLUtils.texSubImage2D( GL10.GL_TEXTURE_2D, /* level= */ 0, /* xoffset= */ 0, /* yoffset= */ 0, overlayBitmap); - GlUtil.checkGlError(); + try { + GlUtil.checkGlError(); + } catch (GlUtil.GlException e) { + Log.e(TAG, "Failed to populate the texture", e); + } // Run the shader program. GlProgram program = checkNotNull(this.program); @@ -124,16 +137,28 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; program.setFloatUniform("uScaleX", bitmapScaleX); program.setFloatUniform("uScaleY", bitmapScaleY); program.setFloatsUniform("uTexTransform", transformMatrix); - program.bindAttributesAndUniforms(); + try { + program.bindAttributesAndUniforms(); + } catch (GlUtil.GlException e) { + Log.e(TAG, "Failed to update the shader program", e); + } GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); - GlUtil.checkGlError(); + try { + GlUtil.checkGlError(); + } catch (GlUtil.GlException e) { + Log.e(TAG, "Failed to draw a frame", e); + } } @Override public void release() { if (program != null) { - program.delete(); + try { + program.delete(); + } catch (GlUtil.GlException e) { + Log.e(TAG, "Failed to delete the shader program", e); + } } } } diff --git a/demos/gl/src/main/java/androidx/media3/demo/gl/MainActivity.java b/demos/gl/src/main/java/androidx/media3/demo/gl/MainActivity.java index bb43d08ced..4923b4a390 100644 --- a/demos/gl/src/main/java/androidx/media3/demo/gl/MainActivity.java +++ b/demos/gl/src/main/java/androidx/media3/demo/gl/MainActivity.java @@ -20,6 +20,7 @@ import android.content.Context; import android.content.Intent; import android.net.Uri; import android.os.Bundle; +import android.text.TextUtils; import android.widget.FrameLayout; import android.widget.Toast; import androidx.annotation.Nullable; @@ -156,13 +157,18 @@ public final class MainActivity extends Activity { DataSource.Factory dataSourceFactory = new DefaultDataSource.Factory(this); MediaSource mediaSource; - @C.ContentType int type = Util.inferContentType(uri, intent.getStringExtra(EXTENSION_EXTRA)); - if (type == C.TYPE_DASH) { + @Nullable String fileExtension = intent.getStringExtra(EXTENSION_EXTRA); + @C.ContentType + int type = + TextUtils.isEmpty(fileExtension) + ? Util.inferContentType(uri) + : Util.inferContentTypeForExtension(fileExtension); + if (type == C.CONTENT_TYPE_DASH) { mediaSource = new DashMediaSource.Factory(dataSourceFactory) .setDrmSessionManagerProvider(unusedMediaItem -> drmSessionManager) .createMediaSource(MediaItem.fromUri(uri)); - } else if (type == C.TYPE_OTHER) { + } else if (type == C.CONTENT_TYPE_OTHER) { mediaSource = new ProgressiveMediaSource.Factory(dataSourceFactory) .setDrmSessionManagerProvider(unusedMediaItem -> drmSessionManager) diff --git a/demos/gl/src/main/java/androidx/media3/demo/gl/VideoProcessingGLSurfaceView.java b/demos/gl/src/main/java/androidx/media3/demo/gl/VideoProcessingGLSurfaceView.java index 9c95122d4b..c7c0aba58a 100644 --- a/demos/gl/src/main/java/androidx/media3/demo/gl/VideoProcessingGLSurfaceView.java +++ b/demos/gl/src/main/java/androidx/media3/demo/gl/VideoProcessingGLSurfaceView.java @@ -28,6 +28,7 @@ import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.Log; import androidx.media3.common.util.TimedValueQueue; import androidx.media3.exoplayer.ExoPlayer; import androidx.media3.exoplayer.video.VideoFrameMetadataListener; @@ -70,6 +71,7 @@ public final class VideoProcessingGLSurfaceView extends GLSurfaceView { } private static final int EGL_PROTECTED_CONTENT_EXT = 0x32C0; + private static final String TAG = "VPGlSurfaceView"; private final VideoRenderer renderer; private final Handler mainHandler; @@ -239,7 +241,11 @@ public final class VideoProcessingGLSurfaceView extends GLSurfaceView { @Override public synchronized void onSurfaceCreated(GL10 gl, EGLConfig config) { - texture = GlUtil.createExternalTexture(); + try { + texture = GlUtil.createExternalTexture(); + } catch (GlUtil.GlException e) { + Log.e(TAG, "Failed to create an external texture", e); + } surfaceTexture = new SurfaceTexture(texture); surfaceTexture.setOnFrameAvailableListener( surfaceTexture -> { diff --git a/demos/main/src/main/AndroidManifest.xml b/demos/main/src/main/AndroidManifest.xml index 3cffe0fa54..76fc35d287 100644 --- a/demos/main/src/main/AndroidManifest.xml +++ b/demos/main/src/main/AndroidManifest.xml @@ -78,6 +78,12 @@ + + + + + + diff --git a/demos/main/src/main/java/androidx/media3/demo/main/IntentUtil.java b/demos/main/src/main/java/androidx/media3/demo/main/IntentUtil.java index cf1fa329ec..f8023cbe63 100644 --- a/demos/main/src/main/java/androidx/media3/demo/main/IntentUtil.java +++ b/demos/main/src/main/java/androidx/media3/demo/main/IntentUtil.java @@ -177,7 +177,7 @@ public class IntentUtil { headers.put(keyRequestPropertiesArray[i], keyRequestPropertiesArray[i + 1]); } } - @Nullable UUID drmUuid = Util.getDrmUuid(Util.castNonNull(drmSchemeExtra)); + @Nullable UUID drmUuid = Util.getDrmUuid(drmSchemeExtra); if (drmUuid != null) { builder.setDrmConfiguration( new MediaItem.DrmConfiguration.Builder(drmUuid) @@ -188,7 +188,7 @@ public class IntentUtil { intent.getBooleanExtra( DRM_FORCE_DEFAULT_LICENSE_URI_EXTRA + extrasKeySuffix, false)) .setLicenseRequestHeaders(headers) - .forceSessionsForAudioAndVideoTracks( + .setForceSessionsForAudioAndVideoTracks( intent.getBooleanExtra(DRM_SESSION_FOR_CLEAR_CONTENT + extrasKeySuffix, false)) .build()); } diff --git a/demos/main/src/main/java/androidx/media3/demo/main/PlayerActivity.java b/demos/main/src/main/java/androidx/media3/demo/main/PlayerActivity.java index df7eb8c40b..22ee82e179 100644 --- a/demos/main/src/main/java/androidx/media3/demo/main/PlayerActivity.java +++ b/demos/main/src/main/java/androidx/media3/demo/main/PlayerActivity.java @@ -38,10 +38,12 @@ import androidx.media3.common.PlaybackException; import androidx.media3.common.Player; import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.Tracks; +import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.datasource.DataSource; import androidx.media3.exoplayer.ExoPlayer; import androidx.media3.exoplayer.RenderersFactory; +import androidx.media3.exoplayer.drm.DefaultDrmSessionManagerProvider; import androidx.media3.exoplayer.drm.FrameworkMediaDrm; import androidx.media3.exoplayer.ima.ImaAdsLoader; import androidx.media3.exoplayer.ima.ImaServerSideAdInsertionMediaSource; @@ -53,7 +55,6 @@ import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.source.ads.AdsLoader; import androidx.media3.exoplayer.util.DebugTextViewHelper; import androidx.media3.exoplayer.util.EventLogger; -import androidx.media3.ui.PlayerControlView; import androidx.media3.ui.PlayerView; import java.util.ArrayList; import java.util.Collections; @@ -62,7 +63,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** An activity that plays media using {@link ExoPlayer}. */ public class PlayerActivity extends AppCompatActivity - implements OnClickListener, PlayerControlView.VisibilityListener { + implements OnClickListener, PlayerView.ControllerVisibilityListener { // Saved instance state keys. @@ -91,7 +92,12 @@ public class PlayerActivity extends AppCompatActivity // For ad playback only. @Nullable private AdsLoader clientSideAdsLoader; + + // TODO: Annotate this and serverSideAdsLoaderState below with @OptIn when it can be applied to + // fields (needs http://r.android.com/2004032 to be released into a version of + // androidx.annotation:annotation-experimental). @Nullable private ImaServerSideAdInsertionMediaSource.AdsLoader serverSideAdsLoader; + private ImaServerSideAdInsertionMediaSource.AdsLoader.@MonotonicNonNull State serverSideAdsLoaderState; @@ -115,17 +121,12 @@ public class PlayerActivity extends AppCompatActivity if (savedInstanceState != null) { trackSelectionParameters = - TrackSelectionParameters.CREATOR.fromBundle( + TrackSelectionParameters.fromBundle( savedInstanceState.getBundle(KEY_TRACK_SELECTION_PARAMETERS)); startAutoPlay = savedInstanceState.getBoolean(KEY_AUTO_PLAY); startItemIndex = savedInstanceState.getInt(KEY_ITEM_INDEX); startPosition = savedInstanceState.getLong(KEY_POSITION); - Bundle adsLoaderStateBundle = savedInstanceState.getBundle(KEY_SERVER_SIDE_ADS_LOADER_STATE); - if (adsLoaderStateBundle != null) { - serverSideAdsLoaderState = - ImaServerSideAdInsertionMediaSource.AdsLoader.State.CREATOR.fromBundle( - adsLoaderStateBundle); - } + restoreServerSideAdsLoaderState(savedInstanceState); } else { trackSelectionParameters = new TrackSelectionParameters.Builder(/* context= */ this).build(); clearStartPosition(); @@ -217,9 +218,7 @@ public class PlayerActivity extends AppCompatActivity outState.putBoolean(KEY_AUTO_PLAY, startAutoPlay); outState.putInt(KEY_ITEM_INDEX, startItemIndex); outState.putLong(KEY_POSITION, startPosition); - if (serverSideAdsLoaderState != null) { - outState.putBundle(KEY_SERVER_SIDE_ADS_LOADER_STATE, serverSideAdsLoaderState.toBundle()); - } + saveServerSideAdsLoaderState(outState); } // Activity input @@ -246,10 +245,10 @@ public class PlayerActivity extends AppCompatActivity } } - // PlayerControlView.VisibilityListener implementation + // PlayerView.ControllerVisibilityListener implementation @Override - public void onVisibilityChange(int visibility) { + public void onVisibilityChanged(int visibility) { debugRootView.setVisibility(visibility); } @@ -271,24 +270,20 @@ public class PlayerActivity extends AppCompatActivity return false; } - boolean preferExtensionDecoders = - intent.getBooleanExtra(IntentUtil.PREFER_EXTENSION_DECODERS_EXTRA, false); - RenderersFactory renderersFactory = - DemoUtil.buildRenderersFactory(/* context= */ this, preferExtensionDecoders); - lastSeenTracks = Tracks.EMPTY; - player = + ExoPlayer.Builder playerBuilder = new ExoPlayer.Builder(/* context= */ this) - .setRenderersFactory(renderersFactory) - .setMediaSourceFactory(createMediaSourceFactory()) - .build(); + .setMediaSourceFactory(createMediaSourceFactory()); + setRenderersFactory( + playerBuilder, intent.getBooleanExtra(IntentUtil.PREFER_EXTENSION_DECODERS_EXTRA, false)); + player = playerBuilder.build(); player.setTrackSelectionParameters(trackSelectionParameters); player.addListener(new PlayerEventListener()); player.addAnalyticsListener(new EventLogger()); player.setAudioAttributes(AudioAttributes.DEFAULT, /* handleAudioFocus= */ true); player.setPlayWhenReady(startAutoPlay); playerView.setPlayer(player); - serverSideAdsLoader.setPlayer(player); + configurePlayerWithServerSideAdsLoader(); debugViewHelper = new DebugTextViewHelper(player, debugTextView); debugViewHelper.start(); } @@ -302,7 +297,12 @@ public class PlayerActivity extends AppCompatActivity return true; } + @OptIn(markerClass = UnstableApi.class) // SSAI configuration private MediaSource.Factory createMediaSourceFactory() { + DefaultDrmSessionManagerProvider drmSessionManagerProvider = + new DefaultDrmSessionManagerProvider(); + drmSessionManagerProvider.setDrmHttpDataSourceFactory( + DemoUtil.getHttpDataSourceFactory(/* context= */ this)); ImaServerSideAdInsertionMediaSource.AdsLoader.Builder serverSideAdLoaderBuilder = new ImaServerSideAdInsertionMediaSource.AdsLoader.Builder(/* context= */ this, playerView); if (serverSideAdsLoaderState != null) { @@ -311,13 +311,30 @@ public class PlayerActivity extends AppCompatActivity serverSideAdsLoader = serverSideAdLoaderBuilder.build(); ImaServerSideAdInsertionMediaSource.Factory imaServerSideAdInsertionMediaSourceFactory = new ImaServerSideAdInsertionMediaSource.Factory( - serverSideAdsLoader, new DefaultMediaSourceFactory(dataSourceFactory)); - return new DefaultMediaSourceFactory(dataSourceFactory) - .setAdsLoaderProvider(this::getClientSideAdsLoader) - .setAdViewProvider(playerView) + serverSideAdsLoader, + new DefaultMediaSourceFactory(/* context= */ this) + .setDataSourceFactory(dataSourceFactory)); + return new DefaultMediaSourceFactory(/* context= */ this) + .setDataSourceFactory(dataSourceFactory) + .setDrmSessionManagerProvider(drmSessionManagerProvider) + .setLocalAdInsertionComponents( + this::getClientSideAdsLoader, /* adViewProvider= */ playerView) .setServerSideAdInsertionMediaSourceFactory(imaServerSideAdInsertionMediaSourceFactory); } + @OptIn(markerClass = UnstableApi.class) + private void setRenderersFactory( + ExoPlayer.Builder playerBuilder, boolean preferExtensionDecoders) { + RenderersFactory renderersFactory = + DemoUtil.buildRenderersFactory(/* context= */ this, preferExtensionDecoders); + playerBuilder.setRenderersFactory(renderersFactory); + } + + @OptIn(markerClass = UnstableApi.class) + private void configurePlayerWithServerSideAdsLoader() { + serverSideAdsLoader.setPlayer(player); + } + private List createMediaItems(Intent intent) { String action = intent.getAction(); boolean actionIsListView = IntentUtil.ACTION_VIEW_LIST.equals(action); @@ -371,8 +388,7 @@ public class PlayerActivity extends AppCompatActivity if (player != null) { updateTrackSelectorParameters(); updateStartPosition(); - serverSideAdsLoaderState = serverSideAdsLoader.release(); - serverSideAdsLoader = null; + releaseServerSideAdsLoader(); debugViewHelper.stop(); debugViewHelper = null; player.release(); @@ -387,6 +403,12 @@ public class PlayerActivity extends AppCompatActivity } } + @OptIn(markerClass = UnstableApi.class) + private void releaseServerSideAdsLoader() { + serverSideAdsLoaderState = serverSideAdsLoader.release(); + serverSideAdsLoader = null; + } + private void releaseClientSideAdsLoader() { if (clientSideAdsLoader != null) { clientSideAdsLoader.release(); @@ -395,6 +417,23 @@ public class PlayerActivity extends AppCompatActivity } } + @OptIn(markerClass = UnstableApi.class) + private void saveServerSideAdsLoaderState(Bundle outState) { + if (serverSideAdsLoaderState != null) { + outState.putBundle(KEY_SERVER_SIDE_ADS_LOADER_STATE, serverSideAdsLoaderState.toBundle()); + } + } + + @OptIn(markerClass = UnstableApi.class) + private void restoreServerSideAdsLoaderState(Bundle savedInstanceState) { + Bundle adsLoaderStateBundle = savedInstanceState.getBundle(KEY_SERVER_SIDE_ADS_LOADER_STATE); + if (adsLoaderStateBundle != null) { + serverSideAdsLoaderState = + ImaServerSideAdInsertionMediaSource.AdsLoader.State.CREATOR.fromBundle( + adsLoaderStateBundle); + } + } + private void updateTrackSelectorParameters() { if (player != null) { trackSelectionParameters = player.getTrackSelectionParameters(); diff --git a/demos/main/src/main/java/androidx/media3/demo/main/SampleChooserActivity.java b/demos/main/src/main/java/androidx/media3/demo/main/SampleChooserActivity.java index fa61a3ba44..fc7144dc91 100644 --- a/demos/main/src/main/java/androidx/media3/demo/main/SampleChooserActivity.java +++ b/demos/main/src/main/java/androidx/media3/demo/main/SampleChooserActivity.java @@ -27,6 +27,7 @@ import android.content.res.AssetManager; import android.net.Uri; import android.os.AsyncTask; import android.os.Bundle; +import android.text.TextUtils; import android.util.JsonReader; import android.view.Menu; import android.view.MenuInflater; @@ -443,7 +444,10 @@ public class SampleChooserActivity extends AppCompatActivity } else { @Nullable String adaptiveMimeType = - Util.getAdaptiveMimeTypeForContentType(Util.inferContentType(uri, extension)); + Util.getAdaptiveMimeTypeForContentType( + TextUtils.isEmpty(extension) + ? Util.inferContentType(uri) + : Util.inferContentTypeForExtension(extension)); mediaItem .setUri(uri) .setMediaMetadata(new MediaMetadata.Builder().setTitle(title).build()) @@ -454,7 +458,7 @@ public class SampleChooserActivity extends AppCompatActivity new MediaItem.DrmConfiguration.Builder(drmUuid) .setLicenseUri(drmLicenseUri) .setLicenseRequestHeaders(drmLicenseRequestHeaders) - .forceSessionsForAudioAndVideoTracks(drmSessionForClearContent) + .setForceSessionsForAudioAndVideoTracks(drmSessionForClearContent) .setMultiSession(drmMultiSession) .setForceDefaultLicenseUri(drmForceDefaultLicenseUri) .build()); diff --git a/demos/main/src/main/java/androidx/media3/demo/main/TrackSelectionDialog.java b/demos/main/src/main/java/androidx/media3/demo/main/TrackSelectionDialog.java index 0c741ec057..d5bee96eae 100644 --- a/demos/main/src/main/java/androidx/media3/demo/main/TrackSelectionDialog.java +++ b/demos/main/src/main/java/androidx/media3/demo/main/TrackSelectionDialog.java @@ -25,6 +25,7 @@ import android.view.View; import android.view.ViewGroup; import android.widget.Button; import androidx.annotation.Nullable; +import androidx.annotation.OptIn; import androidx.appcompat.app.AppCompatDialog; import androidx.fragment.app.DialogFragment; import androidx.fragment.app.Fragment; @@ -36,6 +37,7 @@ import androidx.media3.common.TrackGroup; import androidx.media3.common.TrackSelectionOverride; import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.Tracks; +import androidx.media3.common.util.UnstableApi; import androidx.media3.ui.TrackSelectionView; import androidx.viewpager.widget.ViewPager; import com.google.android.material.tabs.TabLayout; @@ -47,6 +49,7 @@ import java.util.List; import java.util.Map; /** Dialog to select tracks. */ +@OptIn(markerClass = UnstableApi.class) public final class TrackSelectionDialog extends DialogFragment { /** Called when tracks are selected. */ diff --git a/demos/session/src/main/AndroidManifest.xml b/demos/session/src/main/AndroidManifest.xml index f86bcd86bb..e94b527e83 100644 --- a/demos/session/src/main/AndroidManifest.xml +++ b/demos/session/src/main/AndroidManifest.xml @@ -45,6 +45,7 @@ diff --git a/demos/session/src/main/java/androidx/media3/demo/session/PlaybackService.kt b/demos/session/src/main/java/androidx/media3/demo/session/PlaybackService.kt index b4ec76ee5a..cc8291c27d 100644 --- a/demos/session/src/main/java/androidx/media3/demo/session/PlaybackService.kt +++ b/demos/session/src/main/java/androidx/media3/demo/session/PlaybackService.kt @@ -19,35 +19,118 @@ import android.app.PendingIntent.FLAG_IMMUTABLE import android.app.PendingIntent.FLAG_UPDATE_CURRENT import android.app.TaskStackBuilder import android.content.Intent -import android.net.Uri import android.os.Build import android.os.Bundle import androidx.media3.common.AudioAttributes import androidx.media3.common.MediaItem import androidx.media3.exoplayer.ExoPlayer +import androidx.media3.session.CommandButton import androidx.media3.session.LibraryResult import androidx.media3.session.MediaLibraryService import androidx.media3.session.MediaSession +import androidx.media3.session.MediaSession.ControllerInfo +import androidx.media3.session.SessionCommand import androidx.media3.session.SessionResult import com.google.common.collect.ImmutableList import com.google.common.util.concurrent.Futures import com.google.common.util.concurrent.ListenableFuture class PlaybackService : MediaLibraryService() { + private val librarySessionCallback = CustomMediaLibrarySessionCallback() + private lateinit var player: ExoPlayer private lateinit var mediaLibrarySession: MediaLibrarySession - private val librarySessionCallback = CustomMediaLibrarySessionCallback() + private lateinit var customCommands: List + + private var customLayout = ImmutableList.of() companion object { private const val SEARCH_QUERY_PREFIX_COMPAT = "androidx://media3-session/playFromSearch" private const val SEARCH_QUERY_PREFIX = "androidx://media3-session/setMediaUri" + private const val CUSTOM_COMMAND_TOGGLE_SHUFFLE_MODE_ON = + "android.media3.session.demo.SHUFFLE_ON" + private const val CUSTOM_COMMAND_TOGGLE_SHUFFLE_MODE_OFF = + "android.media3.session.demo.SHUFFLE_OFF" } - private inner class CustomMediaLibrarySessionCallback : - MediaLibrarySession.MediaLibrarySessionCallback { + override fun onCreate() { + super.onCreate() + customCommands = + listOf( + getShuffleCommandButton( + SessionCommand(CUSTOM_COMMAND_TOGGLE_SHUFFLE_MODE_ON, Bundle.EMPTY) + ), + getShuffleCommandButton( + SessionCommand(CUSTOM_COMMAND_TOGGLE_SHUFFLE_MODE_OFF, Bundle.EMPTY) + ) + ) + customLayout = ImmutableList.of(customCommands[0]) + initializeSessionAndPlayer() + } + + override fun onGetSession(controllerInfo: ControllerInfo): MediaLibrarySession { + return mediaLibrarySession + } + + override fun onDestroy() { + player.release() + mediaLibrarySession.release() + super.onDestroy() + } + + private inner class CustomMediaLibrarySessionCallback : MediaLibrarySession.Callback { + + override fun onConnect( + session: MediaSession, + controller: ControllerInfo + ): MediaSession.ConnectionResult { + val connectionResult = super.onConnect(session, controller) + val availableSessionCommands = connectionResult.availableSessionCommands.buildUpon() + customCommands.forEach { commandButton -> + // Add custom command to available session commands. + commandButton.sessionCommand?.let { availableSessionCommands.add(it) } + } + return MediaSession.ConnectionResult.accept( + availableSessionCommands.build(), + connectionResult.availablePlayerCommands + ) + } + + override fun onPostConnect(session: MediaSession, controller: ControllerInfo) { + if (!customLayout.isEmpty() && controller.controllerVersion != 0) { + // Let Media3 controller (for instance the MediaNotificationProvider) know about the custom + // layout right after it connected. + ignoreFuture(mediaLibrarySession.setCustomLayout(controller, customLayout)) + } + } + + override fun onCustomCommand( + session: MediaSession, + controller: ControllerInfo, + customCommand: SessionCommand, + args: Bundle + ): ListenableFuture { + if (CUSTOM_COMMAND_TOGGLE_SHUFFLE_MODE_ON == customCommand.customAction) { + // Enable shuffling. + player.shuffleModeEnabled = true + // Change the custom layout to contain the `Disable shuffling` command. + customLayout = ImmutableList.of(customCommands[1]) + // Send the updated custom layout to controllers. + session.setCustomLayout(customLayout) + } else if (CUSTOM_COMMAND_TOGGLE_SHUFFLE_MODE_OFF == customCommand.customAction) { + // Disable shuffling. + player.shuffleModeEnabled = false + // Change the custom layout to contain the `Enable shuffling` command. + customLayout = ImmutableList.of(customCommands[0]) + // Send the updated custom layout to controllers. + session.setCustomLayout(customLayout) + } + return Futures.immediateFuture(SessionResult(SessionResult.RESULT_SUCCESS)) + } + override fun onGetLibraryRoot( session: MediaLibrarySession, - browser: MediaSession.ControllerInfo, + browser: ControllerInfo, params: LibraryParams? ): ListenableFuture> { return Futures.immediateFuture(LibraryResult.ofItem(MediaItemTree.getRootItem(), params)) @@ -55,7 +138,7 @@ class PlaybackService : MediaLibraryService() { override fun onGetItem( session: MediaLibrarySession, - browser: MediaSession.ControllerInfo, + browser: ControllerInfo, mediaId: String ): ListenableFuture> { val item = @@ -66,9 +149,24 @@ class PlaybackService : MediaLibraryService() { return Futures.immediateFuture(LibraryResult.ofItem(item, /* params= */ null)) } + override fun onSubscribe( + session: MediaLibrarySession, + browser: ControllerInfo, + parentId: String, + params: LibraryParams? + ): ListenableFuture> { + val children = + MediaItemTree.getChildren(parentId) + ?: return Futures.immediateFuture( + LibraryResult.ofError(LibraryResult.RESULT_ERROR_BAD_VALUE) + ) + session.notifyChildrenChanged(browser, parentId, children.size, params) + return Futures.immediateFuture(LibraryResult.ofVoid()) + } + override fun onGetChildren( session: MediaLibrarySession, - browser: MediaSession.ControllerInfo, + browser: ControllerInfo, parentId: String, page: Int, pageSize: Int, @@ -83,7 +181,21 @@ class PlaybackService : MediaLibraryService() { return Futures.immediateFuture(LibraryResult.ofItemList(children, params)) } - private fun setMediaItemFromSearchQuery(query: String) { + override fun onAddMediaItems( + mediaSession: MediaSession, + controller: MediaSession.ControllerInfo, + mediaItems: List + ): ListenableFuture> { + val updatedMediaItems: List = + mediaItems.map { mediaItem -> + if (mediaItem.requestMetadata.searchQuery != null) + getMediaItemFromSearchQuery(mediaItem.requestMetadata.searchQuery!!) + else MediaItemTree.getItem(mediaItem.mediaId) ?: mediaItem + } + return Futures.immediateFuture(updatedMediaItems) + } + + private fun getMediaItemFromSearchQuery(query: String): MediaItem { // Only accept query with pattern "play [Title]" or "[Title]" // Where [Title]: must be exactly matched // If no media with exact name found, play a random media instead @@ -94,54 +206,8 @@ class PlaybackService : MediaLibraryService() { query } - val item = MediaItemTree.getItemFromTitle(mediaTitle) ?: MediaItemTree.getRandomItem() - player.setMediaItem(item) + return MediaItemTree.getItemFromTitle(mediaTitle) ?: MediaItemTree.getRandomItem() } - - override fun onSetMediaUri( - session: MediaSession, - controller: MediaSession.ControllerInfo, - uri: Uri, - extras: Bundle - ): Int { - - if (uri.toString().startsWith(SEARCH_QUERY_PREFIX) || - uri.toString().startsWith(SEARCH_QUERY_PREFIX_COMPAT) - ) { - var searchQuery = - uri.getQueryParameter("query") ?: return SessionResult.RESULT_ERROR_NOT_SUPPORTED - setMediaItemFromSearchQuery(searchQuery) - - return SessionResult.RESULT_SUCCESS - } else { - return SessionResult.RESULT_ERROR_NOT_SUPPORTED - } - } - } - - private class CustomMediaItemFiller : MediaSession.MediaItemFiller { - override fun fillInLocalConfiguration( - session: MediaSession, - controller: MediaSession.ControllerInfo, - mediaItem: MediaItem - ): MediaItem { - return MediaItemTree.getItem(mediaItem.mediaId) ?: mediaItem - } - } - - override fun onCreate() { - super.onCreate() - initializeSessionAndPlayer() - } - - override fun onDestroy() { - player.release() - mediaLibrarySession.release() - super.onDestroy() - } - - override fun onGetSession(controllerInfo: MediaSession.ControllerInfo): MediaLibrarySession { - return mediaLibrarySession } private fun initializeSessionAndPlayer() { @@ -151,13 +217,10 @@ class PlaybackService : MediaLibraryService() { .build() MediaItemTree.initialize(assets) - val parentScreenIntent = Intent(this, MainActivity::class.java) - val intent = Intent(this, PlayerActivity::class.java) - - val pendingIntent = + val sessionActivityPendingIntent = TaskStackBuilder.create(this).run { - addNextIntent(parentScreenIntent) - addNextIntent(intent) + addNextIntent(Intent(this@PlaybackService, MainActivity::class.java)) + addNextIntent(Intent(this@PlaybackService, PlayerActivity::class.java)) val immutableFlag = if (Build.VERSION.SDK_INT >= 23) FLAG_IMMUTABLE else 0 getPendingIntent(0, immutableFlag or FLAG_UPDATE_CURRENT) @@ -165,8 +228,29 @@ class PlaybackService : MediaLibraryService() { mediaLibrarySession = MediaLibrarySession.Builder(this, player, librarySessionCallback) - .setMediaItemFiller(CustomMediaItemFiller()) - .setSessionActivity(pendingIntent) + .setSessionActivity(sessionActivityPendingIntent) .build() + if (!customLayout.isEmpty()) { + // Send custom layout to legacy session. + mediaLibrarySession.setCustomLayout(customLayout) + } + } + + private fun getShuffleCommandButton(sessionCommand: SessionCommand): CommandButton { + val isOn = sessionCommand.customAction == CUSTOM_COMMAND_TOGGLE_SHUFFLE_MODE_ON + return CommandButton.Builder() + .setDisplayName( + getString( + if (isOn) R.string.exo_controls_shuffle_on_description + else R.string.exo_controls_shuffle_off_description + ) + ) + .setSessionCommand(sessionCommand) + .setIconResId(if (isOn) R.drawable.exo_icon_shuffle_off else R.drawable.exo_icon_shuffle_on) + .build() + } + + private fun ignoreFuture(customLayout: ListenableFuture) { + /* Do nothing. */ } } diff --git a/demos/surface/src/main/AndroidManifest.xml b/demos/surface/src/main/AndroidManifest.xml index 2c009ed2cc..e33c9e7242 100644 --- a/demos/surface/src/main/AndroidManifest.xml +++ b/demos/surface/src/main/AndroidManifest.xml @@ -43,6 +43,12 @@ + + + + + + diff --git a/demos/surface/src/main/java/androidx/media3/demo/surface/MainActivity.java b/demos/surface/src/main/java/androidx/media3/demo/surface/MainActivity.java index 405f3b4df2..bb3d20c094 100644 --- a/demos/surface/src/main/java/androidx/media3/demo/surface/MainActivity.java +++ b/demos/surface/src/main/java/androidx/media3/demo/surface/MainActivity.java @@ -19,6 +19,7 @@ import android.app.Activity; import android.content.Intent; import android.net.Uri; import android.os.Bundle; +import android.text.TextUtils; import android.view.Surface; import android.view.SurfaceControl; import android.view.SurfaceHolder; @@ -201,13 +202,18 @@ public final class MainActivity extends Activity { DataSource.Factory dataSourceFactory = new DefaultDataSource.Factory(this); MediaSource mediaSource; - @C.ContentType int type = Util.inferContentType(uri, intent.getStringExtra(EXTENSION_EXTRA)); - if (type == C.TYPE_DASH) { + @Nullable String fileExtension = intent.getStringExtra(EXTENSION_EXTRA); + @C.ContentType + int type = + TextUtils.isEmpty(fileExtension) + ? Util.inferContentType(uri) + : Util.inferContentTypeForExtension(fileExtension); + if (type == C.CONTENT_TYPE_DASH) { mediaSource = new DashMediaSource.Factory(dataSourceFactory) .setDrmSessionManagerProvider(unusedMediaItem -> drmSessionManager) .createMediaSource(MediaItem.fromUri(uri)); - } else if (type == C.TYPE_OTHER) { + } else if (type == C.CONTENT_TYPE_OTHER) { mediaSource = new ProgressiveMediaSource.Factory(dataSourceFactory) .setDrmSessionManagerProvider(unusedMediaItem -> drmSessionManager) diff --git a/demos/transformer/BUILD.bazel b/demos/transformer/BUILD.bazel new file mode 100644 index 0000000000..dba4a65315 --- /dev/null +++ b/demos/transformer/BUILD.bazel @@ -0,0 +1,22 @@ +# Build targets for a demo MediaPipe graph. +# See README.md for instructions on using MediaPipe in the demo. + +load("//mediapipe/java/com/google/mediapipe:mediapipe_aar.bzl", "mediapipe_aar") +load( + "//mediapipe/framework/tool:mediapipe_graph.bzl", + "mediapipe_binary_graph", +) + +mediapipe_aar( + name = "edge_detector_mediapipe_aar", + calculators = [ + "//mediapipe/calculators/image:luminance_calculator", + "//mediapipe/calculators/image:sobel_edges_calculator", + ], +) + +mediapipe_binary_graph( + name = "edge_detector_binary_graph", + graph = "edge_detector_mediapipe_graph.pbtxt", + output_name = "edge_detector_mediapipe_graph.binarypb", +) diff --git a/demos/transformer/README.md b/demos/transformer/README.md index fb2657001e..fd767ba6c8 100644 --- a/demos/transformer/README.md +++ b/demos/transformer/README.md @@ -6,4 +6,61 @@ example by removing audio or video. See the [demos README](../README.md) for instructions on how to build and run this demo. +## MediaPipe frame processing demo + +Building the demo app with [MediaPipe][] integration enabled requires some extra +manual steps. + +1. Follow the + [instructions](https://google.github.io/mediapipe/getting_started/install.html) + to install MediaPipe. +1. Copy the Transformer demo's build configuration and MediaPipe graph text + protocol buffer under the MediaPipe source tree. This makes it easy to + [build an AAR][] with bazel by reusing MediaPipe's workspace. + + ```sh + cd "" + MEDIAPIPE_ROOT="$(pwd)" + MEDIAPIPE_TRANSFORMER_ROOT="${MEDIAPIPE_ROOT}/mediapipe/java/com/google/mediapipe/transformer" + cd "" + TRANSFORMER_DEMO_ROOT="$(pwd)" + mkdir -p "${MEDIAPIPE_TRANSFORMER_ROOT}" + mkdir -p "${TRANSFORMER_DEMO_ROOT}/libs" + cp ${TRANSFORMER_DEMO_ROOT}/BUILD.bazel ${MEDIAPIPE_TRANSFORMER_ROOT}/BUILD + cp ${TRANSFORMER_DEMO_ROOT}/src/withMediaPipe/assets/edge_detector_mediapipe_graph.pbtxt \ + ${MEDIAPIPE_TRANSFORMER_ROOT} + ``` + +1. Build the AAR and the binary proto for the demo's MediaPipe graph, then copy + them to Transformer. + + ```sh + cd ${MEDIAPIPE_ROOT} + bazel build -c opt --strip=ALWAYS \ + --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \ + --fat_apk_cpu=arm64-v8a,armeabi-v7a \ + --legacy_whole_archive=0 \ + --features=-legacy_whole_archive \ + --copt=-fvisibility=hidden \ + --copt=-ffunction-sections \ + --copt=-fdata-sections \ + --copt=-fstack-protector \ + --copt=-Oz \ + --copt=-fomit-frame-pointer \ + --copt=-DABSL_MIN_LOG_LEVEL=2 \ + --linkopt=-Wl,--gc-sections,--strip-all \ + mediapipe/java/com/google/mediapipe/transformer:edge_detector_mediapipe_aar.aar + cp bazel-bin/mediapipe/java/com/google/mediapipe/transformer/edge_detector_mediapipe_aar.aar \ + ${TRANSFORMER_DEMO_ROOT}/libs + bazel build mediapipe/java/com/google/mediapipe/transformer:edge_detector_binary_graph + cp bazel-bin/mediapipe/java/com/google/mediapipe/transformer/edge_detector_mediapipe_graph.binarypb \ + ${TRANSFORMER_DEMO_ROOT}/src/withMediaPipe/assets + ``` + +1. In Android Studio, gradle sync and select the `withMediaPipe` build variant + (this will only appear if the AAR is present), then build and run the demo + app and select a MediaPipe-based effect. + [Transformer]: https://exoplayer.dev/transforming-media.html +[MediaPipe]: https://google.github.io/mediapipe/ +[build an AAR]: https://google.github.io/mediapipe/getting_started/android_archive_library.html diff --git a/demos/transformer/build.gradle b/demos/transformer/build.gradle index 0146ec9e7b..a745fcea1f 100644 --- a/demos/transformer/build.gradle +++ b/demos/transformer/build.gradle @@ -45,6 +45,27 @@ android { // This demo app isn't indexed and doesn't have translations. disable 'GoogleAppIndexingWarning','MissingTranslation' } + + flavorDimensions "mediaPipe" + + productFlavors { + noMediaPipe { + dimension "mediaPipe" + } + withMediaPipe { + dimension "mediaPipe" + } + } + + // Ignore the withMediaPipe variant if the MediaPipe AAR is not present. + if (!project.file("libs/edge_detector_mediapipe_aar.aar").exists()) { + variantFilter { variant -> + def names = variant.flavors*.name + if (names.contains("withMediaPipe")) { + setIgnore(true) + } + } + } } dependencies { @@ -56,6 +77,14 @@ dependencies { implementation 'androidx.multidex:multidex:' + androidxMultidexVersion implementation 'com.google.android.material:material:' + androidxMaterialVersion implementation project(modulePrefix + 'lib-exoplayer') + implementation project(modulePrefix + 'lib-exoplayer-dash') implementation project(modulePrefix + 'lib-transformer') implementation project(modulePrefix + 'lib-ui') + + // For MediaPipe and its dependencies: + withMediaPipeImplementation fileTree(dir: 'libs', include: ['*.aar']) + withMediaPipeImplementation 'com.google.flogger:flogger:latest.release' + withMediaPipeImplementation 'com.google.flogger:flogger-system-backend:latest.release' + withMediaPipeImplementation 'com.google.code.findbugs:jsr305:latest.release' + withMediaPipeImplementation 'com.google.protobuf:protobuf-javalite:3.19.1' } diff --git a/demos/transformer/src/main/AndroidManifest.xml b/demos/transformer/src/main/AndroidManifest.xml index 5006e431c1..ff7e08db74 100644 --- a/demos/transformer/src/main/AndroidManifest.xml +++ b/demos/transformer/src/main/AndroidManifest.xml @@ -49,6 +49,12 @@ + + + + + + The bitmap is drawn using an Android {@link Canvas}. */ -// TODO(b/227625365): Delete this class and use a frame processor from the Transformer library, once -// overlaying a bitmap and text is supported in Transformer. -/* package */ final class BitmapOverlayFrameProcessor implements GlFrameProcessor { - static { - GlUtil.glAssertionsEnabled = true; - } +// TODO(b/227625365): Delete this class and use a texture processor from the Transformer library, +// once overlaying a bitmap and text is supported in Transformer. +/* package */ final class BitmapOverlayProcessor extends SingleFrameGlTextureProcessor { private static final String VERTEX_SHADER_PATH = "vertex_shader_copy_es2.glsl"; private static final String FRAGMENT_SHADER_PATH = "fragment_shader_bitmap_overlay_es2.glsl"; @@ -55,16 +53,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final Paint paint; private final Bitmap overlayBitmap; + private final Bitmap logoBitmap; private final Canvas overlayCanvas; + private final GlProgram glProgram; private float bitmapScaleX; private float bitmapScaleY; private int bitmapTexId; - private @MonotonicNonNull Size outputSize; - private @MonotonicNonNull Bitmap logoBitmap; - private @MonotonicNonNull GlProgram glProgram; - public BitmapOverlayFrameProcessor() { + /** + * Creates a new instance. + * + * @throws FrameProcessingException If a problem occurs while reading shader files. + */ + public BitmapOverlayProcessor(Context context) throws FrameProcessingException { paint = new Paint(); paint.setTextSize(64); paint.setAntiAlias(true); @@ -73,19 +75,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; overlayBitmap = Bitmap.createBitmap(BITMAP_WIDTH_HEIGHT, BITMAP_WIDTH_HEIGHT, Bitmap.Config.ARGB_8888); overlayCanvas = new Canvas(overlayBitmap); - } - - @Override - public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) - throws IOException { - if (inputWidth > inputHeight) { - bitmapScaleX = inputWidth / (float) inputHeight; - bitmapScaleY = 1f; - } else { - bitmapScaleX = 1f; - bitmapScaleY = inputHeight / (float) inputWidth; - } - outputSize = new Size(inputWidth, inputHeight); try { logoBitmap = @@ -95,55 +84,77 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } catch (PackageManager.NameNotFoundException e) { throw new IllegalStateException(e); } - bitmapTexId = GlUtil.createTexture(BITMAP_WIDTH_HEIGHT, BITMAP_WIDTH_HEIGHT); - GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, overlayBitmap, /* border= */ 0); + try { + bitmapTexId = GlUtil.createTexture(BITMAP_WIDTH_HEIGHT, BITMAP_WIDTH_HEIGHT); + GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, overlayBitmap, /* border= */ 0); - glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); + glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); + } catch (GlUtil.GlException | IOException e) { + throw new FrameProcessingException(e); + } // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. glProgram.setBufferAttribute( - "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); - glProgram.setBufferAttribute( - "aTexSamplingCoord", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); - glProgram.setSamplerTexIdUniform("uTexSampler0", inputTexId, /* texUnitIndex= */ 0); + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); glProgram.setSamplerTexIdUniform("uTexSampler1", bitmapTexId, /* texUnitIndex= */ 1); + } + + @Override + public Size configure(int inputWidth, int inputHeight) { + if (inputWidth > inputHeight) { + bitmapScaleX = inputWidth / (float) inputHeight; + bitmapScaleY = 1f; + } else { + bitmapScaleX = 1f; + bitmapScaleY = inputHeight / (float) inputWidth; + } + glProgram.setFloatUniform("uScaleX", bitmapScaleX); glProgram.setFloatUniform("uScaleY", bitmapScaleY); + + return new Size(inputWidth, inputHeight); } @Override - public Size getOutputSize() { - return checkStateNotNull(outputSize); + public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { + try { + checkStateNotNull(glProgram).use(); + + // Draw to the canvas and store it in a texture. + String text = + String.format(Locale.US, "%.02f", presentationTimeUs / (float) C.MICROS_PER_SECOND); + overlayBitmap.eraseColor(Color.TRANSPARENT); + overlayCanvas.drawBitmap(checkStateNotNull(logoBitmap), /* left= */ 3, /* top= */ 378, paint); + overlayCanvas.drawText(text, /* x= */ 160, /* y= */ 466, paint); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexId); + GLUtils.texSubImage2D( + GLES20.GL_TEXTURE_2D, + /* level= */ 0, + /* xoffset= */ 0, + /* yoffset= */ 0, + flipBitmapVertically(overlayBitmap)); + GlUtil.checkGlError(); + + glProgram.setSamplerTexIdUniform("uTexSampler0", inputTexId, /* texUnitIndex= */ 0); + glProgram.bindAttributesAndUniforms(); + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + GlUtil.checkGlError(); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e, presentationTimeUs); + } } @Override - public void drawFrame(long presentationTimeUs) { - checkStateNotNull(glProgram); - glProgram.use(); - - // Draw to the canvas and store it in a texture. - String text = - String.format(Locale.US, "%.02f", presentationTimeUs / (float) C.MICROS_PER_SECOND); - overlayBitmap.eraseColor(Color.TRANSPARENT); - overlayCanvas.drawBitmap(checkStateNotNull(logoBitmap), /* left= */ 3, /* top= */ 378, paint); - overlayCanvas.drawText(text, /* x= */ 160, /* y= */ 466, paint); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexId); - GLUtils.texSubImage2D( - GLES20.GL_TEXTURE_2D, - /* level= */ 0, - /* xoffset= */ 0, - /* yoffset= */ 0, - flipBitmapVertically(overlayBitmap)); - GlUtil.checkGlError(); - - glProgram.bindAttributesAndUniforms(); - // The four-vertex triangle strip forms a quad. - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); - } - - @Override - public void release() { + public void release() throws FrameProcessingException { + super.release(); if (glProgram != null) { - glProgram.delete(); + try { + glProgram.delete(); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e); + } } } diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java index 8c078a180d..026f396091 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java @@ -32,6 +32,7 @@ import android.widget.TextView; import androidx.annotation.Nullable; import androidx.appcompat.app.AlertDialog; import androidx.appcompat.app.AppCompatActivity; +import androidx.media3.common.C; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.Util; import com.google.android.material.slider.RangeSlider; @@ -55,44 +56,55 @@ public final class ConfigurationActivity extends AppCompatActivity { public static final String SCALE_X = "scale_x"; public static final String SCALE_Y = "scale_y"; public static final String ROTATE_DEGREES = "rotate_degrees"; + public static final String TRIM_START_MS = "trim_start_ms"; + public static final String TRIM_END_MS = "trim_end_ms"; public static final String ENABLE_FALLBACK = "enable_fallback"; public static final String ENABLE_REQUEST_SDR_TONE_MAPPING = "enable_request_sdr_tone_mapping"; public static final String ENABLE_HDR_EDITING = "enable_hdr_editing"; - public static final String DEMO_FRAME_PROCESSORS_SELECTIONS = "demo_frame_processors_selections"; + public static final String DEMO_EFFECTS_SELECTIONS = "demo_effects_selections"; public static final String PERIODIC_VIGNETTE_CENTER_X = "periodic_vignette_center_x"; public static final String PERIODIC_VIGNETTE_CENTER_Y = "periodic_vignette_center_y"; public static final String PERIODIC_VIGNETTE_INNER_RADIUS = "periodic_vignette_inner_radius"; public static final String PERIODIC_VIGNETTE_OUTER_RADIUS = "periodic_vignette_outer_radius"; private static final String[] INPUT_URIS = { - "https://html5demos.com/assets/dizzy.mp4", - "https://storage.googleapis.com/exoplayer-test-media-1/mp4/1920w_1080h_4s.mp4", + "https://storage.googleapis.com/exoplayer-test-media-1/mp4/android-screens-10s.mp4", "https://storage.googleapis.com/exoplayer-test-media-0/android-block-1080-hevc.mp4", - "https://storage.googleapis.com/exoplayer-test-media-0/BigBuckBunny_320x180.mp4", + "https://html5demos.com/assets/dizzy.mp4", "https://html5demos.com/assets/dizzy.webm", "https://storage.googleapis.com/exoplayer-test-media-1/mp4/portrait_4k60.mp4", "https://storage.googleapis.com/exoplayer-test-media-1/mp4/8k24fps_4s.mp4", + "https://storage.googleapis.com/exoplayer-test-media-1/mp4/1920w_1080h_4s.mp4", + "https://storage.googleapis.com/exoplayer-test-media-0/BigBuckBunny_320x180.mp4", "https://storage.googleapis.com/exoplayer-test-media-1/mp4/portrait_avc_aac.mp4", "https://storage.googleapis.com/exoplayer-test-media-1/mp4/portrait_rotated_avc_aac.mp4", "https://storage.googleapis.com/exoplayer-test-media-1/mp4/slow-motion/slowMotion_stopwatch_240fps_long.mp4", + "https://storage.googleapis.com/exoplayer-test-media-1/gen/screens/dash-vod-single-segment/manifest-baseline.mpd", "https://storage.googleapis.com/exoplayer-test-media-1/mp4/samsung-hdr-hdr10.mp4", }; private static final String[] URI_DESCRIPTIONS = { // same order as INPUT_URIS - "MP4 with H264 video and AAC audio", - "Short MP4 with H265 video and AAC audio", - "MP4 with H265 video and AAC audio", - "Long MP4 with H264 video and AAC audio", - "WebM with VP8 video and Vorbis audio", - "4K 60fps MP4 with H264 video and AAC audio (portrait, timestamps always increase)", - "8k 24fps MP4 with H265 video and AAC audio", - "MP4 with H264 video and AAC audio (portrait, H > W, 0\u00B0)", - "MP4 with H264 video and AAC audio (portrait, H < W, 90\u00B0)", + "720p H264 video and AAC audio", + "1080p H265 video and AAC audio", + "360p H264 video and AAC audio", + "360p VP8 video and Vorbis audio", + "4K H264 video and AAC audio (portrait, no B-frames)", + "8k H265 video and AAC audio", + "Short 1080p H265 video and AAC audio", + "Long 180p H264 video and AAC audio", + "H264 video and AAC audio (portrait, H > W, 0\u00B0)", + "H264 video and AAC audio (portrait, H < W, 90\u00B0)", "SEF slow motion with 240 fps", - "MP4 with HDR (HDR10) H265 video (encoding may fail)", + "480p DASH (non-square pixels)", + "HDR (HDR10) H265 video (encoding may fail)", }; - private static final String[] DEMO_FRAME_PROCESSORS = { - "Dizzy crop", "Periodic vignette", "3D spin", "Overlay logo & timer", "Zoom in start" + private static final String[] DEMO_EFFECTS = { + "Dizzy crop", + "Edge detector (Media Pipe)", + "Periodic vignette", + "3D spin", + "Overlay logo & timer", + "Zoom in start", }; - private static final int PERIODIC_VIGNETTE_INDEX = 1; + private static final int PERIODIC_VIGNETTE_INDEX = 2; private static final String SAME_AS_INPUT_OPTION = "same as input"; private static final float HALF_DIAGONAL = 1f / (float) Math.sqrt(2); @@ -106,12 +118,15 @@ public final class ConfigurationActivity extends AppCompatActivity { private @MonotonicNonNull Spinner resolutionHeightSpinner; private @MonotonicNonNull Spinner scaleSpinner; private @MonotonicNonNull Spinner rotateSpinner; + private @MonotonicNonNull CheckBox trimCheckBox; private @MonotonicNonNull CheckBox enableFallbackCheckBox; private @MonotonicNonNull CheckBox enableRequestSdrToneMappingCheckBox; private @MonotonicNonNull CheckBox enableHdrEditingCheckBox; - private @MonotonicNonNull Button selectDemoFrameProcessorsButton; - private boolean @MonotonicNonNull [] demoFrameProcessorsSelections; + private @MonotonicNonNull Button selectDemoEffectsButton; + private boolean @MonotonicNonNull [] demoEffectsSelections; private int inputUriPosition; + private long trimStartMs; + private long trimEndMs; private float periodicVignetteCenterX; private float periodicVignetteCenterY; private float periodicVignetteInnerRadius; @@ -179,15 +194,20 @@ public final class ConfigurationActivity extends AppCompatActivity { rotateSpinner.setAdapter(rotateAdapter); rotateAdapter.addAll(SAME_AS_INPUT_OPTION, "0", "10", "45", "60", "90", "180"); + trimCheckBox = findViewById(R.id.trim_checkbox); + trimCheckBox.setOnCheckedChangeListener(this::selectTrimBounds); + trimStartMs = C.TIME_UNSET; + trimEndMs = C.TIME_UNSET; + enableFallbackCheckBox = findViewById(R.id.enable_fallback_checkbox); enableRequestSdrToneMappingCheckBox = findViewById(R.id.request_sdr_tone_mapping_checkbox); enableRequestSdrToneMappingCheckBox.setEnabled(isRequestSdrToneMappingSupported()); findViewById(R.id.request_sdr_tone_mapping).setEnabled(isRequestSdrToneMappingSupported()); enableHdrEditingCheckBox = findViewById(R.id.hdr_editing_checkbox); - demoFrameProcessorsSelections = new boolean[DEMO_FRAME_PROCESSORS.length]; - selectDemoFrameProcessorsButton = findViewById(R.id.select_demo_frameprocessors_button); - selectDemoFrameProcessorsButton.setOnClickListener(this::selectFrameProcessors); + demoEffectsSelections = new boolean[DEMO_EFFECTS.length]; + selectDemoEffectsButton = findViewById(R.id.select_demo_effects_button); + selectDemoEffectsButton.setOnClickListener(this::selectDemoEffects); } @Override @@ -215,13 +235,14 @@ public final class ConfigurationActivity extends AppCompatActivity { "resolutionHeightSpinner", "scaleSpinner", "rotateSpinner", + "trimCheckBox", "enableFallbackCheckBox", "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox", - "demoFrameProcessorsSelections" + "demoEffectsSelections" }) private void startTransformation(View view) { - Intent transformerIntent = new Intent(this, TransformerActivity.class); + Intent transformerIntent = new Intent(/* packageContext= */ this, TransformerActivity.class); Bundle bundle = new Bundle(); bundle.putBoolean(SHOULD_REMOVE_AUDIO, removeAudioCheckbox.isChecked()); bundle.putBoolean(SHOULD_REMOVE_VIDEO, removeVideoCheckbox.isChecked()); @@ -249,11 +270,15 @@ public final class ConfigurationActivity extends AppCompatActivity { if (!SAME_AS_INPUT_OPTION.equals(selectedRotate)) { bundle.putFloat(ROTATE_DEGREES, Float.parseFloat(selectedRotate)); } + if (trimCheckBox.isChecked()) { + bundle.putLong(TRIM_START_MS, trimStartMs); + bundle.putLong(TRIM_END_MS, trimEndMs); + } bundle.putBoolean(ENABLE_FALLBACK, enableFallbackCheckBox.isChecked()); bundle.putBoolean( ENABLE_REQUEST_SDR_TONE_MAPPING, enableRequestSdrToneMappingCheckBox.isChecked()); bundle.putBoolean(ENABLE_HDR_EDITING, enableHdrEditingCheckBox.isChecked()); - bundle.putBooleanArray(DEMO_FRAME_PROCESSORS_SELECTIONS, demoFrameProcessorsSelections); + bundle.putBooleanArray(DEMO_EFFECTS_SELECTIONS, demoEffectsSelections); bundle.putFloat(PERIODIC_VIGNETTE_CENTER_X, periodicVignetteCenterX); bundle.putFloat(PERIODIC_VIGNETTE_CENTER_Y, periodicVignetteCenterY); bundle.putFloat(PERIODIC_VIGNETTE_INNER_RADIUS, periodicVignetteInnerRadius); @@ -276,27 +301,46 @@ public final class ConfigurationActivity extends AppCompatActivity { .show(); } - private void selectFrameProcessors(View view) { + private void selectDemoEffects(View view) { new AlertDialog.Builder(/* context= */ this) - .setTitle(R.string.select_demo_frameprocessors) + .setTitle(R.string.select_demo_effects) .setMultiChoiceItems( - DEMO_FRAME_PROCESSORS, - checkNotNull(demoFrameProcessorsSelections), - this::selectFrameProcessor) + DEMO_EFFECTS, checkNotNull(demoEffectsSelections), this::selectDemoEffect) .setPositiveButton(android.R.string.ok, /* listener= */ null) .create() .show(); } + private void selectTrimBounds(View view, boolean isChecked) { + if (!isChecked) { + return; + } + View dialogView = getLayoutInflater().inflate(R.layout.trim_options, /* root= */ null); + RangeSlider radiusRangeSlider = + checkNotNull(dialogView.findViewById(R.id.trim_bounds_range_slider)); + radiusRangeSlider.setValues(0f, 60f); // seconds + new AlertDialog.Builder(/* context= */ this) + .setView(dialogView) + .setPositiveButton( + android.R.string.ok, + (DialogInterface dialogInterface, int i) -> { + List radiusRange = radiusRangeSlider.getValues(); + trimStartMs = 1000 * radiusRange.get(0).longValue(); + trimEndMs = 1000 * radiusRange.get(1).longValue(); + }) + .create() + .show(); + } + @RequiresNonNull("selectedFileTextView") private void selectFileInDialog(DialogInterface dialog, int which) { inputUriPosition = which; selectedFileTextView.setText(URI_DESCRIPTIONS[inputUriPosition]); } - @RequiresNonNull("demoFrameProcessorsSelections") - private void selectFrameProcessor(DialogInterface dialog, int which, boolean isChecked) { - demoFrameProcessorsSelections[which] = isChecked; + @RequiresNonNull("demoEffectsSelections") + private void selectDemoEffect(DialogInterface dialog, int which, boolean isChecked) { + demoEffectsSelections[which] = isChecked; if (!isChecked || which != PERIODIC_VIGNETTE_INDEX) { return; } @@ -335,7 +379,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "rotateSpinner", "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox", - "selectDemoFrameProcessorsButton" + "selectDemoEffectsButton" }) private void onRemoveAudio(View view) { if (((CheckBox) view).isChecked()) { @@ -355,7 +399,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "rotateSpinner", "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox", - "selectDemoFrameProcessorsButton" + "selectDemoEffectsButton" }) private void onRemoveVideo(View view) { if (((CheckBox) view).isChecked()) { @@ -374,7 +418,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "rotateSpinner", "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox", - "selectDemoFrameProcessorsButton" + "selectDemoEffectsButton" }) private void enableTrackSpecificOptions(boolean isAudioEnabled, boolean isVideoEnabled) { audioMimeSpinner.setEnabled(isAudioEnabled); @@ -385,7 +429,7 @@ public final class ConfigurationActivity extends AppCompatActivity { enableRequestSdrToneMappingCheckBox.setEnabled( isRequestSdrToneMappingSupported() && isVideoEnabled); enableHdrEditingCheckBox.setEnabled(isVideoEnabled); - selectDemoFrameProcessorsButton.setEnabled(isVideoEnabled); + selectDemoEffectsButton.setEnabled(isVideoEnabled); findViewById(R.id.audio_mime_text_view).setEnabled(isAudioEnabled); findViewById(R.id.video_mime_text_view).setEnabled(isVideoEnabled); diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/AdvancedFrameProcessorFactory.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/MatrixTransformationFactory.java similarity index 68% rename from demos/transformer/src/main/java/androidx/media3/demo/transformer/AdvancedFrameProcessorFactory.java rename to demos/transformer/src/main/java/androidx/media3/demo/transformer/MatrixTransformationFactory.java index d833273a55..2aded6a470 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/AdvancedFrameProcessorFactory.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/MatrixTransformationFactory.java @@ -18,39 +18,38 @@ package androidx.media3.demo.transformer; import android.graphics.Matrix; import androidx.media3.common.C; import androidx.media3.common.util.Util; -import androidx.media3.transformer.AdvancedFrameProcessor; -import androidx.media3.transformer.GlFrameProcessor; +import androidx.media3.transformer.GlMatrixTransformation; +import androidx.media3.transformer.MatrixTransformation; /** - * Factory for {@link GlFrameProcessor GlFrameProcessors} that create video effects by applying - * transformation matrices to the individual video frames using {@link AdvancedFrameProcessor}. + * Factory for {@link GlMatrixTransformation GlMatrixTransformations} and {@link + * MatrixTransformation MatrixTransformations} that create video effects by applying transformation + * matrices to the individual video frames. */ -/* package */ final class AdvancedFrameProcessorFactory { +/* package */ final class MatrixTransformationFactory { /** - * Returns a {@link GlFrameProcessor} that rescales the frames over the first {@value + * Returns a {@link MatrixTransformation} that rescales the frames over the first {@value * #ZOOM_DURATION_SECONDS} seconds, such that the rectangle filled with the input frame increases * linearly in size from a single point to filling the full output frame. */ - public static GlFrameProcessor createZoomInTransitionFrameProcessor() { - return new AdvancedFrameProcessor( - /* matrixProvider= */ AdvancedFrameProcessorFactory::calculateZoomInTransitionMatrix); + public static MatrixTransformation createZoomInTransition() { + return MatrixTransformationFactory::calculateZoomInTransitionMatrix; } /** - * Returns a {@link GlFrameProcessor} that crops frames to a rectangle that moves on an ellipse. + * Returns a {@link MatrixTransformation} that crops frames to a rectangle that moves on an + * ellipse. */ - public static GlFrameProcessor createDizzyCropFrameProcessor() { - return new AdvancedFrameProcessor( - /* matrixProvider= */ AdvancedFrameProcessorFactory::calculateDizzyCropMatrix); + public static MatrixTransformation createDizzyCropEffect() { + return MatrixTransformationFactory::calculateDizzyCropMatrix; } /** - * Returns a {@link GlFrameProcessor} that rotates a frame in 3D around the y-axis and applies - * perspective projection to 2D. + * Returns a {@link GlMatrixTransformation} that rotates a frame in 3D around the y-axis and + * applies perspective projection to 2D. */ - public static GlFrameProcessor createSpin3dFrameProcessor() { - return new AdvancedFrameProcessor( - /* matrixProvider= */ AdvancedFrameProcessorFactory::calculate3dSpinMatrix); + public static GlMatrixTransformation createSpin3dEffect() { + return MatrixTransformationFactory::calculate3dSpinMatrix; } private static final float ZOOM_DURATION_SECONDS = 2f; diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteFrameProcessor.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java similarity index 54% rename from demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteFrameProcessor.java rename to demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java index 650accf7d3..9ae281b37b 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteFrameProcessor.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/PeriodicVignetteProcessor.java @@ -16,38 +16,29 @@ package androidx.media3.demo.transformer; import static androidx.media3.common.util.Assertions.checkArgument; -import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.content.Context; import android.opengl.GLES20; import android.util.Size; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; -import androidx.media3.transformer.GlFrameProcessor; +import androidx.media3.transformer.FrameProcessingException; +import androidx.media3.transformer.SingleFrameGlTextureProcessor; import java.io.IOException; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** - * A {@link GlFrameProcessor} that periodically dims the frames such that pixels are darker the - * further they are away from the frame center. + * A {@link SingleFrameGlTextureProcessor} that periodically dims the frames such that pixels are + * darker the further they are away from the frame center. */ -/* package */ final class PeriodicVignetteFrameProcessor implements GlFrameProcessor { - static { - GlUtil.glAssertionsEnabled = true; - } +/* package */ final class PeriodicVignetteProcessor extends SingleFrameGlTextureProcessor { private static final String VERTEX_SHADER_PATH = "vertex_shader_copy_es2.glsl"; private static final String FRAGMENT_SHADER_PATH = "fragment_shader_vignette_es2.glsl"; private static final float DIMMING_PERIOD_US = 5_600_000f; - private float centerX; - private float centerY; - private float minInnerRadius; - private float deltaInnerRadius; - private float outerRadius; - - private @MonotonicNonNull Size outputSize; - private @MonotonicNonNull GlProgram glProgram; + private final GlProgram glProgram; + private final float minInnerRadius; + private final float deltaInnerRadius; /** * Creates a new instance. @@ -66,53 +57,65 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param minInnerRadius The lower bound of the radius that is unaffected by the effect. * @param maxInnerRadius The upper bound of the radius that is unaffected by the effect. * @param outerRadius The radius after which all pixels are black. + * @throws FrameProcessingException If a problem occurs while reading shader files. */ - public PeriodicVignetteFrameProcessor( - float centerX, float centerY, float minInnerRadius, float maxInnerRadius, float outerRadius) { + public PeriodicVignetteProcessor( + Context context, + float centerX, + float centerY, + float minInnerRadius, + float maxInnerRadius, + float outerRadius) + throws FrameProcessingException { checkArgument(minInnerRadius <= maxInnerRadius); checkArgument(maxInnerRadius <= outerRadius); - this.centerX = centerX; - this.centerY = centerY; this.minInnerRadius = minInnerRadius; this.deltaInnerRadius = maxInnerRadius - minInnerRadius; - this.outerRadius = outerRadius; - } - - @Override - public void initialize(Context context, int inputTexId, int inputWidth, int inputHeight) - throws IOException { - outputSize = new Size(inputWidth, inputHeight); - glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); - glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); + try { + glProgram = new GlProgram(context, VERTEX_SHADER_PATH, FRAGMENT_SHADER_PATH); + } catch (IOException | GlUtil.GlException e) { + throw new FrameProcessingException(e); + } glProgram.setFloatsUniform("uCenter", new float[] {centerX, centerY}); glProgram.setFloatsUniform("uOuterRadius", new float[] {outerRadius}); // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. glProgram.setBufferAttribute( - "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); - glProgram.setBufferAttribute( - "aTexSamplingCoord", GlUtil.getTextureCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); } @Override - public Size getOutputSize() { - return checkStateNotNull(outputSize); + public Size configure(int inputWidth, int inputHeight) { + return new Size(inputWidth, inputHeight); } @Override - public void drawFrame(long presentationTimeUs) { - checkStateNotNull(glProgram).use(); - double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US; - float innerRadius = minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta)); - glProgram.setFloatsUniform("uInnerRadius", new float[] {innerRadius}); - glProgram.bindAttributesAndUniforms(); - // The four-vertex triangle strip forms a quad. - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + public void drawFrame(int inputTexId, long presentationTimeUs) throws FrameProcessingException { + try { + glProgram.use(); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); + double theta = presentationTimeUs * 2 * Math.PI / DIMMING_PERIOD_US; + float innerRadius = + minInnerRadius + deltaInnerRadius * (0.5f - 0.5f * (float) Math.cos(theta)); + glProgram.setFloatsUniform("uInnerRadius", new float[] {innerRadius}); + glProgram.bindAttributesAndUniforms(); + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e, presentationTimeUs); + } } @Override - public void release() { + public void release() throws FrameProcessingException { + super.release(); if (glProgram != null) { - glProgram.delete(); + try { + glProgram.delete(); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e); + } } } } diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index b6559c6c86..842769f13f 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -19,6 +19,7 @@ import static android.Manifest.permission.READ_EXTERNAL_STORAGE; import static androidx.media3.common.util.Assertions.checkNotNull; import android.app.Activity; +import android.content.Context; import android.content.Intent; import android.content.pm.PackageManager; import android.net.Uri; @@ -31,6 +32,7 @@ import android.view.ViewGroup; import android.widget.TextView; import android.widget.Toast; import androidx.annotation.Nullable; +import androidx.annotation.StringRes; import androidx.appcompat.app.AppCompatActivity; import androidx.media3.common.C; import androidx.media3.common.MediaItem; @@ -40,8 +42,9 @@ import androidx.media3.exoplayer.ExoPlayer; import androidx.media3.exoplayer.util.DebugTextViewHelper; import androidx.media3.transformer.DefaultEncoderFactory; import androidx.media3.transformer.EncoderSelector; -import androidx.media3.transformer.GlFrameProcessor; +import androidx.media3.transformer.GlEffect; import androidx.media3.transformer.ProgressHolder; +import androidx.media3.transformer.SingleFrameGlTextureProcessor; import androidx.media3.transformer.TransformationException; import androidx.media3.transformer.TransformationRequest; import androidx.media3.transformer.TransformationResult; @@ -54,6 +57,7 @@ import com.google.common.base.Ticker; import com.google.common.collect.ImmutableList; import java.io.File; import java.io.IOException; +import java.lang.reflect.Constructor; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -149,9 +153,10 @@ public final class TransformerActivity extends AppCompatActivity { externalCacheFile = createExternalCacheFile("transformer-output.mp4"); String filePath = externalCacheFile.getAbsolutePath(); @Nullable Bundle bundle = intent.getExtras(); + MediaItem mediaItem = createMediaItem(bundle, uri); Transformer transformer = createTransformer(bundle, filePath); transformationStopwatch.start(); - transformer.startTransformation(MediaItem.fromUri(uri), filePath); + transformer.startTransformation(mediaItem, filePath); this.transformer = transformer; } catch (IOException e) { throw new IllegalStateException(e); @@ -178,6 +183,24 @@ public final class TransformerActivity extends AppCompatActivity { }); } + private MediaItem createMediaItem(@Nullable Bundle bundle, Uri uri) { + MediaItem.Builder mediaItemBuilder = new MediaItem.Builder().setUri(uri); + if (bundle != null) { + long trimStartMs = + bundle.getLong(ConfigurationActivity.TRIM_START_MS, /* defaultValue= */ C.TIME_UNSET); + long trimEndMs = + bundle.getLong(ConfigurationActivity.TRIM_END_MS, /* defaultValue= */ C.TIME_UNSET); + if (trimStartMs != C.TIME_UNSET && trimEndMs != C.TIME_UNSET) { + mediaItemBuilder.setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setStartPositionMs(trimStartMs) + .setEndPositionMs(trimEndMs) + .build()); + } + } + return mediaItemBuilder.build(); + } + // Create a cache file, resetting it if it already exists. private File createExternalCacheFile(String fileName) throws IOException { File file = new File(getExternalCacheDir(), fileName); @@ -237,38 +260,64 @@ public final class TransformerActivity extends AppCompatActivity { .setRemoveVideo(bundle.getBoolean(ConfigurationActivity.SHOULD_REMOVE_VIDEO)) .setEncoderFactory( new DefaultEncoderFactory( + /* context= */ this, EncoderSelector.DEFAULT, /* enableFallback= */ bundle.getBoolean(ConfigurationActivity.ENABLE_FALLBACK))); - ImmutableList.Builder frameProcessors = new ImmutableList.Builder<>(); + ImmutableList.Builder effects = new ImmutableList.Builder<>(); @Nullable - boolean[] selectedFrameProcessors = - bundle.getBooleanArray(ConfigurationActivity.DEMO_FRAME_PROCESSORS_SELECTIONS); - if (selectedFrameProcessors != null) { - if (selectedFrameProcessors[0]) { - frameProcessors.add(AdvancedFrameProcessorFactory.createDizzyCropFrameProcessor()); + boolean[] selectedEffects = + bundle.getBooleanArray(ConfigurationActivity.DEMO_EFFECTS_SELECTIONS); + if (selectedEffects != null) { + if (selectedEffects[0]) { + effects.add(MatrixTransformationFactory.createDizzyCropEffect()); } - if (selectedFrameProcessors[1]) { - frameProcessors.add( - new PeriodicVignetteFrameProcessor( - bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X), - bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y), - /* minInnerRadius= */ bundle.getFloat( - ConfigurationActivity.PERIODIC_VIGNETTE_INNER_RADIUS), - /* maxInnerRadius= */ bundle.getFloat( - ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS), - bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS))); + if (selectedEffects[1]) { + try { + Class clazz = Class.forName("androidx.media3.demo.transformer.MediaPipeProcessor"); + Constructor constructor = + clazz.getConstructor(Context.class, String.class, String.class, String.class); + effects.add( + (Context context) -> { + try { + return (SingleFrameGlTextureProcessor) + constructor.newInstance( + context, + /* graphName= */ "edge_detector_mediapipe_graph.binarypb", + /* inputStreamName= */ "input_video", + /* outputStreamName= */ "output_video"); + } catch (Exception e) { + runOnUiThread(() -> showToast(R.string.no_media_pipe_error)); + throw new RuntimeException("Failed to load MediaPipe processor", e); + } + }); + } catch (Exception e) { + showToast(R.string.no_media_pipe_error); + } } - if (selectedFrameProcessors[2]) { - frameProcessors.add(AdvancedFrameProcessorFactory.createSpin3dFrameProcessor()); + if (selectedEffects[2]) { + effects.add( + (Context context) -> + new PeriodicVignetteProcessor( + context, + bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X), + bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y), + /* minInnerRadius= */ bundle.getFloat( + ConfigurationActivity.PERIODIC_VIGNETTE_INNER_RADIUS), + /* maxInnerRadius= */ bundle.getFloat( + ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS), + bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS))); } - if (selectedFrameProcessors[3]) { - frameProcessors.add(new BitmapOverlayFrameProcessor()); + if (selectedEffects[3]) { + effects.add(MatrixTransformationFactory.createSpin3dEffect()); } - if (selectedFrameProcessors[4]) { - frameProcessors.add(AdvancedFrameProcessorFactory.createZoomInTransitionFrameProcessor()); + if (selectedEffects[4]) { + effects.add(BitmapOverlayProcessor::new); } - transformerBuilder.setFrameProcessors(frameProcessors.build()); + if (selectedEffects[5]) { + effects.add(MatrixTransformationFactory.createZoomInTransition()); + } + transformerBuilder.setVideoFrameEffects(effects.build()); } } return transformerBuilder @@ -362,6 +411,10 @@ public final class TransformerActivity extends AppCompatActivity { } } + private void showToast(@StringRes int messageResource) { + Toast.makeText(getApplicationContext(), getString(messageResource), Toast.LENGTH_LONG).show(); + } + private final class DemoDebugViewProvider implements Transformer.DebugViewProvider { @Nullable diff --git a/demos/transformer/src/main/res/layout/configuration_activity.xml b/demos/transformer/src/main/res/layout/configuration_activity.xml index 7d080b7351..2879d6a637 100644 --- a/demos/transformer/src/main/res/layout/configuration_activity.xml +++ b/demos/transformer/src/main/res/layout/configuration_activity.xml @@ -64,7 +64,7 @@ app:layout_constraintEnd_toEndOf="parent" app:layout_constraintStart_toStartOf="parent" app:layout_constraintTop_toBottomOf="@+id/selected_file_text_view" - app:layout_constraintBottom_toTopOf="@+id/select_demo_frameprocessors_button"> + app:layout_constraintBottom_toTopOf="@+id/select_demo_effects_button"> + + + + @@ -192,13 +202,13 @@