diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 204a19e7f6..0232b429ab 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,5 +1,61 @@ # Release notes +### Unreleased changes + +* Core library: + * Enable support for Android platform diagnostics via + `MediaMetricsManager`. ExoPlayer will forward playback events and + performance data to the platform, which helps to provide system + performance and debugging information on the device. This data may also + be collected by Google + [if sharing usage and diagnostics data is enabled](https://support.google.com/accounts/answer/6078260) + by the user of the device. Apps can opt-out of contributing to platform + diagnostics for ExoPlayer with + `ExoPlayer.Builder.setUsePlatformDiagnostics(false)`. +* Track selection: + * Flatten `TrackSelectionOverrides` class into `TrackSelectionParameters`, + and promote `TrackSelectionOverride` to a top level class. +* Audio: + * Use LG AC3 audio decoder advertising non-standard MIME type. +* Extractors: + * Matroska: Parse `DiscardPadding` for Opus tracks. +* UI: + * Fix delivery of events to `OnClickListener`s set on `PlayerView` and + `LegacyPlayerView`, in the case that `useController=false` + ([#9605](https://github.com/google/ExoPlayer/issues/9605)). Also fix + delivery of events to `OnLongClickListener` for all view configurations. + * Fix incorrectly treating a sequence of touch events that exit the bounds + of `PlayerView` and `LegacyPlayerView` before `ACTION_UP` as a click + ([#9861](https://github.com/google/ExoPlayer/issues/9861)). + * Fix `PlayerView` accessibility issue where it was not possible to + tapping would toggle playback rather than hiding the controls + ([#8627](https://github.com/google/ExoPlayer/issues/8627)). + * Rewrite `TrackSelectionView` and `TrackSelectionDialogBuilder` to work + with the `Player` interface rather than `ExoPlayer`. This allows the + views to be used with other `Player` implementations, and removes the + dependency from the UI module to the ExoPlayer module. This is a + breaking change. +* RTSP: + * Add RTP reader for HEVC + ([#36](https://github.com/androidx/media/pull/36)). +* Remove deprecated symbols: + * Remove `Player.Listener.onTracksChanged`. Use + `Player.Listener.onTracksInfoChanged` instead. + * Remove `Player.getCurrentTrackGroups` and + `Player.getCurrentTrackSelections`. Use `Player.getCurrentTracksInfo` + instead. You can also continue to use `ExoPlayer.getCurrentTrackGroups` + and `ExoPlayer.getCurrentTrackSelections`, although these methods remain + deprecated. + * Remove `DownloadHelper` + `DEFAULT_TRACK_SELECTOR_PARAMETERS_WITHOUT_VIEWPORT` and + `DEFAULT_TRACK_SELECTOR_PARAMETERS` constants. Use + `getDefaultTrackSelectorParameters(Context)` instead when possible, and + `DEFAULT_TRACK_SELECTOR_PARAMETERS_WITHOUT_CONTEXT` otherwise. + * FFmpeg extension: + * Update CMake version to `3.21.0+` to avoid a CMake bug causing + AndroidStudio's gradle sync to fail + ([#9933](https://github.com/google/ExoPlayer/issues/9933)). + ### 1.0.0-alpha03 (2022-03-14) This release corresponds to the diff --git a/constants.gradle b/constants.gradle index ba974fd8e9..d016c32571 100644 --- a/constants.gradle +++ b/constants.gradle @@ -26,7 +26,7 @@ project.ext { // https://cs.android.com/android/platform/superproject/+/master:external/guava/METADATA guavaVersion = '31.0.1-android' mockitoVersion = '3.12.4' - robolectricVersion = '4.6.1' + robolectricVersion = '4.8-alpha-1' // Keep this in sync with Google's internal Checker Framework version. checkerframeworkVersion = '3.13.0' checkerframeworkCompatVersion = '2.5.5' diff --git a/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java b/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java index 33e44323a3..a9329b5343 100644 --- a/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java +++ b/demos/gl/src/main/java/androidx/media3/demo/gl/BitmapOverlayVideoProcessor.java @@ -119,8 +119,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // Run the shader program. GlProgram program = checkNotNull(this.program); - program.setSamplerTexIdUniform("uTexSampler0", frameTexture, /* unit= */ 0); - program.setSamplerTexIdUniform("uTexSampler1", textures[0], /* unit= */ 1); + program.setSamplerTexIdUniform("uTexSampler0", frameTexture, /* texUnitIndex= */ 0); + program.setSamplerTexIdUniform("uTexSampler1", textures[0], /* texUnitIndex= */ 1); program.setFloatUniform("uScaleX", bitmapScaleX); program.setFloatUniform("uScaleY", bitmapScaleY); program.setFloatsUniform("uTexTransform", transformMatrix); diff --git a/demos/gl/src/main/java/androidx/media3/demo/gl/MainActivity.java b/demos/gl/src/main/java/androidx/media3/demo/gl/MainActivity.java index 9a4e5c5dc3..bb43d08ced 100644 --- a/demos/gl/src/main/java/androidx/media3/demo/gl/MainActivity.java +++ b/demos/gl/src/main/java/androidx/media3/demo/gl/MainActivity.java @@ -32,7 +32,6 @@ import androidx.media3.common.util.Util; import androidx.media3.datasource.DataSource; import androidx.media3.datasource.DefaultDataSource; import androidx.media3.datasource.DefaultHttpDataSource; -import androidx.media3.datasource.HttpDataSource; import androidx.media3.exoplayer.ExoPlayer; import androidx.media3.exoplayer.dash.DashMediaSource; import androidx.media3.exoplayer.drm.DefaultDrmSessionManager; @@ -144,7 +143,7 @@ public final class MainActivity extends Activity { String drmScheme = Assertions.checkNotNull(intent.getStringExtra(DRM_SCHEME_EXTRA)); String drmLicenseUrl = Assertions.checkNotNull(intent.getStringExtra(DRM_LICENSE_URL_EXTRA)); UUID drmSchemeUuid = Assertions.checkNotNull(Util.getDrmUuid(drmScheme)); - HttpDataSource.Factory licenseDataSourceFactory = new DefaultHttpDataSource.Factory(); + DataSource.Factory licenseDataSourceFactory = new DefaultHttpDataSource.Factory(); HttpMediaDrmCallback drmCallback = new HttpMediaDrmCallback(drmLicenseUrl, licenseDataSourceFactory); drmSessionManager = diff --git a/demos/main/src/main/java/androidx/media3/demo/main/DemoUtil.java b/demos/main/src/main/java/androidx/media3/demo/main/DemoUtil.java index 2e3424697a..2c31ae19ee 100644 --- a/demos/main/src/main/java/androidx/media3/demo/main/DemoUtil.java +++ b/demos/main/src/main/java/androidx/media3/demo/main/DemoUtil.java @@ -21,7 +21,6 @@ import androidx.media3.database.StandaloneDatabaseProvider; import androidx.media3.datasource.DataSource; import androidx.media3.datasource.DefaultDataSource; import androidx.media3.datasource.DefaultHttpDataSource; -import androidx.media3.datasource.HttpDataSource; import androidx.media3.datasource.cache.Cache; import androidx.media3.datasource.cache.CacheDataSource; import androidx.media3.datasource.cache.NoOpCacheEvictor; @@ -59,7 +58,7 @@ public final class DemoUtil { private static final String DOWNLOAD_CONTENT_DIRECTORY = "downloads"; private static DataSource.@MonotonicNonNull Factory dataSourceFactory; - private static HttpDataSource.@MonotonicNonNull Factory httpDataSourceFactory; + private static DataSource.@MonotonicNonNull Factory httpDataSourceFactory; private static @MonotonicNonNull DatabaseProvider databaseProvider; private static @MonotonicNonNull File downloadDirectory; private static @MonotonicNonNull Cache downloadCache; @@ -85,7 +84,7 @@ public final class DemoUtil { .setExtensionRendererMode(extensionRendererMode); } - public static synchronized HttpDataSource.Factory getHttpDataSourceFactory(Context context) { + public static synchronized DataSource.Factory getHttpDataSourceFactory(Context context) { if (httpDataSourceFactory == null) { if (USE_CRONET_FOR_NETWORKING) { context = context.getApplicationContext(); diff --git a/demos/main/src/main/java/androidx/media3/demo/main/DownloadTracker.java b/demos/main/src/main/java/androidx/media3/demo/main/DownloadTracker.java index 4ad82652b4..92349a575e 100644 --- a/demos/main/src/main/java/androidx/media3/demo/main/DownloadTracker.java +++ b/demos/main/src/main/java/androidx/media3/demo/main/DownloadTracker.java @@ -15,8 +15,7 @@ */ package androidx.media3.demo.main; -import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static com.google.common.base.Preconditions.checkNotNull; import android.content.Context; import android.content.DialogInterface; @@ -30,12 +29,11 @@ import androidx.media3.common.DrmInitData; import androidx.media3.common.Format; import androidx.media3.common.MediaItem; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.TracksInfo; import androidx.media3.common.util.Log; import androidx.media3.common.util.Util; -import androidx.media3.datasource.HttpDataSource; +import androidx.media3.datasource.DataSource; import androidx.media3.exoplayer.RenderersFactory; import androidx.media3.exoplayer.drm.DrmSession; import androidx.media3.exoplayer.drm.DrmSessionEventListener; @@ -48,6 +46,7 @@ import androidx.media3.exoplayer.offline.DownloadIndex; import androidx.media3.exoplayer.offline.DownloadManager; import androidx.media3.exoplayer.offline.DownloadRequest; import androidx.media3.exoplayer.offline.DownloadService; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.MappingTrackSelector.MappedTrackInfo; import java.io.IOException; import java.util.HashMap; @@ -66,7 +65,7 @@ public class DownloadTracker { private static final String TAG = "DownloadTracker"; private final Context context; - private final HttpDataSource.Factory httpDataSourceFactory; + private final DataSource.Factory dataSourceFactory; private final CopyOnWriteArraySet listeners; private final HashMap downloads; private final DownloadIndex downloadIndex; @@ -74,11 +73,9 @@ public class DownloadTracker { @Nullable private StartDownloadDialogHelper startDownloadDialogHelper; public DownloadTracker( - Context context, - HttpDataSource.Factory httpDataSourceFactory, - DownloadManager downloadManager) { + Context context, DataSource.Factory dataSourceFactory, DownloadManager downloadManager) { this.context = context.getApplicationContext(); - this.httpDataSourceFactory = httpDataSourceFactory; + this.dataSourceFactory = dataSourceFactory; listeners = new CopyOnWriteArraySet<>(); downloads = new HashMap<>(); downloadIndex = downloadManager.getDownloadIndex(); @@ -87,8 +84,7 @@ public class DownloadTracker { } public void addListener(Listener listener) { - checkNotNull(listener); - listeners.add(listener); + listeners.add(checkNotNull(listener)); } public void removeListener(Listener listener) { @@ -119,8 +115,7 @@ public class DownloadTracker { startDownloadDialogHelper = new StartDownloadDialogHelper( fragmentManager, - DownloadHelper.forMediaItem( - context, mediaItem, renderersFactory, httpDataSourceFactory), + DownloadHelper.forMediaItem(context, mediaItem, renderersFactory, dataSourceFactory), mediaItem); } } @@ -218,7 +213,7 @@ public class DownloadTracker { new WidevineOfflineLicenseFetchTask( format, mediaItem.localConfiguration.drmConfiguration, - httpDataSourceFactory, + dataSourceFactory, /* dialogHelper= */ this, helper); widevineOfflineLicenseFetchTask.execute(); @@ -361,7 +356,7 @@ public class DownloadTracker { private final Format format; private final MediaItem.DrmConfiguration drmConfiguration; - private final HttpDataSource.Factory httpDataSourceFactory; + private final DataSource.Factory dataSourceFactory; private final StartDownloadDialogHelper dialogHelper; private final DownloadHelper downloadHelper; @@ -371,12 +366,12 @@ public class DownloadTracker { public WidevineOfflineLicenseFetchTask( Format format, MediaItem.DrmConfiguration drmConfiguration, - HttpDataSource.Factory httpDataSourceFactory, + DataSource.Factory dataSourceFactory, StartDownloadDialogHelper dialogHelper, DownloadHelper downloadHelper) { this.format = format; this.drmConfiguration = drmConfiguration; - this.httpDataSourceFactory = httpDataSourceFactory; + this.dataSourceFactory = dataSourceFactory; this.dialogHelper = dialogHelper; this.downloadHelper = downloadHelper; } @@ -387,7 +382,7 @@ public class DownloadTracker { OfflineLicenseHelper.newWidevineInstance( drmConfiguration.licenseUri.toString(), drmConfiguration.forceDefaultLicenseUri, - httpDataSourceFactory, + dataSourceFactory, drmConfiguration.licenseRequestHeaders, new DrmSessionEventListener.EventDispatcher()); try { @@ -405,7 +400,7 @@ public class DownloadTracker { if (drmSessionException != null) { dialogHelper.onOfflineLicenseFetchedError(drmSessionException); } else { - dialogHelper.onOfflineLicenseFetched(downloadHelper, checkStateNotNull(keySetId)); + dialogHelper.onOfflineLicenseFetched(downloadHelper, checkNotNull(keySetId)); } } } diff --git a/demos/main/src/main/java/androidx/media3/demo/main/IntentUtil.java b/demos/main/src/main/java/androidx/media3/demo/main/IntentUtil.java index 911dac1ff5..cf1fa329ec 100644 --- a/demos/main/src/main/java/androidx/media3/demo/main/IntentUtil.java +++ b/demos/main/src/main/java/androidx/media3/demo/main/IntentUtil.java @@ -15,8 +15,9 @@ */ package androidx.media3.demo.main; -import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkState; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import android.content.Intent; import android.net.Uri; @@ -26,7 +27,6 @@ import androidx.media3.common.MediaItem; import androidx.media3.common.MediaItem.ClippingConfiguration; import androidx.media3.common.MediaItem.SubtitleConfiguration; import androidx.media3.common.MediaMetadata; -import androidx.media3.common.util.Assertions; import androidx.media3.common.util.Util; import com.google.common.collect.ImmutableList; import java.util.ArrayList; @@ -86,7 +86,7 @@ public class IntentUtil { /** Populates the intent with the given list of {@link MediaItem media items}. */ public static void addToIntent(List mediaItems, Intent intent) { - Assertions.checkArgument(!mediaItems.isEmpty()); + checkArgument(!mediaItems.isEmpty()); if (mediaItems.size() == 1) { MediaItem mediaItem = mediaItems.get(0); MediaItem.LocalConfiguration localConfiguration = checkNotNull(mediaItem.localConfiguration); @@ -241,7 +241,7 @@ public class IntentUtil { drmConfiguration.forcedSessionTrackTypes; if (!forcedDrmSessionTrackTypes.isEmpty()) { // Only video and audio together are supported. - Assertions.checkState( + checkState( forcedDrmSessionTrackTypes.size() == 2 && forcedDrmSessionTrackTypes.contains(C.TRACK_TYPE_VIDEO) && forcedDrmSessionTrackTypes.contains(C.TRACK_TYPE_AUDIO)); diff --git a/demos/main/src/main/java/androidx/media3/demo/main/SampleChooserActivity.java b/demos/main/src/main/java/androidx/media3/demo/main/SampleChooserActivity.java index 058fcec3b0..6b765679ad 100644 --- a/demos/main/src/main/java/androidx/media3/demo/main/SampleChooserActivity.java +++ b/demos/main/src/main/java/androidx/media3/demo/main/SampleChooserActivity.java @@ -15,9 +15,9 @@ */ package androidx.media3.demo.main; -import static androidx.media3.common.util.Assertions.checkArgument; -import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkState; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import android.content.Context; import android.content.Intent; @@ -116,8 +116,11 @@ public class SampleChooserActivity extends AppCompatActivity useExtensionRenderers = DemoUtil.useExtensionRenderers(); downloadTracker = DemoUtil.getDownloadTracker(/* context= */ this); loadSample(); + startDownloadService(); + } - // Start the download service if it should be running but it's not currently. + /** Start the download service if it should be running but it's not currently. */ + private void startDownloadService() { // Starting the service in the foreground causes notification flicker if there is no scheduled // action. Starting it in the background throws an exception if the app is in the background too // (e.g. if device screen is locked). diff --git a/demos/surface/src/main/java/androidx/media3/demo/surface/MainActivity.java b/demos/surface/src/main/java/androidx/media3/demo/surface/MainActivity.java index 649a38dd95..405f3b4df2 100644 --- a/demos/surface/src/main/java/androidx/media3/demo/surface/MainActivity.java +++ b/demos/surface/src/main/java/androidx/media3/demo/surface/MainActivity.java @@ -35,7 +35,6 @@ import androidx.media3.common.util.Util; import androidx.media3.datasource.DataSource; import androidx.media3.datasource.DefaultDataSource; import androidx.media3.datasource.DefaultHttpDataSource; -import androidx.media3.datasource.HttpDataSource; import androidx.media3.exoplayer.ExoPlayer; import androidx.media3.exoplayer.dash.DashMediaSource; import androidx.media3.exoplayer.drm.DefaultDrmSessionManager; @@ -189,7 +188,7 @@ public final class MainActivity extends Activity { String drmScheme = Assertions.checkNotNull(intent.getStringExtra(DRM_SCHEME_EXTRA)); String drmLicenseUrl = Assertions.checkNotNull(intent.getStringExtra(DRM_LICENSE_URL_EXTRA)); UUID drmSchemeUuid = Assertions.checkNotNull(Util.getDrmUuid(drmScheme)); - HttpDataSource.Factory licenseDataSourceFactory = new DefaultHttpDataSource.Factory(); + DataSource.Factory licenseDataSourceFactory = new DefaultHttpDataSource.Factory(); HttpMediaDrmCallback drmCallback = new HttpMediaDrmCallback(drmLicenseUrl, licenseDataSourceFactory); drmSessionManager = diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java index b6a447768c..eaa51847e5 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java @@ -50,8 +50,6 @@ public final class ConfigurationActivity extends AppCompatActivity { public static final String AUDIO_MIME_TYPE = "audio_mime_type"; public static final String VIDEO_MIME_TYPE = "video_mime_type"; public static final String RESOLUTION_HEIGHT = "resolution_height"; - public static final String TRANSLATE_X = "translate_x"; - public static final String TRANSLATE_Y = "translate_y"; public static final String SCALE_X = "scale_x"; public static final String SCALE_Y = "scale_y"; public static final String ROTATE_DEGREES = "rotate_degrees"; @@ -81,7 +79,6 @@ public final class ConfigurationActivity extends AppCompatActivity { private @MonotonicNonNull Spinner audioMimeSpinner; private @MonotonicNonNull Spinner videoMimeSpinner; private @MonotonicNonNull Spinner resolutionHeightSpinner; - private @MonotonicNonNull Spinner translateSpinner; private @MonotonicNonNull Spinner scaleSpinner; private @MonotonicNonNull Spinner rotateSpinner; private @MonotonicNonNull CheckBox enableFallbackCheckBox; @@ -136,14 +133,6 @@ public final class ConfigurationActivity extends AppCompatActivity { resolutionHeightAdapter.addAll( SAME_AS_INPUT_OPTION, "144", "240", "360", "480", "720", "1080", "1440", "2160"); - ArrayAdapter translateAdapter = - new ArrayAdapter<>(/* context= */ this, R.layout.spinner_item); - translateAdapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); - translateSpinner = findViewById(R.id.translate_spinner); - translateSpinner.setAdapter(translateAdapter); - translateAdapter.addAll( - SAME_AS_INPUT_OPTION, "-.1, -.1", "0, 0", ".5, 0", "0, .5", "1, 1", "1.9, 0", "0, 1.9"); - ArrayAdapter scaleAdapter = new ArrayAdapter<>(/* context= */ this, R.layout.spinner_item); scaleAdapter.setDropDownViewResource(android.R.layout.simple_spinner_dropdown_item); @@ -185,7 +174,6 @@ public final class ConfigurationActivity extends AppCompatActivity { "audioMimeSpinner", "videoMimeSpinner", "resolutionHeightSpinner", - "translateSpinner", "scaleSpinner", "rotateSpinner", "enableFallbackCheckBox", @@ -209,13 +197,6 @@ public final class ConfigurationActivity extends AppCompatActivity { if (!SAME_AS_INPUT_OPTION.equals(selectedResolutionHeight)) { bundle.putInt(RESOLUTION_HEIGHT, Integer.parseInt(selectedResolutionHeight)); } - String selectedTranslate = String.valueOf(translateSpinner.getSelectedItem()); - if (!SAME_AS_INPUT_OPTION.equals(selectedTranslate)) { - List translateXY = Arrays.asList(selectedTranslate.split(", ")); - checkState(translateXY.size() == 2); - bundle.putFloat(TRANSLATE_X, Float.parseFloat(translateXY.get(0))); - bundle.putFloat(TRANSLATE_Y, Float.parseFloat(translateXY.get(1))); - } String selectedScale = String.valueOf(scaleSpinner.getSelectedItem()); if (!SAME_AS_INPUT_OPTION.equals(selectedScale)) { List scaleXY = Arrays.asList(selectedScale.split(", ")); @@ -258,7 +239,6 @@ public final class ConfigurationActivity extends AppCompatActivity { "audioMimeSpinner", "videoMimeSpinner", "resolutionHeightSpinner", - "translateSpinner", "scaleSpinner", "rotateSpinner", "enableHdrEditingCheckBox" @@ -277,7 +257,6 @@ public final class ConfigurationActivity extends AppCompatActivity { "audioMimeSpinner", "videoMimeSpinner", "resolutionHeightSpinner", - "translateSpinner", "scaleSpinner", "rotateSpinner", "enableHdrEditingCheckBox" @@ -295,7 +274,6 @@ public final class ConfigurationActivity extends AppCompatActivity { "audioMimeSpinner", "videoMimeSpinner", "resolutionHeightSpinner", - "translateSpinner", "scaleSpinner", "rotateSpinner", "enableHdrEditingCheckBox" @@ -304,7 +282,6 @@ public final class ConfigurationActivity extends AppCompatActivity { audioMimeSpinner.setEnabled(isAudioEnabled); videoMimeSpinner.setEnabled(isVideoEnabled); resolutionHeightSpinner.setEnabled(isVideoEnabled); - translateSpinner.setEnabled(isVideoEnabled); scaleSpinner.setEnabled(isVideoEnabled); rotateSpinner.setEnabled(isVideoEnabled); enableHdrEditingCheckBox.setEnabled(isVideoEnabled); @@ -312,7 +289,6 @@ public final class ConfigurationActivity extends AppCompatActivity { findViewById(R.id.audio_mime_text_view).setEnabled(isAudioEnabled); findViewById(R.id.video_mime_text_view).setEnabled(isVideoEnabled); findViewById(R.id.resolution_height_text_view).setEnabled(isVideoEnabled); - findViewById(R.id.translate).setEnabled(isVideoEnabled); findViewById(R.id.scale).setEnabled(isVideoEnabled); findViewById(R.id.rotate).setEnabled(isVideoEnabled); findViewById(R.id.hdr_editing).setEnabled(isVideoEnabled); diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index 4dc5c0585a..ad83ce75f0 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -21,7 +21,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import android.app.Activity; import android.content.Intent; import android.content.pm.PackageManager; -import android.graphics.Matrix; import android.net.Uri; import android.os.Bundle; import android.os.Handler; @@ -217,10 +216,15 @@ public final class TransformerActivity extends AppCompatActivity { if (resolutionHeight != C.LENGTH_UNSET) { requestBuilder.setResolution(resolutionHeight); } - Matrix transformationMatrix = getTransformationMatrix(bundle); - if (!transformationMatrix.isIdentity()) { - requestBuilder.setTransformationMatrix(transformationMatrix); - } + + float scaleX = bundle.getFloat(ConfigurationActivity.SCALE_X, /* defaultValue= */ 1); + float scaleY = bundle.getFloat(ConfigurationActivity.SCALE_Y, /* defaultValue= */ 1); + requestBuilder.setScale(scaleX, scaleY); + + float rotateDegrees = + bundle.getFloat(ConfigurationActivity.ROTATE_DEGREES, /* defaultValue= */ 0); + requestBuilder.setRotationDegrees(rotateDegrees); + requestBuilder.experimental_setEnableHdrEditing( bundle.getBoolean(ConfigurationActivity.ENABLE_HDR_EDITING)); transformerBuilder @@ -251,27 +255,6 @@ public final class TransformerActivity extends AppCompatActivity { .build(); } - private static Matrix getTransformationMatrix(Bundle bundle) { - Matrix transformationMatrix = new Matrix(); - - float translateX = bundle.getFloat(ConfigurationActivity.TRANSLATE_X, /* defaultValue= */ 0); - float translateY = bundle.getFloat(ConfigurationActivity.TRANSLATE_Y, /* defaultValue= */ 0); - // TODO(b/201293185): Implement an AdvancedFrameEditor to handle translation, as the current - // transformationMatrix is automatically adjusted to focus on the original pixels and - // effectively undo translations. - transformationMatrix.postTranslate(translateX, translateY); - - float scaleX = bundle.getFloat(ConfigurationActivity.SCALE_X, /* defaultValue= */ 1); - float scaleY = bundle.getFloat(ConfigurationActivity.SCALE_Y, /* defaultValue= */ 1); - transformationMatrix.postScale(scaleX, scaleY); - - float rotateDegrees = - bundle.getFloat(ConfigurationActivity.ROTATE_DEGREES, /* defaultValue= */ 0); - transformationMatrix.postRotate(rotateDegrees); - - return transformationMatrix; - } - @RequiresNonNull({ "informationTextView", "progressViewGroup", diff --git a/demos/transformer/src/main/res/layout/configuration_activity.xml b/demos/transformer/src/main/res/layout/configuration_activity.xml index f58f2f7167..1ff3cafc6b 100644 --- a/demos/transformer/src/main/res/layout/configuration_activity.xml +++ b/demos/transformer/src/main/res/layout/configuration_activity.xml @@ -137,17 +137,6 @@ android:layout_gravity="right|center_vertical" android:gravity="right" /> - - - - diff --git a/demos/transformer/src/main/res/values/strings.xml b/demos/transformer/src/main/res/values/strings.xml index 3b27a515ef..8e8f97ecf9 100644 --- a/demos/transformer/src/main/res/values/strings.xml +++ b/demos/transformer/src/main/res/values/strings.xml @@ -24,7 +24,6 @@ Output audio MIME type Output video MIME type Output video resolution - Translate video Scale video Rotate video (degrees) Enable fallback diff --git a/libraries/cast/src/main/java/androidx/media3/cast/CastPlayer.java b/libraries/cast/src/main/java/androidx/media3/cast/CastPlayer.java index 1869c7d7c4..192d8dbabe 100644 --- a/libraries/cast/src/main/java/androidx/media3/cast/CastPlayer.java +++ b/libraries/cast/src/main/java/androidx/media3/cast/CastPlayer.java @@ -111,9 +111,6 @@ public final class CastPlayer extends BasePlayer { private static final String TAG = "CastPlayer"; - private static final int RENDERER_INDEX_VIDEO = 0; - private static final int RENDERER_INDEX_AUDIO = 1; - private static final int RENDERER_INDEX_TEXT = 2; private static final long PROGRESS_REPORT_PERIOD_MS = 1000; private static final long[] EMPTY_TRACK_ID_ARRAY = new long[0]; diff --git a/libraries/common/src/main/java/androidx/media3/common/Player.java b/libraries/common/src/main/java/androidx/media3/common/Player.java index e6329481a3..2828b53e43 100644 --- a/libraries/common/src/main/java/androidx/media3/common/Player.java +++ b/libraries/common/src/main/java/androidx/media3/common/Player.java @@ -683,11 +683,12 @@ public interface Player { /** * Called when the combined {@link MediaMetadata} changes. * - *

The provided {@link MediaMetadata} is a combination of the {@link MediaItem#mediaMetadata} - * and the static and dynamic metadata from the {@link TrackSelection#getFormat(int) track - * selections' formats} and {@link Listener#onMetadata(Metadata)}. If a field is populated in - * the {@link MediaItem#mediaMetadata}, it will be prioritised above the same field coming from - * static or dynamic metadata. + *

The provided {@link MediaMetadata} is a combination of the {@link MediaItem#mediaMetadata + * MediaItem metadata}, the static metadata in the media's {@link Format#metadata Format}, and + * any timed metadata that has been parsed from the media and output via {@link + * Listener#onMetadata(Metadata)}. If a field is populated in the {@link + * MediaItem#mediaMetadata}, it will be prioritised above the same field coming from static or + * timed metadata. * *

This method may be called multiple times in quick succession. * @@ -2123,11 +2124,11 @@ public interface Player { * Returns the current combined {@link MediaMetadata}, or {@link MediaMetadata#EMPTY} if not * supported. * - *

This {@link MediaMetadata} is a combination of the {@link MediaItem#mediaMetadata} and the - * static and dynamic metadata from the {@link TrackSelection#getFormat(int) track selections' - * formats} and {@link Listener#onMetadata(Metadata)}. If a field is populated in the {@link - * MediaItem#mediaMetadata}, it will be prioritised above the same field coming from static or - * dynamic metadata. + *

This {@link MediaMetadata} is a combination of the {@link MediaItem#mediaMetadata MediaItem + * metadata}, the static metadata in the media's {@link Format#metadata Format}, and any timed + * metadata that has been parsed from the media and output via {@link + * Listener#onMetadata(Metadata)}. If a field is populated in the {@link MediaItem#mediaMetadata}, + * it will be prioritised above the same field coming from static or timed metadata. */ MediaMetadata getMediaMetadata(); diff --git a/libraries/common/src/main/java/androidx/media3/common/TrackGroup.java b/libraries/common/src/main/java/androidx/media3/common/TrackGroup.java index 8631aac4ab..dc9867ee30 100644 --- a/libraries/common/src/main/java/androidx/media3/common/TrackGroup.java +++ b/libraries/common/src/main/java/androidx/media3/common/TrackGroup.java @@ -34,7 +34,21 @@ import java.lang.annotation.Target; import java.util.Arrays; import java.util.List; -/** Defines an immutable group of tracks identified by their format identity. */ +/** + * An immutable group of tracks. All tracks in a group present the same content, but their formats + * may differ. + * + *

As an example of how tracks can be grouped, consider an adaptive playback where a main video + * feed is provided in five resolutions, and an alternative video feed (e.g., a different camera + * angle in a sports match) is provided in two resolutions. In this case there will be two video + * track groups, one corresponding to the main video feed containing five tracks, and a second for + * the alternative video feed containing two tracks. + * + *

Note that audio tracks whose languages differ are not grouped, because content in different + * languages is not considered to be the same. Conversely, audio tracks in the same language that + * only differ in properties such as bitrate, sampling rate, channel count and so on can be grouped. + * This also applies to text tracks. + */ public final class TrackGroup implements Bundleable { private static final String TAG = "TrackGroup"; diff --git a/libraries/common/src/main/java/androidx/media3/common/TrackSelectionOverride.java b/libraries/common/src/main/java/androidx/media3/common/TrackSelectionOverride.java index 7bc7b0cd18..e194c5eebf 100644 --- a/libraries/common/src/main/java/androidx/media3/common/TrackSelectionOverride.java +++ b/libraries/common/src/main/java/androidx/media3/common/TrackSelectionOverride.java @@ -31,16 +31,21 @@ import java.lang.annotation.RetentionPolicy; import java.util.List; /** - * Forces the selection of {@link #trackIndices} for a {@link TrackGroup}. + * A track selection override, consisting of a {@link TrackGroup} and the indices of the tracks + * within the group that should be selected. * - *

If multiple tracks in {@link #trackGroup} are overridden, as many as possible will be selected - * depending on the player capabilities. + *

A track selection override is applied during playback if the media being played contains a + * {@link TrackGroup} equal to the one in the override. If a {@link TrackSelectionParameters} + * contains only one override of a given track type that applies to the media, this override will be + * used to control the track selection for that type. If multiple overrides of a given track type + * apply then the player will apply only one of them. * - *

If {@link #trackIndices} is empty, no tracks from {@link #trackGroup} will be played. This is - * similar to {@link TrackSelectionParameters#disabledTrackTypes}, except it will only affect the - * playback of the associated {@link TrackGroup}. For example, if the only {@link - * C#TRACK_TYPE_VIDEO} {@link TrackGroup} is associated with no tracks, no video will play until the - * next video starts. + *

If {@link #trackIndices} is empty then the override specifies that no tracks should be + * selected. Adding an empty override to a {@link TrackSelectionParameters} is similar to {@link + * TrackSelectionParameters.Builder#setTrackTypeDisabled disabling a track type}, except that an + * empty override will only be applied if the media being played contains a {@link TrackGroup} equal + * to the one in the override. Conversely, disabling a track type will prevent selection of tracks + * of that type for all media. */ public final class TrackSelectionOverride implements Bundleable { diff --git a/libraries/common/src/main/java/androidx/media3/common/TrackSelectionParameters.java b/libraries/common/src/main/java/androidx/media3/common/TrackSelectionParameters.java index 232a878f08..534a17c226 100644 --- a/libraries/common/src/main/java/androidx/media3/common/TrackSelectionParameters.java +++ b/libraries/common/src/main/java/androidx/media3/common/TrackSelectionParameters.java @@ -47,10 +47,11 @@ import org.checkerframework.checker.initialization.qual.UnknownInitialization; import org.checkerframework.checker.nullness.qual.EnsuresNonNull; /** - * Constraint parameters for track selection. + * Parameters for controlling track selection. * - *

For example the following code modifies the parameters to restrict video track selections to - * SD, and to select a German audio track if there is one: + *

Parameters can be queried and set on a {@link Player}. For example the following code modifies + * the parameters to restrict video track selections to SD, and to select a German audio track if + * there is one: * *

{@code
  * // Build on the current parameters.
@@ -656,28 +657,26 @@ public class TrackSelectionParameters implements Bundleable {
       return this;
     }
 
-    /** Adds an override for the provided {@link TrackGroup}. */
+    /** Adds an override, replacing any override for the same {@link TrackGroup}. */
     public Builder addOverride(TrackSelectionOverride override) {
       overrides.put(override.trackGroup, override);
       return this;
     }
 
-    /** Removes the override associated with the provided {@link TrackGroup} if present. */
-    public Builder clearOverride(TrackGroup trackGroup) {
-      overrides.remove(trackGroup);
-      return this;
-    }
-
-    /** Set the override for the type of the provided {@link TrackGroup}. */
+    /** Sets an override, replacing all existing overrides with the same track type. */
     public Builder setOverrideForType(TrackSelectionOverride override) {
       clearOverridesOfType(override.getTrackType());
       overrides.put(override.trackGroup, override);
       return this;
     }
 
-    /**
-     * Remove any override associated with {@link TrackGroup TrackGroups} of type {@code trackType}.
-     */
+    /** Removes the override for the provided {@link TrackGroup}, if there is one. */
+    public Builder clearOverride(TrackGroup trackGroup) {
+      overrides.remove(trackGroup);
+      return this;
+    }
+
+    /** Removes all overrides of the provided track type. */
     public Builder clearOverridesOfType(@C.TrackType int trackType) {
       Iterator it = overrides.values().iterator();
       while (it.hasNext()) {
@@ -689,7 +688,7 @@ public class TrackSelectionParameters implements Bundleable {
       return this;
     }
 
-    /** Removes all track overrides. */
+    /** Removes all overrides. */
     public Builder clearOverrides() {
       overrides.clear();
       return this;
diff --git a/libraries/common/src/main/java/androidx/media3/common/TracksInfo.java b/libraries/common/src/main/java/androidx/media3/common/TracksInfo.java
index 793da2a831..4fd69d6407 100644
--- a/libraries/common/src/main/java/androidx/media3/common/TracksInfo.java
+++ b/libraries/common/src/main/java/androidx/media3/common/TracksInfo.java
@@ -41,8 +41,8 @@ public final class TracksInfo implements Bundleable {
 
   /**
    * Information about a single group of tracks, including the underlying {@link TrackGroup}, the
-   * {@link C.TrackType type} of tracks it contains, and the level to which each track is supported
-   * by the player.
+   * level to which each track is supported by the player, and whether any of the tracks are
+   * selected.
    */
   public static final class TrackGroupInfo implements Bundleable {
 
@@ -55,26 +55,26 @@ public final class TracksInfo implements Bundleable {
     private final boolean[] trackSelected;
 
     /**
-     * Constructs a TrackGroupInfo.
+     * Constructs an instance.
      *
-     * @param trackGroup The {@link TrackGroup} described.
-     * @param adaptiveSupported Whether adaptive selections containing more than one track in the
-     *     {@code trackGroup} are supported.
-     * @param trackSupport The {@link C.FormatSupport} of each track in the {@code trackGroup}.
-     * @param tracksSelected Whether each track in the {@code trackGroup} is selected.
+     * @param trackGroup The underlying {@link TrackGroup}.
+     * @param adaptiveSupported Whether the player supports adaptive selections containing more than
+     *     one track in the group.
+     * @param trackSupport The {@link C.FormatSupport} of each track in the group.
+     * @param trackSelected Whether each track in the {@code trackGroup} is selected.
      */
     @UnstableApi
     public TrackGroupInfo(
         TrackGroup trackGroup,
         boolean adaptiveSupported,
         @C.FormatSupport int[] trackSupport,
-        boolean[] tracksSelected) {
+        boolean[] trackSelected) {
       length = trackGroup.length;
-      checkArgument(length == trackSupport.length && length == tracksSelected.length);
+      checkArgument(length == trackSupport.length && length == trackSelected.length);
       this.trackGroup = trackGroup;
       this.adaptiveSupported = adaptiveSupported && length > 1;
       this.trackSupport = trackSupport.clone();
-      this.trackSelected = tracksSelected.clone();
+      this.trackSelected = trackSelected.clone();
     }
 
     /** Returns the underlying {@link TrackGroup}. */
@@ -266,11 +266,11 @@ public final class TracksInfo implements Bundleable {
     }
   }
 
-  private final ImmutableList trackGroupInfos;
-
   /** An {@code TrackInfo} that contains no tracks. */
   @UnstableApi public static final TracksInfo EMPTY = new TracksInfo(ImmutableList.of());
 
+  private final ImmutableList trackGroupInfos;
+
   /**
    * Constructs an instance.
    *
diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlProgram.java b/libraries/common/src/main/java/androidx/media3/common/util/GlProgram.java
index 681ec57433..82b0775a89 100644
--- a/libraries/common/src/main/java/androidx/media3/common/util/GlProgram.java
+++ b/libraries/common/src/main/java/androidx/media3/common/util/GlProgram.java
@@ -147,8 +147,6 @@ public final class GlProgram {
    * 

Call this in the rendering loop to switch between different programs. */ public void use() { - // TODO(b/214975934): When multiple GL programs are supported by Transformer, make sure - // to call use() to switch between programs. GLES20.glUseProgram(programId); GlUtil.checkGlError(); } @@ -175,9 +173,16 @@ public final class GlProgram { checkNotNull(attributeByName.get(name)).setBuffer(values, size); } - /** Sets a texture sampler type uniform. */ - public void setSamplerTexIdUniform(String name, int texId, int unit) { - checkNotNull(uniformByName.get(name)).setSamplerTexId(texId, unit); + /** + * Sets a texture sampler type uniform. + * + * @param name The uniform's name. + * @param texId The texture identifier. + * @param texUnitIndex The texture unit index. Use a different index (0, 1, 2, ...) for each + * texture sampler in the program. + */ + public void setSamplerTexIdUniform(String name, int texId, int texUnitIndex) { + checkNotNull(uniformByName.get(name)).setSamplerTexId(texId, texUnitIndex); } /** Sets a float type uniform. */ @@ -322,7 +327,7 @@ public final class GlProgram { private final float[] value; private int texId; - private int unit; + private int texUnitIndex; private Uniform(String name, int location, int type) { this.name = name; @@ -335,11 +340,11 @@ public final class GlProgram { * Configures {@link #bind()} to use the specified {@code texId} for this sampler uniform. * * @param texId The GL texture identifier from which to sample. - * @param unit The GL texture unit index. + * @param texUnitIndex The GL texture unit index. */ - public void setSamplerTexId(int texId, int unit) { + public void setSamplerTexId(int texId, int texUnitIndex) { this.texId = texId; - this.unit = unit; + this.texUnitIndex = texUnitIndex; } /** Configures {@link #bind()} to use the specified float {@code value} for this uniform. */ @@ -382,7 +387,7 @@ public final class GlProgram { if (texId == 0) { throw new IllegalStateException("No call to setSamplerTexId() before bind."); } - GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + unit); + GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + texUnitIndex); if (type == GLES11Ext.GL_SAMPLER_EXTERNAL_OES || type == GL_SAMPLER_EXTERNAL_2D_Y2Y_EXT) { GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, texId); } else if (type == GLES20.GL_SAMPLER_2D) { @@ -390,7 +395,7 @@ public final class GlProgram { } else { throw new IllegalStateException("Unexpected uniform type: " + type); } - GLES20.glUniform1i(location, unit); + GLES20.glUniform1i(location, texUnitIndex); GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); GLES20.glTexParameteri( diff --git a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java index aa1e0332f6..8fca7eb310 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/GlUtil.java @@ -119,7 +119,8 @@ public final class GlUtil { /** * Returns whether creating a GL context with {@value #EXTENSION_PROTECTED_CONTENT} is possible. - * If {@code true}, the device supports a protected output path for DRM content when using GL. + * + *

If {@code true}, the device supports a protected output path for DRM content when using GL. */ public static boolean isProtectedContentExtensionSupported(Context context) { if (Util.SDK_INT < 24) { @@ -222,6 +223,30 @@ public final class GlUtil { } } + /** + * Asserts the texture size is valid. + * + * @param width The width for a texture. + * @param height The height for a texture. + * @throws GlException If the texture width or height is invalid. + */ + public static void assertValidTextureSize(int width, int height) { + // TODO(b/201293185): Consider handling adjustments for sizes > GL_MAX_TEXTURE_SIZE + // (ex. downscaling appropriately) in a FrameProcessor instead of asserting incorrect values. + + // For valid GL sizes, see: + // https://www.khronos.org/registry/OpenGL-Refpages/es2.0/xhtml/glTexImage2D.xml + int[] maxTextureSizeBuffer = new int[1]; + GLES20.glGetIntegerv(GLES20.GL_MAX_TEXTURE_SIZE, maxTextureSizeBuffer, 0); + int maxTextureSize = maxTextureSizeBuffer[0]; + if (width < 0 || height < 0) { + throwGlException("width or height is less than 0"); + } + if (width > maxTextureSize || height > maxTextureSize) { + throwGlException("width or height is greater than GL_MAX_TEXTURE_SIZE " + maxTextureSize); + } + } + /** * Makes the specified {@code eglSurface} the render target, using a viewport of {@code width} by * {@code height} pixels. @@ -320,6 +345,7 @@ public final class GlUtil { * @param height of the new texture in pixels */ public static int createTexture(int width, int height) { + assertValidTextureSize(width, height); int texId = generateAndBindTexture(GLES20.GL_TEXTURE_2D); ByteBuffer byteBuffer = ByteBuffer.allocateDirect(width * height * 4); GLES20.glTexImage2D( @@ -390,6 +416,11 @@ public final class GlUtil { } } + private static void checkEglException(String errorMessage) { + int error = EGL14.eglGetError(); + checkEglException(error == EGL14.EGL_SUCCESS, errorMessage + ", error code: " + error); + } + @RequiresApi(17) private static final class Api17 { private Api17() {} @@ -438,12 +469,15 @@ public final class GlUtil { Object surface, int[] configAttributes, int[] windowSurfaceAttributes) { - return EGL14.eglCreateWindowSurface( - eglDisplay, - getEglConfig(eglDisplay, configAttributes), - surface, - windowSurfaceAttributes, - /* offset= */ 0); + EGLSurface eglSurface = + EGL14.eglCreateWindowSurface( + eglDisplay, + getEglConfig(eglDisplay, configAttributes), + surface, + windowSurfaceAttributes, + /* offset= */ 0); + checkEglException("Error creating surface"); + return eglSurface; } @DoNotInline @@ -459,8 +493,11 @@ public final class GlUtil { if (boundFramebuffer[0] != framebuffer) { GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, framebuffer); } + checkGlError(); EGL14.eglMakeCurrent(eglDisplay, eglSurface, eglSurface, eglContext); + checkEglException("Error making context current"); GLES20.glViewport(/* x= */ 0, /* y= */ 0, width, height); + checkGlError(); } @DoNotInline @@ -471,19 +508,15 @@ public final class GlUtil { } EGL14.eglMakeCurrent( eglDisplay, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_SURFACE, EGL14.EGL_NO_CONTEXT); - int error = EGL14.eglGetError(); - checkEglException(error == EGL14.EGL_SUCCESS, "Error releasing context: " + error); + checkEglException("Error releasing context"); if (eglContext != null) { EGL14.eglDestroyContext(eglDisplay, eglContext); - error = EGL14.eglGetError(); - checkEglException(error == EGL14.EGL_SUCCESS, "Error destroying context: " + error); + checkEglException("Error destroying context"); } EGL14.eglReleaseThread(); - error = EGL14.eglGetError(); - checkEglException(error == EGL14.EGL_SUCCESS, "Error releasing thread: " + error); + checkEglException("Error releasing thread"); EGL14.eglTerminate(eglDisplay); - error = EGL14.eglGetError(); - checkEglException(error == EGL14.EGL_SUCCESS, "Error terminating display: " + error); + checkEglException("Error terminating display"); } @DoNotInline diff --git a/libraries/common/src/main/java/androidx/media3/common/util/Util.java b/libraries/common/src/main/java/androidx/media3/common/util/Util.java index d45d1d35dc..6cffae7fbd 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/Util.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/Util.java @@ -1143,18 +1143,6 @@ public final class Util { return (timeMs == C.TIME_UNSET || timeMs == C.TIME_END_OF_SOURCE) ? timeMs : (timeMs * 1000); } - /** - * Converts a time in seconds to the corresponding time in microseconds. - * - * @param timeSec The time in seconds. - * @return The corresponding time in microseconds. - */ - public static long secToUs(double timeSec) { - return BigDecimal.valueOf(timeSec) - .multiply(BigDecimal.valueOf(C.MICROS_PER_SECOND)) - .longValue(); - } - /** * Parses an xs:duration attribute value, returning the parsed duration in milliseconds. * diff --git a/libraries/common/src/test/java/androidx/media3/common/util/ColorParserTest.java b/libraries/common/src/test/java/androidx/media3/common/util/ColorParserTest.java index ba4d2b7a75..4791df0905 100644 --- a/libraries/common/src/test/java/androidx/media3/common/util/ColorParserTest.java +++ b/libraries/common/src/test/java/androidx/media3/common/util/ColorParserTest.java @@ -22,6 +22,7 @@ import static android.graphics.Color.argb; import static android.graphics.Color.parseColor; import static androidx.media3.common.util.ColorParser.parseTtmlColor; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; import android.graphics.Color; import androidx.test.ext.junit.runners.AndroidJUnit4; @@ -34,24 +35,26 @@ public final class ColorParserTest { // Negative tests. - @Test(expected = IllegalArgumentException.class) + @Test public void parseUnknownColor() { - ColorParser.parseTtmlColor("colorOfAnElectron"); + assertThrows( + IllegalArgumentException.class, () -> ColorParser.parseTtmlColor("colorOfAnElectron")); } - @Test(expected = IllegalArgumentException.class) + @Test public void parseNull() { - ColorParser.parseTtmlColor(null); + assertThrows(IllegalArgumentException.class, () -> ColorParser.parseTtmlColor(null)); } - @Test(expected = IllegalArgumentException.class) + @Test public void parseEmpty() { - ColorParser.parseTtmlColor(""); + assertThrows(IllegalArgumentException.class, () -> ColorParser.parseTtmlColor("")); } - @Test(expected = IllegalArgumentException.class) + @Test public void rgbColorParsingRgbValuesNegative() { - ColorParser.parseTtmlColor("rgb(-4, 55, 209)"); + assertThrows( + IllegalArgumentException.class, () -> ColorParser.parseTtmlColor("rgb(-4, 55, 209)")); } // Positive tests. diff --git a/libraries/datasource/src/main/java/androidx/media3/datasource/BaseDataSource.java b/libraries/datasource/src/main/java/androidx/media3/datasource/BaseDataSource.java index 4a826df2c2..55d721bce7 100644 --- a/libraries/datasource/src/main/java/androidx/media3/datasource/BaseDataSource.java +++ b/libraries/datasource/src/main/java/androidx/media3/datasource/BaseDataSource.java @@ -48,6 +48,7 @@ public abstract class BaseDataSource implements DataSource { this.listeners = new ArrayList<>(/* initialCapacity= */ 1); } + @UnstableApi @Override public final void addTransferListener(TransferListener transferListener) { checkNotNull(transferListener); diff --git a/libraries/datasource/src/main/java/androidx/media3/datasource/DataSource.java b/libraries/datasource/src/main/java/androidx/media3/datasource/DataSource.java index e1a34d7883..fc93269055 100644 --- a/libraries/datasource/src/main/java/androidx/media3/datasource/DataSource.java +++ b/libraries/datasource/src/main/java/androidx/media3/datasource/DataSource.java @@ -26,13 +26,13 @@ import java.util.List; import java.util.Map; /** Reads data from URI-identified resources. */ -@UnstableApi public interface DataSource extends DataReader { /** A factory for {@link DataSource} instances. */ interface Factory { /** Creates a {@link DataSource} instance. */ + @UnstableApi DataSource createDataSource(); } @@ -41,6 +41,7 @@ public interface DataSource extends DataReader { * * @param transferListener A {@link TransferListener}. */ + @UnstableApi void addTransferListener(TransferListener transferListener); /** @@ -72,6 +73,7 @@ public interface DataSource extends DataReader { * unresolved. For all other requests, the value returned will be equal to the request's * {@link DataSpec#length}. */ + @UnstableApi long open(DataSpec dataSpec) throws IOException; /** @@ -82,6 +84,7 @@ public interface DataSource extends DataReader { * * @return The {@link Uri} from which data is being read, or null if the source is not open. */ + @UnstableApi @Nullable Uri getUri(); @@ -91,6 +94,7 @@ public interface DataSource extends DataReader { * *

Key look-up in the returned map is case-insensitive. */ + @UnstableApi default Map> getResponseHeaders() { return Collections.emptyMap(); } @@ -101,5 +105,6 @@ public interface DataSource extends DataReader { * * @throws IOException If an error occurs closing the source. */ + @UnstableApi void close() throws IOException; } diff --git a/libraries/datasource/src/main/java/androidx/media3/datasource/DefaultDataSource.java b/libraries/datasource/src/main/java/androidx/media3/datasource/DefaultDataSource.java index 0a726541fa..cc179e2d55 100644 --- a/libraries/datasource/src/main/java/androidx/media3/datasource/DefaultDataSource.java +++ b/libraries/datasource/src/main/java/androidx/media3/datasource/DefaultDataSource.java @@ -54,7 +54,6 @@ import java.util.Map; * #DefaultDataSource(Context, DataSource)}. * */ -@UnstableApi public final class DefaultDataSource implements DataSource { /** {@link DataSource.Factory} for {@link DefaultDataSource} instances. */ @@ -98,11 +97,13 @@ public final class DefaultDataSource implements DataSource { * @param transferListener The listener that will be used. * @return This factory. */ + @UnstableApi public Factory setTransferListener(@Nullable TransferListener transferListener) { this.transferListener = transferListener; return this; } + @UnstableApi @Override public DefaultDataSource createDataSource() { DefaultDataSource dataSource = @@ -144,6 +145,7 @@ public final class DefaultDataSource implements DataSource { * * @param context A context. */ + @UnstableApi public DefaultDataSource(Context context, boolean allowCrossProtocolRedirects) { this( context, @@ -162,6 +164,7 @@ public final class DefaultDataSource implements DataSource { * @param allowCrossProtocolRedirects Whether cross-protocol redirects (i.e. redirects from HTTP * to HTTPS and vice versa) are enabled when fetching remote data. */ + @UnstableApi public DefaultDataSource( Context context, @Nullable String userAgent, boolean allowCrossProtocolRedirects) { this( @@ -185,6 +188,7 @@ public final class DefaultDataSource implements DataSource { * @param allowCrossProtocolRedirects Whether cross-protocol redirects (i.e. redirects from HTTP * to HTTPS and vice versa) are enabled when fetching remote data. */ + @UnstableApi public DefaultDataSource( Context context, @Nullable String userAgent, @@ -209,12 +213,14 @@ public final class DefaultDataSource implements DataSource { * @param baseDataSource A {@link DataSource} to use for URI schemes other than file, asset and * content. This {@link DataSource} should normally support at least http(s). */ + @UnstableApi public DefaultDataSource(Context context, DataSource baseDataSource) { this.context = context.getApplicationContext(); this.baseDataSource = Assertions.checkNotNull(baseDataSource); transferListeners = new ArrayList<>(); } + @UnstableApi @Override public void addTransferListener(TransferListener transferListener) { Assertions.checkNotNull(transferListener); @@ -229,6 +235,7 @@ public final class DefaultDataSource implements DataSource { maybeAddListenerToDataSource(rawResourceDataSource, transferListener); } + @UnstableApi @Override public long open(DataSpec dataSpec) throws IOException { Assertions.checkState(dataSource == null); @@ -260,22 +267,26 @@ public final class DefaultDataSource implements DataSource { return dataSource.open(dataSpec); } + @UnstableApi @Override public int read(byte[] buffer, int offset, int length) throws IOException { return Assertions.checkNotNull(dataSource).read(buffer, offset, length); } + @UnstableApi @Override @Nullable public Uri getUri() { return dataSource == null ? null : dataSource.getUri(); } + @UnstableApi @Override public Map> getResponseHeaders() { return dataSource == null ? Collections.emptyMap() : dataSource.getResponseHeaders(); } + @UnstableApi @Override public void close() throws IOException { if (dataSource != null) { diff --git a/libraries/datasource/src/main/java/androidx/media3/datasource/DefaultHttpDataSource.java b/libraries/datasource/src/main/java/androidx/media3/datasource/DefaultHttpDataSource.java index 4b6a9f0772..59d8d4a40f 100644 --- a/libraries/datasource/src/main/java/androidx/media3/datasource/DefaultHttpDataSource.java +++ b/libraries/datasource/src/main/java/androidx/media3/datasource/DefaultHttpDataSource.java @@ -60,7 +60,6 @@ import java.util.zip.GZIPInputStream; * priority) the {@code dataSpec}, {@link #setRequestProperty} and the default properties that can * be passed to {@link HttpDataSource.Factory#setDefaultRequestProperties(Map)}. */ -@UnstableApi public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSource { /** {@link DataSource.Factory} for {@link DefaultHttpDataSource} instances. */ @@ -83,6 +82,7 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou readTimeoutMs = DEFAULT_READ_TIMEOUT_MILLIS; } + @UnstableApi @Override public final Factory setDefaultRequestProperties(Map defaultRequestProperties) { this.defaultRequestProperties.clearAndSet(defaultRequestProperties); @@ -99,6 +99,7 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou * agent of the underlying platform. * @return This factory. */ + @UnstableApi public Factory setUserAgent(@Nullable String userAgent) { this.userAgent = userAgent; return this; @@ -112,6 +113,7 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou * @param connectTimeoutMs The connect timeout, in milliseconds, that will be used. * @return This factory. */ + @UnstableApi public Factory setConnectTimeoutMs(int connectTimeoutMs) { this.connectTimeoutMs = connectTimeoutMs; return this; @@ -125,6 +127,7 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou * @param readTimeoutMs The connect timeout, in milliseconds, that will be used. * @return This factory. */ + @UnstableApi public Factory setReadTimeoutMs(int readTimeoutMs) { this.readTimeoutMs = readTimeoutMs; return this; @@ -138,6 +141,7 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou * @param allowCrossProtocolRedirects Whether to allow cross protocol redirects. * @return This factory. */ + @UnstableApi public Factory setAllowCrossProtocolRedirects(boolean allowCrossProtocolRedirects) { this.allowCrossProtocolRedirects = allowCrossProtocolRedirects; return this; @@ -154,6 +158,7 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou * predicate that was previously set. * @return This factory. */ + @UnstableApi public Factory setContentTypePredicate(@Nullable Predicate contentTypePredicate) { this.contentTypePredicate = contentTypePredicate; return this; @@ -169,6 +174,7 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou * @param transferListener The listener that will be used. * @return This factory. */ + @UnstableApi public Factory setTransferListener(@Nullable TransferListener transferListener) { this.transferListener = transferListener; return this; @@ -178,11 +184,13 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou * Sets whether we should keep the POST method and body when we have HTTP 302 redirects for a * POST request. */ + @UnstableApi public Factory setKeepPostFor302Redirects(boolean keepPostFor302Redirects) { this.keepPostFor302Redirects = keepPostFor302Redirects; return this; } + @UnstableApi @Override public DefaultHttpDataSource createDataSource() { DefaultHttpDataSource dataSource = @@ -202,9 +210,9 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou } /** The default connection timeout, in milliseconds. */ - public static final int DEFAULT_CONNECT_TIMEOUT_MILLIS = 8 * 1000; + @UnstableApi public static final int DEFAULT_CONNECT_TIMEOUT_MILLIS = 8 * 1000; /** The default read timeout, in milliseconds. */ - public static final int DEFAULT_READ_TIMEOUT_MILLIS = 8 * 1000; + @UnstableApi public static final int DEFAULT_READ_TIMEOUT_MILLIS = 8 * 1000; private static final String TAG = "DefaultHttpDataSource"; private static final int MAX_REDIRECTS = 20; // Same limit as okhttp. @@ -232,6 +240,7 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou /** * @deprecated Use {@link DefaultHttpDataSource.Factory} instead. */ + @UnstableApi @SuppressWarnings("deprecation") @Deprecated public DefaultHttpDataSource() { @@ -241,6 +250,7 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou /** * @deprecated Use {@link DefaultHttpDataSource.Factory} instead. */ + @UnstableApi @SuppressWarnings("deprecation") @Deprecated public DefaultHttpDataSource(@Nullable String userAgent) { @@ -250,6 +260,7 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou /** * @deprecated Use {@link DefaultHttpDataSource.Factory} instead. */ + @UnstableApi @SuppressWarnings("deprecation") @Deprecated public DefaultHttpDataSource( @@ -265,6 +276,7 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou /** * @deprecated Use {@link DefaultHttpDataSource.Factory} instead. */ + @UnstableApi @Deprecated public DefaultHttpDataSource( @Nullable String userAgent, @@ -305,22 +317,26 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou * @deprecated Use {@link DefaultHttpDataSource.Factory#setContentTypePredicate(Predicate)} * instead. */ + @UnstableApi @Deprecated public void setContentTypePredicate(@Nullable Predicate contentTypePredicate) { this.contentTypePredicate = contentTypePredicate; } + @UnstableApi @Override @Nullable public Uri getUri() { return connection == null ? null : Uri.parse(connection.getURL().toString()); } + @UnstableApi @Override public int getResponseCode() { return connection == null || responseCode <= 0 ? -1 : responseCode; } + @UnstableApi @Override public Map> getResponseHeaders() { if (connection == null) { @@ -337,6 +353,7 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou return new NullFilteringHeadersMap(connection.getHeaderFields()); } + @UnstableApi @Override public void setRequestProperty(String name, String value) { checkNotNull(name); @@ -344,18 +361,21 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou requestProperties.set(name, value); } + @UnstableApi @Override public void clearRequestProperty(String name) { checkNotNull(name); requestProperties.remove(name); } + @UnstableApi @Override public void clearAllRequestProperties() { requestProperties.clear(); } /** Opens the source to read the specified data. */ + @UnstableApi @Override public long open(DataSpec dataSpec) throws HttpDataSourceException { this.dataSpec = dataSpec; @@ -474,6 +494,7 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou return bytesToRead; } + @UnstableApi @Override public int read(byte[] buffer, int offset, int length) throws HttpDataSourceException { try { @@ -484,6 +505,7 @@ public class DefaultHttpDataSource extends BaseDataSource implements HttpDataSou } } + @UnstableApi @Override public void close() throws HttpDataSourceException { try { diff --git a/libraries/datasource/src/main/java/androidx/media3/datasource/HttpDataSource.java b/libraries/datasource/src/main/java/androidx/media3/datasource/HttpDataSource.java index 967a00b49c..ddc9518a8e 100644 --- a/libraries/datasource/src/main/java/androidx/media3/datasource/HttpDataSource.java +++ b/libraries/datasource/src/main/java/androidx/media3/datasource/HttpDataSource.java @@ -172,6 +172,7 @@ public interface HttpDataSource extends DataSource { } /** A {@link Predicate} that rejects content types often used for pay-walls. */ + @UnstableApi Predicate REJECT_PAYWALL_TYPES = contentType -> { if (contentType == null) { diff --git a/libraries/datasource_cronet/README.md b/libraries/datasource_cronet/README.md index 62599879fb..c64f8b3bae 100644 --- a/libraries/datasource_cronet/README.md +++ b/libraries/datasource_cronet/README.md @@ -38,8 +38,9 @@ If your application only needs to play http(s) content, using the Cronet extension is as simple as updating `DataSource.Factory` instantiations in your application code to use `CronetDataSource.Factory`. If your application also needs to play non-http(s) content such as local files, use: + ``` -new DefaultDataSourceFactory( +new DefaultDataSource.Factory( ... /* baseDataSourceFactory= */ new CronetDataSource.Factory(...) ); ``` diff --git a/libraries/datasource_cronet/src/main/java/androidx/media3/datasource/cronet/CronetDataSource.java b/libraries/datasource_cronet/src/main/java/androidx/media3/datasource/cronet/CronetDataSource.java index 4d5988398e..505a65033c 100644 --- a/libraries/datasource_cronet/src/main/java/androidx/media3/datasource/cronet/CronetDataSource.java +++ b/libraries/datasource_cronet/src/main/java/androidx/media3/datasource/cronet/CronetDataSource.java @@ -68,7 +68,6 @@ import org.chromium.net.UrlResponseInfo; * priority) the {@code dataSpec}, {@link #setRequestProperty} and the default parameters used to * construct the instance. */ -@UnstableApi public class CronetDataSource extends BaseDataSource implements HttpDataSource { static { @@ -132,6 +131,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { * CronetEngine}, or {@link DefaultHttpDataSource} for cases where {@link * CronetEngineWrapper#getCronetEngine()} would have returned {@code null}. */ + @UnstableApi @Deprecated public Factory(CronetEngineWrapper cronetEngineWrapper, Executor executor) { this.cronetEngine = cronetEngineWrapper.getCronetEngine(); @@ -142,6 +142,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { readTimeoutMs = DEFAULT_READ_TIMEOUT_MILLIS; } + @UnstableApi @Override public final Factory setDefaultRequestProperties(Map defaultRequestProperties) { this.defaultRequestProperties.clearAndSet(defaultRequestProperties); @@ -161,6 +162,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { * agent of the underlying {@link CronetEngine}. * @return This factory. */ + @UnstableApi public Factory setUserAgent(@Nullable String userAgent) { this.userAgent = userAgent; if (internalFallbackFactory != null) { @@ -179,6 +181,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { * UrlRequest.Builder#REQUEST_PRIORITY_*} constants. * @return This factory. */ + @UnstableApi public Factory setRequestPriority(int requestPriority) { this.requestPriority = requestPriority; return this; @@ -192,6 +195,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { * @param connectTimeoutMs The connect timeout, in milliseconds, that will be used. * @return This factory. */ + @UnstableApi public Factory setConnectionTimeoutMs(int connectTimeoutMs) { this.connectTimeoutMs = connectTimeoutMs; if (internalFallbackFactory != null) { @@ -208,6 +212,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { * @param resetTimeoutOnRedirects Whether the connect timeout is reset when a redirect occurs. * @return This factory. */ + @UnstableApi public Factory setResetTimeoutOnRedirects(boolean resetTimeoutOnRedirects) { this.resetTimeoutOnRedirects = resetTimeoutOnRedirects; return this; @@ -223,6 +228,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { * to the redirect url in the "Cookie" header. * @return This factory. */ + @UnstableApi public Factory setHandleSetCookieRequests(boolean handleSetCookieRequests) { this.handleSetCookieRequests = handleSetCookieRequests; return this; @@ -236,6 +242,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { * @param readTimeoutMs The connect timeout, in milliseconds, that will be used. * @return This factory. */ + @UnstableApi public Factory setReadTimeoutMs(int readTimeoutMs) { this.readTimeoutMs = readTimeoutMs; if (internalFallbackFactory != null) { @@ -254,6 +261,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { * predicate that was previously set. * @return This factory. */ + @UnstableApi public Factory setContentTypePredicate(@Nullable Predicate contentTypePredicate) { this.contentTypePredicate = contentTypePredicate; if (internalFallbackFactory != null) { @@ -266,6 +274,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { * Sets whether we should keep the POST method and body when we have HTTP 302 redirects for a * POST request. */ + @UnstableApi public Factory setKeepPostFor302Redirects(boolean keepPostFor302Redirects) { this.keepPostFor302Redirects = keepPostFor302Redirects; if (internalFallbackFactory != null) { @@ -284,6 +293,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { * @param transferListener The listener that will be used. * @return This factory. */ + @UnstableApi public Factory setTransferListener(@Nullable TransferListener transferListener) { this.transferListener = transferListener; if (internalFallbackFactory != null) { @@ -303,12 +313,14 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { * @deprecated Do not use {@link CronetDataSource} or its factory in cases where a suitable * {@link CronetEngine} is not available. Use the fallback factory directly in such cases. */ + @UnstableApi @Deprecated public Factory setFallbackFactory(@Nullable HttpDataSource.Factory fallbackFactory) { this.fallbackFactory = fallbackFactory; return this; } + @UnstableApi @Override public HttpDataSource createDataSource() { if (cronetEngine == null) { @@ -337,6 +349,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { } /** Thrown when an error is encountered when trying to open a {@link CronetDataSource}. */ + @UnstableApi public static final class OpenException extends HttpDataSourceException { /** @@ -389,9 +402,9 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { } /** The default connection timeout, in milliseconds. */ - public static final int DEFAULT_CONNECT_TIMEOUT_MILLIS = 8 * 1000; + @UnstableApi public static final int DEFAULT_CONNECT_TIMEOUT_MILLIS = 8 * 1000; /** The default read timeout, in milliseconds. */ - public static final int DEFAULT_READ_TIMEOUT_MILLIS = 8 * 1000; + @UnstableApi public static final int DEFAULT_READ_TIMEOUT_MILLIS = 8 * 1000; /* package */ final UrlRequest.Callback urlRequestCallback; @@ -436,6 +449,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { private volatile long currentConnectTimeoutMs; + @UnstableApi protected CronetDataSource( CronetEngine cronetEngine, Executor executor, @@ -473,6 +487,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { * @param contentTypePredicate The content type {@link Predicate}, or {@code null} to clear a * predicate that was previously set. */ + @UnstableApi @Deprecated public void setContentTypePredicate(@Nullable Predicate contentTypePredicate) { this.contentTypePredicate = contentTypePredicate; @@ -480,21 +495,25 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { // HttpDataSource implementation. + @UnstableApi @Override public void setRequestProperty(String name, String value) { requestProperties.set(name, value); } + @UnstableApi @Override public void clearRequestProperty(String name) { requestProperties.remove(name); } + @UnstableApi @Override public void clearAllRequestProperties() { requestProperties.clear(); } + @UnstableApi @Override public int getResponseCode() { return responseInfo == null || responseInfo.getHttpStatusCode() <= 0 @@ -502,17 +521,20 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { : responseInfo.getHttpStatusCode(); } + @UnstableApi @Override public Map> getResponseHeaders() { return responseInfo == null ? Collections.emptyMap() : responseInfo.getAllHeaders(); } + @UnstableApi @Override @Nullable public Uri getUri() { return responseInfo == null ? null : Uri.parse(responseInfo.getUrl()); } + @UnstableApi @Override public long open(DataSpec dataSpec) throws HttpDataSourceException { Assertions.checkNotNull(dataSpec); @@ -644,6 +666,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { return bytesRemaining; } + @UnstableApi @Override public int read(byte[] buffer, int offset, int length) throws HttpDataSourceException { Assertions.checkState(opened); @@ -715,6 +738,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { * @throws HttpDataSourceException If an error occurs reading from the source. * @throws IllegalArgumentException If {@code buffer} is not a direct ByteBuffer. */ + @UnstableApi public int read(ByteBuffer buffer) throws HttpDataSourceException { Assertions.checkState(opened); @@ -759,6 +783,7 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { return bytesRead; } + @UnstableApi @Override public synchronized void close() { if (currentUrlRequest != null) { @@ -779,17 +804,20 @@ public class CronetDataSource extends BaseDataSource implements HttpDataSource { } /** Returns current {@link UrlRequest}. May be null if the data source is not opened. */ + @UnstableApi @Nullable protected UrlRequest getCurrentUrlRequest() { return currentUrlRequest; } /** Returns current {@link UrlResponseInfo}. May be null if the data source is not opened. */ + @UnstableApi @Nullable protected UrlResponseInfo getCurrentUrlResponseInfo() { return responseInfo; } + @UnstableApi protected UrlRequest.Builder buildRequestBuilder(DataSpec dataSpec) throws IOException { UrlRequest.Builder requestBuilder = cronetEngine diff --git a/libraries/datasource_cronet/src/main/java/androidx/media3/datasource/cronet/CronetUtil.java b/libraries/datasource_cronet/src/main/java/androidx/media3/datasource/cronet/CronetUtil.java index 6afce324f9..03951c0255 100644 --- a/libraries/datasource_cronet/src/main/java/androidx/media3/datasource/cronet/CronetUtil.java +++ b/libraries/datasource_cronet/src/main/java/androidx/media3/datasource/cronet/CronetUtil.java @@ -31,7 +31,6 @@ import org.chromium.net.CronetEngine; import org.chromium.net.CronetProvider; /** Cronet utility methods. */ -@UnstableApi public final class CronetUtil { private static final String TAG = "CronetUtil"; @@ -77,6 +76,7 @@ public final class CronetUtil { * over Cronet Embedded, if both are available. * @return The {@link CronetEngine}, or {@code null} if no suitable engine could be built. */ + @UnstableApi @Nullable public static CronetEngine buildCronetEngine( Context context, @Nullable String userAgent, boolean preferGooglePlayServices) { diff --git a/libraries/datasource_okhttp/src/main/java/androidx/media3/datasource/okhttp/OkHttpDataSource.java b/libraries/datasource_okhttp/src/main/java/androidx/media3/datasource/okhttp/OkHttpDataSource.java index 889c5f6d6a..42ed422550 100644 --- a/libraries/datasource_okhttp/src/main/java/androidx/media3/datasource/okhttp/OkHttpDataSource.java +++ b/libraries/datasource_okhttp/src/main/java/androidx/media3/datasource/okhttp/OkHttpDataSource.java @@ -60,7 +60,6 @@ import okhttp3.ResponseBody; * priority) the {@code dataSpec}, {@link #setRequestProperty} and the default parameters used to * construct the instance. */ -@UnstableApi public class OkHttpDataSource extends BaseDataSource implements HttpDataSource { static { @@ -89,6 +88,7 @@ public class OkHttpDataSource extends BaseDataSource implements HttpDataSource { defaultRequestProperties = new RequestProperties(); } + @UnstableApi @Override public final Factory setDefaultRequestProperties(Map defaultRequestProperties) { this.defaultRequestProperties.clearAndSet(defaultRequestProperties); @@ -105,6 +105,7 @@ public class OkHttpDataSource extends BaseDataSource implements HttpDataSource { * agent of the underlying {@link OkHttpClient}. * @return This factory. */ + @UnstableApi public Factory setUserAgent(@Nullable String userAgent) { this.userAgent = userAgent; return this; @@ -118,6 +119,7 @@ public class OkHttpDataSource extends BaseDataSource implements HttpDataSource { * @param cacheControl The cache control that will be used. * @return This factory. */ + @UnstableApi public Factory setCacheControl(@Nullable CacheControl cacheControl) { this.cacheControl = cacheControl; return this; @@ -134,6 +136,7 @@ public class OkHttpDataSource extends BaseDataSource implements HttpDataSource { * predicate that was previously set. * @return This factory. */ + @UnstableApi public Factory setContentTypePredicate(@Nullable Predicate contentTypePredicate) { this.contentTypePredicate = contentTypePredicate; return this; @@ -149,11 +152,13 @@ public class OkHttpDataSource extends BaseDataSource implements HttpDataSource { * @param transferListener The listener that will be used. * @return This factory. */ + @UnstableApi public Factory setTransferListener(@Nullable TransferListener transferListener) { this.transferListener = transferListener; return this; } + @UnstableApi @Override public OkHttpDataSource createDataSource() { OkHttpDataSource dataSource = @@ -185,6 +190,7 @@ public class OkHttpDataSource extends BaseDataSource implements HttpDataSource { * @deprecated Use {@link OkHttpDataSource.Factory} instead. */ @SuppressWarnings("deprecation") + @UnstableApi @Deprecated public OkHttpDataSource(Call.Factory callFactory) { this(callFactory, /* userAgent= */ null); @@ -194,6 +200,7 @@ public class OkHttpDataSource extends BaseDataSource implements HttpDataSource { * @deprecated Use {@link OkHttpDataSource.Factory} instead. */ @SuppressWarnings("deprecation") + @UnstableApi @Deprecated public OkHttpDataSource(Call.Factory callFactory, @Nullable String userAgent) { this(callFactory, userAgent, /* cacheControl= */ null, /* defaultRequestProperties= */ null); @@ -202,6 +209,7 @@ public class OkHttpDataSource extends BaseDataSource implements HttpDataSource { /** * @deprecated Use {@link OkHttpDataSource.Factory} instead. */ + @UnstableApi @Deprecated public OkHttpDataSource( Call.Factory callFactory, @@ -234,27 +242,32 @@ public class OkHttpDataSource extends BaseDataSource implements HttpDataSource { /** * @deprecated Use {@link OkHttpDataSource.Factory#setContentTypePredicate(Predicate)} instead. */ + @UnstableApi @Deprecated public void setContentTypePredicate(@Nullable Predicate contentTypePredicate) { this.contentTypePredicate = contentTypePredicate; } + @UnstableApi @Override @Nullable public Uri getUri() { return response == null ? null : Uri.parse(response.request().url().toString()); } + @UnstableApi @Override public int getResponseCode() { return response == null ? -1 : response.code(); } + @UnstableApi @Override public Map> getResponseHeaders() { return response == null ? Collections.emptyMap() : response.headers().toMultimap(); } + @UnstableApi @Override public void setRequestProperty(String name, String value) { Assertions.checkNotNull(name); @@ -262,17 +275,20 @@ public class OkHttpDataSource extends BaseDataSource implements HttpDataSource { requestProperties.set(name, value); } + @UnstableApi @Override public void clearRequestProperty(String name) { Assertions.checkNotNull(name); requestProperties.remove(name); } + @UnstableApi @Override public void clearAllRequestProperties() { requestProperties.clear(); } + @UnstableApi @Override public long open(DataSpec dataSpec) throws HttpDataSourceException { this.dataSpec = dataSpec; @@ -358,6 +374,7 @@ public class OkHttpDataSource extends BaseDataSource implements HttpDataSource { return bytesToRead; } + @UnstableApi @Override public int read(byte[] buffer, int offset, int length) throws HttpDataSourceException { try { @@ -368,6 +385,7 @@ public class OkHttpDataSource extends BaseDataSource implements HttpDataSource { } } + @UnstableApi @Override public void close() { if (opened) { diff --git a/libraries/datasource_rtmp/README.md b/libraries/datasource_rtmp/README.md index 8e69321c95..7f52a665b3 100644 --- a/libraries/datasource_rtmp/README.md +++ b/libraries/datasource_rtmp/README.md @@ -39,7 +39,7 @@ injected from application code. `DefaultDataSource` will automatically use the RTMP extension whenever it's available. Hence if your application is using `DefaultDataSource` or -`DefaultDataSourceFactory`, adding support for RTMP streams is as simple as +`DefaultDataSource.Factory`, adding support for RTMP streams is as simple as adding a dependency to the RTMP extension as described above. No changes to your application code are required. Alternatively, if you know that your application doesn't need to handle any other protocols, you can update any diff --git a/libraries/decoder_ffmpeg/build.gradle b/libraries/decoder_ffmpeg/build.gradle index d80695c936..b404aba0d4 100644 --- a/libraries/decoder_ffmpeg/build.gradle +++ b/libraries/decoder_ffmpeg/build.gradle @@ -17,7 +17,8 @@ apply from: "$gradle.ext.androidxMediaSettingsDir/common_library_config.gradle" // failures if ffmpeg hasn't been built according to the README instructions. if (project.file('src/main/jni/ffmpeg').exists()) { android.externalNativeBuild.cmake.path = 'src/main/jni/CMakeLists.txt' - android.externalNativeBuild.cmake.version = '3.7.1+' + // Should match cmake_minimum_required. + android.externalNativeBuild.cmake.version = '3.21.0+' } dependencies { diff --git a/libraries/decoder_ffmpeg/src/main/jni/CMakeLists.txt b/libraries/decoder_ffmpeg/src/main/jni/CMakeLists.txt index c6a89975cf..9b41852481 100644 --- a/libraries/decoder_ffmpeg/src/main/jni/CMakeLists.txt +++ b/libraries/decoder_ffmpeg/src/main/jni/CMakeLists.txt @@ -14,7 +14,7 @@ # limitations under the License. # -cmake_minimum_required(VERSION 3.7.1 FATAL_ERROR) +cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR) # Enable C++11 features. set(CMAKE_CXX_STANDARD 11) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultLoadControl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultLoadControl.java index 978a9bc829..196d26098d 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultLoadControl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultLoadControl.java @@ -21,11 +21,11 @@ import static java.lang.Math.min; import androidx.annotation.Nullable; import androidx.media3.common.C; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.Log; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.exoplayer.upstream.DefaultAllocator; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayer.java index fbb0a1172d..eda8f6c146 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayer.java @@ -38,8 +38,6 @@ import androidx.media3.common.MediaItem; import androidx.media3.common.Player; import androidx.media3.common.PriorityTaskManager; import androidx.media3.common.Timeline; -import androidx.media3.common.TrackGroupArray; -import androidx.media3.common.TrackSelectionArray; import androidx.media3.common.TracksInfo; import androidx.media3.common.VideoSize; import androidx.media3.common.text.Cue; @@ -57,8 +55,10 @@ import androidx.media3.exoplayer.metadata.MetadataRenderer; import androidx.media3.exoplayer.source.DefaultMediaSourceFactory; import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.source.ShuffleOrder; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.text.TextRenderer; import androidx.media3.exoplayer.trackselection.DefaultTrackSelector; +import androidx.media3.exoplayer.trackselection.TrackSelectionArray; import androidx.media3.exoplayer.trackselection.TrackSelector; import androidx.media3.exoplayer.upstream.BandwidthMeter; import androidx.media3.exoplayer.upstream.DefaultBandwidthMeter; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java index 7d5a64f4d8..0e6428c0ee 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImpl.java @@ -69,8 +69,6 @@ import androidx.media3.common.Player; import androidx.media3.common.PriorityTaskManager; import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; -import androidx.media3.common.TrackSelectionArray; import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.TracksInfo; import androidx.media3.common.VideoSize; @@ -93,8 +91,10 @@ import androidx.media3.exoplayer.metadata.MetadataOutput; import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; import androidx.media3.exoplayer.source.ShuffleOrder; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.text.TextOutput; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; +import androidx.media3.exoplayer.trackselection.TrackSelectionArray; import androidx.media3.exoplayer.trackselection.TrackSelector; import androidx.media3.exoplayer.trackselection.TrackSelectorResult; import androidx.media3.exoplayer.upstream.BandwidthMeter; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImplInternal.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImplInternal.java index 84bd8ebbf7..ba32561c63 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImplInternal.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/ExoPlayerImplInternal.java @@ -44,7 +44,6 @@ import androidx.media3.common.Player.PlayWhenReadyChangeReason; import androidx.media3.common.Player.PlaybackSuppressionReason; import androidx.media3.common.Player.RepeatMode; import androidx.media3.common.Timeline; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.Clock; import androidx.media3.common.util.HandlerWrapper; @@ -62,6 +61,7 @@ import androidx.media3.exoplayer.source.MediaPeriod; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; import androidx.media3.exoplayer.source.SampleStream; import androidx.media3.exoplayer.source.ShuffleOrder; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.text.TextRenderer; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; import androidx.media3.exoplayer.trackselection.TrackSelector; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/LoadControl.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/LoadControl.java index 62e41a6ba3..8f8c37c8d4 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/LoadControl.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/LoadControl.java @@ -18,8 +18,8 @@ package androidx.media3.exoplayer; import androidx.media3.common.C; import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.UnstableApi; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; import androidx.media3.exoplayer.upstream.Allocator; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/MediaPeriodHolder.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/MediaPeriodHolder.java index fac91eb349..d16412f219 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/MediaPeriodHolder.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/MediaPeriodHolder.java @@ -21,7 +21,6 @@ import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.Timeline; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.Log; import androidx.media3.exoplayer.source.ClippingMediaPeriod; @@ -29,6 +28,7 @@ import androidx.media3.exoplayer.source.EmptySampleStream; import androidx.media3.exoplayer.source.MediaPeriod; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; import androidx.media3.exoplayer.source.SampleStream; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; import androidx.media3.exoplayer.trackselection.TrackSelector; import androidx.media3.exoplayer.trackselection.TrackSelectorResult; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/MetadataRetriever.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/MetadataRetriever.java index ad49de0654..9a58981f5d 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/MetadataRetriever.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/MetadataRetriever.java @@ -26,7 +26,6 @@ import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; import androidx.media3.common.MediaItem; import androidx.media3.common.Timeline; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Clock; import androidx.media3.common.util.HandlerWrapper; import androidx.media3.common.util.UnstableApi; @@ -34,6 +33,7 @@ import androidx.media3.exoplayer.analytics.PlayerId; import androidx.media3.exoplayer.source.DefaultMediaSourceFactory; import androidx.media3.exoplayer.source.MediaPeriod; import androidx.media3.exoplayer.source.MediaSource; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.exoplayer.upstream.DefaultAllocator; import androidx.media3.extractor.DefaultExtractorsFactory; @@ -157,7 +157,7 @@ public final class MetadataRetriever { mediaPeriod.maybeThrowPrepareError(); } mediaSourceHandler.sendEmptyMessageDelayed( - MESSAGE_CHECK_FOR_FAILURE, /* delayMillis= */ ERROR_POLL_INTERVAL_MS); + MESSAGE_CHECK_FOR_FAILURE, /* delayMs= */ ERROR_POLL_INTERVAL_MS); } catch (Exception e) { trackGroupsFuture.setException(e); mediaSourceHandler.obtainMessage(MESSAGE_RELEASE).sendToTarget(); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/PlaybackInfo.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/PlaybackInfo.java index 6dab072666..f0e104a75a 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/PlaybackInfo.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/PlaybackInfo.java @@ -23,8 +23,8 @@ import androidx.media3.common.PlaybackParameters; import androidx.media3.common.Player; import androidx.media3.common.Player.PlaybackSuppressionReason; import androidx.media3.common.Timeline; -import androidx.media3.common.TrackGroupArray; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.TrackSelectorResult; import com.google.common.collect.ImmutableList; import java.util.List; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/SimpleExoPlayer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/SimpleExoPlayer.java index d791222b9d..861adf766c 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/SimpleExoPlayer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/SimpleExoPlayer.java @@ -35,8 +35,6 @@ import androidx.media3.common.MediaMetadata; import androidx.media3.common.PlaybackParameters; import androidx.media3.common.PriorityTaskManager; import androidx.media3.common.Timeline; -import androidx.media3.common.TrackGroupArray; -import androidx.media3.common.TrackSelectionArray; import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.TracksInfo; import androidx.media3.common.VideoSize; @@ -49,6 +47,8 @@ import androidx.media3.exoplayer.analytics.AnalyticsListener; import androidx.media3.exoplayer.source.DefaultMediaSourceFactory; import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.source.ShuffleOrder; +import androidx.media3.exoplayer.source.TrackGroupArray; +import androidx.media3.exoplayer.trackselection.TrackSelectionArray; import androidx.media3.exoplayer.trackselection.TrackSelector; import androidx.media3.exoplayer.upstream.BandwidthMeter; import androidx.media3.exoplayer.video.VideoFrameMetadataListener; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsListener.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsListener.java index b4900c904a..fad26fe0d8 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsListener.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/analytics/AnalyticsListener.java @@ -45,7 +45,6 @@ import androidx.media3.common.Player.DiscontinuityReason; import androidx.media3.common.Player.PlaybackSuppressionReason; import androidx.media3.common.Player.TimelineChangeReason; import androidx.media3.common.Timeline; -import androidx.media3.common.TrackSelection; import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.TracksInfo; import androidx.media3.common.VideoSize; @@ -60,6 +59,7 @@ import androidx.media3.exoplayer.metadata.MetadataOutput; import androidx.media3.exoplayer.source.LoadEventInfo; import androidx.media3.exoplayer.source.MediaLoadData; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; +import androidx.media3.exoplayer.trackselection.TrackSelection; import androidx.media3.exoplayer.video.VideoDecoderOutputBufferRenderer; import com.google.common.base.Objects; import java.io.IOException; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/DefaultDrmSessionManagerProvider.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/DefaultDrmSessionManagerProvider.java index a67cb15336..01a0f78385 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/DefaultDrmSessionManagerProvider.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/DefaultDrmSessionManagerProvider.java @@ -24,8 +24,8 @@ import androidx.annotation.RequiresApi; import androidx.media3.common.MediaItem; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; +import androidx.media3.datasource.DataSource; import androidx.media3.datasource.DefaultHttpDataSource; -import androidx.media3.datasource.HttpDataSource; import com.google.common.primitives.Ints; import java.util.Map; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -42,7 +42,7 @@ public final class DefaultDrmSessionManagerProvider implements DrmSessionManager @GuardedBy("lock") private @MonotonicNonNull DrmSessionManager manager; - @Nullable private HttpDataSource.Factory drmHttpDataSourceFactory; + @Nullable private DataSource.Factory drmHttpDataSourceFactory; @Nullable private String userAgent; public DefaultDrmSessionManagerProvider() { @@ -50,26 +50,22 @@ public final class DefaultDrmSessionManagerProvider implements DrmSessionManager } /** - * Sets the {@link HttpDataSource.Factory} to be used for creating {@link HttpMediaDrmCallback - * HttpMediaDrmCallbacks} which executes key and provisioning requests over HTTP. If {@code null} - * is passed the {@link DefaultHttpDataSource.Factory} is used. + * Sets the {@link DataSource.Factory} which is used to create {@link HttpMediaDrmCallback} + * instances. If {@code null} is passed a {@link DefaultHttpDataSource.Factory} is used. * - * @param drmHttpDataSourceFactory The HTTP data source factory or {@code null} to use {@link + * @param drmDataSourceFactory The data source factory or {@code null} to use {@link * DefaultHttpDataSource.Factory}. */ - public void setDrmHttpDataSourceFactory( - @Nullable HttpDataSource.Factory drmHttpDataSourceFactory) { - this.drmHttpDataSourceFactory = drmHttpDataSourceFactory; + public void setDrmHttpDataSourceFactory(@Nullable DataSource.Factory drmDataSourceFactory) { + this.drmHttpDataSourceFactory = drmDataSourceFactory; } /** - * Sets the optional user agent to be used for DRM requests. - * - *

In case a factory has been set by {@link - * #setDrmHttpDataSourceFactory(HttpDataSource.Factory)}, this user agent is ignored. - * - * @param userAgent The user agent to be used for DRM requests. + * @deprecated Pass a custom {@link DataSource.Factory} to {@link + * #setDrmHttpDataSourceFactory(DataSource.Factory)} which sets the desired user agent on + * outgoing requests. */ + @Deprecated public void setDrmUserAgent(@Nullable String userAgent) { this.userAgent = userAgent; } @@ -94,7 +90,7 @@ public final class DefaultDrmSessionManagerProvider implements DrmSessionManager @RequiresApi(18) private DrmSessionManager createManager(MediaItem.DrmConfiguration drmConfiguration) { - HttpDataSource.Factory dataSourceFactory = + DataSource.Factory dataSourceFactory = drmHttpDataSourceFactory != null ? drmHttpDataSourceFactory : new DefaultHttpDataSource.Factory().setUserAgent(userAgent); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/FrameworkMediaDrm.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/FrameworkMediaDrm.java index cce784d0f5..9489014fea 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/FrameworkMediaDrm.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/FrameworkMediaDrm.java @@ -15,6 +15,8 @@ */ package androidx.media3.exoplayer.drm; +import static androidx.media3.common.util.Assertions.checkNotNull; + import android.annotation.SuppressLint; import android.media.DeniedByServerException; import android.media.MediaCrypto; @@ -23,6 +25,7 @@ import android.media.MediaDrm; import android.media.MediaDrmException; import android.media.NotProvisionedException; import android.media.UnsupportedSchemeException; +import android.media.metrics.LogSessionId; import android.os.PersistableBundle; import android.text.TextUtils; import androidx.annotation.DoNotInline; @@ -188,7 +191,13 @@ public final class FrameworkMediaDrm implements ExoMediaDrm { @Override public void setPlayerIdForSession(byte[] sessionId, PlayerId playerId) { - // TODO(b/221032172): Implement this when CDM compatibility issues are resolved. + if (Util.SDK_INT >= 31) { + try { + Api31.setLogSessionIdOnMediaDrmSession(mediaDrm, sessionId, playerId); + } catch (UnsupportedOperationException e) { + Log.w(TAG, "setLogSessionId failed."); + } + } } // Return values of MediaDrm.KeyRequest.getRequestType are equal to KeyRequest.RequestType. @@ -518,5 +527,16 @@ public final class FrameworkMediaDrm implements ExoMediaDrm { public static boolean requiresSecureDecoder(MediaDrm mediaDrm, String mimeType) { return mediaDrm.requiresSecureDecoder(mimeType); } + + @DoNotInline + public static void setLogSessionIdOnMediaDrmSession( + MediaDrm mediaDrm, byte[] drmSessionId, PlayerId playerId) { + LogSessionId logSessionId = playerId.getLogSessionId(); + if (!logSessionId.equals(LogSessionId.LOG_SESSION_ID_NONE)) { + MediaDrm.PlaybackComponent playbackComponent = + checkNotNull(mediaDrm.getPlaybackComponent(drmSessionId)); + playbackComponent.setLogSessionId(logSessionId); + } + } } } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/HttpMediaDrmCallback.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/HttpMediaDrmCallback.java index 18a1e63696..95eb55e4d6 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/HttpMediaDrmCallback.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/HttpMediaDrmCallback.java @@ -22,9 +22,9 @@ import androidx.media3.common.C; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; +import androidx.media3.datasource.DataSource; import androidx.media3.datasource.DataSourceInputStream; import androidx.media3.datasource.DataSpec; -import androidx.media3.datasource.HttpDataSource; import androidx.media3.datasource.HttpDataSource.InvalidResponseCodeException; import androidx.media3.datasource.StatsDataSource; import androidx.media3.exoplayer.drm.ExoMediaDrm.KeyRequest; @@ -36,41 +36,47 @@ import java.util.List; import java.util.Map; import java.util.UUID; -/** A {@link MediaDrmCallback} that makes requests using {@link HttpDataSource} instances. */ +/** A {@link MediaDrmCallback} that makes requests using {@link DataSource} instances. */ @UnstableApi public final class HttpMediaDrmCallback implements MediaDrmCallback { private static final int MAX_MANUAL_REDIRECTS = 5; - private final HttpDataSource.Factory dataSourceFactory; + private final DataSource.Factory dataSourceFactory; @Nullable private final String defaultLicenseUrl; private final boolean forceDefaultLicenseUrl; private final Map keyRequestProperties; /** + * Constructs an instance. + * * @param defaultLicenseUrl The default license URL. Used for key requests that do not specify * their own license URL. May be {@code null} if it's known that all key requests will specify * their own URLs. - * @param dataSourceFactory A factory from which to obtain {@link HttpDataSource} instances. + * @param dataSourceFactory A factory from which to obtain {@link DataSource} instances. This will + * usually be an HTTP-based {@link DataSource}. */ public HttpMediaDrmCallback( - @Nullable String defaultLicenseUrl, HttpDataSource.Factory dataSourceFactory) { + @Nullable String defaultLicenseUrl, DataSource.Factory dataSourceFactory) { this(defaultLicenseUrl, /* forceDefaultLicenseUrl= */ false, dataSourceFactory); } /** + * Constructs an instance. + * * @param defaultLicenseUrl The default license URL. Used for key requests that do not specify * their own license URL, or for all key requests if {@code forceDefaultLicenseUrl} is set to * true. May be {@code null} if {@code forceDefaultLicenseUrl} is {@code false} and if it's * known that all key requests will specify their own URLs. * @param forceDefaultLicenseUrl Whether to force use of {@code defaultLicenseUrl} for key * requests that include their own license URL. - * @param dataSourceFactory A factory from which to obtain {@link HttpDataSource} instances. + * @param dataSourceFactory A factory from which to obtain {@link DataSource} instances. This will + * * usually be an HTTP-based {@link DataSource}. */ public HttpMediaDrmCallback( @Nullable String defaultLicenseUrl, boolean forceDefaultLicenseUrl, - HttpDataSource.Factory dataSourceFactory) { + DataSource.Factory dataSourceFactory) { Assertions.checkArgument(!(forceDefaultLicenseUrl && TextUtils.isEmpty(defaultLicenseUrl))); this.dataSourceFactory = dataSourceFactory; this.defaultLicenseUrl = defaultLicenseUrl; @@ -156,7 +162,7 @@ public final class HttpMediaDrmCallback implements MediaDrmCallback { } private static byte[] executePost( - HttpDataSource.Factory dataSourceFactory, + DataSource.Factory dataSourceFactory, String url, @Nullable byte[] httpBody, Map requestProperties) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/OfflineLicenseHelper.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/OfflineLicenseHelper.java index cae26bbf4e..3da5a1dbcd 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/OfflineLicenseHelper.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/drm/OfflineLicenseHelper.java @@ -26,7 +26,7 @@ import androidx.media3.common.DrmInitData; import androidx.media3.common.Format; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.UnstableApi; -import androidx.media3.datasource.HttpDataSource; +import androidx.media3.datasource.DataSource; import androidx.media3.exoplayer.analytics.PlayerId; import androidx.media3.exoplayer.drm.DefaultDrmSessionManager.Mode; import androidx.media3.exoplayer.drm.DrmSession.DrmSessionException; @@ -53,20 +53,17 @@ public final class OfflineLicenseHelper { * * @param defaultLicenseUrl The default license URL. Used for key requests that do not specify * their own license URL. - * @param httpDataSourceFactory A factory from which to obtain {@link HttpDataSource} instances. + * @param dataSourceFactory A factory from which to obtain {@link DataSource} instances. * @param eventDispatcher A {@link DrmSessionEventListener.EventDispatcher} used to distribute * DRM-related events. * @return A new instance which uses Widevine CDM. */ public static OfflineLicenseHelper newWidevineInstance( String defaultLicenseUrl, - HttpDataSource.Factory httpDataSourceFactory, + DataSource.Factory dataSourceFactory, DrmSessionEventListener.EventDispatcher eventDispatcher) { return newWidevineInstance( - defaultLicenseUrl, - /* forceDefaultLicenseUrl= */ false, - httpDataSourceFactory, - eventDispatcher); + defaultLicenseUrl, /* forceDefaultLicenseUrl= */ false, dataSourceFactory, eventDispatcher); } /** @@ -77,7 +74,7 @@ public final class OfflineLicenseHelper { * their own license URL. * @param forceDefaultLicenseUrl Whether to use {@code defaultLicenseUrl} for key requests that * include their own license URL. - * @param httpDataSourceFactory A factory from which to obtain {@link HttpDataSource} instances. + * @param dataSourceFactory A factory from which to obtain {@link DataSource} instances. * @param eventDispatcher A {@link DrmSessionEventListener.EventDispatcher} used to distribute * DRM-related events. * @return A new instance which uses Widevine CDM. @@ -85,12 +82,12 @@ public final class OfflineLicenseHelper { public static OfflineLicenseHelper newWidevineInstance( String defaultLicenseUrl, boolean forceDefaultLicenseUrl, - HttpDataSource.Factory httpDataSourceFactory, + DataSource.Factory dataSourceFactory, DrmSessionEventListener.EventDispatcher eventDispatcher) { return newWidevineInstance( defaultLicenseUrl, forceDefaultLicenseUrl, - httpDataSourceFactory, + dataSourceFactory, /* optionalKeyRequestParameters= */ null, eventDispatcher); } @@ -113,7 +110,7 @@ public final class OfflineLicenseHelper { public static OfflineLicenseHelper newWidevineInstance( String defaultLicenseUrl, boolean forceDefaultLicenseUrl, - HttpDataSource.Factory httpDataSourceFactory, + DataSource.Factory dataSourceFactory, @Nullable Map optionalKeyRequestParameters, DrmSessionEventListener.EventDispatcher eventDispatcher) { return new OfflineLicenseHelper( @@ -121,7 +118,7 @@ public final class OfflineLicenseHelper { .setKeyRequestParameters(optionalKeyRequestParameters) .build( new HttpMediaDrmCallback( - defaultLicenseUrl, forceDefaultLicenseUrl, httpDataSourceFactory)), + defaultLicenseUrl, forceDefaultLicenseUrl, dataSourceFactory)), eventDispatcher); } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/DefaultMediaCodecAdapterFactory.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/DefaultMediaCodecAdapterFactory.java index 8dea9596c8..d5de1d0bdd 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/DefaultMediaCodecAdapterFactory.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/DefaultMediaCodecAdapterFactory.java @@ -96,8 +96,9 @@ public final class DefaultMediaCodecAdapterFactory implements MediaCodecAdapter. @Override public MediaCodecAdapter createAdapter(MediaCodecAdapter.Configuration configuration) throws IOException { - if ((asynchronousMode == MODE_ENABLED && Util.SDK_INT >= 23) - || (asynchronousMode == MODE_DEFAULT && Util.SDK_INT >= 31)) { + if (Util.SDK_INT >= 23 + && (asynchronousMode == MODE_ENABLED + || (asynchronousMode == MODE_DEFAULT && Util.SDK_INT >= 31))) { int trackType = MimeTypes.getTrackType(configuration.format.sampleMimeType); Log.i( TAG, diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecUtil.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecUtil.java index ca24d21b20..c3200150d0 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecUtil.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecUtil.java @@ -443,6 +443,8 @@ public final class MediaCodecUtil { return "audio/x-lg-alac"; } else if (mimeType.equals(MimeTypes.AUDIO_FLAC) && "OMX.lge.flac.decoder".equals(name)) { return "audio/x-lg-flac"; + } else if (mimeType.equals(MimeTypes.AUDIO_AC3) && "OMX.lge.ac3.decoder".equals(name)) { + return "audio/lg-ac3"; } return null; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/offline/DownloadHelper.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/offline/DownloadHelper.java index 41fb472684..7f9b5a9f15 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/offline/DownloadHelper.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/offline/DownloadHelper.java @@ -31,7 +31,6 @@ import androidx.media3.common.MimeTypes; import androidx.media3.common.StreamKey; import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.TrackSelectionOverride; import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.TracksInfo; @@ -52,6 +51,7 @@ import androidx.media3.exoplayer.source.MediaPeriod; import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; import androidx.media3.exoplayer.source.MediaSource.MediaSourceCaller; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.source.chunk.MediaChunk; import androidx.media3.exoplayer.source.chunk.MediaChunkIterator; import androidx.media3.exoplayer.trackselection.BaseTrackSelection; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/ClippingMediaPeriod.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/ClippingMediaPeriod.java index 1744873ab4..4d9e0a1274 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/ClippingMediaPeriod.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/ClippingMediaPeriod.java @@ -19,7 +19,6 @@ import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/DefaultMediaSourceFactory.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/DefaultMediaSourceFactory.java index c64201c8b8..c667c4e7a9 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/DefaultMediaSourceFactory.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/DefaultMediaSourceFactory.java @@ -147,7 +147,6 @@ public final class DefaultMediaSourceFactory implements MediaSourceFactory { * @param dataSourceFactory A {@link DataSource.Factory} to create {@link DataSource} instances * for requesting media data. */ - @UnstableApi public DefaultMediaSourceFactory(DataSource.Factory dataSourceFactory) { this(dataSourceFactory, new DefaultExtractorsFactory()); } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/MaskingMediaPeriod.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/MaskingMediaPeriod.java index 2553a6ab03..0050985cbb 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/MaskingMediaPeriod.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/MaskingMediaPeriod.java @@ -21,7 +21,6 @@ import static androidx.media3.common.util.Util.castNonNull; import androidx.annotation.Nullable; import androidx.media3.common.C; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.UnstableApi; import androidx.media3.exoplayer.SeekParameters; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/MediaPeriod.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/MediaPeriod.java index d625c0033b..c176ebd752 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/MediaPeriod.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/MediaPeriod.java @@ -19,7 +19,6 @@ import androidx.media3.common.C; import androidx.media3.common.StreamKey; import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.UnstableApi; import androidx.media3.exoplayer.ExoPlayer; import androidx.media3.exoplayer.SeekParameters; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/MergingMediaPeriod.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/MergingMediaPeriod.java index cc6b34ebb4..925998ea1c 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/MergingMediaPeriod.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/MergingMediaPeriod.java @@ -23,7 +23,6 @@ import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.StreamKey; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Assertions; import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.exoplayer.FormatHolder; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/ProgressiveMediaPeriod.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/ProgressiveMediaPeriod.java index f20a57caeb..026917b9a0 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/ProgressiveMediaPeriod.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/ProgressiveMediaPeriod.java @@ -28,7 +28,6 @@ import androidx.media3.common.Metadata; import androidx.media3.common.MimeTypes; import androidx.media3.common.ParserException; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.ConditionVariable; import androidx.media3.common.util.ParsableByteArray; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SilenceMediaSource.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SilenceMediaSource.java index d2857100fc..18d52dfa09 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SilenceMediaSource.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SilenceMediaSource.java @@ -26,7 +26,6 @@ import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SingleSampleMediaPeriod.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SingleSampleMediaPeriod.java index 55c2a4c614..16e36635f8 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SingleSampleMediaPeriod.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SingleSampleMediaPeriod.java @@ -20,7 +20,6 @@ import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.Log; import androidx.media3.common.util.Util; diff --git a/libraries/common/src/main/java/androidx/media3/common/TrackGroupArray.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/TrackGroupArray.java similarity index 85% rename from libraries/common/src/main/java/androidx/media3/common/TrackGroupArray.java rename to libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/TrackGroupArray.java index 671d9848fa..d37b8802d3 100644 --- a/libraries/common/src/main/java/androidx/media3/common/TrackGroupArray.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/TrackGroupArray.java @@ -13,13 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package androidx.media3.common; +package androidx.media3.exoplayer.source; import static java.lang.annotation.ElementType.TYPE_USE; import android.os.Bundle; import androidx.annotation.IntDef; import androidx.annotation.Nullable; +import androidx.media3.common.Bundleable; +import androidx.media3.common.C; +import androidx.media3.common.TrackGroup; import androidx.media3.common.util.BundleableUtil; import androidx.media3.common.util.Log; import androidx.media3.common.util.UnstableApi; @@ -30,7 +33,16 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import java.util.List; -/** An immutable array of {@link TrackGroup}s. */ +/** + * An immutable array of {@link TrackGroup}s. + * + *

This class is typically used to represent all of the tracks available in a piece of media. + * Tracks that are known to present the same content are grouped together (e.g., the same video feed + * provided at different resolutions in an adaptive stream). Tracks that are known to present + * different content are in separate track groups (e.g., an audio track will not be in the same + * group as a video track, and an audio track in one language will be in a different group to an + * audio track in another language). + */ @UnstableApi public final class TrackGroupArray implements Bundleable { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/ads/ServerSideAdInsertionMediaSource.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/ads/ServerSideAdInsertionMediaSource.java index e5e29d4c97..c0a1828be6 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/ads/ServerSideAdInsertionMediaSource.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/ads/ServerSideAdInsertionMediaSource.java @@ -36,7 +36,6 @@ import androidx.media3.common.MediaItem; import androidx.media3.common.StreamKey; import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.datasource.TransferListener; @@ -54,6 +53,7 @@ import androidx.media3.exoplayer.source.MediaPeriod; import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.source.MediaSourceEventListener; import androidx.media3.exoplayer.source.SampleStream; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; import androidx.media3.exoplayer.upstream.Allocator; import com.google.common.collect.ArrayListMultimap; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/AdaptiveTrackSelection.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/AdaptiveTrackSelection.java index 0d1c02e80f..6c056a328c 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/AdaptiveTrackSelection.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/AdaptiveTrackSelection.java @@ -24,7 +24,6 @@ import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackSelection; import androidx.media3.common.util.Clock; import androidx.media3.common.util.Log; import androidx.media3.common.util.UnstableApi; @@ -781,7 +780,8 @@ public class AdaptiveTrackSelection extends BaseTrackSelection { } double[] logBitrates = new double[trackBitrates[i].length]; for (int j = 0; j < trackBitrates[i].length; j++) { - logBitrates[j] = trackBitrates[i][j] == Format.NO_VALUE ? 0 : Math.log(trackBitrates[i][j]); + logBitrates[j] = + trackBitrates[i][j] == Format.NO_VALUE ? 0 : Math.log((double) trackBitrates[i][j]); } double totalBitrateDiff = logBitrates[logBitrates.length - 1] - logBitrates[0]; for (int j = 0; j < logBitrates.length - 1; j++) { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/BaseTrackSelection.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/BaseTrackSelection.java index 2e24629e18..0f5f53dde6 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/BaseTrackSelection.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/BaseTrackSelection.java @@ -22,7 +22,6 @@ import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackSelection; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java index 29714b4f35..2d6297635d 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java @@ -36,8 +36,6 @@ import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; -import androidx.media3.common.TrackSelection; import androidx.media3.common.TrackSelectionOverride; import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.util.Assertions; @@ -51,6 +49,7 @@ import androidx.media3.exoplayer.RendererCapabilities.AdaptiveSupport; import androidx.media3.exoplayer.RendererCapabilities.Capabilities; import androidx.media3.exoplayer.RendererConfiguration; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; +import androidx.media3.exoplayer.source.TrackGroupArray; import com.google.common.collect.ComparisonChain; import com.google.common.collect.ImmutableList; import com.google.common.collect.Ordering; @@ -86,11 +85,10 @@ import org.checkerframework.checker.nullness.compatqual.NullableType; * .setMaxVideoSizeSd() * .setPreferredAudioLanguage("de") * .build()); - * * }

* * Some specialized parameters are only available in the extended {@link Parameters} class, which - * can be retrieved and modified in a similar way in this track selector: + * can be retrieved and modified in a similar way by calling methods directly on this class: * *
{@code
  * defaultTrackSelector.setParameters(
@@ -98,7 +96,6 @@ import org.checkerframework.checker.nullness.compatqual.NullableType;
  *         .buildUpon()
  *         .setTunnelingEnabled(true)
  *         .build());
- *
  * }
*/ @UnstableApi diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/ExoTrackSelection.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/ExoTrackSelection.java index d9ace1ebc3..ab7f8dc6de 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/ExoTrackSelection.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/ExoTrackSelection.java @@ -20,7 +20,6 @@ import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackSelection; import androidx.media3.common.util.Log; import androidx.media3.common.util.UnstableApi; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/FixedTrackSelection.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/FixedTrackSelection.java index d6c57bb730..c41b8389f7 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/FixedTrackSelection.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/FixedTrackSelection.java @@ -18,7 +18,6 @@ package androidx.media3.exoplayer.trackselection; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackSelection; import androidx.media3.common.util.UnstableApi; import androidx.media3.exoplayer.source.chunk.MediaChunk; import androidx.media3.exoplayer.source.chunk.MediaChunkIterator; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/MappingTrackSelector.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/MappingTrackSelector.java index 233ac0497d..66bbbb3aee 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/MappingTrackSelector.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/MappingTrackSelector.java @@ -31,7 +31,6 @@ import androidx.media3.common.C; import androidx.media3.common.C.FormatSupport; import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.TracksInfo; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; @@ -42,6 +41,7 @@ import androidx.media3.exoplayer.RendererCapabilities.AdaptiveSupport; import androidx.media3.exoplayer.RendererCapabilities.Capabilities; import androidx.media3.exoplayer.RendererConfiguration; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; +import androidx.media3.exoplayer.source.TrackGroupArray; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; diff --git a/libraries/common/src/main/java/androidx/media3/common/TrackSelection.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/TrackSelection.java similarity index 95% rename from libraries/common/src/main/java/androidx/media3/common/TrackSelection.java rename to libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/TrackSelection.java index a3d73a567e..ff496ca5d9 100644 --- a/libraries/common/src/main/java/androidx/media3/common/TrackSelection.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/TrackSelection.java @@ -13,11 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package androidx.media3.common; +package androidx.media3.exoplayer.trackselection; import static java.lang.annotation.ElementType.TYPE_USE; import androidx.annotation.IntDef; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.TrackGroup; import androidx.media3.common.util.UnstableApi; import java.lang.annotation.Documented; import java.lang.annotation.Retention; diff --git a/libraries/common/src/main/java/androidx/media3/common/TrackSelectionArray.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/TrackSelectionArray.java similarity index 97% rename from libraries/common/src/main/java/androidx/media3/common/TrackSelectionArray.java rename to libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/TrackSelectionArray.java index 4867d8029a..5c9f18b30e 100644 --- a/libraries/common/src/main/java/androidx/media3/common/TrackSelectionArray.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/TrackSelectionArray.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package androidx.media3.common; +package androidx.media3.exoplayer.trackselection; import androidx.annotation.Nullable; import androidx.media3.common.util.UnstableApi; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/TrackSelectionUtil.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/TrackSelectionUtil.java index ed62fa7e63..3cd0b19180 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/TrackSelectionUtil.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/TrackSelectionUtil.java @@ -19,12 +19,11 @@ import android.os.SystemClock; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; -import androidx.media3.common.TrackSelection; import androidx.media3.common.TracksInfo; import androidx.media3.common.TracksInfo.TrackGroupInfo; import androidx.media3.common.util.UnstableApi; import androidx.media3.exoplayer.RendererCapabilities; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.DefaultTrackSelector.SelectionOverride; import androidx.media3.exoplayer.trackselection.ExoTrackSelection.Definition; import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/TrackSelector.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/TrackSelector.java index 07b32ea19b..2fb6ea5ae3 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/TrackSelector.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/TrackSelector.java @@ -17,8 +17,6 @@ package androidx.media3.exoplayer.trackselection; import androidx.annotation.Nullable; import androidx.media3.common.Timeline; -import androidx.media3.common.TrackGroupArray; -import androidx.media3.common.TrackSelection; import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.UnstableApi; @@ -28,6 +26,7 @@ import androidx.media3.exoplayer.Renderer; import androidx.media3.exoplayer.RendererCapabilities; import androidx.media3.exoplayer.RendererConfiguration; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.upstream.BandwidthMeter; /** diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/util/EventLogger.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/util/EventLogger.java index 841a11ed26..758acbedea 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/util/EventLogger.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/util/EventLogger.java @@ -286,19 +286,19 @@ public class EventLogger implements AnalyticsListener { } // TODO: Replace this with an override of onMediaMetadataChanged. // Log metadata for at most one of the selected tracks. - for (int groupIndex = 0; groupIndex < trackGroupInfos.size(); groupIndex++) { + boolean loggedMetadata = false; + for (int groupIndex = 0; !loggedMetadata && groupIndex < trackGroupInfos.size(); groupIndex++) { TracksInfo.TrackGroupInfo trackGroupInfo = trackGroupInfos.get(groupIndex); TrackGroup trackGroup = trackGroupInfo.getTrackGroup(); - for (int trackIndex = 0; trackIndex < trackGroup.length; trackIndex++) { - if (!trackGroupInfo.isTrackSelected(trackIndex)) { - continue; - } - @Nullable Metadata metadata = trackGroup.getFormat(trackIndex).metadata; - if (metadata != null) { - logd(" Metadata ["); - printMetadata(metadata, " "); - logd(" ]"); - break; + for (int trackIndex = 0; !loggedMetadata && trackIndex < trackGroup.length; trackIndex++) { + if (trackGroupInfo.isTrackSelected(trackIndex)) { + @Nullable Metadata metadata = trackGroup.getFormat(trackIndex).metadata; + if (metadata != null && metadata.length() > 0) { + logd(" Metadata ["); + printMetadata(metadata, " "); + logd(" ]"); + loggedMetadata = true; + } } } } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/DefaultLoadControlTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/DefaultLoadControlTest.java index b0fa141e12..2700ab4f85 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/DefaultLoadControlTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/DefaultLoadControlTest.java @@ -18,9 +18,9 @@ package androidx.media3.exoplayer; import static com.google.common.truth.Truth.assertThat; import androidx.media3.common.C; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Util; import androidx.media3.exoplayer.DefaultLoadControl.Builder; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; import androidx.media3.exoplayer.upstream.DefaultAllocator; import androidx.test.ext.junit.runners.AndroidJUnit4; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/ExoPlayerTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/ExoPlayerTest.java index 4fbc9f2ca3..a30b75a4ed 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/ExoPlayerTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/ExoPlayerTest.java @@ -108,7 +108,6 @@ import androidx.media3.common.Player.PositionInfo; import androidx.media3.common.Timeline; import androidx.media3.common.Timeline.Window; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.TracksInfo; import androidx.media3.common.TracksInfo.TrackGroupInfo; import androidx.media3.common.util.Assertions; @@ -127,6 +126,7 @@ import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; import androidx.media3.exoplayer.source.MediaSourceEventListener; import androidx.media3.exoplayer.source.SinglePeriodTimeline; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.source.ads.ServerSideAdInsertionMediaSource; import androidx.media3.exoplayer.trackselection.DefaultTrackSelector; import androidx.media3.exoplayer.upstream.Allocation; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/MetadataRetrieverTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/MetadataRetrieverTest.java index 9567b90184..b9b5756503 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/MetadataRetrieverTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/MetadataRetrieverTest.java @@ -26,7 +26,7 @@ import android.net.Uri; import androidx.media3.common.C; import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; -import androidx.media3.common.TrackGroupArray; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.extractor.metadata.mp4.MdtaMetadataEntry; import androidx.media3.extractor.metadata.mp4.MotionPhotoMetadata; import androidx.media3.extractor.metadata.mp4.SlowMotionData; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/offline/DownloadHelperTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/offline/DownloadHelperTest.java index 409777e1cb..477763d395 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/offline/DownloadHelperTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/offline/DownloadHelperTest.java @@ -27,7 +27,6 @@ import androidx.media3.common.MimeTypes; import androidx.media3.common.StreamKey; import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.TrackSelectionOverride; import androidx.media3.common.TrackSelectionParameters; import androidx.media3.exoplayer.Renderer; @@ -35,6 +34,7 @@ import androidx.media3.exoplayer.RenderersFactory; import androidx.media3.exoplayer.offline.DownloadHelper.Callback; import androidx.media3.exoplayer.source.MediaPeriod; import androidx.media3.exoplayer.source.MediaSourceEventListener.EventDispatcher; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.DefaultTrackSelector; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; import androidx.media3.exoplayer.trackselection.MappingTrackSelector.MappedTrackInfo; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/MergingMediaPeriodTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/MergingMediaPeriodTest.java index 22e21b5d54..02b35f9295 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/MergingMediaPeriodTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/MergingMediaPeriodTest.java @@ -23,7 +23,6 @@ import static com.google.common.truth.Truth.assertThat; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.exoplayer.FormatHolder; import androidx.media3.exoplayer.drm.DrmSessionEventListener; diff --git a/libraries/common/src/test/java/androidx/media3/common/TrackGroupArrayTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/TrackGroupArrayTest.java similarity index 91% rename from libraries/common/src/test/java/androidx/media3/common/TrackGroupArrayTest.java rename to libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/TrackGroupArrayTest.java index 26078cb679..4557f251b9 100644 --- a/libraries/common/src/test/java/androidx/media3/common/TrackGroupArrayTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/TrackGroupArrayTest.java @@ -13,10 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package androidx.media3.common; +package androidx.media3.exoplayer.source; import static com.google.common.truth.Truth.assertThat; +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.TrackGroup; import androidx.test.ext.junit.runners.AndroidJUnit4; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/AdaptiveTrackSelectionTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/AdaptiveTrackSelectionTest.java index b8cc647d9f..6c7061b414 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/AdaptiveTrackSelectionTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/AdaptiveTrackSelectionTest.java @@ -24,7 +24,6 @@ import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackSelection; import androidx.media3.datasource.DataSpec; import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.source.chunk.BaseMediaChunkIterator; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java index 47fa09fba9..1088a23d3e 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java @@ -40,8 +40,6 @@ import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; -import androidx.media3.common.TrackSelection; import androidx.media3.common.TrackSelectionOverride; import androidx.media3.common.TracksInfo; import androidx.media3.common.util.Util; @@ -50,6 +48,7 @@ import androidx.media3.exoplayer.RendererCapabilities; import androidx.media3.exoplayer.RendererCapabilities.Capabilities; import androidx.media3.exoplayer.RendererConfiguration; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.DefaultTrackSelector.Parameters; import androidx.media3.exoplayer.trackselection.DefaultTrackSelector.ParametersBuilder; import androidx.media3.exoplayer.trackselection.DefaultTrackSelector.SelectionOverride; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/MappingTrackSelectorTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/MappingTrackSelectorTest.java index c4ec0a178a..2adf524191 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/MappingTrackSelectorTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/MappingTrackSelectorTest.java @@ -24,7 +24,6 @@ import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Util; import androidx.media3.exoplayer.ExoPlaybackException; import androidx.media3.exoplayer.RendererCapabilities; @@ -32,6 +31,7 @@ import androidx.media3.exoplayer.RendererCapabilities.AdaptiveSupport; import androidx.media3.exoplayer.RendererCapabilities.Capabilities; import androidx.media3.exoplayer.RendererConfiguration; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.test.utils.FakeTimeline; import androidx.test.ext.junit.runners.AndroidJUnit4; import org.junit.BeforeClass; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/TrackSelectionUtilTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/TrackSelectionUtilTest.java index 99cd8bb32a..2491df3a54 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/TrackSelectionUtilTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/TrackSelectionUtilTest.java @@ -32,10 +32,9 @@ import static com.google.common.truth.Truth.assertThat; import androidx.media3.common.Format; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; -import androidx.media3.common.TrackSelection; import androidx.media3.common.TracksInfo; import androidx.media3.common.TracksInfo.TrackGroupInfo; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; import java.util.List; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/TrackSelectorTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/TrackSelectorTest.java index 29f705a099..a1551d0200 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/TrackSelectorTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/TrackSelectorTest.java @@ -20,10 +20,10 @@ import static org.junit.Assert.fail; import androidx.annotation.Nullable; import androidx.media3.common.Timeline; -import androidx.media3.common.TrackGroupArray; import androidx.media3.exoplayer.ExoPlaybackException; import androidx.media3.exoplayer.RendererCapabilities; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.TrackSelector.InvalidationListener; import androidx.media3.exoplayer.upstream.BandwidthMeter; import androidx.test.ext.junit.runners.AndroidJUnit4; diff --git a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaPeriod.java b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaPeriod.java index 23b9262cf0..0021106f10 100644 --- a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaPeriod.java +++ b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaPeriod.java @@ -28,7 +28,6 @@ import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.StreamKey; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Util; import androidx.media3.datasource.TransferListener; import androidx.media3.exoplayer.SeekParameters; @@ -50,6 +49,7 @@ import androidx.media3.exoplayer.source.MediaSourceEventListener; import androidx.media3.exoplayer.source.MediaSourceEventListener.EventDispatcher; import androidx.media3.exoplayer.source.SampleStream; import androidx.media3.exoplayer.source.SequenceableLoader; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.source.chunk.ChunkSampleStream; import androidx.media3.exoplayer.source.chunk.ChunkSampleStream.EmbeddedSampleStream; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; diff --git a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashUtil.java b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashUtil.java index c1a92d22d3..91465ef5ad 100644 --- a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashUtil.java +++ b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashUtil.java @@ -24,7 +24,6 @@ import androidx.media3.common.util.Assertions; import androidx.media3.common.util.UnstableApi; import androidx.media3.datasource.DataSource; import androidx.media3.datasource.DataSpec; -import androidx.media3.datasource.HttpDataSource; import androidx.media3.exoplayer.dash.manifest.DashManifest; import androidx.media3.exoplayer.dash.manifest.DashManifestParser; import androidx.media3.exoplayer.dash.manifest.Period; @@ -85,7 +84,7 @@ public final class DashUtil { /** * Loads a DASH manifest. * - * @param dataSource The {@link HttpDataSource} from which the manifest should be read. + * @param dataSource The {@link DataSource} from which the manifest should be read. * @param uri The {@link Uri} of the manifest to be read. * @return An instance of {@link DashManifest}. * @throws IOException Thrown when there is an error while loading. @@ -97,7 +96,7 @@ public final class DashUtil { /** * Loads a {@link Format} for acquiring keys for a given period in a DASH manifest. * - * @param dataSource The {@link HttpDataSource} from which data should be loaded. + * @param dataSource The {@link DataSource} from which data should be loaded. * @param period The {@link Period}. * @return The loaded {@link Format}, or null if none is defined. * @throws IOException Thrown when there is an error while loading. diff --git a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/manifest/DashManifestParser.java b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/manifest/DashManifestParser.java index 6ee1ab5d6c..9ac84cb9b6 100644 --- a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/manifest/DashManifestParser.java +++ b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/manifest/DashManifestParser.java @@ -1189,41 +1189,41 @@ public class DashManifestParser extends DefaultHandler xpp.nextToken(); while (!XmlPullParserUtil.isEndTag(xpp, "Event")) { switch (xpp.getEventType()) { - case (XmlPullParser.START_DOCUMENT): + case XmlPullParser.START_DOCUMENT: xmlSerializer.startDocument(null, false); break; - case (XmlPullParser.END_DOCUMENT): + case XmlPullParser.END_DOCUMENT: xmlSerializer.endDocument(); break; - case (XmlPullParser.START_TAG): + case XmlPullParser.START_TAG: xmlSerializer.startTag(xpp.getNamespace(), xpp.getName()); for (int i = 0; i < xpp.getAttributeCount(); i++) { xmlSerializer.attribute( xpp.getAttributeNamespace(i), xpp.getAttributeName(i), xpp.getAttributeValue(i)); } break; - case (XmlPullParser.END_TAG): + case XmlPullParser.END_TAG: xmlSerializer.endTag(xpp.getNamespace(), xpp.getName()); break; - case (XmlPullParser.TEXT): + case XmlPullParser.TEXT: xmlSerializer.text(xpp.getText()); break; - case (XmlPullParser.CDSECT): + case XmlPullParser.CDSECT: xmlSerializer.cdsect(xpp.getText()); break; - case (XmlPullParser.ENTITY_REF): + case XmlPullParser.ENTITY_REF: xmlSerializer.entityRef(xpp.getText()); break; - case (XmlPullParser.IGNORABLE_WHITESPACE): + case XmlPullParser.IGNORABLE_WHITESPACE: xmlSerializer.ignorableWhitespace(xpp.getText()); break; - case (XmlPullParser.PROCESSING_INSTRUCTION): + case XmlPullParser.PROCESSING_INSTRUCTION: xmlSerializer.processingInstruction(xpp.getText()); break; - case (XmlPullParser.COMMENT): + case XmlPullParser.COMMENT: xmlSerializer.comment(xpp.getText()); break; - case (XmlPullParser.DOCDECL): + case XmlPullParser.DOCDECL: xmlSerializer.docdecl(xpp.getText()); break; default: // fall out diff --git a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/DashMediaPeriodTest.java b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/DashMediaPeriodTest.java index 1a518729ad..e7c836c0dc 100644 --- a/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/DashMediaPeriodTest.java +++ b/libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/DashMediaPeriodTest.java @@ -21,7 +21,6 @@ import android.net.Uri; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.datasource.TransferListener; import androidx.media3.exoplayer.analytics.PlayerId; import androidx.media3.exoplayer.dash.PlayerEmsgHandler.PlayerEmsgCallback; @@ -33,6 +32,7 @@ import androidx.media3.exoplayer.drm.DrmSessionManager; import androidx.media3.exoplayer.source.CompositeSequenceableLoaderFactory; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; import androidx.media3.exoplayer.source.MediaSourceEventListener; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy; import androidx.media3.exoplayer.upstream.LoaderErrorThrower; diff --git a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java index 0bf8baf717..b5822578f8 100644 --- a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java +++ b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsMediaPeriod.java @@ -25,7 +25,6 @@ import androidx.media3.common.Metadata; import androidx.media3.common.MimeTypes; import androidx.media3.common.StreamKey; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; @@ -45,6 +44,7 @@ import androidx.media3.exoplayer.source.MediaPeriod; import androidx.media3.exoplayer.source.MediaSourceEventListener.EventDispatcher; import androidx.media3.exoplayer.source.SampleStream; import androidx.media3.exoplayer.source.SequenceableLoader; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy; diff --git a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsSampleStreamWrapper.java b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsSampleStreamWrapper.java index 01a1af2027..ac3f1a45cf 100644 --- a/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsSampleStreamWrapper.java +++ b/libraries/exoplayer_hls/src/main/java/androidx/media3/exoplayer/hls/HlsSampleStreamWrapper.java @@ -33,7 +33,6 @@ import androidx.media3.common.Metadata; import androidx.media3.common.MimeTypes; import androidx.media3.common.ParserException; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.Log; import androidx.media3.common.util.ParsableByteArray; @@ -53,6 +52,7 @@ import androidx.media3.exoplayer.source.SampleQueue.UpstreamFormatChangedListene import androidx.media3.exoplayer.source.SampleStream; import androidx.media3.exoplayer.source.SampleStream.ReadFlags; import androidx.media3.exoplayer.source.SequenceableLoader; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.source.chunk.Chunk; import androidx.media3.exoplayer.source.chunk.MediaChunkIterator; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; diff --git a/libraries/exoplayer_ima/src/main/java/androidx/media3/exoplayer/ima/ImaServerSideAdInsertionMediaSource.java b/libraries/exoplayer_ima/src/main/java/androidx/media3/exoplayer/ima/ImaServerSideAdInsertionMediaSource.java index 4bb90ea914..a6d2390860 100644 --- a/libraries/exoplayer_ima/src/main/java/androidx/media3/exoplayer/ima/ImaServerSideAdInsertionMediaSource.java +++ b/libraries/exoplayer_ima/src/main/java/androidx/media3/exoplayer/ima/ImaServerSideAdInsertionMediaSource.java @@ -18,11 +18,12 @@ package androidx.media3.exoplayer.ima; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Util.msToUs; -import static androidx.media3.common.util.Util.secToUs; import static androidx.media3.common.util.Util.sum; import static androidx.media3.common.util.Util.usToMs; import static androidx.media3.exoplayer.ima.ImaUtil.expandAdGroupPlaceholder; import static androidx.media3.exoplayer.ima.ImaUtil.getAdGroupAndIndexInMultiPeriodWindow; +import static androidx.media3.exoplayer.ima.ImaUtil.secToMsRounded; +import static androidx.media3.exoplayer.ima.ImaUtil.secToUsRounded; import static androidx.media3.exoplayer.ima.ImaUtil.splitAdPlaybackStateForPeriods; import static androidx.media3.exoplayer.ima.ImaUtil.updateAdDurationAndPropagate; import static androidx.media3.exoplayer.ima.ImaUtil.updateAdDurationInAdGroup; @@ -661,19 +662,29 @@ public final class ImaServerSideAdInsertionMediaSource extends CompositeMediaSou private static AdPlaybackState setVodAdGroupPlaceholders( List cuePoints, AdPlaybackState adPlaybackState) { + // TODO(b/192231683) Use getEndTimeMs()/getStartTimeMs() after jar target was removed for (int i = 0; i < cuePoints.size(); i++) { CuePoint cuePoint = cuePoints.get(i); + long fromPositionUs = msToUs(secToMsRounded(cuePoint.getStartTime())); adPlaybackState = addAdGroupToAdPlaybackState( adPlaybackState, - /* fromPositionUs= */ secToUs(cuePoint.getStartTime()), + /* fromPositionUs= */ fromPositionUs, /* contentResumeOffsetUs= */ 0, - // TODO(b/192231683) Use getEndTimeMs()/getStartTimeMs() after jar target was removed - /* adDurationsUs...= */ secToUs(cuePoint.getEndTime() - cuePoint.getStartTime())); + /* adDurationsUs...= */ getAdDuration( + /* startTimeSeconds= */ cuePoint.getStartTime(), + /* endTimeSeconds= */ cuePoint.getEndTime())); } return adPlaybackState; } + private static long getAdDuration(double startTimeSeconds, double endTimeSeconds) { + // startTimeSeconds and endTimeSeconds that are coming from the SDK, only have a precision of + // milliseconds so everything that is below a millisecond can be safely considered as coming + // from rounding issues. + return msToUs(secToMsRounded(endTimeSeconds - startTimeSeconds)); + } + private static AdPlaybackState setVodAdInPlaceholder(Ad ad, AdPlaybackState adPlaybackState) { AdPodInfo adPodInfo = ad.getAdPodInfo(); // Handle post rolls that have a podIndex of -1. @@ -685,9 +696,9 @@ public final class ImaServerSideAdInsertionMediaSource extends CompositeMediaSou adPlaybackState = expandAdGroupPlaceholder( adGroupIndex, - /* adGroupDurationUs= */ secToUs(adPodInfo.getMaxDuration()), + /* adGroupDurationUs= */ msToUs(secToMsRounded(adPodInfo.getMaxDuration())), adIndexInAdGroup, - /* adDurationUs= */ secToUs(ad.getDuration()), + /* adDurationUs= */ msToUs(secToMsRounded(ad.getDuration())), /* adsInAdGroupCount= */ adPodInfo.getTotalAds(), adPlaybackState); } else if (adIndexInAdGroup < adGroup.count - 1) { @@ -695,7 +706,7 @@ public final class ImaServerSideAdInsertionMediaSource extends CompositeMediaSou updateAdDurationInAdGroup( adGroupIndex, adIndexInAdGroup, - /* adDurationUs= */ secToUs(ad.getDuration()), + /* adDurationUs= */ msToUs(secToMsRounded(ad.getDuration())), adPlaybackState); } return adPlaybackState; @@ -704,7 +715,7 @@ public final class ImaServerSideAdInsertionMediaSource extends CompositeMediaSou private AdPlaybackState addLiveAdBreak( Ad ad, long currentPeriodPositionUs, AdPlaybackState adPlaybackState) { AdPodInfo adPodInfo = ad.getAdPodInfo(); - long adDurationUs = secToUs(ad.getDuration()); + long adDurationUs = secToUsRounded(ad.getDuration()); int adIndexInAdGroup = adPodInfo.getAdPosition() - 1; // TODO(b/208398934) Support seeking backwards. if (adIndexInAdGroup == 0 || adPlaybackState.adGroupCount == 1) { @@ -718,7 +729,7 @@ public final class ImaServerSideAdInsertionMediaSource extends CompositeMediaSou new long[adCount], adIndexInAdGroup, adDurationUs, - secToUs(adPodInfo.getMaxDuration())); + msToUs(secToMsRounded(adPodInfo.getMaxDuration()))); adPlaybackState = addAdGroupToAdPlaybackState( adPlaybackState, diff --git a/libraries/exoplayer_ima/src/main/java/androidx/media3/exoplayer/ima/ImaUtil.java b/libraries/exoplayer_ima/src/main/java/androidx/media3/exoplayer/ima/ImaUtil.java index cede70d9cb..5619c7f13e 100644 --- a/libraries/exoplayer_ima/src/main/java/androidx/media3/exoplayer/ima/ImaUtil.java +++ b/libraries/exoplayer_ima/src/main/java/androidx/media3/exoplayer/ima/ImaUtil.java @@ -54,7 +54,10 @@ import com.google.ads.interactivemedia.v3.api.player.VideoAdPlayer; import com.google.ads.interactivemedia.v3.api.player.VideoProgressUpdate; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.math.DoubleMath; import java.io.IOException; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; @@ -398,17 +401,13 @@ import java.util.Set; long elapsedAdGroupAdDurationUs = 0; for (int j = periodIndex; j < contentTimeline.getPeriodCount(); j++) { contentTimeline.getPeriod(j, period, /* setIds= */ true); - // TODO(b/192231683) Remove subtracted US from ad group time when we can upgrade the SDK. - // Subtract one microsecond to work around rounding errors with adGroup.timeUs. - if (totalElapsedContentDurationUs < adGroup.timeUs - 1) { + if (totalElapsedContentDurationUs < adGroup.timeUs) { // Period starts before the ad group, so it is a content period. adPlaybackStates.put(checkNotNull(period.uid), contentOnlyAdPlaybackState); totalElapsedContentDurationUs += period.durationUs; } else { long periodStartUs = totalElapsedContentDurationUs + elapsedAdGroupAdDurationUs; - // TODO(b/192231683) Remove additional US when we can upgrade the SDK. - // Add one microsecond to work around rounding errors with adGroup.timeUs. - if (periodStartUs + period.durationUs <= adGroup.timeUs + adGroupDurationUs + 1) { + if (periodStartUs + period.durationUs <= adGroup.timeUs + adGroupDurationUs) { // The period ends before the end of the ad group, so it is an ad period (Note: A VOD ad // reported by the IMA SDK spans multiple periods before the LOADED event arrives). adPlaybackStates.put( @@ -490,16 +489,12 @@ import java.util.Set; long elapsedAdGroupAdDurationUs = 0; for (int j = periodIndex; j < contentTimeline.getPeriodCount(); j++) { contentTimeline.getPeriod(j, period, /* setIds= */ true); - // TODO(b/192231683) Remove subtracted US from ad group time when we can upgrade the SDK. - // Subtract one microsecond to work around rounding errors with adGroup.timeUs. - if (totalElapsedContentDurationUs < adGroup.timeUs - 1) { + if (totalElapsedContentDurationUs < adGroup.timeUs) { // Period starts before the ad group, so it is a content period. totalElapsedContentDurationUs += period.durationUs; } else { long periodStartUs = totalElapsedContentDurationUs + elapsedAdGroupAdDurationUs; - // TODO(b/192231683) Remove additional US when we can upgrade the SDK. - // Add one microsecond to work around rounding errors with adGroup.timeUs. - if (periodStartUs + period.durationUs <= adGroup.timeUs + adGroupDurationUs + 1) { + if (periodStartUs + period.durationUs <= adGroup.timeUs + adGroupDurationUs) { // The period ends before the end of the ad group, so it is an ad period. if (j == adPeriodIndex) { return new Pair<>(/* adGroupIndex= */ i, adIndexInAdGroup); @@ -518,5 +513,31 @@ import java.util.Set; throw new IllegalStateException(); } + /** + * Converts a time in seconds to the corresponding time in microseconds. + * + *

Fractional values are rounded to the nearest microsecond using {@link RoundingMode#HALF_UP}. + * + * @param timeSec The time in seconds. + * @return The corresponding time in microseconds. + */ + public static long secToUsRounded(double timeSec) { + return DoubleMath.roundToLong( + BigDecimal.valueOf(timeSec).scaleByPowerOfTen(6).doubleValue(), RoundingMode.HALF_UP); + } + + /** + * Converts a time in seconds to the corresponding time in milliseconds. + * + *

Fractional values are rounded to the nearest millisecond using {@link RoundingMode#HALF_UP}. + * + * @param timeSec The time in seconds. + * @return The corresponding time in milliseconds. + */ + public static long secToMsRounded(double timeSec) { + return DoubleMath.roundToLong( + BigDecimal.valueOf(timeSec).scaleByPowerOfTen(3).doubleValue(), RoundingMode.HALF_UP); + } + private ImaUtil() {} } diff --git a/libraries/exoplayer_ima/src/test/java/androidx/media3/exoplayer/ima/FakeExoPlayer.java b/libraries/exoplayer_ima/src/test/java/androidx/media3/exoplayer/ima/FakeExoPlayer.java index 6a9602c976..8974a062a1 100644 --- a/libraries/exoplayer_ima/src/test/java/androidx/media3/exoplayer/ima/FakeExoPlayer.java +++ b/libraries/exoplayer_ima/src/test/java/androidx/media3/exoplayer/ima/FakeExoPlayer.java @@ -23,13 +23,13 @@ import androidx.media3.common.MediaItem; import androidx.media3.common.PlaybackException; import androidx.media3.common.Player; import androidx.media3.common.Timeline; -import androidx.media3.common.TrackSelectionArray; import androidx.media3.common.TrackSelectionParameters; import androidx.media3.common.TracksInfo; import androidx.media3.common.util.Clock; import androidx.media3.common.util.ListenerSet; import androidx.media3.common.util.Util; import androidx.media3.exoplayer.ExoPlayer; +import androidx.media3.exoplayer.trackselection.TrackSelectionArray; import androidx.media3.test.utils.StubExoPlayer; /** A fake {@link ExoPlayer} for testing content/ad playback. */ @@ -79,7 +79,7 @@ import androidx.media3.test.utils.StubExoPlayer; PositionInfo oldPosition = new PositionInfo( windowUid, - /* windowIndex= */ 0, + /* mediaItemIndex= */ 0, mediaItem, periodUid, /* periodIndex= */ 0, @@ -97,7 +97,7 @@ import androidx.media3.test.utils.StubExoPlayer; PositionInfo newPosition = new PositionInfo( windowUid, - /* windowIndex= */ 0, + /* mediaItemIndex= */ 0, mediaItem, periodUid, /* periodIndex= */ 0, @@ -128,7 +128,7 @@ import androidx.media3.test.utils.StubExoPlayer; PositionInfo oldPosition = new PositionInfo( windowUid, - /* windowIndex= */ 0, + /* mediaItemIndex= */ 0, mediaItem, periodUid, /* periodIndex= */ 0, @@ -146,7 +146,7 @@ import androidx.media3.test.utils.StubExoPlayer; PositionInfo newPosition = new PositionInfo( windowUid, - /* windowIndex= */ 0, + /* mediaItemIndex= */ 0, mediaItem, periodUid, /* periodIndex= */ 0, diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaPeriod.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaPeriod.java index 28084cdef6..2d23137700 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaPeriod.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaPeriod.java @@ -28,8 +28,6 @@ import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.StreamKey; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; -import androidx.media3.common.TrackSelection; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; @@ -44,7 +42,9 @@ import androidx.media3.exoplayer.source.SampleQueue.UpstreamFormatChangedListene import androidx.media3.exoplayer.source.SampleStream; import androidx.media3.exoplayer.source.SampleStream.ReadDataResult; import androidx.media3.exoplayer.source.SampleStream.ReadFlags; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; +import androidx.media3.exoplayer.trackselection.TrackSelection; import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.exoplayer.upstream.Loader; import androidx.media3.exoplayer.upstream.Loader.Loadable; diff --git a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriod.java b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriod.java index a634fbd54f..0ff9a96246 100644 --- a/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriod.java +++ b/libraries/exoplayer_smoothstreaming/src/main/java/androidx/media3/exoplayer/smoothstreaming/SsMediaPeriod.java @@ -20,7 +20,6 @@ import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.StreamKey; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.datasource.TransferListener; import androidx.media3.exoplayer.SeekParameters; import androidx.media3.exoplayer.drm.DrmSessionEventListener; @@ -31,6 +30,7 @@ import androidx.media3.exoplayer.source.MediaPeriod; import androidx.media3.exoplayer.source.MediaSourceEventListener; import androidx.media3.exoplayer.source.SampleStream; import androidx.media3.exoplayer.source.SequenceableLoader; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.source.chunk.ChunkSampleStream; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; import androidx.media3.exoplayer.upstream.Allocator; diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/NalUnitUtil.java b/libraries/extractor/src/main/java/androidx/media3/extractor/NalUnitUtil.java index b795b79fed..c61d7eaba5 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/NalUnitUtil.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/NalUnitUtil.java @@ -31,6 +31,21 @@ public final class NalUnitUtil { private static final String TAG = "NalUnitUtil"; + /** Coded slice of a non-IDR picture. */ + public static final int NAL_UNIT_TYPE_NON_IDR = 1; + /** Coded slice data partition A. */ + public static final int NAL_UNIT_TYPE_PARTITION_A = 2; + /** Coded slice of an IDR picture. */ + public static final int NAL_UNIT_TYPE_IDR = 5; + /** Supplemental enhancement information. */ + public static final int NAL_UNIT_TYPE_SEI = 6; + /** Sequence parameter set. */ + public static final int NAL_UNIT_TYPE_SPS = 7; + /** Picture parameter set. */ + public static final int NAL_UNIT_TYPE_PPS = 8; + /** Access unit delimiter. */ + public static final int NAL_UNIT_TYPE_AUD = 9; + /** Holds data parsed from a H.264 sequence parameter set NAL unit. */ public static final class SpsData { @@ -38,6 +53,7 @@ public final class NalUnitUtil { public final int constraintsFlagsAndReservedZero2Bits; public final int levelIdc; public final int seqParameterSetId; + public final int maxNumRefFrames; public final int width; public final int height; public final float pixelWidthHeightRatio; @@ -53,6 +69,7 @@ public final class NalUnitUtil { int constraintsFlagsAndReservedZero2Bits, int levelIdc, int seqParameterSetId, + int maxNumRefFrames, int width, int height, float pixelWidthHeightRatio, @@ -66,6 +83,7 @@ public final class NalUnitUtil { this.constraintsFlagsAndReservedZero2Bits = constraintsFlagsAndReservedZero2Bits; this.levelIdc = levelIdc; this.seqParameterSetId = seqParameterSetId; + this.maxNumRefFrames = maxNumRefFrames; this.width = width; this.height = height; this.pixelWidthHeightRatio = pixelWidthHeightRatio; @@ -372,7 +390,7 @@ public final class NalUnitUtil { data.readUnsignedExpGolombCodedInt(); // offset_for_ref_frame[i] } } - data.readUnsignedExpGolombCodedInt(); // max_num_ref_frames + int maxNumRefFrames = data.readUnsignedExpGolombCodedInt(); // max_num_ref_frames data.skipBit(); // gaps_in_frame_num_value_allowed_flag int picWidthInMbs = data.readUnsignedExpGolombCodedInt() + 1; @@ -432,6 +450,7 @@ public final class NalUnitUtil { constraintsFlagsAndReservedZero2Bits, levelIdc, seqParameterSetId, + maxNumRefFrames, frameWidth, frameHeight, pixelWidthHeightRatio, diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java index e839cf1e9e..f56a7dd7c0 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java @@ -15,7 +15,6 @@ */ package androidx.media3.extractor.mp4; -import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Util.castNonNull; import static androidx.media3.extractor.mp4.AtomParsers.parseTraks; import static androidx.media3.extractor.mp4.Sniffer.BRAND_HEIC; @@ -165,8 +164,8 @@ public final class Mp4Extractor implements Extractor, SeekMap { private int sampleCurrentNalBytesRemaining; // Extractor outputs. - private @MonotonicNonNull ExtractorOutput extractorOutput; - private Mp4Track @MonotonicNonNull [] tracks; + private ExtractorOutput extractorOutput; + private Mp4Track[] tracks; private long @MonotonicNonNull [][] accumulatedSampleSizes; private int firstVideoTrackIndex; @@ -197,6 +196,8 @@ public final class Mp4Extractor implements Extractor, SeekMap { nalLength = new ParsableByteArray(4); scratch = new ParsableByteArray(); sampleTrackIndex = C.INDEX_UNSET; + extractorOutput = ExtractorOutput.PLACEHOLDER; + tracks = new Mp4Track[0]; } @Override @@ -227,7 +228,7 @@ public final class Mp4Extractor implements Extractor, SeekMap { sefReader.reset(); slowMotionMetadataEntries.clear(); } - } else if (tracks != null) { + } else { for (Mp4Track track : tracks) { updateSampleIndex(track, timeUs); if (track.trueHdSampleRechunker != null) { @@ -280,7 +281,7 @@ public final class Mp4Extractor implements Extractor, SeekMap { @Override public SeekPoints getSeekPoints(long timeUs) { - if (checkNotNull(tracks).length == 0) { + if (tracks.length == 0) { return new SeekPoints(SeekPoint.START); } @@ -502,7 +503,6 @@ public final class Mp4Extractor implements Extractor, SeekMap { isQuickTime, /* modifyTrackFunction= */ track -> track); - ExtractorOutput extractorOutput = checkNotNull(this.extractorOutput); int trackCount = trackSampleTables.size(); for (int i = 0; i < trackCount; i++) { TrackSampleTable trackSampleTable = trackSampleTables.get(i); @@ -582,7 +582,7 @@ public final class Mp4Extractor implements Extractor, SeekMap { return RESULT_END_OF_INPUT; } } - Mp4Track track = castNonNull(tracks)[sampleTrackIndex]; + Mp4Track track = tracks[sampleTrackIndex]; TrackOutput trackOutput = track.trackOutput; int sampleIndex = track.sampleIndex; long position = track.sampleTable.offsets[sampleIndex]; @@ -699,7 +699,7 @@ public final class Mp4Extractor implements Extractor, SeekMap { long minAccumulatedBytes = Long.MAX_VALUE; boolean minAccumulatedBytesRequiresReload = true; int minAccumulatedBytesTrackIndex = C.INDEX_UNSET; - for (int trackIndex = 0; trackIndex < castNonNull(tracks).length; trackIndex++) { + for (int trackIndex = 0; trackIndex < tracks.length; trackIndex++) { Mp4Track track = tracks[trackIndex]; int sampleIndex = track.sampleIndex; if (sampleIndex == track.sampleTable.sampleCount) { @@ -744,7 +744,6 @@ public final class Mp4Extractor implements Extractor, SeekMap { private void processEndOfStreamReadingAtomHeader() { if (fileType == FILE_TYPE_HEIC && (flags & FLAG_READ_MOTION_PHOTO_METADATA) != 0) { // Add image track and prepare media. - ExtractorOutput extractorOutput = checkNotNull(this.extractorOutput); TrackOutput trackOutput = extractorOutput.track(/* id= */ 0, C.TRACK_TYPE_IMAGE); @Nullable Metadata metadata = motionPhotoMetadata == null ? null : new Metadata(motionPhotoMetadata); diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H264Reader.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H264Reader.java index 33f4e99547..cd4d3416d1 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H264Reader.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/H264Reader.java @@ -44,10 +44,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @UnstableApi public final class H264Reader implements ElementaryStreamReader { - private static final int NAL_UNIT_TYPE_SEI = 6; // Supplemental enhancement information - private static final int NAL_UNIT_TYPE_SPS = 7; // Sequence parameter set - private static final int NAL_UNIT_TYPE_PPS = 8; // Picture parameter set - private final SeiReader seiReader; private final boolean allowNonIdrKeyframes; private final boolean detectAccessUnits; @@ -85,9 +81,9 @@ public final class H264Reader implements ElementaryStreamReader { this.allowNonIdrKeyframes = allowNonIdrKeyframes; this.detectAccessUnits = detectAccessUnits; prefixFlags = new boolean[3]; - sps = new NalUnitTargetBuffer(NAL_UNIT_TYPE_SPS, 128); - pps = new NalUnitTargetBuffer(NAL_UNIT_TYPE_PPS, 128); - sei = new NalUnitTargetBuffer(NAL_UNIT_TYPE_SEI, 128); + sps = new NalUnitTargetBuffer(NalUnitUtil.NAL_UNIT_TYPE_SPS, 128); + pps = new NalUnitTargetBuffer(NalUnitUtil.NAL_UNIT_TYPE_PPS, 128); + sei = new NalUnitTargetBuffer(NalUnitUtil.NAL_UNIT_TYPE_SEI, 128); pesTimeUs = C.TIME_UNSET; seiWrapper = new ParsableByteArray(); } @@ -266,11 +262,6 @@ public final class H264Reader implements ElementaryStreamReader { private static final int DEFAULT_BUFFER_SIZE = 128; - private static final int NAL_UNIT_TYPE_NON_IDR = 1; // Coded slice of a non-IDR picture - private static final int NAL_UNIT_TYPE_PARTITION_A = 2; // Coded slice data partition A - private static final int NAL_UNIT_TYPE_IDR = 5; // Coded slice of an IDR picture - private static final int NAL_UNIT_TYPE_AUD = 9; // Access unit delimiter - private final TrackOutput output; private final boolean allowNonIdrKeyframes; private final boolean detectAccessUnits; @@ -331,11 +322,11 @@ public final class H264Reader implements ElementaryStreamReader { nalUnitType = type; nalUnitTimeUs = pesTimeUs; nalUnitStartPosition = position; - if ((allowNonIdrKeyframes && nalUnitType == NAL_UNIT_TYPE_NON_IDR) + if ((allowNonIdrKeyframes && nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_NON_IDR) || (detectAccessUnits - && (nalUnitType == NAL_UNIT_TYPE_IDR - || nalUnitType == NAL_UNIT_TYPE_NON_IDR - || nalUnitType == NAL_UNIT_TYPE_PARTITION_A))) { + && (nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_IDR + || nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_NON_IDR + || nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_PARTITION_A))) { // Store the previous header and prepare to populate the new one. SliceHeaderData newSliceHeader = previousSliceHeader; previousSliceHeader = sliceHeader; @@ -425,7 +416,7 @@ public final class H264Reader implements ElementaryStreamReader { bottomFieldFlagPresent = true; } } - boolean idrPicFlag = nalUnitType == NAL_UNIT_TYPE_IDR; + boolean idrPicFlag = nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_IDR; int idrPicId = 0; if (idrPicFlag) { if (!bitArray.canReadExpGolombCodedNum()) { @@ -480,7 +471,7 @@ public final class H264Reader implements ElementaryStreamReader { public boolean endNalUnit( long position, int offset, boolean hasOutputFormat, boolean randomAccessIndicator) { - if (nalUnitType == NAL_UNIT_TYPE_AUD + if (nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_AUD || (detectAccessUnits && sliceHeader.isFirstVclNalUnitOfPicture(previousSliceHeader))) { // If the NAL unit ending is the start of a new sample, output the previous one. if (hasOutputFormat && readingSample) { @@ -495,8 +486,8 @@ public final class H264Reader implements ElementaryStreamReader { boolean treatIFrameAsKeyframe = allowNonIdrKeyframes ? sliceHeader.isISlice() : randomAccessIndicator; sampleIsKeyframe |= - nalUnitType == NAL_UNIT_TYPE_IDR - || (treatIFrameAsKeyframe && nalUnitType == NAL_UNIT_TYPE_NON_IDR); + nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_IDR + || (treatIFrameAsKeyframe && nalUnitType == NalUnitUtil.NAL_UNIT_TYPE_NON_IDR); return sampleIsKeyframe; } diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/NalUnitUtilTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/NalUnitUtilTest.java index ebd2ccc45f..01d7fe15f9 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/NalUnitUtilTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/NalUnitUtilTest.java @@ -126,6 +126,7 @@ public final class NalUnitUtilTest { public void parseSpsNalUnit() { NalUnitUtil.SpsData data = NalUnitUtil.parseSpsNalUnit(SPS_TEST_DATA, SPS_TEST_DATA_OFFSET, SPS_TEST_DATA.length); + assertThat(data.maxNumRefFrames).isEqualTo(4); assertThat(data.width).isEqualTo(640); assertThat(data.height).isEqualTo(360); assertThat(data.deltaPicOrderAlwaysZeroFlag).isFalse(); diff --git a/libraries/session/src/main/java/androidx/media3/session/MediaNotificationManager.java b/libraries/session/src/main/java/androidx/media3/session/MediaNotificationManager.java index 7d3ceac3bb..397d0e2bcb 100644 --- a/libraries/session/src/main/java/androidx/media3/session/MediaNotificationManager.java +++ b/libraries/session/src/main/java/androidx/media3/session/MediaNotificationManager.java @@ -15,6 +15,8 @@ */ package androidx.media3.session; +import static androidx.media3.common.util.Assertions.checkStateNotNull; + import android.app.Notification; import android.content.Intent; import android.os.Bundle; @@ -25,6 +27,7 @@ import androidx.core.app.NotificationManagerCompat; import androidx.core.content.ContextCompat; import androidx.media3.common.Player; import androidx.media3.common.util.Util; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import java.util.HashMap; import java.util.List; @@ -50,6 +53,7 @@ import java.util.concurrent.TimeoutException; private final Map> controllerMap; private int totalNotificationCount; + @Nullable private MediaNotification mediaNotification; public MediaNotificationManager( MediaSessionService mediaSessionService, @@ -122,13 +126,13 @@ import java.util.concurrent.TimeoutException; MediaController mediaController; try { - mediaController = controllerFuture.get(0, TimeUnit.MILLISECONDS); - } catch (ExecutionException | InterruptedException | TimeoutException e) { + mediaController = checkStateNotNull(Futures.getDone(controllerFuture)); + } catch (ExecutionException e) { // We should never reach this point. throw new IllegalStateException(e); } - int notificationSequence = ++this.totalNotificationCount; + int notificationSequence = ++totalNotificationCount; MediaNotification.Provider.Callback callback = notification -> mainExecutor.execute( @@ -141,45 +145,68 @@ import java.util.concurrent.TimeoutException; private void onNotificationUpdated( int notificationSequence, MediaSession session, MediaNotification mediaNotification) { - if (notificationSequence == this.totalNotificationCount) { + if (notificationSequence == totalNotificationCount) { updateNotification(session, mediaNotification); } } private void updateNotification(MediaSession session, MediaNotification mediaNotification) { - int id = mediaNotification.notificationId; - Notification notification = mediaNotification.notification; - if (Util.SDK_INT >= 21) { // Call Notification.MediaStyle#setMediaSession() indirectly. android.media.session.MediaSession.Token fwkToken = (android.media.session.MediaSession.Token) session.getSessionCompat().getSessionToken().getToken(); - notification.extras.putParcelable(Notification.EXTRA_MEDIA_SESSION, fwkToken); + mediaNotification.notification.extras.putParcelable( + Notification.EXTRA_MEDIA_SESSION, fwkToken); } + this.mediaNotification = mediaNotification; Player player = session.getPlayer(); - if (player.getPlayWhenReady()) { + if (player.getPlayWhenReady() && canStartPlayback(player)) { ContextCompat.startForegroundService(mediaSessionService, startSelfIntent); - mediaSessionService.startForeground(id, notification); + mediaSessionService.startForeground( + mediaNotification.notificationId, mediaNotification.notification); } else { - stopForegroundServiceIfNeeded(); - notificationManagerCompat.notify(id, notification); + maybeStopForegroundService(/* removeNotifications= */ false); + notificationManagerCompat.notify( + mediaNotification.notificationId, mediaNotification.notification); } } - private void stopForegroundServiceIfNeeded() { + /** + * Stops the service from the foreground, if no player is actively playing content. + * + * @param removeNotifications Whether to remove notifications, if the service is stopped from the + * foreground. + */ + private void maybeStopForegroundService(boolean removeNotifications) { List sessions = mediaSessionService.getSessions(); for (int i = 0; i < sessions.size(); i++) { Player player = sessions.get(i).getPlayer(); - if (player.getPlayWhenReady()) { + if (player.getPlayWhenReady() && canStartPlayback(player)) { return; } } - // Calling stopForeground(true) is a workaround for pre-L devices which prevents - // the media notification from being undismissable. - boolean shouldRemoveNotification = Util.SDK_INT < 21; - mediaSessionService.stopForeground(shouldRemoveNotification); + // To hide the notification on all API levels, we need to call both Service.stopForeground(true) + // and notificationManagerCompat.cancelAll(). For pre-L devices, we must also call + // Service.stopForeground(true) anyway as a workaround that prevents the media notification from + // being undismissable. + mediaSessionService.stopForeground(removeNotifications || Util.SDK_INT < 21); + if (removeNotifications && mediaNotification != null) { + notificationManagerCompat.cancel(mediaNotification.notificationId); + // Update the notification count so that if a pending notification callback arrives (e.g., a + // bitmap is loaded), we don't show the notification. + totalNotificationCount++; + mediaNotification = null; + } + } + + /** + * Returns whether {@code player} can start playback and therefore we should present a + * notification for this player. + */ + private static boolean canStartPlayback(Player player) { + return player.getPlaybackState() != Player.STATE_IDLE && !player.getCurrentTimeline().isEmpty(); } private final class MediaControllerListener implements MediaController.Listener, Player.Listener { @@ -190,11 +217,20 @@ import java.util.concurrent.TimeoutException; } public void onConnected() { - updateNotification(session); + if (canStartPlayback(session.getPlayer())) { + updateNotification(session); + } } @Override public void onEvents(Player player, Player.Events events) { + if (!canStartPlayback(player)) { + maybeStopForegroundService(/* removeNotifications= */ true); + return; + } + + // Limit the events on which we may update the notification to ensure we don't update the + // notification too frequently, otherwise the system may suppress notifications. if (events.containsAny( Player.EVENT_PLAYBACK_STATE_CHANGED, Player.EVENT_PLAY_WHEN_READY_CHANGED, @@ -206,7 +242,7 @@ import java.util.concurrent.TimeoutException; @Override public void onDisconnected(MediaController controller) { mediaSessionService.removeSession(session); - stopForegroundServiceIfNeeded(); + maybeStopForegroundService(/* removeNotifications= */ true); } } } diff --git a/libraries/session/src/main/java/androidx/media3/session/PlayerWrapper.java b/libraries/session/src/main/java/androidx/media3/session/PlayerWrapper.java index c9ceca0f89..cdac091033 100644 --- a/libraries/session/src/main/java/androidx/media3/session/PlayerWrapper.java +++ b/libraries/session/src/main/java/androidx/media3/session/PlayerWrapper.java @@ -808,8 +808,8 @@ import java.util.List; return new PositionInfo( /* windowUid= */ null, getCurrentMediaItemIndex(), - /* periodUid= */ null, getCurrentMediaItem(), + /* periodUid= */ null, getCurrentPeriodIndex(), getCurrentPosition(), getContentPosition(), diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame_request_output_height.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame_request_output_height.png new file mode 100644 index 0000000000..ed03445385 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame_request_output_height.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame_rotate_45_scale_to_fit.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame_rotate_45_scale_to_fit.png new file mode 100644 index 0000000000..1dee08c552 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame_rotate_45_scale_to_fit.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame_rotate_then_translate.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame_rotate_then_translate.png new file mode 100644 index 0000000000..5cd7a9e989 Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame_rotate_then_translate.png differ diff --git a/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame_translate_then_rotate.png b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame_translate_then_rotate.png new file mode 100644 index 0000000000..a2efe9118c Binary files /dev/null and b/libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame_translate_then_rotate.png differ diff --git a/libraries/test_exoplayer_playback/src/androidTest/java/androidx/media3/test/exoplayer/playback/gts/DashTestRunner.java b/libraries/test_exoplayer_playback/src/androidTest/java/androidx/media3/test/exoplayer/playback/gts/DashTestRunner.java index f2b3ec716c..16429406d3 100644 --- a/libraries/test_exoplayer_playback/src/androidTest/java/androidx/media3/test/exoplayer/playback/gts/DashTestRunner.java +++ b/libraries/test_exoplayer_playback/src/androidTest/java/androidx/media3/test/exoplayer/playback/gts/DashTestRunner.java @@ -27,7 +27,6 @@ import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MediaItem; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.Log; import androidx.media3.common.util.Util; @@ -46,6 +45,7 @@ import androidx.media3.exoplayer.drm.MediaDrmCallback; import androidx.media3.exoplayer.drm.UnsupportedDrmException; import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import androidx.media3.exoplayer.source.MediaSource; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.DefaultTrackSelector; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; import androidx.media3.exoplayer.trackselection.MappingTrackSelector; diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeAdaptiveMediaPeriod.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeAdaptiveMediaPeriod.java index fbf23dee12..d104d00fb3 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeAdaptiveMediaPeriod.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeAdaptiveMediaPeriod.java @@ -23,7 +23,6 @@ import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.MimeTypes; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; @@ -38,6 +37,7 @@ import androidx.media3.exoplayer.source.MediaPeriod; import androidx.media3.exoplayer.source.MediaSourceEventListener; import androidx.media3.exoplayer.source.SampleStream; import androidx.media3.exoplayer.source.SequenceableLoader; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.source.chunk.ChunkSampleStream; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; import androidx.media3.exoplayer.upstream.Allocator; diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeAdaptiveMediaSource.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeAdaptiveMediaSource.java index ca6e264daa..eeca86404c 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeAdaptiveMediaSource.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeAdaptiveMediaSource.java @@ -18,7 +18,6 @@ package androidx.media3.test.utils; import androidx.annotation.Nullable; import androidx.media3.common.Timeline; import androidx.media3.common.Timeline.Period; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.datasource.TransferListener; @@ -27,6 +26,7 @@ import androidx.media3.exoplayer.drm.DrmSessionManager; import androidx.media3.exoplayer.source.MediaPeriod; import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.source.MediaSourceEventListener; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.upstream.Allocator; /** diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeMediaPeriod.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeMediaPeriod.java index ae7b68389b..52fb79a3bd 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeMediaPeriod.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeMediaPeriod.java @@ -28,7 +28,6 @@ import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.datasource.DataSpec; @@ -40,6 +39,7 @@ import androidx.media3.exoplayer.source.MediaPeriod; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; import androidx.media3.exoplayer.source.MediaSourceEventListener; import androidx.media3.exoplayer.source.SampleStream; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem; diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeMediaSource.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeMediaSource.java index 1779b3ef71..c1c206ebf1 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeMediaSource.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeMediaSource.java @@ -30,7 +30,6 @@ import androidx.media3.common.MediaItem; import androidx.media3.common.Timeline; import androidx.media3.common.Timeline.Period; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.Assertions; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; @@ -45,6 +44,7 @@ import androidx.media3.exoplayer.source.MediaLoadData; import androidx.media3.exoplayer.source.MediaPeriod; import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.source.MediaSourceEventListener; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.test.utils.FakeMediaPeriod.TrackDataFactory; import com.google.common.collect.ImmutableMap; diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeTrackSelector.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeTrackSelector.java index a71025f785..1fa7bf119c 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeTrackSelector.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/FakeTrackSelector.java @@ -17,11 +17,11 @@ package androidx.media3.test.utils; import androidx.media3.common.Timeline; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.UnstableApi; import androidx.media3.exoplayer.RendererCapabilities.AdaptiveSupport; import androidx.media3.exoplayer.RendererCapabilities.Capabilities; import androidx.media3.exoplayer.source.MediaSource.MediaPeriodId; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.trackselection.DefaultTrackSelector; import androidx.media3.exoplayer.trackselection.ExoTrackSelection; import androidx.media3.exoplayer.trackselection.MappingTrackSelector; diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/MediaPeriodAsserts.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/MediaPeriodAsserts.java index 71e0a49fe4..c794f731d8 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/MediaPeriodAsserts.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/MediaPeriodAsserts.java @@ -22,12 +22,12 @@ import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.StreamKey; import androidx.media3.common.TrackGroup; -import androidx.media3.common.TrackGroupArray; import androidx.media3.common.util.ConditionVariable; import androidx.media3.common.util.UnstableApi; import androidx.media3.exoplayer.offline.FilterableManifest; import androidx.media3.exoplayer.source.MediaPeriod; import androidx.media3.exoplayer.source.MediaPeriod.Callback; +import androidx.media3.exoplayer.source.TrackGroupArray; import androidx.media3.exoplayer.source.chunk.MediaChunk; import androidx.media3.exoplayer.source.chunk.MediaChunkIterator; import androidx.media3.exoplayer.trackselection.BaseTrackSelection; diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/StubExoPlayer.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/StubExoPlayer.java index 9b7470c7c8..ee75dc9d73 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/StubExoPlayer.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/StubExoPlayer.java @@ -21,8 +21,6 @@ import androidx.media3.common.AudioAttributes; import androidx.media3.common.AuxEffectInfo; import androidx.media3.common.Format; import androidx.media3.common.PriorityTaskManager; -import androidx.media3.common.TrackGroupArray; -import androidx.media3.common.TrackSelectionArray; import androidx.media3.common.util.Clock; import androidx.media3.common.util.UnstableApi; import androidx.media3.exoplayer.DecoderCounters; @@ -35,6 +33,8 @@ import androidx.media3.exoplayer.analytics.AnalyticsCollector; import androidx.media3.exoplayer.analytics.AnalyticsListener; import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.source.ShuffleOrder; +import androidx.media3.exoplayer.source.TrackGroupArray; +import androidx.media3.exoplayer.trackselection.TrackSelectionArray; import androidx.media3.exoplayer.trackselection.TrackSelector; import androidx.media3.exoplayer.video.VideoFrameMetadataListener; import androidx.media3.exoplayer.video.spherical.CameraMotionListener; diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformationFrameProcessorTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AdvancedFrameProcessorPixelTest.java similarity index 77% rename from libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformationFrameProcessorTest.java rename to libraries/transformer/src/androidTest/java/androidx/media3/transformer/AdvancedFrameProcessorPixelTest.java index 6ad1c188c7..895d10af18 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformationFrameProcessorTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AdvancedFrameProcessorPixelTest.java @@ -39,15 +39,15 @@ import org.junit.Test; import org.junit.runner.RunWith; /** - * Pixel test for frame processing via {@link TransformationFrameProcessor}. + * Pixel test for frame processing via {@link AdvancedFrameProcessor}. * *

Expected images are taken from an emulator, so tests on different emulators or physical * devices may fail. To test on other devices, please increase the {@link * BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps - * as recommended in {@link FrameEditorDataProcessingTest}. + * as recommended in {@link FrameProcessorChainPixelTest}. */ @RunWith(AndroidJUnit4.class) -public final class TransformationFrameProcessorTest { +public final class AdvancedFrameProcessorPixelTest { static { GlUtil.glAssertionsEnabled = true; @@ -55,11 +55,9 @@ public final class TransformationFrameProcessorTest { private final EGLDisplay eglDisplay = GlUtil.createEglDisplay(); private final EGLContext eglContext = GlUtil.createEglContext(eglDisplay); - private @MonotonicNonNull GlFrameProcessor transformationFrameProcessor; + private @MonotonicNonNull GlFrameProcessor advancedFrameProcessor; private int inputTexId; private int outputTexId; - // TODO(b/214975934): Once the frame processors are allowed to have different input and output - // dimensions, get the output dimensions from the frame processor. private int width; private int height; @@ -82,22 +80,21 @@ public final class TransformationFrameProcessorTest { @After public void release() { - if (transformationFrameProcessor != null) { - transformationFrameProcessor.release(); + if (advancedFrameProcessor != null) { + advancedFrameProcessor.release(); } GlUtil.destroyEglContext(eglDisplay, eglContext); } @Test public void updateProgramAndDraw_noEdits_producesExpectedOutput() throws Exception { - final String testId = "updateProgramAndDraw_noEdits"; + String testId = "updateProgramAndDraw_noEdits"; Matrix identityMatrix = new Matrix(); - transformationFrameProcessor = - new TransformationFrameProcessor(getApplicationContext(), identityMatrix); - transformationFrameProcessor.initialize(); + advancedFrameProcessor = new AdvancedFrameProcessor(getApplicationContext(), identityMatrix); + advancedFrameProcessor.initialize(inputTexId); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING); - transformationFrameProcessor.updateProgramAndDraw(inputTexId, /* presentationTimeNs= */ 0); + advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); @@ -112,16 +109,16 @@ public final class TransformationFrameProcessorTest { @Test public void updateProgramAndDraw_translateRight_producesExpectedOutput() throws Exception { - final String testId = "updateProgramAndDraw_translateRight"; + String testId = "updateProgramAndDraw_translateRight"; Matrix translateRightMatrix = new Matrix(); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); - transformationFrameProcessor = - new TransformationFrameProcessor(getApplicationContext(), translateRightMatrix); - transformationFrameProcessor.initialize(); + advancedFrameProcessor = + new AdvancedFrameProcessor(getApplicationContext(), translateRightMatrix); + advancedFrameProcessor.initialize(inputTexId); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING); - transformationFrameProcessor.updateProgramAndDraw(inputTexId, /* presentationTimeNs= */ 0); + advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); @@ -136,16 +133,15 @@ public final class TransformationFrameProcessorTest { @Test public void updateProgramAndDraw_scaleNarrow_producesExpectedOutput() throws Exception { - final String testId = "updateProgramAndDraw_scaleNarrow"; + String testId = "updateProgramAndDraw_scaleNarrow"; Matrix scaleNarrowMatrix = new Matrix(); scaleNarrowMatrix.postScale(.5f, 1.2f); - transformationFrameProcessor = - new TransformationFrameProcessor(getApplicationContext(), scaleNarrowMatrix); - transformationFrameProcessor.initialize(); + advancedFrameProcessor = new AdvancedFrameProcessor(getApplicationContext(), scaleNarrowMatrix); + advancedFrameProcessor.initialize(inputTexId); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING); - transformationFrameProcessor.updateProgramAndDraw(inputTexId, /* presentationTimeNs= */ 0); + advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); @@ -160,18 +156,14 @@ public final class TransformationFrameProcessorTest { @Test public void updateProgramAndDraw_rotate90_producesExpectedOutput() throws Exception { - final String testId = "updateProgramAndDraw_rotate90"; - // TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline - // resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can - // test that rotation doesn't distort the image. + String testId = "updateProgramAndDraw_rotate90"; Matrix rotate90Matrix = new Matrix(); rotate90Matrix.postRotate(/* degrees= */ 90); - transformationFrameProcessor = - new TransformationFrameProcessor(getApplicationContext(), rotate90Matrix); - transformationFrameProcessor.initialize(); + advancedFrameProcessor = new AdvancedFrameProcessor(getApplicationContext(), rotate90Matrix); + advancedFrameProcessor.initialize(inputTexId); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING); - transformationFrameProcessor.updateProgramAndDraw(inputTexId, /* presentationTimeNs= */ 0); + advancedFrameProcessor.updateProgramAndDraw(/* presentationTimeNs= */ 0); Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromCurrentGlFramebuffer(width, height); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/BitmapTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/BitmapTestUtil.java index 1fe9566fb2..db2df0c07e 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/BitmapTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/BitmapTestUtil.java @@ -39,8 +39,8 @@ import java.io.InputStream; import java.nio.ByteBuffer; /** - * Utilities for instrumentation tests for the {@link FrameEditor} and {@link GlFrameProcessor - * GlFrameProcessors}. + * Utilities for instrumentation tests for the {@link FrameProcessorChain} and {@link + * GlFrameProcessor GlFrameProcessors}. */ public class BitmapTestUtil { @@ -53,19 +53,29 @@ public class BitmapTestUtil { "media/bitmap/sample_mp4_first_frame_translate_right.png"; public static final String SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING = "media/bitmap/sample_mp4_first_frame_scale_narrow.png"; + public static final String ROTATE_THEN_TRANSLATE_EXPECTED_OUTPUT_PNG_ASSET_STRING = + "media/bitmap/sample_mp4_first_frame_rotate_then_translate.png"; + public static final String TRANSLATE_THEN_ROTATE_EXPECTED_OUTPUT_PNG_ASSET_STRING = + "media/bitmap/sample_mp4_first_frame_translate_then_rotate.png"; public static final String ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING = "media/bitmap/sample_mp4_first_frame_rotate90.png"; + public static final String REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING = + "media/bitmap/sample_mp4_first_frame_request_output_height.png"; + public static final String ROTATE45_SCALE_TO_FIT_EXPECTED_OUTPUT_PNG_ASSET_STRING = + "media/bitmap/sample_mp4_first_frame_rotate_45_scale_to_fit.png"; /** * Maximum allowed average pixel difference between the expected and actual edited images for the * test to pass. The value is chosen so that differences in decoder behavior across emulator * versions don't affect whether the test passes for most emulators, but substantial distortions - * introduced by changes in the behavior of the frame editor will cause the test to fail. + * introduced by changes in the behavior of the {@link GlFrameProcessor GlFrameProcessors} will + * cause the test to fail. * *

To run this test on physical devices, please use a value of 5f, rather than 0.1f. This * higher value will ignore some very small errors, but will allow for some differences caused by * graphics implementations to be ignored. When the difference is close to the threshold, manually * inspect expected/actual bitmaps to confirm failure, as it's possible this is caused by a - * difference in the codec or graphics implementation as opposed to a FrameEditor issue. + * difference in the codec or graphics implementation as opposed to a {@link GlFrameProcessor} + * issue. */ public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 0.1f; diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java deleted file mode 100644 index a35b2985a4..0000000000 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2021 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.transformer; - -import static androidx.test.core.app.ApplicationProvider.getApplicationContext; -import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.assertThrows; - -import android.content.Context; -import android.graphics.Matrix; -import android.graphics.SurfaceTexture; -import android.view.Surface; -import androidx.test.ext.junit.runners.AndroidJUnit4; -import org.junit.Test; -import org.junit.runner.RunWith; - -/** - * Test for {@link FrameEditor#create(Context, int, int, float, GlFrameProcessor, Surface, boolean, - * Transformer.DebugViewProvider) creating} a {@link FrameEditor}. - */ -@RunWith(AndroidJUnit4.class) -public final class FrameEditorTest { - // TODO(b/212539951): Make this a robolectric test by e.g. updating shadows or adding a - // wrapper around GlUtil to allow the usage of mocks or fakes which don't need (Shadow)GLES20. - - @Test - public void create_withSupportedPixelWidthHeightRatio_completesSuccessfully() - throws TransformationException { - Context context = getApplicationContext(); - - FrameEditor.create( - context, - /* outputWidth= */ 200, - /* outputHeight= */ 100, - /* pixelWidthHeightRatio= */ 1, - new TransformationFrameProcessor(context, new Matrix()), - new Surface(new SurfaceTexture(false)), - /* enableExperimentalHdrEditing= */ false, - Transformer.DebugViewProvider.NONE); - } - - @Test - public void create_withUnsupportedPixelWidthHeightRatio_throwsException() { - Context context = getApplicationContext(); - - TransformationException exception = - assertThrows( - TransformationException.class, - () -> - FrameEditor.create( - context, - /* outputWidth= */ 200, - /* outputHeight= */ 100, - /* pixelWidthHeightRatio= */ 2, - new TransformationFrameProcessor(context, new Matrix()), - new Surface(new SurfaceTexture(false)), - /* enableExperimentalHdrEditing= */ false, - Transformer.DebugViewProvider.NONE)); - - assertThat(exception).hasCauseThat().isInstanceOf(UnsupportedOperationException.class); - assertThat(exception).hasCauseThat().hasMessageThat().contains("pixelWidthHeightRatio"); - } -} diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java similarity index 52% rename from libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java rename to libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java index ccbb5928b0..00a9dafa6a 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameEditorDataProcessingTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java @@ -18,11 +18,14 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.transformer.BitmapTestUtil.FIRST_FRAME_PNG_ASSET_STRING; import static androidx.media3.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; -import static androidx.media3.transformer.BitmapTestUtil.ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING; -import static androidx.media3.transformer.BitmapTestUtil.SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING; +import static androidx.media3.transformer.BitmapTestUtil.REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING; +import static androidx.media3.transformer.BitmapTestUtil.ROTATE45_SCALE_TO_FIT_EXPECTED_OUTPUT_PNG_ASSET_STRING; +import static androidx.media3.transformer.BitmapTestUtil.ROTATE_THEN_TRANSLATE_EXPECTED_OUTPUT_PNG_ASSET_STRING; import static androidx.media3.transformer.BitmapTestUtil.TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING; +import static androidx.media3.transformer.BitmapTestUtil.TRANSLATE_THEN_ROTATE_EXPECTED_OUTPUT_PNG_ASSET_STRING; import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; +import static java.util.Arrays.asList; import android.content.Context; import android.content.res.AssetFileDescriptor; @@ -34,6 +37,7 @@ import android.media.ImageReader; import android.media.MediaCodec; import android.media.MediaExtractor; import android.media.MediaFormat; +import android.util.Size; import androidx.annotation.Nullable; import androidx.media3.common.MimeTypes; import androidx.test.ext.junit.runners.AndroidJUnit4; @@ -44,7 +48,7 @@ import org.junit.Test; import org.junit.runner.RunWith; /** - * Pixel test for frame processing via {@link FrameEditor}. + * Pixel test for frame processing via {@link FrameProcessorChain}. * *

Expected images are taken from an emulator, so tests on different emulators or physical * devices may fail. To test on other devices, please increase the {@link @@ -52,39 +56,35 @@ import org.junit.runner.RunWith; * bitmaps. */ @RunWith(AndroidJUnit4.class) -public final class FrameEditorDataProcessingTest { - // TODO(b/214975934): Once FrameEditor is converted to a FrameProcessorChain, replace these tests - // with a test for a few example combinations of GlFrameProcessors rather than testing all use - // cases of TransformationFrameProcessor. +public final class FrameProcessorChainPixelTest { /** Input video of which we only use the first frame. */ private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4"; /** Timeout for dequeueing buffers from the codec, in microseconds. */ private static final int DEQUEUE_TIMEOUT_US = 5_000_000; /** - * Time to wait for the decoded frame to populate the frame editor's input surface and the frame - * editor to finish processing the frame, in milliseconds. + * Time to wait for the decoded frame to populate the {@link FrameProcessorChain}'s input surface + * and the {@link FrameProcessorChain} to finish processing the frame, in milliseconds. */ - private static final int FRAME_PROCESSING_WAIT_MS = 1000; + private static final int FRAME_PROCESSING_WAIT_MS = 5000; /** The ratio of width over height, for each pixel in a frame. */ private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1; - private @MonotonicNonNull FrameEditor frameEditor; - private @MonotonicNonNull ImageReader frameEditorOutputImageReader; + private @MonotonicNonNull FrameProcessorChain frameProcessorChain; + private @MonotonicNonNull ImageReader outputImageReader; private @MonotonicNonNull MediaFormat mediaFormat; @After public void release() { - if (frameEditor != null) { - frameEditor.release(); + if (frameProcessorChain != null) { + frameProcessorChain.release(); } } @Test public void processData_noEdits_producesExpectedOutput() throws Exception { - final String testId = "processData_noEdits"; - Matrix identityMatrix = new Matrix(); - setUpAndPrepareFirstFrame(identityMatrix); + String testId = "processData_noEdits"; + setUpAndPrepareFirstFrame(); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING); Bitmap actualBitmap = processFirstFrameAndEnd(); @@ -99,11 +99,14 @@ public final class FrameEditorDataProcessingTest { } @Test - public void processData_translateRight_producesExpectedOutput() throws Exception { - final String testId = "processData_translateRight"; + public void processData_withAdvancedFrameProcessor_translateRight_producesExpectedOutput() + throws Exception { + String testId = "processData_withAdvancedFrameProcessor_translateRight"; Matrix translateRightMatrix = new Matrix(); translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); - setUpAndPrepareFirstFrame(translateRightMatrix); + GlFrameProcessor glFrameProcessor = + new AdvancedFrameProcessor(getApplicationContext(), translateRightMatrix); + setUpAndPrepareFirstFrame(glFrameProcessor); Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING); @@ -119,13 +122,20 @@ public final class FrameEditorDataProcessingTest { } @Test - public void processData_scaleNarrow_producesExpectedOutput() throws Exception { - final String testId = "processData_scaleNarrow"; - Matrix scaleNarrowMatrix = new Matrix(); - scaleNarrowMatrix.postScale(.5f, 1.2f); - setUpAndPrepareFirstFrame(scaleNarrowMatrix); + public void processData_withAdvancedAndScaleToFitFrameProcessors_producesExpectedOutput() + throws Exception { + String testId = "processData_withAdvancedAndScaleToFitFrameProcessors"; + Matrix translateRightMatrix = new Matrix(); + translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); + GlFrameProcessor translateRightFrameProcessor = + new AdvancedFrameProcessor(getApplicationContext(), translateRightMatrix); + GlFrameProcessor rotate45FrameProcessor = + new ScaleToFitFrameProcessor.Builder(getApplicationContext()) + .setRotationDegrees(45) + .build(); + setUpAndPrepareFirstFrame(translateRightFrameProcessor, rotate45FrameProcessor); Bitmap expectedBitmap = - BitmapTestUtil.readBitmap(SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING); + BitmapTestUtil.readBitmap(TRANSLATE_THEN_ROTATE_EXPECTED_OUTPUT_PNG_ASSET_STRING); Bitmap actualBitmap = processFirstFrameAndEnd(); @@ -139,15 +149,20 @@ public final class FrameEditorDataProcessingTest { } @Test - public void processData_rotate90_producesExpectedOutput() throws Exception { - final String testId = "processData_rotate90"; - // TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline - // resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can - // test that rotation doesn't distort the image. - Matrix rotate90Matrix = new Matrix(); - rotate90Matrix.postRotate(/* degrees= */ 90); - setUpAndPrepareFirstFrame(rotate90Matrix); - Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING); + public void processData_withScaleToFitAndAdvancedFrameProcessors_producesExpectedOutput() + throws Exception { + String testId = "processData_withScaleToFitAndAdvancedFrameProcessors"; + GlFrameProcessor rotate45FrameProcessor = + new ScaleToFitFrameProcessor.Builder(getApplicationContext()) + .setRotationDegrees(45) + .build(); + Matrix translateRightMatrix = new Matrix(); + translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); + GlFrameProcessor translateRightFrameProcessor = + new AdvancedFrameProcessor(getApplicationContext(), translateRightMatrix); + setUpAndPrepareFirstFrame(rotate45FrameProcessor, translateRightFrameProcessor); + Bitmap expectedBitmap = + BitmapTestUtil.readBitmap(ROTATE_THEN_TRANSLATE_EXPECTED_OUTPUT_PNG_ASSET_STRING); Bitmap actualBitmap = processFirstFrameAndEnd(); @@ -160,12 +175,65 @@ public final class FrameEditorDataProcessingTest { assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } - private void setUpAndPrepareFirstFrame(Matrix transformationMatrix) throws Exception { + @Test + public void + processData_withPresentationFrameProcessor_requestOutputHeight_producesExpectedOutput() + throws Exception { + String testId = "processData_withPresentationFrameProcessor_requestOutputHeight"; + GlFrameProcessor glFrameProcessor = + new PresentationFrameProcessor.Builder(getApplicationContext()).setResolution(480).build(); + setUpAndPrepareFirstFrame(glFrameProcessor); + Bitmap expectedBitmap = + BitmapTestUtil.readBitmap(REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING); + + Bitmap actualBitmap = processFirstFrameAndEnd(); + + // TODO(b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + BitmapTestUtil.saveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void processData_withScaleToFitFrameProcessor_rotate45_producesExpectedOutput() + throws Exception { + String testId = "processData_withScaleToFitFrameProcessor_rotate45"; + GlFrameProcessor glFrameProcessor = + new ScaleToFitFrameProcessor.Builder(getApplicationContext()) + .setRotationDegrees(45) + .build(); + setUpAndPrepareFirstFrame(glFrameProcessor); + Bitmap expectedBitmap = + BitmapTestUtil.readBitmap(ROTATE45_SCALE_TO_FIT_EXPECTED_OUTPUT_PNG_ASSET_STRING); + + Bitmap actualBitmap = processFirstFrameAndEnd(); + + // TODO(b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + BitmapTestUtil.getAveragePixelAbsoluteDifferenceArgb8888( + expectedBitmap, actualBitmap, testId); + BitmapTestUtil.saveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "actual", actualBitmap, /* throwOnFailure= */ false); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + /** + * Set up and prepare the first frame from an input video, as well as relevant test + * infrastructure. The frame will be sent towards the {@link FrameProcessorChain}, and may be + * accessed on the {@link FrameProcessorChain}'s output {@code outputImageReader}. + * + * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} that will apply changes + * to the input frame. + */ + private void setUpAndPrepareFirstFrame(GlFrameProcessor... frameProcessors) throws Exception { // Set up the extractor to read the first video frame and get its format. MediaExtractor mediaExtractor = new MediaExtractor(); @Nullable MediaCodec mediaCodec = null; - try (AssetFileDescriptor afd = - getApplicationContext().getAssets().openFd(INPUT_MP4_ASSET_STRING)) { + Context context = getApplicationContext(); + try (AssetFileDescriptor afd = context.getAssets().openFd(INPUT_MP4_ASSET_STRING)) { mediaExtractor.setDataSource(afd.getFileDescriptor(), afd.getStartOffset(), afd.getLength()); for (int i = 0; i < mediaExtractor.getTrackCount(); i++) { if (MimeTypes.isVideo(mediaExtractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME))) { @@ -175,28 +243,35 @@ public final class FrameEditorDataProcessingTest { } } - int width = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH); - int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); - frameEditorOutputImageReader = - ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); - Context context = getApplicationContext(); - frameEditor = - FrameEditor.create( + int inputWidth = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH); + int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); + frameProcessorChain = + new FrameProcessorChain( context, - width, - height, PIXEL_WIDTH_HEIGHT_RATIO, - new TransformationFrameProcessor(context, transformationMatrix), - frameEditorOutputImageReader.getSurface(), - /* enableExperimentalHdrEditing= */ false, - Transformer.DebugViewProvider.NONE); - frameEditor.registerInputFrame(); + inputWidth, + inputHeight, + asList(frameProcessors), + /* enableExperimentalHdrEditing= */ false); + Size outputSize = frameProcessorChain.getOutputSize(); + outputImageReader = + ImageReader.newInstance( + outputSize.getWidth(), + outputSize.getHeight(), + PixelFormat.RGBA_8888, + /* maxImages= */ 1); + frameProcessorChain.configure( + outputImageReader.getSurface(), + outputSize.getWidth(), + outputSize.getHeight(), + /* debugSurfaceView= */ null); + frameProcessorChain.registerInputFrame(); // Queue the first video frame from the extractor. String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME)); mediaCodec = MediaCodec.createDecoderByType(mimeType); mediaCodec.configure( - mediaFormat, frameEditor.createInputSurface(), /* crypto= */ null, /* flags= */ 0); + mediaFormat, frameProcessorChain.getInputSurface(), /* crypto= */ null, /* flags= */ 0); mediaCodec.start(); int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US); assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER); @@ -237,14 +312,15 @@ public final class FrameEditorDataProcessingTest { } private Bitmap processFirstFrameAndEnd() throws InterruptedException, TransformationException { - checkNotNull(frameEditor).signalEndOfInputStream(); + checkNotNull(frameProcessorChain).signalEndOfInputStream(); Thread.sleep(FRAME_PROCESSING_WAIT_MS); - assertThat(frameEditor.isEnded()).isTrue(); - frameEditor.getAndRethrowBackgroundExceptions(); + assertThat(frameProcessorChain.isEnded()).isTrue(); + frameProcessorChain.getAndRethrowBackgroundExceptions(); - Image editorOutputImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage(); - Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromRgba8888Image(editorOutputImage); - editorOutputImage.close(); + Image frameProcessorChainOutputImage = checkNotNull(outputImageReader).acquireLatestImage(); + Bitmap actualBitmap = + BitmapTestUtil.createArgb8888BitmapFromRgba8888Image(frameProcessorChainOutputImage); + frameProcessorChainOutputImage.close(); return actualBitmap; } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SsimHelper.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SsimHelper.java index e6c43d062f..5b00cd21b4 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SsimHelper.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SsimHelper.java @@ -20,6 +20,7 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; import static java.lang.Math.pow; import android.content.Context; @@ -33,15 +34,12 @@ import android.media.MediaExtractor; import android.media.MediaFormat; import android.os.Handler; import androidx.annotation.Nullable; -import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.ConditionVariable; import androidx.media3.common.util.Util; import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * A helper for calculating SSIM score for transcoded videos. @@ -58,26 +56,8 @@ public final class SsimHelper { /** The default comparison interval. */ public static final int DEFAULT_COMPARISON_INTERVAL = 11; - private static final int SURFACE_WAIT_MS = 10; + private static final int IMAGE_AVAILABLE_TIMEOUT_MS = 10_000; private static final int DECODED_IMAGE_CHANNEL_COUNT = 3; - private static final int[] EMPTY_BUFFER = new int[0]; - - private final Context context; - private final String expectedVideoPath; - private final String actualVideoPath; - private final int comparisonInterval; - - private @MonotonicNonNull VideoDecodingWrapper expectedDecodingWrapper; - private @MonotonicNonNull VideoDecodingWrapper actualDecodingWrapper; - private double accumulatedSsim; - private int comparedImagesCount; - - // These atomic fields are read on both test thread (where MediaCodec is controlled) and set on - // the main thread (where ImageReader invokes its callback). - private final AtomicReference expectedLumaBuffer; - private final AtomicReference actualLumaBuffer; - private final AtomicInteger width; - private final AtomicInteger height; /** * Returns the mean SSIM score between the expected and the actual video. @@ -92,95 +72,49 @@ public final class SsimHelper { */ public static double calculate(Context context, String expectedVideoPath, String actualVideoPath) throws IOException, InterruptedException { - return new SsimHelper(context, expectedVideoPath, actualVideoPath, DEFAULT_COMPARISON_INTERVAL) - .calculateSsim(); - } - - private SsimHelper( - Context context, String expectedVideoPath, String actualVideoPath, int comparisonInterval) { - this.context = context; - this.expectedVideoPath = expectedVideoPath; - this.actualVideoPath = actualVideoPath; - this.comparisonInterval = comparisonInterval; - this.expectedLumaBuffer = new AtomicReference<>(EMPTY_BUFFER); - this.actualLumaBuffer = new AtomicReference<>(EMPTY_BUFFER); - this.width = new AtomicInteger(Format.NO_VALUE); - this.height = new AtomicInteger(Format.NO_VALUE); - } - - /** Calculates the SSIM score between the two videos. */ - private double calculateSsim() throws InterruptedException, IOException { - // The test thread has no looper, so a handler is created on which the - // ImageReader.OnImageAvailableListener is called. - Handler mainThreadHandler = Util.createHandlerForCurrentOrMainLooper(); - ImageReader.OnImageAvailableListener onImageAvailableListener = this::onImageAvailableListener; - expectedDecodingWrapper = - new VideoDecodingWrapper( - context, - expectedVideoPath, - onImageAvailableListener, - mainThreadHandler, - comparisonInterval); - actualDecodingWrapper = - new VideoDecodingWrapper( - context, - actualVideoPath, - onImageAvailableListener, - mainThreadHandler, - comparisonInterval); - + VideoDecodingWrapper expectedDecodingWrapper = + new VideoDecodingWrapper(context, expectedVideoPath, DEFAULT_COMPARISON_INTERVAL); + VideoDecodingWrapper actualDecodingWrapper = + new VideoDecodingWrapper(context, actualVideoPath, DEFAULT_COMPARISON_INTERVAL); + double accumulatedSsim = 0.0; + int comparedImagesCount = 0; try { - while (!expectedDecodingWrapper.hasEnded() && !actualDecodingWrapper.hasEnded()) { - if (!expectedDecodingWrapper.runUntilComparisonFrameOrEnded() - || !actualDecodingWrapper.runUntilComparisonFrameOrEnded()) { - continue; + while (true) { + @Nullable Image expectedImage = expectedDecodingWrapper.runUntilComparisonFrameOrEnded(); + @Nullable Image actualImage = actualDecodingWrapper.runUntilComparisonFrameOrEnded(); + if (expectedImage == null) { + assertThat(actualImage).isNull(); + break; } + checkNotNull(actualImage); - while (expectedLumaBuffer.get() == EMPTY_BUFFER || actualLumaBuffer.get() == EMPTY_BUFFER) { - // Wait for the ImageReader to call onImageAvailable and process the luma channel on the - // main thread. - Thread.sleep(SURFACE_WAIT_MS); + int width = expectedImage.getWidth(); + int height = expectedImage.getHeight(); + assertThat(actualImage.getWidth()).isEqualTo(width); + assertThat(actualImage.getHeight()).isEqualTo(height); + try { + accumulatedSsim += + SsimCalculator.calculate( + extractLumaChannelBuffer(expectedImage), + extractLumaChannelBuffer(actualImage), + /* offset= */ 0, + /* stride= */ width, + width, + height); + } finally { + expectedImage.close(); + actualImage.close(); } - accumulatedSsim += - SsimCalculator.calculate( - expectedLumaBuffer.get(), - actualLumaBuffer.get(), - /* offset= */ 0, - /* stride= */ width.get(), - width.get(), - height.get()); comparedImagesCount++; - expectedLumaBuffer.set(EMPTY_BUFFER); - actualLumaBuffer.set(EMPTY_BUFFER); } } finally { expectedDecodingWrapper.close(); actualDecodingWrapper.close(); } - - if (comparedImagesCount == 0) { - throw new IOException("Input had no frames."); - } + assertWithMessage("Input had no frames.").that(comparedImagesCount).isGreaterThan(0); return accumulatedSsim / comparedImagesCount; } - private void onImageAvailableListener(ImageReader imageReader) { - // This method is invoked on the main thread. - Image image = imageReader.acquireLatestImage(); - int[] lumaBuffer = extractLumaChannelBuffer(image); - width.set(image.getWidth()); - height.set(image.getHeight()); - image.close(); - - if (imageReader == checkNotNull(expectedDecodingWrapper).imageReader) { - expectedLumaBuffer.set(lumaBuffer); - } else if (imageReader == checkNotNull(actualDecodingWrapper).imageReader) { - actualLumaBuffer.set(lumaBuffer); - } else { - throw new IllegalStateException("Unexpected ImageReader."); - } - } - /** * Returns the buffer of the luma (Y) channel of the image. * @@ -206,6 +140,10 @@ public final class SsimHelper { return lumaChannelBuffer; } + private SsimHelper() { + // Prevent instantiation. + } + private static final class VideoDecodingWrapper implements Closeable { // Use ExoPlayer's 10ms timeout setting. In practise, the test durations from using timeouts of // 1/10/100ms don't differ significantly. @@ -221,11 +159,12 @@ public final class SsimHelper { private final MediaExtractor mediaExtractor; private final MediaCodec.BufferInfo bufferInfo; private final ImageReader imageReader; + private final ConditionVariable imageAvailableConditionVariable; private final int comparisonInterval; private boolean isCurrentFrameComparisonFrame; private boolean hasReadEndOfInputStream; - private boolean queuedEndOfStreamToEncoder; + private boolean queuedEndOfStreamToDecoder; private boolean dequeuedAllDecodedFrames; private int dequeuedFramesCount; @@ -234,18 +173,11 @@ public final class SsimHelper { * * @param context The {@link Context}. * @param filePath The path to the video file. - * @param imageAvailableListener An {@link ImageReader.OnImageAvailableListener} implementation. - * @param handler The {@link Handler} on which the {@code imageAvailableListener} is called. * @param comparisonInterval The number of frames between the frames selected for comparison by * SSIM. * @throws IOException When failed to open the video file. */ - public VideoDecodingWrapper( - Context context, - String filePath, - ImageReader.OnImageAvailableListener imageAvailableListener, - Handler handler, - int comparisonInterval) + public VideoDecodingWrapper(Context context, String filePath, int comparisonInterval) throws IOException { this.comparisonInterval = comparisonInterval; mediaExtractor = new MediaExtractor(); @@ -275,9 +207,14 @@ public final class SsimHelper { checkState(mediaFormat.containsKey(MediaFormat.KEY_HEIGHT)); int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); + // Create a handler for the main thread to receive image available notifications. The current + // (test) thread blocks until this callback is received. + Handler mainThreadHandler = Util.createHandlerForCurrentOrMainLooper(); + imageAvailableConditionVariable = new ConditionVariable(); imageReader = ImageReader.newInstance(width, height, IMAGE_READER_COLOR_SPACE, MAX_IMAGES_ALLOWED); - imageReader.setOnImageAvailableListener(imageAvailableListener, handler); + imageReader.setOnImageAvailableListener( + imageReader -> imageAvailableConditionVariable.open(), mainThreadHandler); String sampleMimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME)); mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MEDIA_CODEC_COLOR_SPACE); @@ -288,34 +225,33 @@ public final class SsimHelper { } /** - * Run decoding until a comparison frame is rendered, or decoding has ended. - * - *

The method returns after rendering the comparison frame. There is no guarantee that the - * frame is available for processing at this time. - * - * @return {@code true} when a comparison frame is encountered, or {@code false} if decoding - * {@link #hasEnded() had ended}. + * Returns the next decoded comparison frame, or {@code null} if the stream has ended. The + * caller takes ownership of any returned image and is responsible for closing it before calling + * this method again. */ - public boolean runUntilComparisonFrameOrEnded() { + @Nullable + public Image runUntilComparisonFrameOrEnded() throws InterruptedException { while (!hasEnded() && !isCurrentFrameComparisonFrame) { while (dequeueOneFrameFromDecoder()) {} - while (queueOneFrameToEncoder()) {} + while (queueOneFrameToDecoder()) {} } if (isCurrentFrameComparisonFrame) { isCurrentFrameComparisonFrame = false; - return true; + assertThat(imageAvailableConditionVariable.block(IMAGE_AVAILABLE_TIMEOUT_MS)).isTrue(); + imageAvailableConditionVariable.close(); + return imageReader.acquireLatestImage(); } - return false; + return null; } /** Returns whether decoding has ended. */ - public boolean hasEnded() { - return queuedEndOfStreamToEncoder && dequeuedAllDecodedFrames; + private boolean hasEnded() { + return dequeuedAllDecodedFrames; } /** Returns whether a frame is queued to the {@link MediaCodec decoder}. */ - private boolean queueOneFrameToEncoder() { - if (queuedEndOfStreamToEncoder) { + private boolean queueOneFrameToDecoder() { + if (queuedEndOfStreamToDecoder) { return false; } @@ -331,7 +267,7 @@ public final class SsimHelper { /* size= */ 0, /* presentationTimeUs= */ 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); - queuedEndOfStreamToEncoder = true; + queuedEndOfStreamToDecoder = true; return false; } @@ -343,7 +279,10 @@ public final class SsimHelper { sampleSize, mediaExtractor.getSampleTime(), mediaExtractor.getSampleFlags()); - hasReadEndOfInputStream = !mediaExtractor.advance(); + // MediaExtractor.advance does not reliably return false for end-of-stream, so check sample + // metadata instead as a more reliable signal. See [internal: b/121204004]. + mediaExtractor.advance(); + hasReadEndOfInputStream = mediaExtractor.getSampleTime() == -1; return true; } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformationTestResult.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformationTestResult.java index 7e5a42cf92..06cd6f27ce 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformationTestResult.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformationTestResult.java @@ -15,31 +15,117 @@ */ package androidx.media3.transformer; +import androidx.annotation.Nullable; +import androidx.media3.common.C; + /** A test only class for holding the details of a test transformation. */ public class TransformationTestResult { /** Represents an unset or unknown SSIM score. */ public static final double SSIM_UNSET = -1.0d; + /** A builder for {@link TransformationTestResult}. */ + public static class Builder { + private final TransformationResult transformationResult; + + @Nullable private String filePath; + @Nullable private Exception analysisException; + + private long elapsedTimeMs; + private double ssim; + + /** Creates a new {@link Builder}. */ + public Builder(TransformationResult transformationResult) { + this.transformationResult = transformationResult; + this.elapsedTimeMs = C.TIME_UNSET; + this.ssim = SSIM_UNSET; + } + + /** + * Sets the file path of the output file. + * + *

{@code null} represents an unset or unknown value. + * + * @param filePath The path. + * @return This {@link Builder}. + */ + public Builder setFilePath(@Nullable String filePath) { + this.filePath = filePath; + return this; + } + + /** + * Sets the amount of time taken to perform the transformation in milliseconds. {@link + * C#TIME_UNSET} if unset. + * + *

{@link C#TIME_UNSET} represents an unset or unknown value. + * + * @param elapsedTimeMs The time, in ms. + * @return This {@link Builder}. + */ + public Builder setElapsedTimeMs(long elapsedTimeMs) { + this.elapsedTimeMs = elapsedTimeMs; + return this; + } + + /** + * Sets the SSIM of the output file, compared to input file. + * + *

{@link #SSIM_UNSET} represents an unset or unknown value. + * + * @param ssim The structural similarity index. + * @return This {@link Builder}. + */ + public Builder setSsim(double ssim) { + this.ssim = ssim; + return this; + } + + /** + * Sets an {@link Exception} that occurred during post-transformation analysis. + * + *

{@code null} represents an unset or unknown value. + * + * @param analysisException The {@link Exception} thrown during analysis. + * @return This {@link Builder}. + */ + public Builder setAnalysisException(@Nullable Exception analysisException) { + this.analysisException = analysisException; + return this; + } + + /** Builds the {@link TransformationTestResult} instance. */ + public TransformationTestResult build() { + return new TransformationTestResult( + transformationResult, filePath, elapsedTimeMs, ssim, analysisException); + } + } + public final TransformationResult transformationResult; - public final String filePath; - /** The amount of time taken to perform the transformation in milliseconds. */ - public final long transformationDurationMs; + + @Nullable public final String filePath; + /** + * The amount of time taken to perform the transformation in milliseconds. {@link C#TIME_UNSET} if + * unset. + */ + public final long elapsedTimeMs; /** The SSIM score of the transformation, {@link #SSIM_UNSET} if unavailable. */ public final double ssim; + /** + * The {@link Exception} that was thrown during post-tranformation analysis, or {@code null} if + * nothing was thrown. + */ + @Nullable public final Exception analysisException; - public TransformationTestResult( - TransformationResult transformationResult, String filePath, long transformationDurationMs) { - this(transformationResult, filePath, transformationDurationMs, /* ssim= */ SSIM_UNSET); - } - - public TransformationTestResult( + private TransformationTestResult( TransformationResult transformationResult, - String filePath, - long transformationDurationMs, - double ssim) { + @Nullable String filePath, + long elapsedTimeMs, + double ssim, + @Nullable Exception analysisException) { this.transformationResult = transformationResult; this.filePath = filePath; - this.transformationDurationMs = transformationDurationMs; + this.elapsedTimeMs = elapsedTimeMs; this.ssim = ssim; + this.analysisException = analysisException; } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java index 3479e10fc4..499a015696 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java @@ -39,6 +39,7 @@ import org.json.JSONObject; /** An android instrumentation test runner for {@link Transformer}. */ public class TransformerAndroidTestRunner { + private static final String TAG_PREFIX = "TransformerAndroidTest_"; /** The default transformation timeout value. */ public static final int DEFAULT_TIMEOUT_SECONDS = 120; @@ -49,6 +50,7 @@ public class TransformerAndroidTestRunner { private final Transformer transformer; private boolean calculateSsim; private int timeoutSeconds; + private boolean suppressAnalysisExceptions; /** * Creates a {@link Builder}. @@ -93,9 +95,29 @@ public class TransformerAndroidTestRunner { return this; } + /** + * Sets whether the runner should suppress any {@link Exception} that occurs as a result of + * post-transformation analysis, such as SSIM calculation. + * + *

Regardless of this value, analysis exceptions are attached to the analysis file. + * + *

It's recommended to add a comment explaining why this suppression is needed, ideally with + * a bug number. + * + *

The default value is {@code false}. + * + * @param suppressAnalysisExceptions Whether to suppress analysis exceptions. + * @return This {@link Builder}. + */ + public Builder setSuppressAnalysisExceptions(boolean suppressAnalysisExceptions) { + this.suppressAnalysisExceptions = suppressAnalysisExceptions; + return this; + } + /** Builds the {@link TransformerAndroidTestRunner}. */ public TransformerAndroidTestRunner build() { - return new TransformerAndroidTestRunner(context, transformer, timeoutSeconds, calculateSsim); + return new TransformerAndroidTestRunner( + context, transformer, timeoutSeconds, calculateSsim, suppressAnalysisExceptions); } } @@ -103,13 +125,19 @@ public class TransformerAndroidTestRunner { private final Transformer transformer; private final int timeoutSeconds; private final boolean calculateSsim; + private final boolean suppressAnalysisExceptions; private TransformerAndroidTestRunner( - Context context, Transformer transformer, int timeoutSeconds, boolean calculateSsim) { + Context context, + Transformer transformer, + int timeoutSeconds, + boolean calculateSsim, + boolean suppressAnalysisExceptions) { this.context = context; this.transformer = transformer; this.timeoutSeconds = timeoutSeconds; this.calculateSsim = calculateSsim; + this.suppressAnalysisExceptions = suppressAnalysisExceptions; } /** @@ -126,6 +154,9 @@ public class TransformerAndroidTestRunner { try { TransformationTestResult transformationTestResult = runInternal(testId, uriString); resultJson.put("transformationResult", getTestResultJson(transformationTestResult)); + if (!suppressAnalysisExceptions && transformationTestResult.analysisException != null) { + throw transformationTestResult.analysisException; + } return transformationTestResult; } catch (Exception e) { resultJson.put("exception", getExceptionJson(e)); @@ -147,11 +178,10 @@ public class TransformerAndroidTestRunner { * complete. * @throws TransformationException If an exception occurs as a result of the transformation. * @throws IllegalArgumentException If the path is invalid. - * @throws IllegalStateException If this method is called from the wrong thread. - * @throws IllegalStateException If a transformation is already in progress. - * @throws Exception If the transformation did not complete. + * @throws IllegalStateException If an unexpected exception occurs when starting a transformation. */ - private TransformationTestResult runInternal(String testId, String uriString) throws Exception { + private TransformationTestResult runInternal(String testId, String uriString) + throws InterruptedException, IOException, TimeoutException, TransformationException { AtomicReference<@NullableType TransformationException> transformationExceptionReference = new AtomicReference<>(); AtomicReference<@NullableType Exception> unexpectedExceptionReference = new AtomicReference<>(); @@ -201,11 +231,12 @@ public class TransformerAndroidTestRunner { if (!countDownLatch.await(timeoutSeconds, SECONDS)) { throw new TimeoutException("Transformer timed out after " + timeoutSeconds + " seconds."); } - long transformationDurationMs = SystemClock.DEFAULT.elapsedRealtime() - startTimeMs; + long elapsedTimeMs = SystemClock.DEFAULT.elapsedRealtime() - startTimeMs; @Nullable Exception unexpectedException = unexpectedExceptionReference.get(); if (unexpectedException != null) { - throw unexpectedException; + throw new IllegalStateException( + "Unexpected exception starting the transformer.", unexpectedException); } @Nullable @@ -222,16 +253,31 @@ public class TransformerAndroidTestRunner { .setFileSizeBytes(outputVideoFile.length()) .build(); - if (!calculateSsim) { - return new TransformationTestResult( - transformationResult, outputVideoFile.getPath(), transformationDurationMs); + TransformationTestResult.Builder resultBuilder = + new TransformationTestResult.Builder(transformationResult) + .setFilePath(outputVideoFile.getPath()) + .setElapsedTimeMs(elapsedTimeMs); + + try { + if (calculateSsim) { + double ssim = + SsimHelper.calculate( + context, /* expectedVideoPath= */ uriString, outputVideoFile.getPath()); + resultBuilder.setSsim(ssim); + } + } catch (InterruptedException interruptedException) { + // InterruptedException is a special unexpected case because it is not related to Ssim + // calculation, so it should be thrown, rather than processed as part of the + // TransformationTestResult. + throw interruptedException; + } catch (Exception analysisException) { + // Catch all (checked and unchecked) exceptions throw by the SsimHelper and process them as + // part of the TransformationTestResult. + resultBuilder.setAnalysisException(analysisException); + Log.e(TAG_PREFIX + testId, "SSIM calculation failed.", analysisException); } - double ssim = - SsimHelper.calculate( - context, /* expectedVideoPath= */ uriString, outputVideoFile.getPath()); - return new TransformationTestResult( - transformationResult, outputVideoFile.getPath(), transformationDurationMs, ssim); + return resultBuilder.build(); } private static void writeTestSummaryToFile(Context context, String testId, JSONObject resultJson) @@ -241,7 +287,7 @@ public class TransformerAndroidTestRunner { String analysisContents = resultJson.toString(/* indentSpaces= */ 2); // Log contents as well as writing to file, for easier visibility on individual device testing. - Log.i("TransformerAndroidTest_" + testId, analysisContents); + Log.i(TAG_PREFIX + testId, analysisContents); File analysisFile = AndroidTestUtil.createExternalCacheFile(context, /* fileName= */ testId + "-result.txt"); @@ -272,10 +318,16 @@ public class TransformerAndroidTestRunner { if (transformationResult.averageVideoBitrate != C.RATE_UNSET_INT) { transformationResultJson.put("averageVideoBitrate", transformationResult.averageVideoBitrate); } + if (testResult.elapsedTimeMs != C.TIME_UNSET) { + transformationResultJson.put("elapsedTimeMs", testResult.elapsedTimeMs); + } if (testResult.ssim != TransformationTestResult.SSIM_UNSET) { transformationResultJson.put("ssim", testResult.ssim); } - transformationResultJson.put("transformationDurationMs", testResult.transformationDurationMs); + if (testResult.analysisException != null) { + transformationResultJson.put( + "analysisException", getExceptionJson(testResult.analysisException)); + } return transformationResultJson; } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 0a4044d08e..d44f514b9d 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -19,8 +19,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import static com.google.common.truth.Truth.assertThat; import android.content.Context; -import android.graphics.Matrix; -import androidx.media3.common.MimeTypes; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import org.junit.Test; @@ -33,50 +31,17 @@ import org.junit.runner.RunWith; @RunWith(AndroidJUnit4.class) public class TransformerEndToEndTest { - private static final String VP9_VIDEO_URI_STRING = "asset:///media/vp9/bear-vp9.webm"; private static final String AVC_VIDEO_URI_STRING = "asset:///media/mp4/sample.mp4"; - @Test - public void videoTranscoding_completesWithConsistentFrameCount() throws Exception { - Context context = ApplicationProvider.getApplicationContext(); - FrameCountingMuxer.Factory muxerFactory = - new FrameCountingMuxer.Factory(new FrameworkMuxer.Factory()); - Transformer transformer = - new Transformer.Builder(context) - .setTransformationRequest( - new TransformationRequest.Builder().setVideoMimeType(MimeTypes.VIDEO_H264).build()) - .setMuxerFactory(muxerFactory) - .setEncoderFactory( - new DefaultEncoderFactory(EncoderSelector.DEFAULT, /* enableFallback= */ false)) - .build(); - // Result of the following command: - // ffprobe -count_frames -select_streams v:0 -show_entries stream=nb_read_frames bear-vp9.webm - int expectedFrameCount = 82; - - new TransformerAndroidTestRunner.Builder(context, transformer) - .build() - .run( - /* testId= */ "videoTranscoding_completesWithConsistentFrameCount", - VP9_VIDEO_URI_STRING); - - FrameCountingMuxer frameCountingMuxer = - checkNotNull(muxerFactory.getLastFrameCountingMuxerCreated()); - assertThat(frameCountingMuxer.getFrameCount()).isEqualTo(expectedFrameCount); - } - @Test public void videoEditing_completesWithConsistentFrameCount() throws Exception { Context context = ApplicationProvider.getApplicationContext(); - Matrix transformationMatrix = new Matrix(); - transformationMatrix.postTranslate(/* dx= */ .2f, /* dy= */ .1f); FrameCountingMuxer.Factory muxerFactory = new FrameCountingMuxer.Factory(new FrameworkMuxer.Factory()); Transformer transformer = new Transformer.Builder(context) .setTransformationRequest( - new TransformationRequest.Builder() - .setTransformationMatrix(transformationMatrix) - .build()) + new TransformationRequest.Builder().setResolution(480).build()) .setMuxerFactory(muxerFactory) .setEncoderFactory( new DefaultEncoderFactory(EncoderSelector.DEFAULT, /* enableFallback= */ false)) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java index 8245e86c06..5b77a783be 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java @@ -19,7 +19,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import static com.google.common.truth.Truth.assertWithMessage; import android.content.Context; -import android.graphics.Matrix; import androidx.media3.common.MimeTypes; import androidx.media3.transformer.AndroidTestUtil; import androidx.media3.transformer.TransformationRequest; @@ -41,13 +40,11 @@ public final class RepeatedTranscodeTransformationTest { @Test public void repeatedTranscode_givesConsistentLengthOutput() throws Exception { Context context = ApplicationProvider.getApplicationContext(); - Matrix transformationMatrix = new Matrix(); - transformationMatrix.postTranslate(/* dx= */ 0.1f, /* dy= */ 0.1f); Transformer transformer = new Transformer.Builder(context) .setTransformationRequest( new TransformationRequest.Builder() - .setTransformationMatrix(transformationMatrix) + .setRotationDegrees(45) // Video MIME type is H264. .setAudioMimeType(MimeTypes.AUDIO_AAC) .build()) @@ -74,15 +71,13 @@ public final class RepeatedTranscodeTransformationTest { @Test public void repeatedTranscodeNoAudio_givesConsistentLengthOutput() throws Exception { Context context = ApplicationProvider.getApplicationContext(); - Matrix transformationMatrix = new Matrix(); - transformationMatrix.postTranslate(/* dx= */ 0.1f, /* dy= */ 0.1f); Transformer transformer = new Transformer.Builder(context) .setRemoveAudio(true) .setTransformationRequest( new TransformationRequest.Builder() // Video MIME type is H264. - .setTransformationMatrix(transformationMatrix) + .setRotationDegrees(45) .build()) .build(); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetTransformationMatrixTransformationTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetFrameEditTransformationTest.java similarity index 69% rename from libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetTransformationMatrixTransformationTest.java rename to libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetFrameEditTransformationTest.java index 79d1322111..92bb7b3d16 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetTransformationMatrixTransformationTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetFrameEditTransformationTest.java @@ -18,7 +18,6 @@ package androidx.media3.transformer.mh; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING; import android.content.Context; -import android.graphics.Matrix; import androidx.media3.transformer.TransformationRequest; import androidx.media3.transformer.Transformer; import androidx.media3.transformer.TransformerAndroidTestRunner; @@ -27,26 +26,21 @@ import androidx.test.ext.junit.runners.AndroidJUnit4; import org.junit.Test; import org.junit.runner.RunWith; -/** {@link Transformer} instrumentation test for setting a transformation matrix. */ +/** {@link Transformer} instrumentation test for applying a frame edit. */ @RunWith(AndroidJUnit4.class) -public class SetTransformationMatrixTransformationTest { +public class SetFrameEditTransformationTest { @Test - public void setTransformationMatrixTransform() throws Exception { + public void setFrameEditTransform() throws Exception { Context context = ApplicationProvider.getApplicationContext(); - Matrix transformationMatrix = new Matrix(); - transformationMatrix.postTranslate(/* dx= */ .2f, /* dy= */ .1f); Transformer transformer = new Transformer.Builder(context) .setTransformationRequest( - new TransformationRequest.Builder() - .setTransformationMatrix(transformationMatrix) - .build()) + new TransformationRequest.Builder().setRotationDegrees(45).build()) .build(); new TransformerAndroidTestRunner.Builder(context, transformer) .build() .run( - /* testId= */ "setTransformationMatrixTransform", - MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + /* testId= */ "SetFrameEditTransform", MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeQualityTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeQualityTest.java index 795fefaa8f..2db30956a9 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeQualityTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeQualityTest.java @@ -39,10 +39,8 @@ public final class TranscodeQualityTest { Transformer transformer = new Transformer.Builder(context) .setTransformationRequest( - new TransformationRequest.Builder() - .setVideoMimeType(MimeTypes.VIDEO_H265) - .setAudioMimeType(MimeTypes.AUDIO_AMR_NB) - .build()) + new TransformationRequest.Builder().setVideoMimeType(MimeTypes.VIDEO_H265).build()) + .setRemoveAudio(true) .build(); TransformationTestResult result = diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformationTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformationTest.java index e22be1c145..67ae26d8f0 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformationTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformationTest.java @@ -25,10 +25,13 @@ import androidx.media3.common.Format; import androidx.media3.common.util.Log; import androidx.media3.common.util.Util; import androidx.media3.transformer.Codec; +import androidx.media3.transformer.DefaultEncoderFactory; +import androidx.media3.transformer.EncoderSelector; import androidx.media3.transformer.TransformationException; import androidx.media3.transformer.TransformationRequest; import androidx.media3.transformer.Transformer; import androidx.media3.transformer.TransformerAndroidTestRunner; +import androidx.media3.transformer.VideoEncoderSettings; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import java.util.List; @@ -43,20 +46,18 @@ public class TransformationTest { @Test public void transform() throws Exception { - final String testId = TAG + "_transform"; - + String testId = TAG + "_transform"; Context context = ApplicationProvider.getApplicationContext(); Transformer transformer = new Transformer.Builder(context).build(); - // TODO(b/223381524): Enable Ssim calculation after fixing queueInputBuffer exception. new TransformerAndroidTestRunner.Builder(context, transformer) + .setCalculateSsim(true) .build() .run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); } @Test public void transformWithDecodeEncode() throws Exception { - final String testId = TAG + "_transformForceCodecUse"; - + String testId = TAG + "_transformForceCodecUse"; Context context = ApplicationProvider.getApplicationContext(); Transformer transformer = new Transformer.Builder(context) @@ -87,40 +88,56 @@ public class TransformationTest { } }) .build(); - // TODO(b/223381524): Enable Ssim calculation after fixing queueInputBuffer exception. new TransformerAndroidTestRunner.Builder(context, transformer) + .setCalculateSsim(true) + .build() + .run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + } + + @Test + public void transformToSpecificBitrate() throws Exception { + String testId = TAG + "_transformWithSpecificBitrate"; + Context context = ApplicationProvider.getApplicationContext(); + Transformer transformer = + new Transformer.Builder(context) + .setRemoveAudio(true) + .setEncoderFactory( + new DefaultEncoderFactory( + EncoderSelector.DEFAULT, + new VideoEncoderSettings.Builder().setBitrate(5_000_000).build(), + /* enableFallback= */ true)) + .build(); + new TransformerAndroidTestRunner.Builder(context, transformer) + .setCalculateSsim(true) .build() .run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); } @Test public void transform4K60() throws Exception { - final String testId = TAG + "_transform4K60"; - - // TODO(b/223381524): Enable Ssim calculation after fixing queueInputBuffer exception. + String testId = TAG + "_transform4K60"; Context context = ApplicationProvider.getApplicationContext(); Transformer transformer = new Transformer.Builder(context).build(); new TransformerAndroidTestRunner.Builder(context, transformer) + .setCalculateSsim(true) .build() .run(testId, MP4_REMOTE_4K60_PORTRAIT_URI_STRING); } @Test public void transformNoAudio() throws Exception { - final String testId = TAG + "_transformNoAudio"; - - // TODO(b/223381524): Enable Ssim calculation after fixing queueInputBuffer exception. + String testId = TAG + "_transformNoAudio"; Context context = ApplicationProvider.getApplicationContext(); Transformer transformer = new Transformer.Builder(context).setRemoveAudio(true).build(); new TransformerAndroidTestRunner.Builder(context, transformer) + .setCalculateSsim(true) .build() .run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); } @Test public void transformNoVideo() throws Exception { - final String testId = TAG + "_transformNoVideo"; - + String testId = TAG + "_transformNoVideo"; Context context = ApplicationProvider.getApplicationContext(); Transformer transformer = new Transformer.Builder(context).setRemoveVideo(true).build(); new TransformerAndroidTestRunner.Builder(context, transformer) @@ -130,7 +147,7 @@ public class TransformationTest { @Test public void transformSef() throws Exception { - final String testId = TAG + "_transformSef"; + String testId = TAG + "_transformSef"; if (Util.SDK_INT < 25) { // TODO(b/210593256): Remove test skipping after removing the MediaMuxer dependency. @@ -144,7 +161,6 @@ public class TransformationTest { .setTransformationRequest( new TransformationRequest.Builder().setFlattenForSlowMotion(true).build()) .build(); - // TODO(b/223381524): Enable Ssim calculation after fixing queueInputBuffer exception. new TransformerAndroidTestRunner.Builder(context, transformer) .build() .run(testId, MP4_ASSET_SEF_URI_STRING); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AdvancedFrameProcessor.java similarity index 77% rename from libraries/transformer/src/main/java/androidx/media3/transformer/TransformationFrameProcessor.java rename to libraries/transformer/src/main/java/androidx/media3/transformer/AdvancedFrameProcessor.java index 75c2c131c0..c450b61e44 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AdvancedFrameProcessor.java @@ -20,13 +20,21 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.content.Context; import android.graphics.Matrix; import android.opengl.GLES20; +import android.util.Size; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.UnstableApi; import java.io.IOException; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -/** Applies a transformation matrix in the vertex shader. */ -/* package */ class TransformationFrameProcessor implements GlFrameProcessor { +/** + * Applies a transformation matrix in the vertex shader. Operations are done on normalized device + * coordinates (-1 to 1 on x and y axes). No automatic adjustments (like done in {@link + * ScaleToFitFrameProcessor}) are applied on the transformation. Width and height are not modified. + * The background color will default to black. + */ +@UnstableApi +public final class AdvancedFrameProcessor implements GlFrameProcessor { static { GlUtil.glAssertionsEnabled = true; @@ -37,8 +45,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final String FRAGMENT_SHADER_PATH = "shaders/fragment_shader_copy_es2.glsl"; /** - * Returns a 4x4, column-major Matrix float array, from an input {@link Matrix}. This is useful - * for converting to the 4x4 column-major format commonly used in OpenGL. + * Returns a 4x4, column-major Matrix float array, from an input {@link Matrix}. + * + *

This is useful for converting to the 4x4 column-major format commonly used in OpenGL. */ private static float[] getGlMatrixArray(Matrix matrix) { float[] matrix3x3Array = new float[9]; @@ -85,18 +94,26 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * Creates a new instance. * * @param context The {@link Context}. - * @param transformationMatrix The transformation matrix to apply to each frame. + * @param transformationMatrix The transformation matrix to apply to each frame. Operations are + * done on normalized device coordinates (-1 to 1 on x and y), and no automatic adjustments + * are applied on the transformation matrix. */ - public TransformationFrameProcessor(Context context, Matrix transformationMatrix) { + public AdvancedFrameProcessor(Context context, Matrix transformationMatrix) { this.context = context; - this.transformationMatrix = transformationMatrix; + this.transformationMatrix = new Matrix(transformationMatrix); } @Override - public void initialize() throws IOException { + public Size configureOutputSize(int inputWidth, int inputHeight) { + return new Size(inputWidth, inputHeight); + } + + @Override + public void initialize(int inputTexId) throws IOException { // TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms // expected in the code. glProgram = new GlProgram(context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_PATH); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. glProgram.setBufferAttribute( "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); @@ -106,15 +123,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void updateProgramAndDraw(int inputTexId, long presentationTimeNs) { + public void updateProgramAndDraw(long presentationTimeNs) { checkStateNotNull(glProgram); - glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* unit= */ 0); glProgram.use(); glProgram.bindAttributesAndUniforms(); GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0); GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + GlUtil.checkGlError(); // The four-vertex triangle strip forms a quad. GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + GlUtil.checkGlError(); } @Override diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java index 968849f44d..104a48ccbd 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java @@ -97,8 +97,10 @@ public interface Codec { * * @param format The {@link Format} (of the output data) used to determine the underlying * encoder and its configuration values. {@link Format#sampleMimeType}, {@link Format#width} - * and {@link Format#height} must be set to those of the desired output video format. {@link - * Format#rotationDegrees} should be 0. The video should always be in landscape orientation. + * and {@link Format#height} are set to those of the desired output video format. {@link + * Format#rotationDegrees} is 0 and {@link Format#width} {@code >=} {@link Format#height}, + * therefore the video is always in landscape orientation. {@link Format#frameRate} is set + * to the output video's frame rate, if available. * @param allowedMimeTypes The non-empty list of allowed output sample {@link MimeTypes MIME * types}. * @return A {@link Codec} for video encoding. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodec.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodec.java index 72c3155424..3280df2da4 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodec.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodec.java @@ -68,16 +68,14 @@ public final class DefaultCodec implements Codec { * {@code null}. * @param configurationMediaFormat The {@link MediaFormat} to configure the underlying {@link * MediaCodec}. - * @param mediaCodecName The name of a specific {@link MediaCodec} to instantiate. If {@code - * null}, {@code DefaultCodec} uses {@link Format#sampleMimeType - * configurationFormat.sampleMimeType} to create the underlying {@link MediaCodec codec}. + * @param mediaCodecName The name of a specific {@link MediaCodec} to instantiate. * @param isDecoder Whether the {@code DefaultCodec} is intended as a decoder. * @param outputSurface The output {@link Surface} if the {@link MediaCodec} outputs to a surface. */ public DefaultCodec( Format configurationFormat, MediaFormat configurationMediaFormat, - @Nullable String mediaCodecName, + String mediaCodecName, boolean isDecoder, @Nullable Surface outputSurface) throws TransformationException { @@ -87,17 +85,11 @@ public final class DefaultCodec implements Codec { inputBufferIndex = C.INDEX_UNSET; outputBufferIndex = C.INDEX_UNSET; - String sampleMimeType = checkNotNull(configurationFormat.sampleMimeType); - boolean isVideo = MimeTypes.isVideo(sampleMimeType); + boolean isVideo = MimeTypes.isVideo(checkNotNull(configurationFormat.sampleMimeType)); @Nullable MediaCodec mediaCodec = null; @Nullable Surface inputSurface = null; try { - mediaCodec = - mediaCodecName != null - ? MediaCodec.createByCodecName(mediaCodecName) - : isDecoder - ? MediaCodec.createDecoderByType(sampleMimeType) - : MediaCodec.createEncoderByType(sampleMimeType); + mediaCodec = MediaCodec.createByCodecName(mediaCodecName); configureCodec(mediaCodec, configurationMediaFormat, isDecoder, outputSurface); if (isVideo && !isDecoder) { inputSurface = mediaCodec.createInputSurface(); @@ -108,7 +100,6 @@ public final class DefaultCodec implements Codec { inputSurface.release(); } if (mediaCodec != null) { - mediaCodecName = mediaCodec.getName(); mediaCodec.release(); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java index 5e951a6847..a044351682 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java @@ -21,11 +21,15 @@ import static androidx.media3.common.util.Util.SDK_INT; import android.media.MediaFormat; import android.view.Surface; +import androidx.annotation.Nullable; import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; import androidx.media3.common.util.MediaFormatUtil; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; /** A default implementation of {@link Codec.DecoderFactory}. */ /* package */ final class DefaultDecoderFactory implements Codec.DecoderFactory { + @Override public Codec createForAudioDecoding(Format format) throws TransformationException { MediaFormat mediaFormat = @@ -35,12 +39,13 @@ import androidx.media3.common.util.MediaFormatUtil; mediaFormat, MediaFormat.KEY_MAX_INPUT_SIZE, format.maxInputSize); MediaFormatUtil.setCsdBuffers(mediaFormat, format.initializationData); + @Nullable + String mediaCodecName = EncoderUtil.findCodecForFormat(mediaFormat, /* isDecoder= */ true); + if (mediaCodecName == null) { + throw createTransformationException(format); + } return new DefaultCodec( - format, - mediaFormat, - /* mediaCodecName= */ null, - /* isDecoder= */ true, - /* outputSurface= */ null); + format, mediaFormat, mediaCodecName, /* isDecoder= */ true, /* outputSurface= */ null); } @Override @@ -59,7 +64,23 @@ import androidx.media3.common.util.MediaFormatUtil; mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0); } + @Nullable + String mediaCodecName = EncoderUtil.findCodecForFormat(mediaFormat, /* isDecoder= */ true); + if (mediaCodecName == null) { + throw createTransformationException(format); + } return new DefaultCodec( - format, mediaFormat, /* mediaCodecName= */ null, /* isDecoder= */ true, outputSurface); + format, mediaFormat, mediaCodecName, /* isDecoder= */ true, outputSurface); + } + + @RequiresNonNull("#1.sampleMimeType") + private static TransformationException createTransformationException(Format format) { + return TransformationException.createForCodec( + new IllegalArgumentException("The requested decoding format is not supported."), + MimeTypes.isVideo(format.sampleMimeType), + /* isDecoder= */ true, + format, + /* mediaCodecName= */ null, + TransformationException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java index 3471357521..b59f6ca2bb 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java @@ -26,45 +26,70 @@ import static java.lang.Math.abs; import android.media.MediaCodecInfo; import android.media.MediaFormat; import android.util.Pair; +import android.util.Size; import androidx.annotation.Nullable; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.Log; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; -import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.List; import org.checkerframework.checker.nullness.qual.RequiresNonNull; /** A default implementation of {@link Codec.EncoderFactory}. */ +// TODO(b/224949986) Split audio and video encoder factory. @UnstableApi public final class DefaultEncoderFactory implements Codec.EncoderFactory { - private static final int DEFAULT_COLOR_FORMAT = - MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; - private static final int DEFAULT_FRAME_RATE = 60; - private static final int DEFAULT_I_FRAME_INTERVAL_SECS = 1; + private static final int DEFAULT_FRAME_RATE = 30; + private static final String TAG = "DefaultEncoderFactory"; - @Nullable private final EncoderSelector videoEncoderSelector; + private final EncoderSelector videoEncoderSelector; + private final VideoEncoderSettings requestedVideoEncoderSettings; private final boolean enableFallback; /** - * Creates a new instance using the {@link EncoderSelector#DEFAULT default encoder selector}, and - * format fallback enabled. - * - *

With format fallback enabled, and when the requested {@link Format} is not supported, {@code - * DefaultEncoderFactory} finds a format that is supported by the device and configures the {@link - * Codec} with it. The fallback process may change the requested {@link Format#sampleMimeType MIME - * type}, resolution, {@link Format#bitrate bitrate}, {@link Format#codecs profile/level}, etc. + * Creates a new instance using the {@link EncoderSelector#DEFAULT default encoder selector}, a + * default {@link VideoEncoderSettings}, and with format fallback enabled. */ public DefaultEncoderFactory() { this(EncoderSelector.DEFAULT, /* enableFallback= */ true); } - /** Creates a new instance. */ + /** Creates a new instance using a default {@link VideoEncoderSettings}. */ + public DefaultEncoderFactory(EncoderSelector videoEncoderSelector, boolean enableFallback) { + this(videoEncoderSelector, VideoEncoderSettings.DEFAULT, enableFallback); + } + + /** + * Creates a new instance. + * + *

With format fallback enabled, when the requested {@link Format} is not supported, {@code + * DefaultEncoderFactory} finds a format that is supported by the device and configures the {@link + * Codec} with it. The fallback process may change the requested {@link Format#sampleMimeType MIME + * type}, resolution, {@link Format#bitrate bitrate}, {@link Format#codecs profile/level} etc. + * + *

Values in {@code requestedVideoEncoderSettings} could be adjusted to improve encoding + * quality and/or reduce failures. Specifically, {@link VideoEncoderSettings#profile} and {@link + * VideoEncoderSettings#level} are ignored for {@link MimeTypes#VIDEO_H264}. Consider implementing + * {@link Codec.EncoderFactory} if such adjustments are unwanted. + * + *

{@code requestedVideoEncoderSettings} should be handled with care because there is no + * fallback support for it. For example, using incompatible {@link VideoEncoderSettings#profile} + * and {@link VideoEncoderSettings#level} can cause codec configuration failure. Setting an + * unsupported {@link VideoEncoderSettings#bitrateMode} may cause encoder instantiation failure. + * + * @param videoEncoderSelector The {@link EncoderSelector}. + * @param requestedVideoEncoderSettings The {@link VideoEncoderSettings}. + * @param enableFallback Whether to enable fallback. + */ public DefaultEncoderFactory( - @Nullable EncoderSelector videoEncoderSelector, boolean enableFallback) { + EncoderSelector videoEncoderSelector, + VideoEncoderSettings requestedVideoEncoderSettings, + boolean enableFallback) { this.videoEncoderSelector = videoEncoderSelector; + this.requestedVideoEncoderSettings = requestedVideoEncoderSettings; this.enableFallback = enableFallback; } @@ -73,19 +98,14 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { throws TransformationException { // TODO(b/210591626) Add encoder selection for audio. checkArgument(!allowedMimeTypes.isEmpty()); + checkNotNull(format.sampleMimeType); if (!allowedMimeTypes.contains(format.sampleMimeType)) { if (enableFallback) { // TODO(b/210591626): Pick fallback MIME type using same strategy as for encoder // capabilities limitations. format = format.buildUpon().setSampleMimeType(allowedMimeTypes.get(0)).build(); } else { - throw TransformationException.createForCodec( - new IllegalArgumentException("The requested output format is not supported."), - /* isVideo= */ false, - /* isDecoder= */ false, - format, - /* mediaCodecName= */ null, - TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); + throw createTransformationException(format); } } MediaFormat mediaFormat = @@ -93,17 +113,21 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount); mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.bitrate); + @Nullable + String mediaCodecName = EncoderUtil.findCodecForFormat(mediaFormat, /* isDecoder= */ false); + if (mediaCodecName == null) { + throw createTransformationException(format); + } return new DefaultCodec( - format, - mediaFormat, - /* mediaCodecName= */ null, - /* isDecoder= */ false, - /* outputSurface= */ null); + format, mediaFormat, mediaCodecName, /* isDecoder= */ false, /* outputSurface= */ null); } @Override public Codec createForVideoEncoding(Format format, List allowedMimeTypes) throws TransformationException { + if (format.frameRate == Format.NO_VALUE) { + format = format.buildUpon().setFrameRate(DEFAULT_FRAME_RATE).build(); + } checkArgument(format.width != Format.NO_VALUE); checkArgument(format.height != Format.NO_VALUE); // According to interface Javadoc, format.rotationDegrees should be 0. The video should always @@ -115,84 +139,51 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { checkStateNotNull(videoEncoderSelector); @Nullable - Pair encoderAndClosestFormatSupport = + VideoEncoderQueryResult encoderAndClosestFormatSupport = findEncoderWithClosestFormatSupport( - format, videoEncoderSelector, allowedMimeTypes, enableFallback); + format, + requestedVideoEncoderSettings, + videoEncoderSelector, + allowedMimeTypes, + enableFallback); + if (encoderAndClosestFormatSupport == null) { - throw TransformationException.createForCodec( - new IllegalArgumentException("The requested output format is not supported."), - /* isVideo= */ true, - /* isDecoder= */ false, - format, - /* mediaCodecName= */ null, - TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); + throw createTransformationException(format); } - MediaCodecInfo encoderInfo = encoderAndClosestFormatSupport.first; - format = encoderAndClosestFormatSupport.second; + MediaCodecInfo encoderInfo = encoderAndClosestFormatSupport.encoder; + format = encoderAndClosestFormatSupport.supportedFormat; + VideoEncoderSettings supportedVideoEncoderSettings = + encoderAndClosestFormatSupport.supportedEncoderSettings; + String mimeType = checkNotNull(format.sampleMimeType); MediaFormat mediaFormat = MediaFormat.createVideoFormat(mimeType, format.width, format.height); mediaFormat.setFloat(MediaFormat.KEY_FRAME_RATE, format.frameRate); - mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.averageBitrate); + mediaFormat.setInteger( + MediaFormat.KEY_BIT_RATE, + supportedVideoEncoderSettings.bitrate != VideoEncoderSettings.NO_VALUE + ? supportedVideoEncoderSettings.bitrate + : getSuggestedBitrate(format.width, format.height, format.frameRate)); - @Nullable - Pair codecProfileAndLevel = MediaCodecUtil.getCodecProfileAndLevel(format); - if (codecProfileAndLevel != null) { - // The codecProfileAndLevel is supported by the encoder. - mediaFormat.setInteger(MediaFormat.KEY_PROFILE, codecProfileAndLevel.first); - if (SDK_INT >= 23) { - mediaFormat.setInteger(MediaFormat.KEY_LEVEL, codecProfileAndLevel.second); - } + mediaFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, supportedVideoEncoderSettings.bitrateMode); + + if (supportedVideoEncoderSettings.profile != VideoEncoderSettings.NO_VALUE + && supportedVideoEncoderSettings.level != VideoEncoderSettings.NO_VALUE + && SDK_INT >= 23) { + // Set profile and level at the same time to maximize compatibility, or the encoder will pick + // the values. + mediaFormat.setInteger(MediaFormat.KEY_PROFILE, supportedVideoEncoderSettings.profile); + mediaFormat.setInteger(MediaFormat.KEY_LEVEL, supportedVideoEncoderSettings.level); } - // TODO(b/210593256): Remove overriding profile/level (before API 29) after switching to in-app - // muxing. if (mimeType.equals(MimeTypes.VIDEO_H264)) { - // Applying suggested profile/level settings from - // https://developer.android.com/guide/topics/media/sharing-video#b-frames_and_encoding_profiles - if (Util.SDK_INT >= 29) { - int supportedEncodingLevel = - EncoderUtil.findHighestSupportedEncodingLevel( - encoderInfo, mimeType, MediaCodecInfo.CodecProfileLevel.AVCProfileHigh); - if (supportedEncodingLevel != EncoderUtil.LEVEL_UNSET) { - // Use the highest supported profile and use B-frames. - mediaFormat.setInteger( - MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileHigh); - mediaFormat.setInteger(MediaFormat.KEY_LEVEL, supportedEncodingLevel); - mediaFormat.setInteger(MediaFormat.KEY_MAX_B_FRAMES, 1); - } - } else if (Util.SDK_INT >= 26) { - int supportedEncodingLevel = - EncoderUtil.findHighestSupportedEncodingLevel( - encoderInfo, mimeType, MediaCodecInfo.CodecProfileLevel.AVCProfileHigh); - if (supportedEncodingLevel != EncoderUtil.LEVEL_UNSET) { - // Use the highest-supported profile, but disable the generation of B-frames using - // MediaFormat.KEY_LATENCY. This accommodates some limitations in the MediaMuxer in these - // system versions. - mediaFormat.setInteger( - MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileHigh); - mediaFormat.setInteger(MediaFormat.KEY_LEVEL, supportedEncodingLevel); - // TODO(b/210593256): Set KEY_LATENCY to 2 to enable B-frame production after switching to - // in-app muxing. - mediaFormat.setInteger(MediaFormat.KEY_LATENCY, 1); - } - } else if (Util.SDK_INT >= 24) { - int supportedLevel = - EncoderUtil.findHighestSupportedEncodingLevel( - encoderInfo, mimeType, MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline); - checkState(supportedLevel != EncoderUtil.LEVEL_UNSET); - // Use the baseline profile for safest results, as encoding in baseline is required per - // https://source.android.com/compatibility/5.0/android-5.0-cdd#5_2_video_encoding - mediaFormat.setInteger( - MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline); - mediaFormat.setInteger(MediaFormat.KEY_LEVEL, supportedLevel); - } - // For API levels below 24, setting profile and level can lead to failures in MediaCodec - // configuration. The encoder selects the profile/level when we don't set them. + adjustMediaFormatForH264EncoderSettings(mediaFormat, encoderInfo); } - mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, DEFAULT_COLOR_FORMAT); - mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, DEFAULT_I_FRAME_INTERVAL_SECS); + mediaFormat.setInteger( + MediaFormat.KEY_COLOR_FORMAT, supportedVideoEncoderSettings.colorProfile); + mediaFormat.setFloat( + MediaFormat.KEY_I_FRAME_INTERVAL, supportedVideoEncoderSettings.iFrameIntervalSeconds); return new DefaultCodec( format, @@ -202,15 +193,22 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { /* outputSurface= */ null); } + @Override + public boolean videoNeedsEncoding() { + return !requestedVideoEncoderSettings.equals(VideoEncoderSettings.DEFAULT); + } + /** - * Finds a {@link MediaCodecInfo encoder} that supports the requested format most closely. Returns - * the {@link MediaCodecInfo encoder} and the supported {@link Format} in a {@link Pair}, or - * {@code null} if none is found. + * Finds an {@link MediaCodecInfo encoder} that supports the requested format most closely. + * + *

Returns the {@link MediaCodecInfo encoder} and the supported {@link Format} in a {@link + * Pair}, or {@code null} if none is found. */ @RequiresNonNull("#1.sampleMimeType") @Nullable - private static Pair findEncoderWithClosestFormatSupport( + private static VideoEncoderQueryResult findEncoderWithClosestFormatSupport( Format requestedFormat, + VideoEncoderSettings videoEncoderSettings, EncoderSelector encoderSelector, List allowedMimeTypes, boolean enableFallback) { @@ -226,81 +224,179 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { return null; } if (!enableFallback) { - return Pair.create(encodersForMimeType.get(0), requestedFormat); + return new VideoEncoderQueryResult( + encodersForMimeType.get(0), requestedFormat, videoEncoderSettings); } + ImmutableList filteredEncoders = - filterEncoders( - encodersForMimeType, - /* cost= */ (encoderInfo) -> { - @Nullable - Pair closestSupportedResolution = - EncoderUtil.getSupportedResolution( - encoderInfo, mimeType, requestedFormat.width, requestedFormat.height); - if (closestSupportedResolution == null) { - // Drops encoder. - return Integer.MAX_VALUE; - } - return abs( - requestedFormat.width * requestedFormat.height - - closestSupportedResolution.first * closestSupportedResolution.second); - }); + filterEncodersByResolution( + encodersForMimeType, mimeType, requestedFormat.width, requestedFormat.height); if (filteredEncoders.isEmpty()) { return null; } // The supported resolution is the same for all remaining encoders. - Pair finalResolution = + Size finalResolution = checkNotNull( EncoderUtil.getSupportedResolution( filteredEncoders.get(0), mimeType, requestedFormat.width, requestedFormat.height)); int requestedBitrate = - requestedFormat.averageBitrate == Format.NO_VALUE - ? getSuggestedBitrate( - /* width= */ finalResolution.first, - /* height= */ finalResolution.second, - requestedFormat.frameRate == Format.NO_VALUE - ? DEFAULT_FRAME_RATE - : requestedFormat.frameRate) - : requestedFormat.averageBitrate; + videoEncoderSettings.bitrate != VideoEncoderSettings.NO_VALUE + ? videoEncoderSettings.bitrate + : getSuggestedBitrate( + finalResolution.getWidth(), finalResolution.getHeight(), requestedFormat.frameRate); + filteredEncoders = filterEncodersByBitrate(filteredEncoders, mimeType, requestedBitrate); + if (filteredEncoders.isEmpty()) { + return null; + } + filteredEncoders = - filterEncoders( - filteredEncoders, - /* cost= */ (encoderInfo) -> { - int achievableBitrate = - EncoderUtil.getClosestSupportedBitrate(encoderInfo, mimeType, requestedBitrate); - return abs(achievableBitrate - requestedBitrate); - }); + filterEncodersByBitrateMode(filteredEncoders, mimeType, videoEncoderSettings.bitrateMode); if (filteredEncoders.isEmpty()) { return null; } MediaCodecInfo pickedEncoder = filteredEncoders.get(0); - @Nullable - Pair profileLevel = MediaCodecUtil.getCodecProfileAndLevel(requestedFormat); - @Nullable String codecs = null; - if (profileLevel != null - && requestedFormat.sampleMimeType.equals(mimeType) - && profileLevel.second - <= EncoderUtil.findHighestSupportedEncodingLevel( - pickedEncoder, mimeType, /* profile= */ profileLevel.first)) { - codecs = requestedFormat.codecs; + int closestSupportedBitrate = + EncoderUtil.getClosestSupportedBitrate(pickedEncoder, mimeType, requestedBitrate); + VideoEncoderSettings.Builder supportedEncodingSettingBuilder = + videoEncoderSettings.buildUpon().setBitrate(closestSupportedBitrate); + + if (videoEncoderSettings.profile == VideoEncoderSettings.NO_VALUE + || videoEncoderSettings.level == VideoEncoderSettings.NO_VALUE + || videoEncoderSettings.level + > EncoderUtil.findHighestSupportedEncodingLevel( + pickedEncoder, mimeType, videoEncoderSettings.profile)) { + supportedEncodingSettingBuilder.setEncodingProfileLevel( + VideoEncoderSettings.NO_VALUE, VideoEncoderSettings.NO_VALUE); } - Format encoderSupportedFormat = + Format supportedEncoderFormat = requestedFormat .buildUpon() .setSampleMimeType(mimeType) - .setCodecs(codecs) - .setWidth(finalResolution.first) - .setHeight(finalResolution.second) - .setFrameRate( - requestedFormat.frameRate != Format.NO_VALUE - ? requestedFormat.frameRate - : DEFAULT_FRAME_RATE) - .setAverageBitrate( - EncoderUtil.getClosestSupportedBitrate(pickedEncoder, mimeType, requestedBitrate)) + .setWidth(finalResolution.getWidth()) + .setHeight(finalResolution.getHeight()) + .setAverageBitrate(closestSupportedBitrate) .build(); - return Pair.create(pickedEncoder, encoderSupportedFormat); + return new VideoEncoderQueryResult( + pickedEncoder, supportedEncoderFormat, supportedEncodingSettingBuilder.build()); + } + + /** Returns a list of encoders that support the requested resolution most closely. */ + private static ImmutableList filterEncodersByResolution( + List encoders, String mimeType, int requestedWidth, int requestedHeight) { + return filterEncoders( + encoders, + /* cost= */ (encoderInfo) -> { + @Nullable + Size closestSupportedResolution = + EncoderUtil.getSupportedResolution( + encoderInfo, mimeType, requestedWidth, requestedHeight); + if (closestSupportedResolution == null) { + // Drops encoder. + return Integer.MAX_VALUE; + } + return abs( + requestedWidth * requestedHeight + - closestSupportedResolution.getWidth() * closestSupportedResolution.getHeight()); + }, + /* filterName= */ "resolution"); + } + + /** Returns a list of encoders that support the requested bitrate most closely. */ + private static ImmutableList filterEncodersByBitrate( + List encoders, String mimeType, int requestedBitrate) { + return filterEncoders( + encoders, + /* cost= */ (encoderInfo) -> { + int achievableBitrate = + EncoderUtil.getClosestSupportedBitrate(encoderInfo, mimeType, requestedBitrate); + return abs(achievableBitrate - requestedBitrate); + }, + /* filterName= */ "bitrate"); + } + + /** Returns a list of encoders that support the requested bitrate mode. */ + private static ImmutableList filterEncodersByBitrateMode( + List encoders, String mimeType, int requestedBitrateMode) { + return filterEncoders( + encoders, + /* cost= */ (encoderInfo) -> + EncoderUtil.isBitrateModeSupported(encoderInfo, mimeType, requestedBitrateMode) + ? 0 + : Integer.MAX_VALUE, // Drops encoder. + /* filterName= */ "bitrate mode"); + } + + private static final class VideoEncoderQueryResult { + public final MediaCodecInfo encoder; + public final Format supportedFormat; + public final VideoEncoderSettings supportedEncoderSettings; + + public VideoEncoderQueryResult( + MediaCodecInfo encoder, + Format supportedFormat, + VideoEncoderSettings supportedEncoderSettings) { + this.encoder = encoder; + this.supportedFormat = supportedFormat; + this.supportedEncoderSettings = supportedEncoderSettings; + } + } + + /** + * Applying suggested profile/level settings from + * https://developer.android.com/guide/topics/media/sharing-video#b-frames_and_encoding_profiles + * + *

The adjustment is applied in-place to {@code mediaFormat}. + */ + private static void adjustMediaFormatForH264EncoderSettings( + MediaFormat mediaFormat, MediaCodecInfo encoderInfo) { + // TODO(b/210593256): Remove overriding profile/level (before API 29) after switching to in-app + // muxing. + String mimeType = MimeTypes.VIDEO_H264; + if (Util.SDK_INT >= 29) { + int expectedEncodingProfile = MediaCodecInfo.CodecProfileLevel.AVCProfileHigh; + int supportedEncodingLevel = + EncoderUtil.findHighestSupportedEncodingLevel( + encoderInfo, mimeType, expectedEncodingProfile); + if (supportedEncodingLevel != EncoderUtil.LEVEL_UNSET) { + // Use the highest supported profile and use B-frames. + mediaFormat.setInteger(MediaFormat.KEY_PROFILE, expectedEncodingProfile); + mediaFormat.setInteger(MediaFormat.KEY_LEVEL, supportedEncodingLevel); + mediaFormat.setInteger(MediaFormat.KEY_MAX_B_FRAMES, 1); + } + } else if (Util.SDK_INT >= 26) { + int expectedEncodingProfile = MediaCodecInfo.CodecProfileLevel.AVCProfileHigh; + int supportedEncodingLevel = + EncoderUtil.findHighestSupportedEncodingLevel( + encoderInfo, mimeType, expectedEncodingProfile); + if (supportedEncodingLevel != EncoderUtil.LEVEL_UNSET) { + // Use the highest-supported profile, but disable the generation of B-frames using + // MediaFormat.KEY_LATENCY. This accommodates some limitations in the MediaMuxer in these + // system versions. + mediaFormat.setInteger(MediaFormat.KEY_PROFILE, expectedEncodingProfile); + mediaFormat.setInteger(MediaFormat.KEY_LEVEL, supportedEncodingLevel); + // TODO(b/210593256): Set KEY_LATENCY to 2 to enable B-frame production after switching to + // in-app muxing. + mediaFormat.setInteger(MediaFormat.KEY_LATENCY, 1); + } + } else if (Util.SDK_INT >= 24) { + int expectedEncodingProfile = MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline; + int supportedLevel = + EncoderUtil.findHighestSupportedEncodingLevel( + encoderInfo, mimeType, expectedEncodingProfile); + checkState(supportedLevel != EncoderUtil.LEVEL_UNSET); + // Use the baseline profile for safest results, as encoding in baseline is required per + // https://source.android.com/compatibility/5.0/android-5.0-cdd#5_2_video_encoding + mediaFormat.setInteger(MediaFormat.KEY_PROFILE, expectedEncodingProfile); + mediaFormat.setInteger(MediaFormat.KEY_LEVEL, supportedLevel); + } else { + // For API levels below 24, setting profile and level can lead to failures in MediaCodec + // configuration. The encoder selects the profile/level when we don't set them. + mediaFormat.setString(MediaFormat.KEY_PROFILE, null); + mediaFormat.setString(MediaFormat.KEY_LEVEL, null); + } } private interface EncoderFallbackCost { @@ -314,8 +410,17 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { int getParameterSupportGap(MediaCodecInfo encoderInfo); } + /** + * Filters a list of {@link MediaCodecInfo encoders} by a {@link EncoderFallbackCost cost + * function}. + * + * @param encoders A list of {@link MediaCodecInfo encoders}. + * @param cost A {@link EncoderFallbackCost cost function}. + * @return A list of {@link MediaCodecInfo encoders} with the lowest costs, empty if the costs of + * all encoders are {@link Integer#MAX_VALUE}. + */ private static ImmutableList filterEncoders( - List encoders, EncoderFallbackCost cost) { + List encoders, EncoderFallbackCost cost, String filterName) { List filteredEncoders = new ArrayList<>(encoders.size()); int minGap = Integer.MAX_VALUE; @@ -334,9 +439,24 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { filteredEncoders.add(encoderInfo); } } + + List removedEncoders = new ArrayList<>(encoders); + removedEncoders.removeAll(filteredEncoders); + StringBuilder stringBuilder = + new StringBuilder("Encoders removed for ").append(filterName).append(":\n"); + for (int i = 0; i < removedEncoders.size(); i++) { + MediaCodecInfo encoderInfo = removedEncoders.get(i); + stringBuilder.append(Util.formatInvariant(" %s\n", encoderInfo.getName())); + } + Log.d(TAG, stringBuilder.toString()); + return ImmutableList.copyOf(filteredEncoders); } + /** + * Finds a {@link MimeTypes MIME type} that is supported by the encoder and in the {@code + * allowedMimeTypes}. + */ @Nullable private static String findFallbackMimeType( EncoderSelector encoderSelector, String requestedMimeType, List allowedMimeTypes) { @@ -369,4 +489,15 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory { // 1080p30 -> 6.2Mbps, 720p30 -> 2.7Mbps. return (int) (width * height * frameRate * 0.1); } + + @RequiresNonNull("#1.sampleMimeType") + private static TransformationException createTransformationException(Format format) { + return TransformationException.createForCodec( + new IllegalArgumentException("The requested encoding format is not supported."), + MimeTypes.isVideo(format.sampleMimeType), + /* isDecoder= */ false, + format, + /* mediaCodecName= */ null, + TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java index 777bcc7674..8dc0bafc36 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/EncoderUtil.java @@ -22,12 +22,14 @@ import static java.lang.Math.round; import android.media.MediaCodec; import android.media.MediaCodecInfo; import android.media.MediaCodecList; -import android.util.Pair; +import android.media.MediaFormat; +import android.util.Size; import androidx.annotation.DoNotInline; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.MediaFormatUtil; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import com.google.common.base.Ascii; @@ -80,10 +82,10 @@ public final class EncoderUtil { * @param mimeType The output MIME type. * @param width The original width. * @param height The original height. - * @return A {@link Pair} of width and height, or {@code null} if unable to find a fix. + * @return A {@link Size supported resolution}, or {@code null} if unable to find a fallback. */ @Nullable - public static Pair getSupportedResolution( + public static Size getSupportedResolution( MediaCodecInfo encoderInfo, String mimeType, int width, int height) { MediaCodecInfo.VideoCapabilities videoEncoderCapabilities = encoderInfo.getCapabilitiesForType(mimeType).getVideoCapabilities(); @@ -94,28 +96,28 @@ public final class EncoderUtil { width = alignResolution(width, widthAlignment); height = alignResolution(height, heightAlignment); if (videoEncoderCapabilities.isSizeSupported(width, height)) { - return Pair.create(width, height); + return new Size(width, height); } // Try three-fourths (e.g. 1440 -> 1080). int newWidth = alignResolution(width * 3 / 4, widthAlignment); int newHeight = alignResolution(height * 3 / 4, heightAlignment); if (videoEncoderCapabilities.isSizeSupported(newWidth, newHeight)) { - return Pair.create(newWidth, newHeight); + return new Size(newWidth, newHeight); } // Try two-thirds (e.g. 4k -> 1440). newWidth = alignResolution(width * 2 / 3, widthAlignment); newHeight = alignResolution(height * 2 / 3, heightAlignment); if (videoEncoderCapabilities.isSizeSupported(newWidth, newHeight)) { - return Pair.create(newWidth, newHeight); + return new Size(newWidth, newHeight); } // Try half (e.g. 4k -> 1080). newWidth = alignResolution(width / 2, widthAlignment); newHeight = alignResolution(height / 2, heightAlignment); if (videoEncoderCapabilities.isSizeSupported(newWidth, newHeight)) { - return Pair.create(newWidth, newHeight); + return new Size(newWidth, newHeight); } // Fix frame being too wide or too tall. @@ -127,9 +129,7 @@ public final class EncoderUtil { height = alignResolution(adjustedHeight, heightAlignment); } - return videoEncoderCapabilities.isSizeSupported(width, height) - ? Pair.create(width, height) - : null; + return videoEncoderCapabilities.isSizeSupported(width, height) ? new Size(width, height) : null; } /** @@ -156,6 +156,32 @@ public final class EncoderUtil { return maxSupportedLevel; } + /** + * Finds a {@link MediaCodec codec} that supports the {@link MediaFormat}, or {@code null} if none + * is found. + */ + @Nullable + public static String findCodecForFormat(MediaFormat format, boolean isDecoder) { + MediaCodecList mediaCodecList = new MediaCodecList(MediaCodecList.ALL_CODECS); + // Format must not include KEY_FRAME_RATE on API21. + // https://developer.android.com/reference/android/media/MediaCodecList#findDecoderForFormat(android.media.MediaFormat) + @Nullable String frameRate = null; + if (Util.SDK_INT == 21 && format.containsKey(MediaFormat.KEY_FRAME_RATE)) { + frameRate = format.getString(MediaFormat.KEY_FRAME_RATE); + format.setString(MediaFormat.KEY_FRAME_RATE, null); + } + + String mediaCodecName = + isDecoder + ? mediaCodecList.findDecoderForFormat(format) + : mediaCodecList.findEncoderForFormat(format); + + if (Util.SDK_INT == 21) { + MediaFormatUtil.maybeSetString(format, MediaFormat.KEY_FRAME_RATE, frameRate); + } + return mediaCodecName; + } + /** * Finds the {@link MediaCodecInfo encoder}'s closest supported bitrate from the given bitrate. */ @@ -168,6 +194,15 @@ public final class EncoderUtil { .clamp(bitrate); } + /** Returns whether the bitrate mode is supported by the encoder. */ + public static boolean isBitrateModeSupported( + MediaCodecInfo encoderInfo, String mimeType, int bitrateMode) { + return encoderInfo + .getCapabilitiesForType(mimeType) + .getEncoderCapabilities() + .isBitrateModeSupported(bitrateMode); + } + /** Checks if a {@link MediaCodecInfo codec} is hardware-accelerated. */ public static boolean isHardwareAccelerated(MediaCodecInfo encoderInfo, String mimeType) { // TODO(b/214964116): Merge into MediaCodecUtil. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java index e1314a2b62..64100097e4 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExternalCopyFrameProcessor.java @@ -19,6 +19,7 @@ import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.content.Context; import android.opengl.GLES20; +import android.util.Size; import androidx.media3.common.util.GlProgram; import androidx.media3.common.util.GlUtil; import java.io.IOException; @@ -58,7 +59,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void initialize() throws IOException { + public Size configureOutputSize(int inputWidth, int inputHeight) { + return new Size(inputWidth, inputHeight); + } + + @Override + public void initialize(int inputTexId) throws IOException { // TODO(b/205002913): check the loaded program is consistent with the attributes and uniforms // expected in the code. String vertexShaderFilePath = @@ -70,6 +76,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH; glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath); + glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); // Draw the frame on the entire normalized device coordinate space, from -1 to 1, for x and y. glProgram.setBufferAttribute( "aFramePosition", GlUtil.getNormalizedCoordinateBounds(), GlUtil.RECTANGLE_VERTICES_COUNT); @@ -94,9 +101,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void updateProgramAndDraw(int inputTexId, long presentationTimeNs) { + public void updateProgramAndDraw(long presentationTimeNs) { checkStateNotNull(glProgram); - glProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* unit= */ 0); glProgram.use(); glProgram.bindAttributesAndUniforms(); GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java deleted file mode 100644 index 7b00b1351b..0000000000 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java +++ /dev/null @@ -1,424 +0,0 @@ -/* - * Copyright 2021 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package androidx.media3.transformer; - -import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkState; - -import android.content.Context; -import android.graphics.SurfaceTexture; -import android.opengl.EGL14; -import android.opengl.EGLContext; -import android.opengl.EGLDisplay; -import android.opengl.EGLExt; -import android.opengl.EGLSurface; -import android.opengl.GLES20; -import android.view.Surface; -import android.view.SurfaceView; -import androidx.annotation.Nullable; -import androidx.media3.common.C; -import androidx.media3.common.util.GlUtil; -import androidx.media3.common.util.Util; -import java.io.IOException; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.atomic.AtomicInteger; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; - -/** - * {@code FrameEditor} applies changes to individual video frames. - * - *

Input becomes available on its {@link #createInputSurface() input surface} asynchronously and - * is processed on a background thread as it becomes available. All input frames should be {@link - * #registerInputFrame() registered} before they are rendered to the input surface. {@link - * #hasPendingFrames()} can be used to check whether there are frames that have not been fully - * processed yet. Output is written to its {@link #create(Context, int, int, float, - * GlFrameProcessor, Surface, boolean, Transformer.DebugViewProvider) output surface}. - */ -/* package */ final class FrameEditor { - - static { - GlUtil.glAssertionsEnabled = true; - } - - /** - * Returns a new {@code FrameEditor} for applying changes to individual frames. - * - * @param context A {@link Context}. - * @param outputWidth The output width in pixels. - * @param outputHeight The output height in pixels. - * @param pixelWidthHeightRatio The ratio of width over height, for each pixel. - * @param transformationFrameProcessor The {@link GlFrameProcessor} to apply to each frame. - * @param outputSurface The {@link Surface}. - * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. - * @param debugViewProvider Provider for optional debug views to show intermediate output. - * @return A configured {@code FrameEditor}. - * @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1, reading shader - * files fails, or an OpenGL error occurs while creating and configuring the OpenGL - * components. - */ - // TODO(b/214975934): Take a List as input and rename FrameEditor to - // FrameProcessorChain. - public static FrameEditor create( - Context context, - int outputWidth, - int outputHeight, - float pixelWidthHeightRatio, - GlFrameProcessor transformationFrameProcessor, - Surface outputSurface, - boolean enableExperimentalHdrEditing, - Transformer.DebugViewProvider debugViewProvider) - throws TransformationException { - if (pixelWidthHeightRatio != 1.0f) { - // TODO(b/211782176): Consider implementing support for non-square pixels. - throw TransformationException.createForFrameEditor( - new UnsupportedOperationException( - "Transformer's frame editor currently does not support frame edits on non-square" - + " pixels. The pixelWidthHeightRatio is: " - + pixelWidthHeightRatio), - TransformationException.ERROR_CODE_GL_INIT_FAILED); - } - - @Nullable - SurfaceView debugSurfaceView = - debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight); - int debugPreviewWidth; - int debugPreviewHeight; - if (debugSurfaceView != null) { - debugPreviewWidth = debugSurfaceView.getWidth(); - debugPreviewHeight = debugSurfaceView.getHeight(); - } else { - debugPreviewWidth = C.LENGTH_UNSET; - debugPreviewHeight = C.LENGTH_UNSET; - } - - ExternalCopyFrameProcessor externalCopyFrameProcessor = - new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing); - - ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); - Future frameEditorFuture = - singleThreadExecutorService.submit( - () -> - createOpenGlObjectsAndFrameEditor( - singleThreadExecutorService, - externalCopyFrameProcessor, - transformationFrameProcessor, - outputSurface, - outputWidth, - outputHeight, - enableExperimentalHdrEditing, - debugSurfaceView, - debugPreviewWidth, - debugPreviewHeight)); - try { - return frameEditorFuture.get(); - } catch (ExecutionException e) { - throw TransformationException.createForFrameEditor( - e, TransformationException.ERROR_CODE_GL_INIT_FAILED); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw TransformationException.createForFrameEditor( - e, TransformationException.ERROR_CODE_GL_INIT_FAILED); - } - } - - /** - * Creates a {@code FrameEditor} and its OpenGL objects. - * - *

As the {@code FrameEditor} will call OpenGL commands on the {@code - * singleThreadExecutorService}'s thread, the OpenGL context and objects also need to be created - * on that thread. So this method should only be called on the {@code - * singleThreadExecutorService}'s thread. - */ - private static FrameEditor createOpenGlObjectsAndFrameEditor( - ExecutorService singleThreadExecutorService, - ExternalCopyFrameProcessor externalCopyFrameProcessor, - GlFrameProcessor transformationFrameProcessor, - Surface outputSurface, - int outputWidth, - int outputHeight, - boolean enableExperimentalHdrEditing, - @Nullable SurfaceView debugSurfaceView, - int debugPreviewWidth, - int debugPreviewHeight) - throws IOException { - EGLDisplay eglDisplay = GlUtil.createEglDisplay(); - - final EGLContext eglContext; - final EGLSurface eglSurface; - @Nullable EGLSurface debugPreviewEglSurface = null; - if (enableExperimentalHdrEditing) { - eglContext = GlUtil.createEglContextEs3Rgba1010102(eglDisplay); - // TODO(b/209404935): Don't assume BT.2020 PQ input/output. - eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurface); - if (debugSurfaceView != null) { - debugPreviewEglSurface = - GlUtil.getEglSurfaceBt2020Pq(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); - } - } else { - eglContext = GlUtil.createEglContext(eglDisplay); - eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface); - if (debugSurfaceView != null) { - debugPreviewEglSurface = - GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); - } - } - - GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); - int inputExternalTexId = GlUtil.createExternalTexture(); - int intermediateTexId = GlUtil.createTexture(outputWidth, outputHeight); - int frameBuffer = GlUtil.createFboForTexture(intermediateTexId); - externalCopyFrameProcessor.initialize(); - transformationFrameProcessor.initialize(); - - return new FrameEditor( - singleThreadExecutorService, - eglDisplay, - eglContext, - eglSurface, - externalCopyFrameProcessor, - transformationFrameProcessor, - inputExternalTexId, - intermediateTexId, - frameBuffer, - outputWidth, - outputHeight, - debugPreviewEglSurface, - debugPreviewWidth, - debugPreviewHeight); - } - - private static final String THREAD_NAME = "Transformer:FrameEditor"; - - /** Some OpenGL commands may block, so all OpenGL commands are run on a background thread. */ - private final ExecutorService singleThreadExecutorService; - /** Futures corresponding to the executor service's pending tasks. */ - private final ConcurrentLinkedQueue> futures; - /** Number of frames {@link #registerInputFrame() registered} but not fully processed. */ - private final AtomicInteger pendingFrameCount; - // TODO(b/214975934): Write javadoc for fields where the purpose might be unclear to someone less - // familiar with this class and consider grouping some of these fields into new classes to - // reduce the number of constructor parameters. - private final EGLDisplay eglDisplay; - private final EGLContext eglContext; - private final EGLSurface eglSurface; - private final ExternalCopyFrameProcessor externalCopyFrameProcessor; - private final GlFrameProcessor transformationFrameProcessor; - - /** Identifier of the external texture the {@code FrameEditor} reads its input from. */ - private final int inputExternalTexId; - /** Transformation matrix associated with the surface texture. */ - private final float[] textureTransformMatrix; - - /** - * Identifier of the texture where the output of the {@link ExternalCopyFrameProcessor} is written - * to and the {@link TransformationFrameProcessor} reads its input from. - */ - private final int intermediateTexId; - /** Identifier of a framebuffer object associated with the intermediate texture. */ - private final int frameBuffer; - - private final int outputWidth; - private final int outputHeight; - - @Nullable private final EGLSurface debugPreviewEglSurface; - private final int debugPreviewWidth; - private final int debugPreviewHeight; - - private @MonotonicNonNull SurfaceTexture inputSurfaceTexture; - private @MonotonicNonNull Surface inputSurface; - private boolean inputStreamEnded; - private volatile boolean releaseRequested; - - private FrameEditor( - ExecutorService singleThreadExecutorService, - EGLDisplay eglDisplay, - EGLContext eglContext, - EGLSurface eglSurface, - ExternalCopyFrameProcessor externalCopyFrameProcessor, - GlFrameProcessor transformationFrameProcessor, - int inputExternalTexId, - int intermediateTexId, - int frameBuffer, - int outputWidth, - int outputHeight, - @Nullable EGLSurface debugPreviewEglSurface, - int debugPreviewWidth, - int debugPreviewHeight) { - this.singleThreadExecutorService = singleThreadExecutorService; - this.eglDisplay = eglDisplay; - this.eglContext = eglContext; - this.eglSurface = eglSurface; - this.externalCopyFrameProcessor = externalCopyFrameProcessor; - this.transformationFrameProcessor = transformationFrameProcessor; - this.inputExternalTexId = inputExternalTexId; - this.intermediateTexId = intermediateTexId; - this.frameBuffer = frameBuffer; - this.outputWidth = outputWidth; - this.outputHeight = outputHeight; - this.debugPreviewEglSurface = debugPreviewEglSurface; - this.debugPreviewWidth = debugPreviewWidth; - this.debugPreviewHeight = debugPreviewHeight; - - futures = new ConcurrentLinkedQueue<>(); - pendingFrameCount = new AtomicInteger(); - textureTransformMatrix = new float[16]; - } - - /** - * Creates the input {@link Surface} and configures it to process frames. - * - *

This method must not be called again after creating an input surface. - * - * @return The configured input {@link Surface}. - * @throws IllegalStateException If an input {@link Surface} has already been created. - */ - public Surface createInputSurface() { - checkState(inputSurface == null, "The input surface has already been created."); - inputSurfaceTexture = new SurfaceTexture(inputExternalTexId); - inputSurfaceTexture.setOnFrameAvailableListener( - surfaceTexture -> { - if (releaseRequested) { - // Frames can still become available after a transformation is cancelled but they can be - // ignored. - return; - } - try { - futures.add(singleThreadExecutorService.submit(this::processFrame)); - } catch (RejectedExecutionException e) { - if (!releaseRequested) { - throw e; - } - } - }); - inputSurface = new Surface(inputSurfaceTexture); - return inputSurface; - } - - /** - * Informs the frame editor that a frame will be queued to its input surface. - * - *

Should be called before rendering a frame to the frame editor's input surface. - * - * @throws IllegalStateException If called after {@link #signalEndOfInputStream()}. - */ - public void registerInputFrame() { - checkState(!inputStreamEnded); - pendingFrameCount.incrementAndGet(); - } - - /** - * Checks whether any exceptions occurred during asynchronous frame processing and rethrows the - * first exception encountered. - */ - public void getAndRethrowBackgroundExceptions() throws TransformationException { - @Nullable Future oldestGlProcessingFuture = futures.peek(); - while (oldestGlProcessingFuture != null && oldestGlProcessingFuture.isDone()) { - futures.poll(); - try { - oldestGlProcessingFuture.get(); - } catch (ExecutionException e) { - throw TransformationException.createForFrameEditor( - e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw TransformationException.createForFrameEditor( - e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED); - } - oldestGlProcessingFuture = futures.peek(); - } - } - - /** - * Returns whether there are input frames that have been {@link #registerInputFrame() registered} - * but not completely processed yet. - */ - public boolean hasPendingFrames() { - return pendingFrameCount.get() > 0; - } - - /** Returns whether all frames have been processed. */ - public boolean isEnded() { - return inputStreamEnded && !hasPendingFrames(); - } - - /** Informs the {@code FrameEditor} that no further input frames should be accepted. */ - public void signalEndOfInputStream() { - inputStreamEnded = true; - } - - /** - * Releases all resources. - * - *

If the frame editor is released before it has {@link #isEnded() ended}, it will attempt to - * cancel processing any input frames that have already become available. Input frames that become - * available after release are ignored. - */ - public void release() { - releaseRequested = true; - while (!futures.isEmpty()) { - checkNotNull(futures.poll()).cancel(/* mayInterruptIfRunning= */ true); - } - futures.add( - singleThreadExecutorService.submit( - () -> { - externalCopyFrameProcessor.release(); - transformationFrameProcessor.release(); - GlUtil.destroyEglContext(eglDisplay, eglContext); - })); - if (inputSurfaceTexture != null) { - inputSurfaceTexture.release(); - } - if (inputSurface != null) { - inputSurface.release(); - } - singleThreadExecutorService.shutdown(); - } - - /** Processes an input frame. */ - @RequiresNonNull("inputSurfaceTexture") - private void processFrame() { - inputSurfaceTexture.updateTexImage(); - inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); - long presentationTimeNs = inputSurfaceTexture.getTimestamp(); - - GlUtil.focusFramebuffer( - eglDisplay, eglContext, eglSurface, frameBuffer, outputWidth, outputHeight); - externalCopyFrameProcessor.setTextureTransformMatrix(textureTransformMatrix); - externalCopyFrameProcessor.updateProgramAndDraw(inputExternalTexId, presentationTimeNs); - - GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); - transformationFrameProcessor.updateProgramAndDraw(intermediateTexId, presentationTimeNs); - EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs); - EGL14.eglSwapBuffers(eglDisplay, eglSurface); - - if (debugPreviewEglSurface != null) { - GlUtil.focusEglSurface( - eglDisplay, eglContext, debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight); - GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0); - GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); - // The four-vertex triangle strip forms a quad. - GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); - EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface); - } - - checkState(pendingFrameCount.getAndDecrement() > 0); - } -} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java new file mode 100644 index 0000000000..fd96d9c5a6 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java @@ -0,0 +1,477 @@ +/* + * Copyright 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static com.google.common.collect.Iterables.getLast; + +import android.content.Context; +import android.graphics.SurfaceTexture; +import android.opengl.EGL14; +import android.opengl.EGLContext; +import android.opengl.EGLDisplay; +import android.opengl.EGLExt; +import android.opengl.EGLSurface; +import android.opengl.GLES20; +import android.util.Pair; +import android.util.Size; +import android.view.Surface; +import android.view.SurfaceView; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.Util; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicInteger; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; + +/** + * {@code FrameProcessorChain} applies changes to individual video frames. + * + *

Input becomes available on its {@link #getInputSurface() input surface} asynchronously and is + * processed on a background thread as it becomes available. All input frames should be {@link + * #registerInputFrame() registered} before they are rendered to the input surface. {@link + * #getPendingFrameCount()} can be used to check whether there are frames that have not been fully + * processed yet. Output is written to its {@link #configure(Surface, int, int, SurfaceView) output + * surface}. + */ +/* package */ final class FrameProcessorChain { + + static { + GlUtil.glAssertionsEnabled = true; + } + + private static final String THREAD_NAME = "Transformer:FrameProcessorChain"; + + private final boolean enableExperimentalHdrEditing; + private final EGLDisplay eglDisplay; + private final EGLContext eglContext; + /** Some OpenGL commands may block, so all OpenGL commands are run on a background thread. */ + private final ExecutorService singleThreadExecutorService; + /** The {@link #singleThreadExecutorService} thread. */ + private @MonotonicNonNull Thread glThread; + /** Futures corresponding to the executor service's pending tasks. */ + private final ConcurrentLinkedQueue> futures; + /** Number of frames {@link #registerInputFrame() registered} but not fully processed. */ + private final AtomicInteger pendingFrameCount; + /** Prevents further frame processing tasks from being scheduled after {@link #release()}. */ + private volatile boolean releaseRequested; + + private boolean inputStreamEnded; + /** Wraps the {@link #inputSurfaceTexture}. */ + private @MonotonicNonNull Surface inputSurface; + /** Associated with an OpenGL external texture. */ + private @MonotonicNonNull SurfaceTexture inputSurfaceTexture; + /** + * Identifier of the external texture the {@link ExternalCopyFrameProcessor} reads its input from. + */ + private int inputExternalTexId; + /** Transformation matrix associated with the {@link #inputSurfaceTexture}. */ + private final float[] textureTransformMatrix; + + private final ExternalCopyFrameProcessor externalCopyFrameProcessor; + private final List frameProcessors; + /** + * Identifiers of a framebuffer object associated with the intermediate textures that receive + * output from the previous {@link GlFrameProcessor}, and provide input for the following {@link + * GlFrameProcessor}. + * + *

The {@link ExternalCopyFrameProcessor} writes to the first framebuffer. + */ + private final int[] framebuffers; + /** The input {@link Size} of each of the {@code frameProcessors}. */ + private final ImmutableList inputSizes; + + private int outputWidth; + private int outputHeight; + /** + * Wraps the output {@link Surface} that is populated with the output of the final {@link + * GlFrameProcessor} for each frame. + */ + private @MonotonicNonNull EGLSurface eglSurface; + + private int debugPreviewWidth; + private int debugPreviewHeight; + /** + * Wraps a debug {@link SurfaceView} that is populated with the output of the final {@link + * GlFrameProcessor} for each frame. + */ + private @MonotonicNonNull EGLSurface debugPreviewEglSurface; + + /** + * Creates a new instance. + * + * @param context A {@link Context}. + * @param pixelWidthHeightRatio The ratio of width over height, for each pixel. + * @param inputWidth The input frame width, in pixels. + * @param inputHeight The input frame height, in pixels. + * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame. + * @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal. + * @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1. + */ + public FrameProcessorChain( + Context context, + float pixelWidthHeightRatio, + int inputWidth, + int inputHeight, + List frameProcessors, + boolean enableExperimentalHdrEditing) + throws TransformationException { + if (pixelWidthHeightRatio != 1.0f) { + // TODO(b/211782176): Consider implementing support for non-square pixels. + throw TransformationException.createForFrameProcessorChain( + new UnsupportedOperationException( + "Transformer's FrameProcessorChain currently does not support frame edits on" + + " non-square pixels. The pixelWidthHeightRatio is: " + + pixelWidthHeightRatio), + TransformationException.ERROR_CODE_GL_INIT_FAILED); + } + + this.enableExperimentalHdrEditing = enableExperimentalHdrEditing; + this.frameProcessors = ImmutableList.copyOf(frameProcessors); + + try { + eglDisplay = GlUtil.createEglDisplay(); + eglContext = + enableExperimentalHdrEditing + ? GlUtil.createEglContextEs3Rgba1010102(eglDisplay) + : GlUtil.createEglContext(eglDisplay); + } catch (GlUtil.GlException e) { + throw TransformationException.createForFrameProcessorChain( + e, TransformationException.ERROR_CODE_GL_INIT_FAILED); + } + singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); + futures = new ConcurrentLinkedQueue<>(); + pendingFrameCount = new AtomicInteger(); + textureTransformMatrix = new float[16]; + externalCopyFrameProcessor = + new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing); + framebuffers = new int[frameProcessors.size()]; + Pair, Size> sizes = + configureFrameProcessorSizes(inputWidth, inputHeight, frameProcessors); + inputSizes = sizes.first; + outputWidth = sizes.second.getWidth(); + outputHeight = sizes.second.getHeight(); + debugPreviewWidth = C.LENGTH_UNSET; + debugPreviewHeight = C.LENGTH_UNSET; + } + + /** Returns the output {@link Size}. */ + public Size getOutputSize() { + return new Size(outputWidth, outputHeight); + } + + /** + * Configures the {@code FrameProcessorChain} to process frames to the specified output targets. + * + *

This method may only be called once and may override the {@link + * GlFrameProcessor#configureOutputSize(int, int) output size} of the final {@link + * GlFrameProcessor}. + * + * @param outputSurface The output {@link Surface}. + * @param outputWidth The output width, in pixels. + * @param outputHeight The output height, in pixels. + * @param debugSurfaceView Optional debug {@link SurfaceView} to show output. + * @throws IllegalStateException If the {@code FrameProcessorChain} has already been configured. + * @throws TransformationException If reading shader files fails, or an OpenGL error occurs while + * creating and configuring the OpenGL components. + */ + public void configure( + Surface outputSurface, + int outputWidth, + int outputHeight, + @Nullable SurfaceView debugSurfaceView) + throws TransformationException { + checkState(inputSurface == null, "The FrameProcessorChain has already been configured."); + // TODO(b/218488308): Don't override output size for encoder fallback. Instead allow the final + // GlFrameProcessor to be re-configured or append another GlFrameProcessor. + this.outputWidth = outputWidth; + this.outputHeight = outputHeight; + + if (debugSurfaceView != null) { + debugPreviewWidth = debugSurfaceView.getWidth(); + debugPreviewHeight = debugSurfaceView.getHeight(); + } + + try { + // Wait for task to finish to be able to use inputExternalTexId to create the SurfaceTexture. + singleThreadExecutorService + .submit( + () -> + createOpenGlObjectsAndInitializeFrameProcessors(outputSurface, debugSurfaceView)) + .get(); + } catch (ExecutionException e) { + throw TransformationException.createForFrameProcessorChain( + e, TransformationException.ERROR_CODE_GL_INIT_FAILED); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw TransformationException.createForFrameProcessorChain( + e, TransformationException.ERROR_CODE_GL_INIT_FAILED); + } + + inputSurfaceTexture = new SurfaceTexture(inputExternalTexId); + inputSurfaceTexture.setOnFrameAvailableListener( + surfaceTexture -> { + if (releaseRequested) { + // Frames can still become available after a transformation is cancelled but they can be + // ignored. + return; + } + try { + futures.add(singleThreadExecutorService.submit(this::processFrame)); + } catch (RejectedExecutionException e) { + if (!releaseRequested) { + throw e; + } + } + }); + inputSurface = new Surface(inputSurfaceTexture); + } + + /** + * Returns the input {@link Surface}. + * + *

The {@code FrameProcessorChain} must be {@link #configure(Surface, int, int, SurfaceView) + * configured}. + */ + public Surface getInputSurface() { + checkStateNotNull(inputSurface, "The FrameProcessorChain must be configured."); + return inputSurface; + } + + /** + * Informs the {@code FrameProcessorChain} that a frame will be queued to its input surface. + * + *

Should be called before rendering a frame to the frame processor chain's input surface. + * + * @throws IllegalStateException If called after {@link #signalEndOfInputStream()}. + */ + public void registerInputFrame() { + checkState(!inputStreamEnded); + pendingFrameCount.incrementAndGet(); + } + + /** + * Checks whether any exceptions occurred during asynchronous frame processing and rethrows the + * first exception encountered. + */ + public void getAndRethrowBackgroundExceptions() throws TransformationException { + @Nullable Future oldestGlProcessingFuture = futures.peek(); + while (oldestGlProcessingFuture != null && oldestGlProcessingFuture.isDone()) { + futures.poll(); + try { + oldestGlProcessingFuture.get(); + } catch (ExecutionException e) { + throw TransformationException.createForFrameProcessorChain( + e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw TransformationException.createForFrameProcessorChain( + e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED); + } + oldestGlProcessingFuture = futures.peek(); + } + } + + /** + * Returns the number of input frames that have been {@link #registerInputFrame() registered} but + * not completely processed yet. + */ + public int getPendingFrameCount() { + return pendingFrameCount.get(); + } + + /** Returns whether all frames have been processed. */ + public boolean isEnded() { + return inputStreamEnded && getPendingFrameCount() == 0; + } + + /** Informs the {@code FrameProcessorChain} that no further input frames should be accepted. */ + public void signalEndOfInputStream() { + inputStreamEnded = true; + } + + /** + * Releases all resources. + * + *

If the frame processor chain is released before it has {@link #isEnded() ended}, it will + * attempt to cancel processing any input frames that have already become available. Input frames + * that become available after release are ignored. + */ + public void release() { + releaseRequested = true; + while (!futures.isEmpty()) { + checkNotNull(futures.poll()).cancel(/* mayInterruptIfRunning= */ true); + } + futures.add( + singleThreadExecutorService.submit( + () -> { + externalCopyFrameProcessor.release(); + for (int i = 0; i < frameProcessors.size(); i++) { + frameProcessors.get(i).release(); + } + GlUtil.destroyEglContext(eglDisplay, eglContext); + })); + if (inputSurfaceTexture != null) { + inputSurfaceTexture.release(); + } + if (inputSurface != null) { + inputSurface.release(); + } + singleThreadExecutorService.shutdown(); + } + + /** + * Creates the OpenGL textures, framebuffers, surfaces, and initializes the {@link + * GlFrameProcessor GlFrameProcessors}. + * + *

This method must by executed on the same thread as {@link #processFrame()}, i.e., executed + * by the {@link #singleThreadExecutorService}. + */ + @EnsuresNonNull("eglSurface") + private Void createOpenGlObjectsAndInitializeFrameProcessors( + Surface outputSurface, @Nullable SurfaceView debugSurfaceView) throws IOException { + glThread = Thread.currentThread(); + if (enableExperimentalHdrEditing) { + // TODO(b/209404935): Don't assume BT.2020 PQ input/output. + eglSurface = GlUtil.getEglSurfaceBt2020Pq(eglDisplay, outputSurface); + if (debugSurfaceView != null) { + debugPreviewEglSurface = + GlUtil.getEglSurfaceBt2020Pq(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); + } + } else { + eglSurface = GlUtil.getEglSurface(eglDisplay, outputSurface); + if (debugSurfaceView != null) { + debugPreviewEglSurface = + GlUtil.getEglSurface(eglDisplay, checkNotNull(debugSurfaceView.getHolder())); + } + } + GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); + + inputExternalTexId = GlUtil.createExternalTexture(); + Size inputSize = inputSizes.get(0); + externalCopyFrameProcessor.configureOutputSize(inputSize.getWidth(), inputSize.getHeight()); + externalCopyFrameProcessor.initialize(inputExternalTexId); + + for (int i = 0; i < frameProcessors.size(); i++) { + inputSize = inputSizes.get(i); + int inputTexId = GlUtil.createTexture(inputSize.getWidth(), inputSize.getHeight()); + framebuffers[i] = GlUtil.createFboForTexture(inputTexId); + frameProcessors.get(i).initialize(inputTexId); + } + // Return something because only Callables not Runnables can throw checked exceptions. + return null; + } + + /** + * Processes an input frame. + * + *

This method must by executed on the same thread as {@link + * #createOpenGlObjectsAndInitializeFrameProcessors(Surface,SurfaceView)}, i.e., executed by the + * {@link #singleThreadExecutorService}. + */ + @RequiresNonNull({"inputSurfaceTexture", "eglSurface"}) + private void processFrame() { + checkState(Thread.currentThread().equals(glThread)); + + Size outputSize = inputSizes.get(0); + if (frameProcessors.isEmpty()) { + GlUtil.focusEglSurface( + eglDisplay, eglContext, eglSurface, outputSize.getWidth(), outputSize.getHeight()); + } else { + GlUtil.focusFramebuffer( + eglDisplay, + eglContext, + eglSurface, + framebuffers[0], + outputSize.getWidth(), + outputSize.getHeight()); + } + inputSurfaceTexture.updateTexImage(); + inputSurfaceTexture.getTransformMatrix(textureTransformMatrix); + externalCopyFrameProcessor.setTextureTransformMatrix(textureTransformMatrix); + long presentationTimeNs = inputSurfaceTexture.getTimestamp(); + externalCopyFrameProcessor.updateProgramAndDraw(presentationTimeNs); + + for (int i = 0; i < frameProcessors.size() - 1; i++) { + outputSize = inputSizes.get(i + 1); + GlUtil.focusFramebuffer( + eglDisplay, + eglContext, + eglSurface, + framebuffers[i + 1], + outputSize.getWidth(), + outputSize.getHeight()); + frameProcessors.get(i).updateProgramAndDraw(presentationTimeNs); + } + if (!frameProcessors.isEmpty()) { + GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight); + getLast(frameProcessors).updateProgramAndDraw(presentationTimeNs); + } + + EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs); + EGL14.eglSwapBuffers(eglDisplay, eglSurface); + + if (debugPreviewEglSurface != null) { + GlUtil.focusEglSurface( + eglDisplay, eglContext, debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight); + GLES20.glClearColor(/* red= */ 0, /* green= */ 0, /* blue= */ 0, /* alpha= */ 0); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + // The four-vertex triangle strip forms a quad. + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, /* first= */ 0, /* count= */ 4); + EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface); + } + + checkState(pendingFrameCount.getAndDecrement() > 0); + } + + /** + * Configures the input and output {@link Size sizes} of a list of {@link GlFrameProcessor + * GlFrameProcessors}. + * + * @param inputWidth The width of frames passed to the first {@link GlFrameProcessor}, in pixels. + * @param inputHeight The height of frames passed to the first {@link GlFrameProcessor}, in + * pixels. + * @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors}. + * @return The input {@link Size} of each {@link GlFrameProcessor} and the output {@link Size} of + * the final {@link GlFrameProcessor}. + */ + private static Pair, Size> configureFrameProcessorSizes( + int inputWidth, int inputHeight, List frameProcessors) { + Size size = new Size(inputWidth, inputHeight); + if (frameProcessors.isEmpty()) { + return Pair.create(ImmutableList.of(size), size); + } + + ImmutableList.Builder inputSizes = new ImmutableList.Builder<>(); + for (int i = 0; i < frameProcessors.size(); i++) { + inputSizes.add(size); + size = frameProcessors.get(i).configureOutputSize(size.getWidth(), size.getHeight()); + } + return Pair.create(inputSizes.build(), size); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java index 18be0193e4..9bf254965a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/GlFrameProcessor.java @@ -15,13 +15,36 @@ */ package androidx.media3.transformer; +import android.util.Size; +import androidx.media3.common.util.UnstableApi; import java.io.IOException; -/** Manages a GLSL shader program for processing a frame. */ -/* package */ interface GlFrameProcessor { +/** + * Manages a GLSL shader program for processing a frame. + * + *

Methods must be called in the following order: + * + *

    + *
  1. The constructor, for implementation-specific arguments. + *
  2. {@link #configureOutputSize(int, int)}, to configure based on input dimensions. + *
  3. {@link #initialize(int)}, to set up graphics initialization. + *
  4. {@link #updateProgramAndDraw(long)}, to process one frame. + *
  5. {@link #release()}, upon conclusion of processing. + *
+ */ +@UnstableApi +public interface GlFrameProcessor { + // TODO(b/213313666): Investigate whether all configuration can be moved to initialize by + // using a placeholder surface until the encoder surface is known. If so, convert + // configureOutputSize to a simple getter. - // TODO(b/214975934): Add getOutputDimensions(inputWidth, inputHeight) and move output dimension - // calculations out of the VideoTranscodingSamplePipeline into the frame processors. + /** + * Returns the output {@link Size} of frames processed through {@link + * #updateProgramAndDraw(long)}. + * + *

This method must be called before {@link #initialize(int)} and does not use OpenGL. + */ + Size configureOutputSize(int inputWidth, int inputHeight); /** * Does any initialization necessary such as loading and compiling a GLSL shader programs. @@ -29,18 +52,17 @@ import java.io.IOException; *

This method may only be called after creating the OpenGL context and focusing a render * target. */ - void initialize() throws IOException; + void initialize(int inputTexId) throws IOException; /** * Updates the shader program's vertex attributes and uniforms, binds them, and draws. * - *

The frame processor must be {@link #initialize() initialized}. The caller is responsible for - * focussing the correct render target before calling this method. + *

The frame processor must be {@link #initialize(int) initialized}. The caller is responsible + * for focussing the correct render target before calling this method. * - * @param inputTexId The identifier of an OpenGL texture that the fragment shader can sample from. * @param presentationTimeNs The presentation timestamp of the current frame, in nanoseconds. */ - void updateProgramAndDraw(int inputTexId, long presentationTimeNs); + void updateProgramAndDraw(long presentationTimeNs); /** Releases all resources. */ void release(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java new file mode 100644 index 0000000000..f6941f2e41 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/PresentationFrameProcessor.java @@ -0,0 +1,165 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Assertions.checkStateNotNull; + +import android.content.Context; +import android.graphics.Matrix; +import android.util.Size; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.UnstableApi; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** Controls how a frame is viewed, by changing resolution. */ +// TODO(b/213190310): Implement crop, aspect ratio changes, etc. +@UnstableApi +public final class PresentationFrameProcessor implements GlFrameProcessor { + + /** A builder for {@link PresentationFrameProcessor} instances. */ + public static final class Builder { + // Mandatory field. + private final Context context; + + // Optional field. + private int outputHeight; + + /** + * Creates a builder with default values. + * + * @param context The {@link Context}. + */ + public Builder(Context context) { + this.context = context; + outputHeight = C.LENGTH_UNSET; + } + + /** + * Sets the output resolution using the output height. + * + *

The default value {@link C#LENGTH_UNSET} corresponds to using the same height as the + * input. Output width of the displayed frame will scale to preserve the frame's aspect ratio + * after other transformations. + * + *

For example, a 1920x1440 frame can be scaled to 640x480 by calling setResolution(480). + * + * @param outputHeight The output height of the displayed frame, in pixels. + * @return This builder. + */ + public Builder setResolution(int outputHeight) { + this.outputHeight = outputHeight; + return this; + } + + public PresentationFrameProcessor build() { + return new PresentationFrameProcessor(context, outputHeight); + } + } + + static { + GlUtil.glAssertionsEnabled = true; + } + + private final Context context; + private final int requestedHeight; + + private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor; + private int inputWidth; + private int inputHeight; + private int outputRotationDegrees; + private @MonotonicNonNull Matrix transformationMatrix; + + /** + * Creates a new instance. + * + * @param context The {@link Context}. + * @param requestedHeight The height of the output frame, in pixels. + */ + private PresentationFrameProcessor(Context context, int requestedHeight) { + this.context = context; + this.requestedHeight = requestedHeight; + + inputWidth = C.LENGTH_UNSET; + inputHeight = C.LENGTH_UNSET; + outputRotationDegrees = C.LENGTH_UNSET; + } + + /** + * Returns {@link Format#rotationDegrees} for the output frame. + * + *

Return values may be {@code 0} or {@code 90} degrees. + * + *

This method can only be called after {@link #configureOutputSize(int, int)}. + */ + public int getOutputRotationDegrees() { + checkState(outputRotationDegrees != C.LENGTH_UNSET); + return outputRotationDegrees; + } + + @Override + public Size configureOutputSize(int inputWidth, int inputHeight) { + this.inputWidth = inputWidth; + this.inputHeight = inputHeight; + transformationMatrix = new Matrix(); + + int displayWidth = inputWidth; + int displayHeight = inputHeight; + + // Scale width and height to desired requestedHeight, preserving aspect ratio. + if (requestedHeight != C.LENGTH_UNSET && requestedHeight != displayHeight) { + displayWidth = Math.round((float) requestedHeight * displayWidth / displayHeight); + displayHeight = requestedHeight; + } + + // Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded + // frame before encoding, so the encoded frame's width >= height, and set + // outputRotationDegrees to ensure the frame is displayed in the correct orientation. + if (displayHeight > displayWidth) { + outputRotationDegrees = 90; + // TODO(b/201293185): After fragment shader transformations are implemented, put + // postRotate in a later GlFrameProcessor. + transformationMatrix.postRotate(outputRotationDegrees); + return new Size(displayHeight, displayWidth); + } else { + outputRotationDegrees = 0; + return new Size(displayWidth, displayHeight); + } + } + + @Override + public void initialize(int inputTexId) throws IOException { + checkStateNotNull(transformationMatrix); + advancedFrameProcessor = new AdvancedFrameProcessor(context, transformationMatrix); + advancedFrameProcessor.configureOutputSize(inputWidth, inputHeight); + advancedFrameProcessor.initialize(inputTexId); + } + + @Override + public void updateProgramAndDraw(long presentationTimeNs) { + checkStateNotNull(advancedFrameProcessor).updateProgramAndDraw(presentationTimeNs); + } + + @Override + public void release() { + if (advancedFrameProcessor != null) { + advancedFrameProcessor.release(); + } + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java index b2aa387111..63ec1e8c75 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java @@ -40,8 +40,8 @@ import androidx.media3.decoder.DecoderInputBuffer; void queueInputBuffer() throws TransformationException; /** - * Processes the input data and returns whether more data can be processed by calling this method - * again. + * Processes the input data and returns whether it may be possible to process more data by calling + * this method again. */ boolean processData() throws TransformationException; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java new file mode 100644 index 0000000000..0875e706c4 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ScaleToFitFrameProcessor.java @@ -0,0 +1,189 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static java.lang.Math.max; +import static java.lang.Math.min; + +import android.content.Context; +import android.graphics.Matrix; +import android.util.Size; +import androidx.media3.common.C; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.UnstableApi; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Applies a simple rotation and/or scale in the vertex shader. All input frames' pixels will be + * preserved, potentially changing the width and height of the frame by scaling dimensions to fit. + * The background color will default to black. + */ +@UnstableApi +public final class ScaleToFitFrameProcessor implements GlFrameProcessor { + + /** A builder for {@link ScaleToFitFrameProcessor} instances. */ + public static final class Builder { + // Mandatory field. + private final Context context; + + // Optional fields. + private float scaleX; + private float scaleY; + private float rotationDegrees; + + /** + * Creates a builder with default values. + * + * @param context The {@link Context}. + */ + public Builder(Context context) { + this.context = context; + + scaleX = 1; + scaleY = 1; + rotationDegrees = 0; + } + + /** + * Sets the x and y axis scaling factors to apply to each frame's width and height. + * + *

The values default to 1, which corresponds to not scaling along both axes. + * + * @param scaleX The multiplier by which the frame will scale horizontally, along the x-axis. + * @param scaleY The multiplier by which the frame will scale vertically, along the y-axis. + * @return This builder. + */ + public Builder setScale(float scaleX, float scaleY) { + this.scaleX = scaleX; + this.scaleY = scaleY; + return this; + } + + /** + * Sets the counterclockwise rotation degrees. + * + *

The default value, 0, corresponds to not applying any rotation. + * + * @param rotationDegrees The counterclockwise rotation, in degrees. + * @return This builder. + */ + public Builder setRotationDegrees(float rotationDegrees) { + this.rotationDegrees = rotationDegrees; + return this; + } + + public ScaleToFitFrameProcessor build() { + return new ScaleToFitFrameProcessor(context, scaleX, scaleY, rotationDegrees); + } + } + + static { + GlUtil.glAssertionsEnabled = true; + } + + private final Context context; + private final Matrix transformationMatrix; + + private @MonotonicNonNull AdvancedFrameProcessor advancedFrameProcessor; + private int inputWidth; + private int inputHeight; + private @MonotonicNonNull Matrix adjustedTransformationMatrix; + + /** + * Creates a new instance. + * + * @param context The {@link Context}. + * @param scaleX The multiplier by which the frame will scale horizontally, along the x-axis. + * @param scaleY The multiplier by which the frame will scale vertically, along the y-axis. + * @param rotationDegrees How much to rotate the frame counterclockwise, in degrees. + */ + private ScaleToFitFrameProcessor( + Context context, float scaleX, float scaleY, float rotationDegrees) { + + this.context = context; + this.transformationMatrix = new Matrix(); + this.transformationMatrix.postScale(scaleX, scaleY); + this.transformationMatrix.postRotate(rotationDegrees); + + inputWidth = C.LENGTH_UNSET; + inputHeight = C.LENGTH_UNSET; + } + + @Override + public Size configureOutputSize(int inputWidth, int inputHeight) { + this.inputWidth = inputWidth; + this.inputHeight = inputHeight; + adjustedTransformationMatrix = new Matrix(transformationMatrix); + + if (transformationMatrix.isIdentity()) { + return new Size(inputWidth, inputHeight); + } + + float inputAspectRatio = (float) inputWidth / inputHeight; + // Scale frames by inputAspectRatio, to account for OpenGL's normalized device + // coordinates (NDC) (a square from -1 to 1 for both x and y) and preserve rectangular + // display of input pixels during transformations (ex. rotations). With scaling, + // transformationMatrix operations operate on a rectangle for x from -inputAspectRatio to + // inputAspectRatio, and y from -1 to 1. + adjustedTransformationMatrix.preScale(/* sx= */ inputAspectRatio, /* sy= */ 1f); + adjustedTransformationMatrix.postScale(/* sx= */ 1f / inputAspectRatio, /* sy= */ 1f); + + // Modify transformationMatrix to keep input pixels. + float[][] transformOnNdcPoints = {{-1, -1, 0, 1}, {-1, 1, 0, 1}, {1, -1, 0, 1}, {1, 1, 0, 1}}; + float xMin = Float.MAX_VALUE; + float xMax = Float.MIN_VALUE; + float yMin = Float.MAX_VALUE; + float yMax = Float.MIN_VALUE; + for (float[] transformOnNdcPoint : transformOnNdcPoints) { + adjustedTransformationMatrix.mapPoints(transformOnNdcPoint); + xMin = min(xMin, transformOnNdcPoint[0]); + xMax = max(xMax, transformOnNdcPoint[0]); + yMin = min(yMin, transformOnNdcPoint[1]); + yMax = max(yMax, transformOnNdcPoint[1]); + } + + float ndcWidthAndHeight = 2f; // Length from -1 to 1. + float xScale = (xMax - xMin) / ndcWidthAndHeight; + float yScale = (yMax - yMin) / ndcWidthAndHeight; + adjustedTransformationMatrix.postScale(1f / xScale, 1f / yScale); + + int outputWidth = Math.round(inputWidth * xScale); + int outputHeight = Math.round(inputHeight * yScale); + return new Size(outputWidth, outputHeight); + } + + @Override + public void initialize(int inputTexId) throws IOException { + checkStateNotNull(adjustedTransformationMatrix); + advancedFrameProcessor = new AdvancedFrameProcessor(context, adjustedTransformationMatrix); + advancedFrameProcessor.configureOutputSize(inputWidth, inputHeight); + advancedFrameProcessor.initialize(inputTexId); + } + + @Override + public void updateProgramAndDraw(long presentationTimeNs) { + checkStateNotNull(advancedFrameProcessor).updateProgramAndDraw(presentationTimeNs); + } + + @Override + public void release() { + if (advancedFrameProcessor != null) { + advancedFrameProcessor.release(); + } + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java index bf4bfa7ebb..8361d1d7d0 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java @@ -274,15 +274,15 @@ public final class TransformationException extends Exception { } /** - * Creates an instance for a {@link FrameEditor} related exception. + * Creates an instance for a {@link FrameProcessorChain} related exception. * * @param cause The cause of the failure. * @param errorCode See {@link #errorCode}. * @return The created instance. */ - /* package */ static TransformationException createForFrameEditor( + /* package */ static TransformationException createForFrameProcessorChain( Throwable cause, int errorCode) { - return new TransformationException("FrameEditor error", cause, errorCode); + return new TransformationException("FrameProcessorChain error", cause, errorCode); } /** diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java index 7d5b08604f..94c3454ddd 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java @@ -18,7 +18,6 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkArgument; -import android.graphics.Matrix; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.MimeTypes; @@ -34,8 +33,10 @@ public final class TransformationRequest { /** A builder for {@link TransformationRequest} instances. */ public static final class Builder { - private Matrix transformationMatrix; private boolean flattenForSlowMotion; + private float scaleX; + private float scaleY; + private float rotationDegrees; private int outputHeight; @Nullable private String audioMimeType; @Nullable private String videoMimeType; @@ -48,13 +49,17 @@ public final class TransformationRequest { * {@link TransformationRequest}. */ public Builder() { - transformationMatrix = new Matrix(); + scaleX = 1; + scaleY = 1; + rotationDegrees = 0; outputHeight = C.LENGTH_UNSET; } private Builder(TransformationRequest transformationRequest) { - this.transformationMatrix = new Matrix(transformationRequest.transformationMatrix); this.flattenForSlowMotion = transformationRequest.flattenForSlowMotion; + this.scaleX = transformationRequest.scaleX; + this.scaleY = transformationRequest.scaleY; + this.rotationDegrees = transformationRequest.rotationDegrees; this.outputHeight = transformationRequest.outputHeight; this.audioMimeType = transformationRequest.audioMimeType; this.videoMimeType = transformationRequest.videoMimeType; @@ -62,32 +67,11 @@ public final class TransformationRequest { } /** - * Sets the transformation matrix. The default value is to apply no change. + * Sets whether the input should be flattened for media containing slow motion markers. * - *

This can be used to perform operations supported by {@link Matrix}, like scaling and - * rotating the video. - * - *

The video dimensions will be on the x axis, from -aspectRatio to aspectRatio, and on the y - * axis, from -1 to 1. - * - *

For now, resolution will not be affected by this method. - * - * @param transformationMatrix The transformation to apply to video frames. - * @return This builder. - */ - public Builder setTransformationMatrix(Matrix transformationMatrix) { - // TODO(b/201293185): Implement an AdvancedFrameEditor to handle translation, as the current - // transformationMatrix is automatically adjusted to focus on the original pixels and - // effectively undo translations. - this.transformationMatrix = new Matrix(transformationMatrix); - return this; - } - - /** - * Sets whether the input should be flattened for media containing slow motion markers. The - * transformed output is obtained by removing the slow motion metadata and by actually slowing - * down the parts of the video and audio streams defined in this metadata. The default value for - * {@code flattenForSlowMotion} is {@code false}. + *

The transformed output is obtained by removing the slow motion metadata and by actually + * slowing down the parts of the video and audio streams defined in this metadata. The default + * value for {@code flattenForSlowMotion} is {@code false}. * *

Only Samsung Extension Format (SEF) slow motion metadata type is supported. The * transformation has no effect if the input does not contain this metadata type. @@ -114,9 +98,41 @@ public final class TransformationRequest { } /** - * Sets the output resolution using the output height. The default value {@link C#LENGTH_UNSET} - * corresponds to using the same height as the input. Output width of the displayed video will - * scale to preserve the video's aspect ratio after other transformations. + * Sets the x and y axis scaling factors to apply to each frame's width and height, stretching + * the video along these axes appropriately. + * + *

The values default to 1, which corresponds to not scaling along both axes. + * + * @param scaleX The multiplier by which the frame will scale horizontally, along the x-axis. + * @param scaleY The multiplier by which the frame will scale vertically, along the y-axis. + * @return This builder. + */ + public Builder setScale(float scaleX, float scaleY) { + this.scaleX = scaleX; + this.scaleY = scaleY; + return this; + } + + /** + * Sets the rotation, in degrees, counterclockwise, to apply to each frame, automatically + * adjusting the frame's width and height to preserve all input pixels. + * + *

The default value, 0, corresponds to not applying any rotation. + * + * @param rotationDegrees The counterclockwise rotation, in degrees. + * @return This builder. + */ + public Builder setRotationDegrees(float rotationDegrees) { + this.rotationDegrees = rotationDegrees; + return this; + } + + /** + * Sets the output resolution using the output height. + * + *

The default value {@link C#LENGTH_UNSET} corresponds to using the same height as the + * input. Output width of the displayed video will scale to preserve the video's aspect ratio + * after other transformations. * *

For example, a 1920x1440 video can be scaled to 640x480 by calling setResolution(480). * @@ -124,14 +140,15 @@ public final class TransformationRequest { * @return This builder. */ public Builder setResolution(int outputHeight) { - // TODO(b/201293185): Restructure to input a Presentation class. this.outputHeight = outputHeight; return this; } /** - * Sets the video MIME type of the output. The default value is {@code null} which corresponds - * to using the same MIME type as the input. Supported MIME types are: + * Sets the video MIME type of the output. + * + *

The default value is {@code null} which corresponds to using the same MIME type as the + * input. Supported MIME types are: * *

    *
  • {@link MimeTypes#VIDEO_H263} @@ -154,8 +171,10 @@ public final class TransformationRequest { } /** - * Sets the audio MIME type of the output. The default value is {@code null} which corresponds - * to using the same MIME type as the input. Supported MIME types are: + * Sets the audio MIME type of the output. + * + *

    The default value is {@code null} which corresponds to using the same MIME type as the + * input. Supported MIME types are: * *

      *
    • {@link MimeTypes#AUDIO_AAC} @@ -196,8 +215,10 @@ public final class TransformationRequest { /** Builds a {@link TransformationRequest} instance. */ public TransformationRequest build() { return new TransformationRequest( - transformationMatrix, flattenForSlowMotion, + scaleX, + scaleY, + rotationDegrees, outputHeight, audioMimeType, videoMimeType, @@ -205,18 +226,32 @@ public final class TransformationRequest { } } - /** - * A {@link Matrix transformation matrix} to apply to video frames. - * - * @see Builder#setTransformationMatrix(Matrix) - */ - public final Matrix transformationMatrix; /** * Whether the input should be flattened for media containing slow motion markers. * * @see Builder#setFlattenForSlowMotion(boolean) */ public final boolean flattenForSlowMotion; + /** + * The requested scale factor, on the x-axis, of the output video, or 1 if inferred from the + * input. + * + * @see Builder#setScale(float, float) + */ + public final float scaleX; + /** + * The requested scale factor, on the y-axis, of the output video, or 1 if inferred from the + * input. + * + * @see Builder#setScale(float, float) + */ + public final float scaleY; + /** + * The requested rotation, in degrees, of the output video, or 0 if inferred from the input. + * + * @see Builder#setRotationDegrees(float) + */ + public final float rotationDegrees; /** * The requested height of the output video, or {@link C#LENGTH_UNSET} if inferred from the input. * @@ -245,14 +280,18 @@ public final class TransformationRequest { public final boolean enableHdrEditing; private TransformationRequest( - Matrix transformationMatrix, boolean flattenForSlowMotion, + float scaleX, + float scaleY, + float rotationDegrees, int outputHeight, @Nullable String audioMimeType, @Nullable String videoMimeType, boolean enableHdrEditing) { - this.transformationMatrix = transformationMatrix; this.flattenForSlowMotion = flattenForSlowMotion; + this.scaleX = scaleX; + this.scaleY = scaleY; + this.rotationDegrees = rotationDegrees; this.outputHeight = outputHeight; this.audioMimeType = audioMimeType; this.videoMimeType = videoMimeType; @@ -268,8 +307,10 @@ public final class TransformationRequest { return false; } TransformationRequest that = (TransformationRequest) o; - return transformationMatrix.equals(that.transformationMatrix) - && flattenForSlowMotion == that.flattenForSlowMotion + return flattenForSlowMotion == that.flattenForSlowMotion + && scaleX == that.scaleX + && scaleY == that.scaleY + && rotationDegrees == that.rotationDegrees && outputHeight == that.outputHeight && Util.areEqual(audioMimeType, that.audioMimeType) && Util.areEqual(videoMimeType, that.videoMimeType) @@ -278,8 +319,10 @@ public final class TransformationRequest { @Override public int hashCode() { - int result = transformationMatrix.hashCode(); - result = 31 * result + (flattenForSlowMotion ? 1 : 0); + int result = (flattenForSlowMotion ? 1 : 0); + result = 31 * result + Float.floatToIntBits(scaleX); + result = 31 * result + Float.floatToIntBits(scaleY); + result = 31 * result + Float.floatToIntBits(rotationDegrees); result = 31 * result + outputHeight; result = 31 * result + (audioMimeType != null ? audioMimeType.hashCode() : 0); result = 31 * result + (videoMimeType != null ? videoMimeType.hashCode() : 0); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 20511ec634..f642399d17 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -59,11 +59,13 @@ import androidx.media3.exoplayer.trackselection.DefaultTrackSelector; import androidx.media3.exoplayer.video.VideoRendererEventListener; import androidx.media3.extractor.DefaultExtractorsFactory; import androidx.media3.extractor.mp4.Mp4Extractor; +import com.google.common.collect.ImmutableList; import java.io.IOException; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** @@ -102,6 +104,7 @@ public final class Transformer { private boolean removeVideo; private String containerMimeType; private TransformationRequest transformationRequest; + private ImmutableList frameProcessors; private ListenerSet listeners; private DebugViewProvider debugViewProvider; private Looper looper; @@ -121,6 +124,7 @@ public final class Transformer { debugViewProvider = DebugViewProvider.NONE; containerMimeType = MimeTypes.VIDEO_MP4; transformationRequest = new TransformationRequest.Builder().build(); + frameProcessors = ImmutableList.of(); } /** @@ -137,7 +141,8 @@ public final class Transformer { encoderFactory = Codec.EncoderFactory.DEFAULT; debugViewProvider = DebugViewProvider.NONE; containerMimeType = MimeTypes.VIDEO_MP4; - this.transformationRequest = new TransformationRequest.Builder().build(); + transformationRequest = new TransformationRequest.Builder().build(); + frameProcessors = ImmutableList.of(); } /** Creates a builder with the values of the provided {@link Transformer}. */ @@ -149,6 +154,7 @@ public final class Transformer { this.removeVideo = transformer.removeVideo; this.containerMimeType = transformer.containerMimeType; this.transformationRequest = transformer.transformationRequest; + this.frameProcessors = transformer.frameProcessors; this.listeners = transformer.listeners; this.looper = transformer.looper; this.encoderFactory = transformer.encoderFactory; @@ -168,6 +174,10 @@ public final class Transformer { /** * Sets the {@link TransformationRequest} which configures the editing and transcoding options. * + *

      Actual applied values may differ, per device capabilities. {@link + * Listener#onFallbackApplied(MediaItem, TransformationRequest, TransformationRequest)} will be + * invoked with the actual applied values. + * * @param transformationRequest The {@link TransformationRequest}. * @return This builder. */ @@ -177,9 +187,28 @@ public final class Transformer { } /** - * Sets the {@link MediaSource.Factory} to be used to retrieve the inputs to transform. The - * default value is a {@link DefaultMediaSourceFactory} built with the context provided in - * {@link #Builder(Context) the constructor}. + * Sets the {@linkplain GlFrameProcessor frame processors} to apply to each frame. + * + *

      The {@linkplain GlFrameProcessor frame processors} are applied before any {@linkplain + * TransformationRequest.Builder#setScale(float, float) scale}, {@linkplain + * TransformationRequest.Builder#setRotationDegrees(float) rotation}, or {@linkplain + * TransformationRequest.Builder#setResolution(int) resolution} changes specified in the {@link + * #setTransformationRequest(TransformationRequest) TransformationRequest} but after {@linkplain + * TransformationRequest.Builder#setFlattenForSlowMotion(boolean) slow-motion flattening}. + * + * @param frameProcessors The {@linkplain GlFrameProcessor frame processors}. + * @return This builder. + */ + public Builder setFrameProcessors(List frameProcessors) { + this.frameProcessors = ImmutableList.copyOf(frameProcessors); + return this; + } + + /** + * Sets the {@link MediaSource.Factory} to be used to retrieve the inputs to transform. + * + *

      The default value is a {@link DefaultMediaSourceFactory} built with the context provided + * in {@link #Builder(Context) the constructor}. * * @param mediaSourceFactory A {@link MediaSource.Factory}. * @return This builder. @@ -190,7 +219,9 @@ public final class Transformer { } /** - * Sets whether to remove the audio from the output. The default value is {@code false}. + * Sets whether to remove the audio from the output. + * + *

      The default value is {@code false}. * *

      The audio and video cannot both be removed because the output would not contain any * samples. @@ -204,7 +235,9 @@ public final class Transformer { } /** - * Sets whether to remove the video from the output. The default value is {@code false}. + * Sets whether to remove the video from the output. + * + *

      The default value is {@code false}. * *

      The audio and video cannot both be removed because the output would not contain any * samples. @@ -289,9 +322,10 @@ public final class Transformer { /** * Sets the {@link Looper} that must be used for all calls to the transformer and that is used - * to call listeners on. The default value is the Looper of the thread that this builder was - * created on, or if that thread does not have a Looper, the Looper of the application's main - * thread. + * to call listeners on. + * + *

      The default value is the Looper of the thread that this builder was created on, or if that + * thread does not have a Looper, the Looper of the application's main thread. * * @param looper A {@link Looper}. * @return This builder. @@ -303,8 +337,9 @@ public final class Transformer { } /** - * Sets the {@link Codec.EncoderFactory} that will be used by the transformer. The default value - * is {@link Codec.EncoderFactory#DEFAULT}. + * Sets the {@link Codec.EncoderFactory} that will be used by the transformer. + * + *

      The default value is {@link Codec.EncoderFactory#DEFAULT}. * * @param encoderFactory The {@link Codec.EncoderFactory} instance. * @return This builder. @@ -316,8 +351,10 @@ public final class Transformer { /** * Sets a provider for views to show diagnostic information (if available) during - * transformation. This is intended for debugging. The default value is {@link - * DebugViewProvider#NONE}, which doesn't show any debug info. + * transformation. + * + *

      This is intended for debugging. The default value is {@link DebugViewProvider#NONE}, which + * doesn't show any debug info. * *

      Not all transformations will result in debug views being populated. * @@ -330,8 +367,9 @@ public final class Transformer { } /** - * Sets the {@link Clock} that will be used by the transformer. The default value is {@link - * Clock#DEFAULT}. + * Sets the {@link Clock} that will be used by the transformer. + * + *

      The default value is {@link Clock#DEFAULT}. * * @param clock The {@link Clock} instance. * @return This builder. @@ -344,8 +382,9 @@ public final class Transformer { } /** - * Sets the factory for muxers that write the media container. The default value is a {@link - * FrameworkMuxer.Factory}. + * Sets the factory for muxers that write the media container. + * + *

      The default value is a {@link FrameworkMuxer.Factory}. * * @param muxerFactory A {@link Muxer.Factory}. * @return This builder. @@ -393,6 +432,7 @@ public final class Transformer { removeVideo, containerMimeType, transformationRequest, + frameProcessors, listeners, looper, clock, @@ -514,6 +554,7 @@ public final class Transformer { private final boolean removeVideo; private final String containerMimeType; private final TransformationRequest transformationRequest; + private final ImmutableList frameProcessors; private final Looper looper; private final Clock clock; private final Codec.EncoderFactory encoderFactory; @@ -534,6 +575,7 @@ public final class Transformer { boolean removeVideo, String containerMimeType, TransformationRequest transformationRequest, + ImmutableList frameProcessors, ListenerSet listeners, Looper looper, Clock clock, @@ -548,6 +590,7 @@ public final class Transformer { this.removeVideo = removeVideo; this.containerMimeType = containerMimeType; this.transformationRequest = transformationRequest; + this.frameProcessors = frameProcessors; this.listeners = listeners; this.looper = looper; this.clock = clock; @@ -688,6 +731,7 @@ public final class Transformer { removeAudio, removeVideo, transformationRequest, + frameProcessors, encoderFactory, decoderFactory, new FallbackListener(mediaItem, listeners, transformationRequest), @@ -799,6 +843,7 @@ public final class Transformer { private final boolean removeAudio; private final boolean removeVideo; private final TransformationRequest transformationRequest; + private final ImmutableList frameProcessors; private final Codec.EncoderFactory encoderFactory; private final Codec.DecoderFactory decoderFactory; private final FallbackListener fallbackListener; @@ -810,6 +855,7 @@ public final class Transformer { boolean removeAudio, boolean removeVideo, TransformationRequest transformationRequest, + ImmutableList frameProcessors, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, FallbackListener fallbackListener, @@ -819,6 +865,7 @@ public final class Transformer { this.removeAudio = removeAudio; this.removeVideo = removeVideo; this.transformationRequest = transformationRequest; + this.frameProcessors = frameProcessors; this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; this.fallbackListener = fallbackListener; @@ -854,6 +901,7 @@ public final class Transformer { muxerWrapper, mediaClock, transformationRequest, + frameProcessors, encoderFactory, decoderFactory, fallbackListener, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java index bcb5541d73..b3042a922b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java @@ -25,6 +25,7 @@ import androidx.media3.common.Format; import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.exoplayer.FormatHolder; import androidx.media3.exoplayer.source.SampleStream.ReadDataResult; +import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -34,6 +35,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private static final String TAG = "TVideoRenderer"; private final Context context; + private final ImmutableList frameProcessors; private final Codec.EncoderFactory encoderFactory; private final Codec.DecoderFactory decoderFactory; private final Transformer.DebugViewProvider debugViewProvider; @@ -46,12 +48,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; MuxerWrapper muxerWrapper, TransformerMediaClock mediaClock, TransformationRequest transformationRequest, + ImmutableList frameProcessors, Codec.EncoderFactory encoderFactory, Codec.DecoderFactory decoderFactory, FallbackListener fallbackListener, Transformer.DebugViewProvider debugViewProvider) { super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener); this.context = context; + this.frameProcessors = frameProcessors; this.encoderFactory = encoderFactory; this.decoderFactory = decoderFactory; this.debugViewProvider = debugViewProvider; @@ -86,6 +90,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; context, inputFormat, transformationRequest, + frameProcessors, decoderFactory, encoderFactory, muxerWrapper.getSupportedSampleMimeTypes(getTrackType()), @@ -113,14 +118,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) { return false; } + if (transformationRequest.rotationDegrees != 0f) { + return false; + } + if (transformationRequest.scaleX != 1f) { + return false; + } + if (transformationRequest.scaleY != 1f) { + return false; + } if (transformationRequest.outputHeight != C.LENGTH_UNSET && transformationRequest.outputHeight != inputFormat.height) { return false; } - if (!transformationRequest.transformationMatrix.isIdentity()) { - // TODO(b/201293185, b/214010296): Move FrameProcessor transformationMatrix calculation / - // adjustments out of the VideoTranscodingSamplePipeline, so that we can skip transcoding when - // adjustments result in identity matrices. + if (!frameProcessors.isEmpty()) { return false; } return true; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoEncoderSettings.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoEncoderSettings.java new file mode 100644 index 0000000000..f00fc7f433 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoEncoderSettings.java @@ -0,0 +1,245 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkArgument; +import static java.lang.annotation.ElementType.TYPE_USE; + +import android.annotation.SuppressLint; +import android.media.MediaCodecInfo; +import androidx.annotation.IntDef; +import androidx.annotation.Nullable; +import androidx.media3.common.Format; +import androidx.media3.common.util.UnstableApi; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** Represents the video encoder settings. */ +@UnstableApi +public final class VideoEncoderSettings { + + /** A value for various fields to indicate that the field's value is unknown or not applicable. */ + public static final int NO_VALUE = Format.NO_VALUE; + /** The default encoding color profile. */ + public static final int DEFAULT_COLOR_PROFILE = + MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; + /** The default I-frame interval in seconds. */ + public static final float DEFAULT_I_FRAME_INTERVAL_SECONDS = 1.0f; + + /** A default {@link VideoEncoderSettings}. */ + public static final VideoEncoderSettings DEFAULT = new Builder().build(); + + /** + * The allowed values for {@code bitrateMode}, one of + * + *

        + *
      • Constant quality: {@link MediaCodecInfo.EncoderCapabilities#BITRATE_MODE_CQ}. + *
      • Variable bitrate: {@link MediaCodecInfo.EncoderCapabilities#BITRATE_MODE_VBR}. + *
      • Constant bitrate: {@link MediaCodecInfo.EncoderCapabilities#BITRATE_MODE_CBR}. + *
      • Constant bitrate with frame drops: {@link + * MediaCodecInfo.EncoderCapabilities#BITRATE_MODE_CBR_FD}, available from API31. + *
      + */ + @SuppressLint("InlinedApi") + @Documented + @Retention(RetentionPolicy.SOURCE) + @Target(TYPE_USE) + @IntDef({ + MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CQ, + MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR, + MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR, + MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR_FD + }) + public @interface BitrateMode {} + + /** Builds {@link VideoEncoderSettings} instances. */ + public static final class Builder { + private int bitrate; + private @BitrateMode int bitrateMode; + private int profile; + private int level; + private int colorProfile; + private float iFrameIntervalSeconds; + + /** Creates a new instance. */ + public Builder() { + this.bitrate = NO_VALUE; + this.bitrateMode = MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR; + this.profile = NO_VALUE; + this.level = NO_VALUE; + this.colorProfile = DEFAULT_COLOR_PROFILE; + this.iFrameIntervalSeconds = DEFAULT_I_FRAME_INTERVAL_SECONDS; + } + + private Builder(VideoEncoderSettings videoEncoderSettings) { + this.bitrate = videoEncoderSettings.bitrate; + this.bitrateMode = videoEncoderSettings.bitrateMode; + this.profile = videoEncoderSettings.profile; + this.level = videoEncoderSettings.level; + this.colorProfile = videoEncoderSettings.colorProfile; + this.iFrameIntervalSeconds = videoEncoderSettings.iFrameIntervalSeconds; + } + + /** + * Sets {@link VideoEncoderSettings#bitrate}. The default value is {@link #NO_VALUE}. + * + * @param bitrate The {@link VideoEncoderSettings#bitrate}. + * @return This builder. + */ + public Builder setBitrate(int bitrate) { + this.bitrate = bitrate; + return this; + } + + /** + * Sets {@link VideoEncoderSettings#bitrateMode}. The default value is {@code + * MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR}. + * + *

      Only {@link MediaCodecInfo.EncoderCapabilities#BITRATE_MODE_VBR} and {@link + * MediaCodecInfo.EncoderCapabilities#BITRATE_MODE_CBR} are allowed. + * + * @param bitrateMode The {@link VideoEncoderSettings#bitrateMode}. + * @return This builder. + */ + public Builder setBitrateMode(@BitrateMode int bitrateMode) { + checkArgument( + bitrateMode == MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR + || bitrateMode == MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR); + this.bitrateMode = bitrateMode; + return this; + } + + /** + * Sets {@link VideoEncoderSettings#profile} and {@link VideoEncoderSettings#level}. The default + * values are both {@link #NO_VALUE}. + * + *

      The value must be one of the values defined in {@link MediaCodecInfo.CodecProfileLevel}, + * or {@link #NO_VALUE}. + * + *

      Profile and level settings will be ignored when using {@link DefaultEncoderFactory} and + * encoding to H264. + * + * @param encodingProfile The {@link VideoEncoderSettings#profile}. + * @param encodingLevel The {@link VideoEncoderSettings#level}. + * @return This builder. + */ + public Builder setEncodingProfileLevel(int encodingProfile, int encodingLevel) { + this.profile = encodingProfile; + this.level = encodingLevel; + return this; + } + + /** + * Sets {@link VideoEncoderSettings#colorProfile}. The default value is {@link + * #DEFAULT_COLOR_PROFILE}. + * + *

      The value must be one of the {@code COLOR_*} constants defined in {@link + * MediaCodecInfo.CodecCapabilities}. + * + * @param colorProfile The {@link VideoEncoderSettings#colorProfile}. + * @return This builder. + */ + public Builder setColorProfile(int colorProfile) { + this.colorProfile = colorProfile; + return this; + } + + /** + * Sets {@link VideoEncoderSettings#iFrameIntervalSeconds}. The default value is {@link + * #DEFAULT_I_FRAME_INTERVAL_SECONDS}. + * + * @param iFrameIntervalSeconds The {@link VideoEncoderSettings#iFrameIntervalSeconds}. + * @return This builder. + */ + public Builder setiFrameIntervalSeconds(float iFrameIntervalSeconds) { + this.iFrameIntervalSeconds = iFrameIntervalSeconds; + return this; + } + + /** Builds the instance. */ + public VideoEncoderSettings build() { + return new VideoEncoderSettings( + bitrate, bitrateMode, profile, level, colorProfile, iFrameIntervalSeconds); + } + } + + /** The encoding bitrate. */ + public final int bitrate; + /** One of {@link BitrateMode the allowed modes}. */ + public final @BitrateMode int bitrateMode; + /** The encoding profile. */ + public final int profile; + /** The encoding level. */ + public final int level; + /** The encoding color profile. */ + public final int colorProfile; + /** The encoding I-Frame interval in seconds. */ + public final float iFrameIntervalSeconds; + + private VideoEncoderSettings( + int bitrate, + int bitrateMode, + int profile, + int level, + int colorProfile, + float iFrameIntervalSeconds) { + this.bitrate = bitrate; + this.bitrateMode = bitrateMode; + this.profile = profile; + this.level = level; + this.colorProfile = colorProfile; + this.iFrameIntervalSeconds = iFrameIntervalSeconds; + } + + /** + * Returns a {@link VideoEncoderSettings.Builder} initialized with the values of this instance. + */ + public Builder buildUpon() { + return new Builder(this); + } + + @Override + public boolean equals(@Nullable Object o) { + if (this == o) { + return true; + } + if (!(o instanceof VideoEncoderSettings)) { + return false; + } + VideoEncoderSettings that = (VideoEncoderSettings) o; + return bitrate == that.bitrate + && bitrateMode == that.bitrateMode + && profile == that.profile + && level == that.level + && colorProfile == that.colorProfile + && iFrameIntervalSeconds == that.iFrameIntervalSeconds; + } + + @Override + public int hashCode() { + int result = 7; + result = 31 * result + bitrate; + result = 31 * result + bitrateMode; + result = 31 * result + profile; + result = 31 * result + level; + result = 31 * result + colorProfile; + result = 31 * result + Float.floatToIntBits(iFrameIntervalSeconds); + return result; + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 070223d934..5b4da873c7 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -17,20 +17,15 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Util.SDK_INT; -import static java.lang.Math.max; -import static java.lang.Math.min; import android.content.Context; -import android.graphics.Matrix; import android.media.MediaCodec; -import android.media.MediaFormat; +import android.util.Size; import androidx.annotation.Nullable; -import androidx.annotation.RequiresApi; -import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; +import com.google.common.collect.ImmutableList; import java.util.List; import org.checkerframework.dataflow.qual.Pure; @@ -39,11 +34,14 @@ import org.checkerframework.dataflow.qual.Pure; */ /* package */ final class VideoTranscodingSamplePipeline implements SamplePipeline { + private static final int FRAME_COUNT_UNLIMITED = -1; + private final int outputRotationDegrees; private final DecoderInputBuffer decoderInputBuffer; private final Codec decoder; + private final int maxPendingFrameCount; - @Nullable private final FrameEditor frameEditor; + private final FrameProcessorChain frameProcessorChain; private final Codec encoder; private final DecoderInputBuffer encoderOutputBuffer; @@ -54,6 +52,7 @@ import org.checkerframework.dataflow.qual.Pure; Context context, Format inputFormat, TransformationRequest transformationRequest, + ImmutableList frameProcessors, Codec.DecoderFactory decoderFactory, Codec.EncoderFactory encoderFactory, List allowedOutputMimeTypes, @@ -70,114 +69,63 @@ import org.checkerframework.dataflow.qual.Pure; (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.width : inputFormat.height; int decodedHeight = (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; - float decodedAspectRatio = (float) decodedWidth / decodedHeight; - Matrix transformationMatrix = new Matrix(transformationRequest.transformationMatrix); - - int outputWidth = decodedWidth; - int outputHeight = decodedHeight; - if (!transformationMatrix.isIdentity()) { - // Scale frames by decodedAspectRatio, to account for FrameEditor's normalized device - // coordinates (NDC) (a square from -1 to 1 for both x and y) and preserve rectangular display - // of input pixels during transformations (ex. rotations). With scaling, transformationMatrix - // operations operate on a rectangle for x from -decodedAspectRatio to decodedAspectRatio, and - // y from -1 to 1. - transformationMatrix.preScale(/* sx= */ decodedAspectRatio, /* sy= */ 1f); - transformationMatrix.postScale(/* sx= */ 1f / decodedAspectRatio, /* sy= */ 1f); - - float[][] transformOnNdcPoints = {{-1, -1, 0, 1}, {-1, 1, 0, 1}, {1, -1, 0, 1}, {1, 1, 0, 1}}; - float xMin = Float.MAX_VALUE; - float xMax = Float.MIN_VALUE; - float yMin = Float.MAX_VALUE; - float yMax = Float.MIN_VALUE; - for (float[] transformOnNdcPoint : transformOnNdcPoints) { - transformationMatrix.mapPoints(transformOnNdcPoint); - xMin = min(xMin, transformOnNdcPoint[0]); - xMax = max(xMax, transformOnNdcPoint[0]); - yMin = min(yMin, transformOnNdcPoint[1]); - yMax = max(yMax, transformOnNdcPoint[1]); - } - - float xCenter = (xMax + xMin) / 2f; - float yCenter = (yMax + yMin) / 2f; - transformationMatrix.postTranslate(-xCenter, -yCenter); - - float ndcWidthAndHeight = 2f; // Length from -1 to 1. - float xScale = (xMax - xMin) / ndcWidthAndHeight; - float yScale = (yMax - yMin) / ndcWidthAndHeight; - transformationMatrix.postScale(1f / xScale, 1f / yScale); - outputWidth = Math.round(decodedWidth * xScale); - outputHeight = Math.round(decodedHeight * yScale); - } - // Scale width and height to desired transformationRequest.outputHeight, preserving - // aspect ratio. - if (transformationRequest.outputHeight != C.LENGTH_UNSET - && transformationRequest.outputHeight != outputHeight) { - outputWidth = - Math.round((float) transformationRequest.outputHeight * outputWidth / outputHeight); - outputHeight = transformationRequest.outputHeight; - } - - // Encoders commonly support higher maximum widths than maximum heights. Rotate the decoded - // video before encoding, so the encoded video's width >= height, and set outputRotationDegrees - // to ensure the video is displayed in the correct orientation. - int requestedEncoderWidth; - int requestedEncoderHeight; - boolean swapEncodingDimensions = outputHeight > outputWidth; - if (swapEncodingDimensions) { - outputRotationDegrees = 90; - requestedEncoderWidth = outputHeight; - requestedEncoderHeight = outputWidth; - // TODO(b/201293185): After fragment shader transformations are implemented, put - // postRotate in a later vertex shader. - transformationMatrix.postRotate(outputRotationDegrees); - } else { - outputRotationDegrees = 0; - requestedEncoderWidth = outputWidth; - requestedEncoderHeight = outputHeight; - } + // TODO(b/213190310): Don't create a ScaleToFitFrameProcessor if scale and rotation are unset. + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor.Builder(context) + .setScale(transformationRequest.scaleX, transformationRequest.scaleY) + .setRotationDegrees(transformationRequest.rotationDegrees) + .build(); + PresentationFrameProcessor presentationFrameProcessor = + new PresentationFrameProcessor.Builder(context) + .setResolution(transformationRequest.outputHeight) + .build(); + frameProcessorChain = + new FrameProcessorChain( + context, + inputFormat.pixelWidthHeightRatio, + /* inputWidth= */ decodedWidth, + /* inputHeight= */ decodedHeight, + new ImmutableList.Builder() + .addAll(frameProcessors) + .add(scaleToFitFrameProcessor) + .add(presentationFrameProcessor) + .build(), + transformationRequest.enableHdrEditing); + Size requestedEncoderSize = frameProcessorChain.getOutputSize(); + outputRotationDegrees = presentationFrameProcessor.getOutputRotationDegrees(); Format requestedEncoderFormat = new Format.Builder() - .setWidth(requestedEncoderWidth) - .setHeight(requestedEncoderHeight) + .setWidth(requestedEncoderSize.getWidth()) + .setHeight(requestedEncoderSize.getHeight()) .setRotationDegrees(0) + .setFrameRate(inputFormat.frameRate) .setSampleMimeType( transformationRequest.videoMimeType != null ? transformationRequest.videoMimeType : inputFormat.sampleMimeType) .build(); + encoder = encoderFactory.createForVideoEncoding(requestedEncoderFormat, allowedOutputMimeTypes); Format encoderSupportedFormat = encoder.getConfigurationFormat(); fallbackListener.onTransformationRequestFinalized( createFallbackTransformationRequest( transformationRequest, - /* resolutionIsHeight= */ !swapEncodingDimensions, + /* hasOutputFormatRotation= */ outputRotationDegrees == 0, requestedEncoderFormat, encoderSupportedFormat)); - if (transformationRequest.enableHdrEditing - || inputFormat.height != encoderSupportedFormat.height - || inputFormat.width != encoderSupportedFormat.width - || !transformationMatrix.isIdentity()) { - frameEditor = - FrameEditor.create( - context, - encoderSupportedFormat.width, - encoderSupportedFormat.height, - inputFormat.pixelWidthHeightRatio, - new TransformationFrameProcessor(context, transformationMatrix), - /* outputSurface= */ encoder.getInputSurface(), - transformationRequest.enableHdrEditing, - debugViewProvider); - } else { - frameEditor = null; - } + frameProcessorChain.configure( + /* outputSurface= */ encoder.getInputSurface(), + /* outputWidth= */ encoderSupportedFormat.width, + /* outputHeight= */ encoderSupportedFormat.height, + debugViewProvider.getDebugPreviewSurfaceView( + encoderSupportedFormat.width, encoderSupportedFormat.height)); decoder = - decoderFactory.createForVideoDecoding( - inputFormat, - frameEditor == null ? encoder.getInputSurface() : frameEditor.createInputSurface()); + decoderFactory.createForVideoDecoding(inputFormat, frameProcessorChain.getInputSurface()); + maxPendingFrameCount = getMaxPendingFrameCount(); } @Override @@ -193,71 +141,27 @@ import org.checkerframework.dataflow.qual.Pure; @Override public boolean processData() throws TransformationException { - if (frameEditor != null) { - frameEditor.getAndRethrowBackgroundExceptions(); - if (frameEditor.isEnded()) { - if (!signaledEndOfStreamToEncoder) { - encoder.signalEndOfInputStream(); - signaledEndOfStreamToEncoder = true; - } - return false; - } - } - if (decoder.isEnded()) { - return false; - } - - boolean canProcessMoreDataImmediately = false; - if (SDK_INT >= 29 - && !(("samsung".equals(Util.MANUFACTURER) || "OnePlus".equals(Util.MANUFACTURER)) - && SDK_INT < 31)) { - // TODO(b/213455700): Fix Samsung and OnePlus devices filling the decoder in processDataV29(). - processDataV29(); - } else { - canProcessMoreDataImmediately = processDataDefault(); - } - if (decoder.isEnded()) { - if (frameEditor != null) { - frameEditor.signalEndOfInputStream(); - } else { + frameProcessorChain.getAndRethrowBackgroundExceptions(); + if (frameProcessorChain.isEnded()) { + if (!signaledEndOfStreamToEncoder) { encoder.signalEndOfInputStream(); signaledEndOfStreamToEncoder = true; - return false; } - } - return canProcessMoreDataImmediately; - } - - /** - * Processes input data from API 29. - * - *

      In this method the decoder could decode multiple frames in one invocation; as compared to - * {@link #processDataDefault()}, in which one frame is decoded in each invocation. Consequently, - * if {@link FrameEditor} processes frames slower than the decoder, decoded frames are queued up - * in the decoder's output surface. - * - *

      Prior to API 29, decoders may drop frames to keep their output surface from growing out of - * bound; while after API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame - * dropping even when the surface is full. As dropping random frames is not acceptable in {@code - * Transformer}, using this method requires API level 29 or higher. - */ - @RequiresApi(29) - private void processDataV29() throws TransformationException { - while (maybeProcessDecoderOutput()) {} - } - - /** - * Processes at most one input frame and returns whether a frame was processed. - * - *

      Only renders decoder output to the {@link FrameEditor}'s input surface if the {@link - * FrameEditor} has finished processing the previous frame. - */ - private boolean processDataDefault() throws TransformationException { - // TODO(b/214975934): Check whether this can be converted to a while-loop like processDataV29. - if (frameEditor != null && frameEditor.hasPendingFrames()) { return false; } - return maybeProcessDecoderOutput(); + if (decoder.isEnded()) { + return false; + } + + boolean processedData = false; + while (maybeProcessDecoderOutput()) { + processedData = true; + } + if (decoder.isEnded()) { + frameProcessorChain.signalEndOfInputStream(); + } + // If the decoder produced output, signal that it may be possible to process data again. + return processedData; } @Override @@ -294,30 +198,38 @@ import org.checkerframework.dataflow.qual.Pure; @Override public void release() { - if (frameEditor != null) { - frameEditor.release(); - } + frameProcessorChain.release(); decoder.release(); encoder.release(); } + /** + * Creates a fallback transformation request to execute, based on device-specific support. + * + * @param transformationRequest The requested transformation. + * @param hasOutputFormatRotation Whether the input video will be rotated to landscape during + * processing, with {@link Format#rotationDegrees} of 90 added to the output format. + * @param requestedFormat The requested format. + * @param supportedFormat A format supported by the device. + */ @Pure private static TransformationRequest createFallbackTransformationRequest( TransformationRequest transformationRequest, - boolean resolutionIsHeight, + boolean hasOutputFormatRotation, Format requestedFormat, - Format actualFormat) { + Format supportedFormat) { // TODO(b/210591626): Also update bitrate etc. once encoder configuration and fallback are // implemented. - if (Util.areEqual(requestedFormat.sampleMimeType, actualFormat.sampleMimeType) - && ((!resolutionIsHeight && requestedFormat.width == actualFormat.width) - || (resolutionIsHeight && requestedFormat.height == actualFormat.height))) { + if (Util.areEqual(requestedFormat.sampleMimeType, supportedFormat.sampleMimeType) + && (hasOutputFormatRotation + ? requestedFormat.width == supportedFormat.width + : requestedFormat.height == supportedFormat.height)) { return transformationRequest; } return transformationRequest .buildUpon() - .setVideoMimeType(actualFormat.sampleMimeType) - .setResolution(resolutionIsHeight ? requestedFormat.height : requestedFormat.width) + .setVideoMimeType(supportedFormat.sampleMimeType) + .setResolution(hasOutputFormatRotation ? requestedFormat.width : requestedFormat.height) .build(); } @@ -332,10 +244,40 @@ import org.checkerframework.dataflow.qual.Pure; return false; } - if (frameEditor != null) { - frameEditor.registerInputFrame(); + if (maxPendingFrameCount != FRAME_COUNT_UNLIMITED + && frameProcessorChain.getPendingFrameCount() == maxPendingFrameCount) { + return false; } + + frameProcessorChain.registerInputFrame(); decoder.releaseOutputBuffer(/* render= */ true); return true; } + + /** + * Returns the maximum number of frames that may be pending in the output {@link + * FrameProcessorChain} at a time, or {@link #FRAME_COUNT_UNLIMITED} if it's not necessary to + * enforce a limit. + */ + private static int getMaxPendingFrameCount() { + if (Util.SDK_INT < 29) { + // Prior to API 29, decoders may drop frames to keep their output surface from growing out of + // bounds, while from API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame + // dropping even when the surface is full. We never want frame dropping so allow a maximum of + // one frame to be pending at a time. + // TODO(b/226330223): Investigate increasing this limit. + return 1; + } + if (Util.SDK_INT < 31 + && ("OnePlus".equals(Util.MANUFACTURER) || "samsung".equals(Util.MANUFACTURER))) { + // Some OMX decoders don't correctly track their number of output buffers available, and get + // stuck if too many frames are rendered without being processed, so we limit the number of + // pending frames to avoid getting stuck. This value is experimentally determined. See also + // b/213455700. + return 10; + } + // Otherwise don't limit the number of frames that can be pending at a time, to maximize + // throughput. + return FRAME_COUNT_UNLIMITED; + } } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/AdvancedFrameProcessorTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/AdvancedFrameProcessorTest.java new file mode 100644 index 0000000000..802c4edabe --- /dev/null +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/AdvancedFrameProcessorTest.java @@ -0,0 +1,64 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.common.truth.Truth.assertThat; + +import android.graphics.Matrix; +import android.util.Size; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Unit tests for {@link AdvancedFrameProcessor}. + * + *

      See {@link AdvancedFrameProcessorPixelTest} for pixel tests testing {@link + * AdvancedFrameProcessor} given a transformation matrix. + */ +@RunWith(AndroidJUnit4.class) +public final class AdvancedFrameProcessorTest { + @Test + public void getOutputDimensions_withIdentityMatrix_leavesDimensionsUnchanged() { + Matrix identityMatrix = new Matrix(); + int inputWidth = 200; + int inputHeight = 150; + AdvancedFrameProcessor advancedFrameProcessor = + new AdvancedFrameProcessor(getApplicationContext(), identityMatrix); + + Size outputSize = advancedFrameProcessor.configureOutputSize(inputWidth, inputHeight); + + assertThat(outputSize.getWidth()).isEqualTo(inputWidth); + assertThat(outputSize.getHeight()).isEqualTo(inputHeight); + } + + @Test + public void getOutputDimensions_withTransformationMatrix_leavesDimensionsUnchanged() { + Matrix transformationMatrix = new Matrix(); + transformationMatrix.postRotate(/* degrees= */ 90); + transformationMatrix.postScale(/* sx= */ .5f, /* sy= */ 1.2f); + int inputWidth = 200; + int inputHeight = 150; + AdvancedFrameProcessor advancedFrameProcessor = + new AdvancedFrameProcessor(getApplicationContext(), transformationMatrix); + + Size outputSize = advancedFrameProcessor.configureOutputSize(inputWidth, inputHeight); + + assertThat(outputSize.getWidth()).isEqualTo(inputWidth); + assertThat(outputSize.getHeight()).isEqualTo(inputHeight); + } +} diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java new file mode 100644 index 0000000000..0051978bdc --- /dev/null +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/DefaultEncoderFactoryTest.java @@ -0,0 +1,149 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import android.media.MediaCodecInfo; +import android.media.MediaFormat; +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.shadows.MediaCodecInfoBuilder; +import org.robolectric.shadows.ShadowMediaCodecList; + +/** Unit test for {@link DefaultEncoderFactory}. */ +@RunWith(AndroidJUnit4.class) +public class DefaultEncoderFactoryTest { + + @Before + public void setUp() { + MediaFormat avcFormat = new MediaFormat(); + avcFormat.setString(MediaFormat.KEY_MIME, MediaFormat.MIMETYPE_VIDEO_AVC); + MediaCodecInfo.CodecProfileLevel profileLevel = new MediaCodecInfo.CodecProfileLevel(); + profileLevel.profile = MediaCodecInfo.CodecProfileLevel.AVCProfileHigh; + // Using Level4 gives us 8192 16x16 blocks. If using width 1920 uses 120 blocks, 8192 / 120 = 68 + // blocks will be left for encoding height 1088. + profileLevel.level = MediaCodecInfo.CodecProfileLevel.AVCLevel4; + + ShadowMediaCodecList.addCodec( + MediaCodecInfoBuilder.newBuilder() + .setName("test.transformer.avc.encoder") + .setIsEncoder(true) + .setCapabilities( + MediaCodecInfoBuilder.CodecCapabilitiesBuilder.newBuilder() + .setMediaFormat(avcFormat) + .setIsEncoder(true) + .setColorFormats( + new int[] {MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible}) + .setProfileLevels(new MediaCodecInfo.CodecProfileLevel[] {profileLevel}) + .build()) + .build()); + } + + @Test + public void createForVideoEncoding_withFallbackOnAndSupportedInputFormat_configuresEncoder() + throws Exception { + Format requestedVideoFormat = createVideoFormat(MimeTypes.VIDEO_H264, 1920, 1080, 30); + Format actualVideoFormat = + new DefaultEncoderFactory() + .createForVideoEncoding( + requestedVideoFormat, + /* allowedMimeTypes= */ ImmutableList.of(MimeTypes.VIDEO_H264)) + .getConfigurationFormat(); + + assertThat(actualVideoFormat.sampleMimeType).isEqualTo(MimeTypes.VIDEO_H264); + assertThat(actualVideoFormat.width).isEqualTo(1920); + assertThat(actualVideoFormat.height).isEqualTo(1080); + // 1920 * 1080 * 30 * 0.1 + assertThat(actualVideoFormat.averageBitrate).isEqualTo(6_220_800); + } + + @Test + public void createForVideoEncoding_withFallbackOnAndUnsupportedMimeType_configuresEncoder() + throws Exception { + Format requestedVideoFormat = createVideoFormat(MimeTypes.VIDEO_H265, 1920, 1080, 30); + Format actualVideoFormat = + new DefaultEncoderFactory() + .createForVideoEncoding( + requestedVideoFormat, + /* allowedMimeTypes= */ ImmutableList.of(MimeTypes.VIDEO_H264)) + .getConfigurationFormat(); + + assertThat(actualVideoFormat.sampleMimeType).isEqualTo(MimeTypes.VIDEO_H264); + assertThat(actualVideoFormat.width).isEqualTo(1920); + assertThat(actualVideoFormat.height).isEqualTo(1080); + // 1920 * 1080 * 30 * 0.1 + assertThat(actualVideoFormat.averageBitrate).isEqualTo(6_220_800); + } + + @Test + public void createForVideoEncoding_withFallbackOnAndUnsupportedResolution_configuresEncoder() + throws Exception { + Format requestedVideoFormat = createVideoFormat(MimeTypes.VIDEO_H264, 3840, 2160, 60); + Format actualVideoFormat = + new DefaultEncoderFactory() + .createForVideoEncoding( + requestedVideoFormat, + /* allowedMimeTypes= */ ImmutableList.of(MimeTypes.VIDEO_H264)) + .getConfigurationFormat(); + + assertThat(actualVideoFormat.width).isEqualTo(1920); + assertThat(actualVideoFormat.height).isEqualTo(1080); + } + + @Test + public void createForVideoEncoding_withNoSupportedEncoder_throws() { + Format requestedVideoFormat = createVideoFormat(MimeTypes.VIDEO_H264, 1920, 1080, 30); + assertThrows( + TransformationException.class, + () -> + new DefaultEncoderFactory() + .createForVideoEncoding( + requestedVideoFormat, + /* allowedMimeTypes= */ ImmutableList.of(MimeTypes.VIDEO_H265))); + } + + @Test + public void createForVideoEncoding_withNoAvailableEncoderFromEncoderSelector_throws() { + Format requestedVideoFormat = createVideoFormat(MimeTypes.VIDEO_H264, 1920, 1080, 30); + assertThrows( + TransformationException.class, + () -> + new DefaultEncoderFactory( + /* videoEncoderSelector= */ mimeType -> ImmutableList.of(), + /* enableFallback= */ true) + .createForVideoEncoding( + requestedVideoFormat, + /* allowedMimeTypes= */ ImmutableList.of(MimeTypes.VIDEO_H264))); + } + + private static Format createVideoFormat(String mimeType, int width, int height, int frameRate) { + return new Format.Builder() + .setWidth(width) + .setHeight(height) + .setFrameRate(frameRate) + .setRotationDegrees(0) + .setSampleMimeType(mimeType) + .build(); + } +} diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/EncoderUtilTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/EncoderUtilTest.java index ca1de6f82f..4e6824f439 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/EncoderUtilTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/EncoderUtilTest.java @@ -20,7 +20,7 @@ import static com.google.common.truth.Truth.assertThat; import android.media.MediaCodecInfo; import android.media.MediaFormat; -import android.util.Pair; +import android.util.Size; import androidx.annotation.Nullable; import androidx.media3.common.MimeTypes; import androidx.test.ext.junit.runners.AndroidJUnit4; @@ -67,11 +67,12 @@ public class EncoderUtilTest { MediaCodecInfo encoderInfo = supportedEncoders.get(0); @Nullable - Pair closestSupportedResolution = + Size closestSupportedResolution = EncoderUtil.getSupportedResolution(encoderInfo, MIME_TYPE, 1920, 1080); assertThat(closestSupportedResolution).isNotNull(); - assertThat(closestSupportedResolution).isEqualTo(Pair.create(1920, 1080)); + assertThat(closestSupportedResolution.getWidth()).isEqualTo(1920); + assertThat(closestSupportedResolution.getHeight()).isEqualTo(1080); } @Test @@ -80,11 +81,12 @@ public class EncoderUtilTest { MediaCodecInfo encoderInfo = supportedEncoders.get(0); @Nullable - Pair closestSupportedResolution = + Size closestSupportedResolution = EncoderUtil.getSupportedResolution(encoderInfo, MIME_TYPE, 1919, 1081); assertThat(closestSupportedResolution).isNotNull(); - assertThat(closestSupportedResolution).isEqualTo(Pair.create(1920, 1080)); + assertThat(closestSupportedResolution.getWidth()).isEqualTo(1920); + assertThat(closestSupportedResolution.getHeight()).isEqualTo(1080); } @Test @@ -93,11 +95,12 @@ public class EncoderUtilTest { MediaCodecInfo encoderInfo = supportedEncoders.get(0); @Nullable - Pair closestSupportedResolution = + Size closestSupportedResolution = EncoderUtil.getSupportedResolution(encoderInfo, MIME_TYPE, 1920, 1920); assertThat(closestSupportedResolution).isNotNull(); - assertThat(closestSupportedResolution).isEqualTo(Pair.create(1440, 1440)); + assertThat(closestSupportedResolution.getWidth()).isEqualTo(1440); + assertThat(closestSupportedResolution.getHeight()).isEqualTo(1440); } @Test @@ -106,10 +109,11 @@ public class EncoderUtilTest { MediaCodecInfo encoderInfo = supportedEncoders.get(0); @Nullable - Pair closestSupportedResolution = + Size closestSupportedResolution = EncoderUtil.getSupportedResolution(encoderInfo, MIME_TYPE, 3840, 2160); assertThat(closestSupportedResolution).isNotNull(); - assertThat(closestSupportedResolution).isEqualTo(Pair.create(1920, 1080)); + assertThat(closestSupportedResolution.getWidth()).isEqualTo(1920); + assertThat(closestSupportedResolution.getHeight()).isEqualTo(1080); } } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java new file mode 100644 index 0000000000..8749225434 --- /dev/null +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/FrameProcessorChainTest.java @@ -0,0 +1,155 @@ +/* + * Copyright 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import android.content.Context; +import android.util.Size; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Robolectric tests for {@link FrameProcessorChain}. + * + *

      See {@code FrameProcessorChainPixelTest} in the androidTest directory for instrumentation + * tests. + */ +@RunWith(AndroidJUnit4.class) +public final class FrameProcessorChainTest { + + @Test + public void construct_withSupportedPixelWidthHeightRatio_completesSuccessfully() + throws TransformationException { + Context context = getApplicationContext(); + + new FrameProcessorChain( + context, + /* pixelWidthHeightRatio= */ 1, + /* inputWidth= */ 200, + /* inputHeight= */ 100, + /* frameProcessors= */ ImmutableList.of(), + /* enableExperimentalHdrEditing= */ false); + } + + @Test + public void construct_withUnsupportedPixelWidthHeightRatio_throwsException() { + Context context = getApplicationContext(); + + TransformationException exception = + assertThrows( + TransformationException.class, + () -> + new FrameProcessorChain( + context, + /* pixelWidthHeightRatio= */ 2, + /* inputWidth= */ 200, + /* inputHeight= */ 100, + /* frameProcessors= */ ImmutableList.of(), + /* enableExperimentalHdrEditing= */ false)); + + assertThat(exception).hasCauseThat().isInstanceOf(UnsupportedOperationException.class); + assertThat(exception).hasCauseThat().hasMessageThat().contains("pixelWidthHeightRatio"); + } + + @Test + public void getOutputSize_withoutFrameProcessors_returnsInputSize() + throws TransformationException { + Size inputSize = new Size(200, 100); + FrameProcessorChain frameProcessorChain = + createFrameProcessorChainWithFakeFrameProcessors( + inputSize, /* frameProcessorOutputSizes= */ ImmutableList.of()); + + Size outputSize = frameProcessorChain.getOutputSize(); + + assertThat(outputSize).isEqualTo(inputSize); + } + + @Test + public void getOutputSize_withOneFrameProcessor_returnsItsOutputSize() + throws TransformationException { + Size inputSize = new Size(200, 100); + Size frameProcessorOutputSize = new Size(300, 250); + FrameProcessorChain frameProcessorChain = + createFrameProcessorChainWithFakeFrameProcessors( + inputSize, /* frameProcessorOutputSizes= */ ImmutableList.of(frameProcessorOutputSize)); + + Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize(); + + assertThat(frameProcessorChainOutputSize).isEqualTo(frameProcessorOutputSize); + } + + @Test + public void getOutputSize_withThreeFrameProcessors_returnsLastOutputSize() + throws TransformationException { + Size inputSize = new Size(200, 100); + Size outputSize1 = new Size(300, 250); + Size outputSize2 = new Size(400, 244); + Size outputSize3 = new Size(150, 160); + FrameProcessorChain frameProcessorChain = + createFrameProcessorChainWithFakeFrameProcessors( + inputSize, + /* frameProcessorOutputSizes= */ ImmutableList.of( + outputSize1, outputSize2, outputSize3)); + + Size frameProcessorChainOutputSize = frameProcessorChain.getOutputSize(); + + assertThat(frameProcessorChainOutputSize).isEqualTo(outputSize3); + } + + private static FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors( + Size inputSize, List frameProcessorOutputSizes) throws TransformationException { + ImmutableList.Builder frameProcessors = new ImmutableList.Builder<>(); + for (Size element : frameProcessorOutputSizes) { + frameProcessors.add(new FakeFrameProcessor(element)); + } + return new FrameProcessorChain( + getApplicationContext(), + /* pixelWidthHeightRatio= */ 1, + inputSize.getWidth(), + inputSize.getHeight(), + frameProcessors.build(), + /* enableExperimentalHdrEditing= */ false); + } + + private static class FakeFrameProcessor implements GlFrameProcessor { + + private final Size outputSize; + + private FakeFrameProcessor(Size outputSize) { + this.outputSize = outputSize; + } + + @Override + public Size configureOutputSize(int inputWidth, int inputHeight) { + return outputSize; + } + + @Override + public void initialize(int inputTexId) {} + + @Override + public void updateProgramAndDraw(long presentationTimeNs) {} + + @Override + public void release() {} + } +} diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java new file mode 100644 index 0000000000..0703a2468f --- /dev/null +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/PresentationFrameProcessorTest.java @@ -0,0 +1,102 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import android.util.Size; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Unit tests for {@link PresentationFrameProcessor}. + * + *

      See {@code AdvancedFrameProcessorPixelTest} for pixel tests testing {@link + * AdvancedFrameProcessor} given a transformation matrix. + */ +@RunWith(AndroidJUnit4.class) +public final class PresentationFrameProcessorTest { + @Test + public void configureOutputSize_noEditsLandscape_leavesFramesUnchanged() { + int inputWidth = 200; + int inputHeight = 150; + PresentationFrameProcessor presentationFrameProcessor = + new PresentationFrameProcessor.Builder(getApplicationContext()).build(); + + Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); + + assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); + assertThat(outputSize.getWidth()).isEqualTo(inputWidth); + assertThat(outputSize.getHeight()).isEqualTo(inputHeight); + } + + @Test + public void configureOutputSize_noEditsSquare_leavesFramesUnchanged() { + int inputWidth = 150; + int inputHeight = 150; + PresentationFrameProcessor presentationFrameProcessor = + new PresentationFrameProcessor.Builder(getApplicationContext()).build(); + + Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); + + assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); + assertThat(outputSize.getWidth()).isEqualTo(inputWidth); + assertThat(outputSize.getHeight()).isEqualTo(inputHeight); + } + + @Test + public void configureOutputSize_noEditsPortrait_flipsOrientation() { + int inputWidth = 150; + int inputHeight = 200; + PresentationFrameProcessor presentationFrameProcessor = + new PresentationFrameProcessor.Builder(getApplicationContext()).build(); + + Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); + + assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(90); + assertThat(outputSize.getWidth()).isEqualTo(inputHeight); + assertThat(outputSize.getHeight()).isEqualTo(inputWidth); + } + + @Test + public void configureOutputSize_setResolution_changesDimensions() { + int inputWidth = 200; + int inputHeight = 150; + int requestedHeight = 300; + PresentationFrameProcessor presentationFrameProcessor = + new PresentationFrameProcessor.Builder(getApplicationContext()) + .setResolution(requestedHeight) + .build(); + + Size outputSize = presentationFrameProcessor.configureOutputSize(inputWidth, inputHeight); + + assertThat(presentationFrameProcessor.getOutputRotationDegrees()).isEqualTo(0); + assertThat(outputSize.getWidth()).isEqualTo(requestedHeight * inputWidth / inputHeight); + assertThat(outputSize.getHeight()).isEqualTo(requestedHeight); + } + + @Test + public void getOutputRotationDegreesBeforeConfigure_throwsIllegalStateException() { + PresentationFrameProcessor presentationFrameProcessor = + new PresentationFrameProcessor.Builder(getApplicationContext()).build(); + + // configureOutputSize not called before initialize. + assertThrows(IllegalStateException.class, presentationFrameProcessor::getOutputRotationDegrees); + } +} diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/ScaleToFitFrameProcessorTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/ScaleToFitFrameProcessorTest.java new file mode 100644 index 0000000000..6fd582b74f --- /dev/null +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/ScaleToFitFrameProcessorTest.java @@ -0,0 +1,135 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import android.util.Size; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Unit tests for {@link ScaleToFitFrameProcessor}. + * + *

      See {@code AdvancedFrameProcessorPixelTest} for pixel tests testing {@link + * AdvancedFrameProcessor} given a transformation matrix. + */ +@RunWith(AndroidJUnit4.class) +public final class ScaleToFitFrameProcessorTest { + + @Test + public void configureOutputSize_noEdits_leavesFramesUnchanged() { + int inputWidth = 200; + int inputHeight = 150; + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor.Builder(getApplicationContext()).build(); + + Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); + + assertThat(outputSize.getWidth()).isEqualTo(inputWidth); + assertThat(outputSize.getHeight()).isEqualTo(inputHeight); + } + + @Test + public void initializeBeforeConfigure_throwsIllegalStateException() { + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor.Builder(getApplicationContext()).build(); + + // configureOutputSize not called before initialize. + assertThrows( + IllegalStateException.class, + () -> scaleToFitFrameProcessor.initialize(/* inputTexId= */ 0)); + } + + @Test + public void configureOutputSize_scaleNarrow_decreasesWidth() { + int inputWidth = 200; + int inputHeight = 150; + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor.Builder(getApplicationContext()) + .setScale(/* scaleX= */ .5f, /* scaleY= */ 1f) + .build(); + + Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); + + assertThat(outputSize.getWidth()).isEqualTo(Math.round(inputWidth * .5f)); + assertThat(outputSize.getHeight()).isEqualTo(inputHeight); + } + + @Test + public void configureOutputSize_scaleWide_increasesWidth() { + int inputWidth = 200; + int inputHeight = 150; + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor.Builder(getApplicationContext()) + .setScale(/* scaleX= */ 2f, /* scaleY= */ 1f) + .build(); + + Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); + + assertThat(outputSize.getWidth()).isEqualTo(inputWidth * 2); + assertThat(outputSize.getHeight()).isEqualTo(inputHeight); + } + + @Test + public void configureOutputDimensions_scaleTall_increasesHeight() { + int inputWidth = 200; + int inputHeight = 150; + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor.Builder(getApplicationContext()) + .setScale(/* scaleX= */ 1f, /* scaleY= */ 2f) + .build(); + + Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); + + assertThat(outputSize.getWidth()).isEqualTo(inputWidth); + assertThat(outputSize.getHeight()).isEqualTo(inputHeight * 2); + } + + @Test + public void configureOutputSize_rotate90_swapsDimensions() { + int inputWidth = 200; + int inputHeight = 150; + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor.Builder(getApplicationContext()) + .setRotationDegrees(90) + .build(); + + Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); + + assertThat(outputSize.getWidth()).isEqualTo(inputHeight); + assertThat(outputSize.getHeight()).isEqualTo(inputWidth); + } + + @Test + public void configureOutputSize_rotate45_changesDimensions() { + int inputWidth = 200; + int inputHeight = 150; + ScaleToFitFrameProcessor scaleToFitFrameProcessor = + new ScaleToFitFrameProcessor.Builder(getApplicationContext()) + .setRotationDegrees(45) + .build(); + long expectedOutputWidthHeight = 247; + + Size outputSize = scaleToFitFrameProcessor.configureOutputSize(inputWidth, inputHeight); + + assertThat(outputSize.getWidth()).isEqualTo(expectedOutputWidthHeight); + assertThat(outputSize.getHeight()).isEqualTo(expectedOutputWidthHeight); + } +} diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformationRequestTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformationRequestTest.java index 848e5a0c73..ff90ef809b 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformationRequestTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformationRequestTest.java @@ -18,7 +18,6 @@ package androidx.media3.transformer; import static com.google.common.truth.Truth.assertThat; -import android.graphics.Matrix; import androidx.media3.common.MimeTypes; import androidx.test.ext.junit.runners.AndroidJUnit4; import org.junit.Test; @@ -35,15 +34,12 @@ public class TransformationRequestTest { } private static TransformationRequest createTestTransformationRequest() { - Matrix transformationMatrix = new Matrix(); - transformationMatrix.preRotate(36); - transformationMatrix.postTranslate((float) 0.5, (float) -0.2); - return new TransformationRequest.Builder() .setFlattenForSlowMotion(true) .setAudioMimeType(MimeTypes.AUDIO_AAC) .setVideoMimeType(MimeTypes.VIDEO_H264) - .setTransformationMatrix(transformationMatrix) + .setRotationDegrees(45) + .setScale(/* scaleX= */ 1f, /* scaleY= */ 2f) .build(); } } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java index 000f8c83ed..ef47ce3433 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -30,6 +30,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import android.content.Context; +import android.media.MediaCodecInfo; import android.media.MediaCrypto; import android.media.MediaFormat; import android.os.Handler; @@ -49,6 +50,7 @@ import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.primitives.Ints; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; @@ -64,7 +66,9 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.robolectric.shadows.MediaCodecInfoBuilder; import org.robolectric.shadows.ShadowMediaCodec; +import org.robolectric.shadows.ShadowMediaCodecList; /** End-to-end test for {@link Transformer}. */ @RunWith(AndroidJUnit4.class) @@ -750,10 +754,26 @@ public final class TransformerEndToEndTest { /* inputBufferSize= */ 10_000, /* outputBufferSize= */ 10_000, /* codec= */ (in, out) -> out.put(in)); - ShadowMediaCodec.addDecoder(MimeTypes.AUDIO_AAC, codecConfig); - ShadowMediaCodec.addDecoder(MimeTypes.AUDIO_AC3, codecConfig); - ShadowMediaCodec.addDecoder(MimeTypes.AUDIO_AMR_NB, codecConfig); - ShadowMediaCodec.addEncoder(MimeTypes.AUDIO_AAC, codecConfig); + addCodec( + MimeTypes.AUDIO_AAC, + codecConfig, + /* colorFormats= */ ImmutableList.of(), + /* isDecoder= */ true); + addCodec( + MimeTypes.AUDIO_AC3, + codecConfig, + /* colorFormats= */ ImmutableList.of(), + /* isDecoder= */ true); + addCodec( + MimeTypes.AUDIO_AMR_NB, + codecConfig, + /* colorFormats= */ ImmutableList.of(), + /* isDecoder= */ true); + addCodec( + MimeTypes.AUDIO_AAC, + codecConfig, + /* colorFormats= */ ImmutableList.of(), + /* isDecoder= */ false); ShadowMediaCodec.CodecConfig throwingCodecConfig = new ShadowMediaCodec.CodecConfig( @@ -776,9 +796,54 @@ public final class TransformerEndToEndTest { } }); - ShadowMediaCodec.addDecoder(MimeTypes.AUDIO_AMR_WB, throwingCodecConfig); - ShadowMediaCodec.addEncoder(MimeTypes.AUDIO_AMR_NB, throwingCodecConfig); - ShadowMediaCodec.addEncoder(MimeTypes.VIDEO_H263, throwingCodecConfig); + addCodec( + MimeTypes.AUDIO_AMR_WB, + throwingCodecConfig, + /* colorFormats= */ ImmutableList.of(), + /* isDecoder= */ true); + addCodec( + MimeTypes.VIDEO_H263, + throwingCodecConfig, + ImmutableList.of(MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible), + /* isDecoder= */ false); + addCodec( + MimeTypes.AUDIO_AMR_NB, + throwingCodecConfig, + /* colorFormats= */ ImmutableList.of(), + /* isDecoder= */ false); + } + + private static void addCodec( + String mimeType, + ShadowMediaCodec.CodecConfig codecConfig, + List colorFormats, + boolean isDecoder) { + String codecName = + Util.formatInvariant( + isDecoder ? "exo.%s.decoder" : "exo.%s.encoder", mimeType.replace('/', '-')); + if (isDecoder) { + ShadowMediaCodec.addDecoder(codecName, codecConfig); + } else { + ShadowMediaCodec.addEncoder(codecName, codecConfig); + } + + MediaFormat mediaFormat = new MediaFormat(); + mediaFormat.setString(MediaFormat.KEY_MIME, mimeType); + MediaCodecInfoBuilder.CodecCapabilitiesBuilder codecCapabilities = + MediaCodecInfoBuilder.CodecCapabilitiesBuilder.newBuilder() + .setMediaFormat(mediaFormat) + .setIsEncoder(!isDecoder); + + if (!colorFormats.isEmpty()) { + codecCapabilities.setColorFormats(Ints.toArray(colorFormats)); + } + + ShadowMediaCodecList.addCodec( + MediaCodecInfoBuilder.newBuilder() + .setName(codecName) + .setIsEncoder(!isDecoder) + .setCapabilities(codecCapabilities.build()) + .build()); } private static void removeEncodersAndDecoders() { diff --git a/libraries/ui/src/main/java/androidx/media3/ui/PlayerControlViewLayoutManager.java b/libraries/ui/src/main/java/androidx/media3/ui/PlayerControlViewLayoutManager.java index 2bee086110..e9c5b2e082 100644 --- a/libraries/ui/src/main/java/androidx/media3/ui/PlayerControlViewLayoutManager.java +++ b/libraries/ui/src/main/java/androidx/media3/ui/PlayerControlViewLayoutManager.java @@ -604,13 +604,15 @@ import java.util.List; } if (timeBar != null) { - MarginLayoutParams timeBarParams = (MarginLayoutParams) timeBar.getLayoutParams(); int timeBarMarginBottom = playerControlView .getResources() .getDimensionPixelSize(R.dimen.exo_styled_progress_margin_bottom); - timeBarParams.bottomMargin = (isMinimalMode ? 0 : timeBarMarginBottom); - timeBar.setLayoutParams(timeBarParams); + @Nullable MarginLayoutParams timeBarParams = (MarginLayoutParams) timeBar.getLayoutParams(); + if (timeBarParams != null) { + timeBarParams.bottomMargin = (isMinimalMode ? 0 : timeBarMarginBottom); + timeBar.setLayoutParams(timeBarParams); + } if (timeBar instanceof DefaultTimeBar) { DefaultTimeBar defaultTimeBar = (DefaultTimeBar) timeBar; if (isMinimalMode) { diff --git a/libraries/ui/src/main/java/androidx/media3/ui/PlayerView.java b/libraries/ui/src/main/java/androidx/media3/ui/PlayerView.java index be9ed95f61..b88da6bafa 100644 --- a/libraries/ui/src/main/java/androidx/media3/ui/PlayerView.java +++ b/libraries/ui/src/main/java/androidx/media3/ui/PlayerView.java @@ -431,6 +431,9 @@ public class PlayerView extends FrameLayout implements AdViewProvider { controller.hideImmediately(); controller.addVisibilityListener(/* listener= */ componentListener); } + if (useController) { + setClickable(true); + } updateContentDescription(); } @@ -593,10 +596,14 @@ public class PlayerView extends FrameLayout implements AdViewProvider { * Sets whether the playback controls can be shown. If set to {@code false} the playback controls * are never visible and are disconnected from the player. * + *

      This call will update whether the view is clickable. After the call, the view will be + * clickable if playback controls can be shown or if the view has a registered click listener. + * * @param useController Whether the playback controls can be shown. */ public void setUseController(boolean useController) { Assertions.checkState(!useController || controller != null); + setClickable(useController || hasOnClickListeners()); if (this.useController == useController) { return; } @@ -1015,30 +1022,10 @@ public class PlayerView extends FrameLayout implements AdViewProvider { return subtitleView; } - @Override - public boolean onTouchEvent(MotionEvent event) { - if (!useController() || player == null) { - return false; - } - switch (event.getAction()) { - case MotionEvent.ACTION_DOWN: - isTouching = true; - return true; - case MotionEvent.ACTION_UP: - if (isTouching) { - isTouching = false; - return performClick(); - } - return false; - default: - return false; - } - } - @Override public boolean performClick() { - super.performClick(); - return toggleControllerVisibility(); + toggleControllerVisibility(); + return super.performClick(); } @Override @@ -1134,18 +1121,15 @@ public class PlayerView extends FrameLayout implements AdViewProvider { return false; } - private boolean toggleControllerVisibility() { + private void toggleControllerVisibility() { if (!useController() || player == null) { - return false; + return; } if (!controller.isFullyVisible()) { maybeShowController(true); - return true; } else if (controllerHideOnTouch) { controller.hide(); - return true; } - return false; } /** Shows the playback controls, but only if forced or shown indefinitely. */ diff --git a/libraries/ui/src/main/res/layout/exo_legacy_player_view.xml b/libraries/ui/src/main/res/layout/exo_legacy_player_view.xml deleted file mode 100644 index 4ef9caefb3..0000000000 --- a/libraries/ui/src/main/res/layout/exo_legacy_player_view.xml +++ /dev/null @@ -1,67 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - -