From 0e5a5e029455ea5e456cbb6028e9be1e87cdfe09 Mon Sep 17 00:00:00 2001 From: ibaker Date: Wed, 15 May 2024 05:52:35 -0700 Subject: [PATCH] Publish `CompositionPlayer` for playing compositions This class is not ready for production app usage yet, so it is still marked `@RestrictTo(LIBRARY_GROUP)` for now. Apps can experiment with it in a non-prod context by suppressing the associated lint error. * Issue: androidx/media#1014 * Issue: androidx/media#1185 * Issue: androidx/media#816 PiperOrigin-RevId: 633921353 --- demos/composition/README.md | 6 + demos/composition/build.gradle | 63 ++ demos/composition/proguard-rules.txt | 1 + .../composition/src/main/AndroidManifest.xml | 51 + .../demo/composition/AssetItemAdapter.java | 69 ++ .../CompositionPreviewActivity.java | 354 +++++++ .../MatrixTransformationFactory.java | 93 ++ .../media3/demo/composition/package-info.java | 22 + .../layout/composition_preview_activity.xml | 117 +++ .../src/main/res/layout/preset_item.xml | 30 + .../src/main/res/mipmap-hdpi/ic_launcher.png | Bin 0 -> 3394 bytes .../src/main/res/mipmap-mdpi/ic_launcher.png | Bin 0 -> 2184 bytes .../src/main/res/mipmap-xhdpi/ic_launcher.png | Bin 0 -> 4886 bytes .../main/res/mipmap-xxhdpi/ic_launcher.png | Bin 0 -> 7492 bytes .../main/res/mipmap-xxxhdpi/ic_launcher.png | Bin 0 -> 10801 bytes .../src/main/res/values-night/themes.xml | 31 + .../src/main/res/values/arrays.xml | 74 ++ .../src/main/res/values/colors.xml | 24 + .../src/main/res/values/strings.xml | 27 + .../src/main/res/values/themes.xml | 31 + .../transformer/CompositionPlayerTest.java | 529 ++++++++++ .../performance/CompositionPlaybackTest.java | 154 +++ .../CompositionPlayerSeekTest.java | 380 +++++++ ...ideoCompositionPreviewPerformanceTest.java | 141 +++ .../transformer/AudioGraphInputAudioSink.java | 298 ++++++ .../media3/transformer/CompositionPlayer.java | 955 ++++++++++++++++++ .../CompositionPlayerInternal.java | 237 +++++ .../transformer/PreviewAudioPipeline.java | 211 ++++ .../SequencePlayerRenderersWrapper.java | 433 ++++++++ .../CompositionPlayerAudioPlaybackTest.java | 326 ++++++ .../transformer/CompositionPlayerTest.java | 592 +++++++++++ .../transformer/PreviewAudioPipelineTest.java | 194 ++++ 32 files changed, 5443 insertions(+) create mode 100644 demos/composition/README.md create mode 100644 demos/composition/build.gradle create mode 100644 demos/composition/proguard-rules.txt create mode 100644 demos/composition/src/main/AndroidManifest.xml create mode 100644 demos/composition/src/main/java/androidx/media3/demo/composition/AssetItemAdapter.java create mode 100644 demos/composition/src/main/java/androidx/media3/demo/composition/CompositionPreviewActivity.java create mode 100644 demos/composition/src/main/java/androidx/media3/demo/composition/MatrixTransformationFactory.java create mode 100644 demos/composition/src/main/java/androidx/media3/demo/composition/package-info.java create mode 100644 demos/composition/src/main/res/layout/composition_preview_activity.xml create mode 100644 demos/composition/src/main/res/layout/preset_item.xml create mode 100644 demos/composition/src/main/res/mipmap-hdpi/ic_launcher.png create mode 100644 demos/composition/src/main/res/mipmap-mdpi/ic_launcher.png create mode 100644 demos/composition/src/main/res/mipmap-xhdpi/ic_launcher.png create mode 100644 demos/composition/src/main/res/mipmap-xxhdpi/ic_launcher.png create mode 100644 demos/composition/src/main/res/mipmap-xxxhdpi/ic_launcher.png create mode 100644 demos/composition/src/main/res/values-night/themes.xml create mode 100644 demos/composition/src/main/res/values/arrays.xml create mode 100644 demos/composition/src/main/res/values/colors.xml create mode 100644 demos/composition/src/main/res/values/strings.xml create mode 100644 demos/composition/src/main/res/values/themes.xml create mode 100644 libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerTest.java create mode 100644 libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlaybackTest.java create mode 100644 libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlayerSeekTest.java create mode 100644 libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/VideoCompositionPreviewPerformanceTest.java create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraphInputAudioSink.java create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayerInternal.java create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/PreviewAudioPipeline.java create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java create mode 100644 libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerAudioPlaybackTest.java create mode 100644 libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerTest.java create mode 100644 libraries/transformer/src/test/java/androidx/media3/transformer/PreviewAudioPipelineTest.java diff --git a/demos/composition/README.md b/demos/composition/README.md new file mode 100644 index 0000000000..48ca854449 --- /dev/null +++ b/demos/composition/README.md @@ -0,0 +1,6 @@ +# Composition demo + +This app is an experimental demo app to demonstrate how to use Composition and CompositionPlayer APIs. + +See the [demos README](../README.md) for instructions on how to build and run +this demo. diff --git a/demos/composition/build.gradle b/demos/composition/build.gradle new file mode 100644 index 0000000000..e26187cbd2 --- /dev/null +++ b/demos/composition/build.gradle @@ -0,0 +1,63 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +apply from: '../../constants.gradle' +apply plugin: 'com.android.application' + +android { + namespace 'androidx.media3.demo.composition' + + compileSdk project.ext.compileSdkVersion + + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } + + defaultConfig { + versionName project.ext.releaseVersion + versionCode project.ext.releaseVersionCode + minSdkVersion 21 + targetSdkVersion project.ext.appTargetSdkVersion + multiDexEnabled true + } + + buildTypes { + release { + shrinkResources true + minifyEnabled true + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.txt' + signingConfig signingConfigs.debug + } + } + + lintOptions { + // This demo app isn't indexed and doesn't have translations. + disable 'GoogleAppIndexingWarning','MissingTranslation' + } +} + +dependencies { + implementation 'androidx.appcompat:appcompat:1.6.1' + implementation 'com.google.android.material:material:' + androidxMaterialVersion + implementation project(modulePrefix + 'lib-effect') + implementation project(modulePrefix + 'lib-exoplayer') + implementation project(modulePrefix + 'lib-exoplayer-dash') + implementation project(modulePrefix + 'lib-transformer') + implementation project(modulePrefix + 'lib-ui') + implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion + implementation 'androidx.multidex:multidex:' + androidxMultidexVersion + compileOnly 'org.checkerframework:checker-qual:' + checkerframeworkVersion +} diff --git a/demos/composition/proguard-rules.txt b/demos/composition/proguard-rules.txt new file mode 100644 index 0000000000..cd85b36a6c --- /dev/null +++ b/demos/composition/proguard-rules.txt @@ -0,0 +1 @@ +# Proguard rules specific to the composition demo app. diff --git a/demos/composition/src/main/AndroidManifest.xml b/demos/composition/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..a521883c12 --- /dev/null +++ b/demos/composition/src/main/AndroidManifest.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/demos/composition/src/main/java/androidx/media3/demo/composition/AssetItemAdapter.java b/demos/composition/src/main/java/androidx/media3/demo/composition/AssetItemAdapter.java new file mode 100644 index 0000000000..620fdf6903 --- /dev/null +++ b/demos/composition/src/main/java/androidx/media3/demo/composition/AssetItemAdapter.java @@ -0,0 +1,69 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.demo.composition; + +import android.view.LayoutInflater; +import android.view.View; +import android.view.ViewGroup; +import android.widget.TextView; +import androidx.recyclerview.widget.RecyclerView; +import java.util.List; + +/** A {@link RecyclerView.Adapter} that displays assets in a sequence in a {@link RecyclerView}. */ +public final class AssetItemAdapter extends RecyclerView.Adapter { + private static final String TAG = "AssetItemAdapter"; + + private final List data; + + /** + * Creates a new instance + * + * @param data A list of items to populate RecyclerView with. + */ + public AssetItemAdapter(List data) { + this.data = data; + } + + @Override + public ViewHolder onCreateViewHolder(ViewGroup parent, int viewType) { + View v = LayoutInflater.from(parent.getContext()).inflate(R.layout.preset_item, parent, false); + return new ViewHolder(v); + } + + @Override + public void onBindViewHolder(ViewHolder holder, int position) { + holder.getTextView().setText(data.get(position)); + } + + @Override + public int getItemCount() { + return data.size(); + } + + /** A {@link RecyclerView.ViewHolder} used to build {@link AssetItemAdapter}. */ + public static final class ViewHolder extends RecyclerView.ViewHolder { + private final TextView textView; + + private ViewHolder(View view) { + super(view); + textView = view.findViewById(R.id.preset_name_text); + } + + private TextView getTextView() { + return textView; + } + } +} diff --git a/demos/composition/src/main/java/androidx/media3/demo/composition/CompositionPreviewActivity.java b/demos/composition/src/main/java/androidx/media3/demo/composition/CompositionPreviewActivity.java new file mode 100644 index 0000000000..5c46c5c53a --- /dev/null +++ b/demos/composition/src/main/java/androidx/media3/demo/composition/CompositionPreviewActivity.java @@ -0,0 +1,354 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.demo.composition; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.transformer.ImageUtil.getCommonImageMimeTypeFromExtension; + +import android.app.Activity; +import android.content.DialogInterface; +import android.net.Uri; +import android.os.Bundle; +import android.view.View; +import android.widget.Toast; +import androidx.annotation.Nullable; +import androidx.appcompat.app.AlertDialog; +import androidx.appcompat.app.AppCompatActivity; +import androidx.appcompat.widget.AppCompatButton; +import androidx.appcompat.widget.AppCompatTextView; +import androidx.media3.common.Effect; +import androidx.media3.common.MediaItem; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.PlaybackException; +import androidx.media3.common.Player; +import androidx.media3.common.audio.SonicAudioProcessor; +import androidx.media3.common.util.Clock; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.Util; +import androidx.media3.effect.RgbFilter; +import androidx.media3.transformer.Composition; +import androidx.media3.transformer.CompositionPlayer; +import androidx.media3.transformer.EditedMediaItem; +import androidx.media3.transformer.EditedMediaItemSequence; +import androidx.media3.transformer.Effects; +import androidx.media3.transformer.ExportException; +import androidx.media3.transformer.ExportResult; +import androidx.media3.transformer.JsonUtil; +import androidx.media3.transformer.Transformer; +import androidx.media3.ui.PlayerView; +import androidx.recyclerview.widget.DividerItemDecoration; +import androidx.recyclerview.widget.LinearLayoutManager; +import androidx.recyclerview.widget.RecyclerView; +import com.google.common.base.Stopwatch; +import com.google.common.base.Ticker; +import com.google.common.collect.ImmutableList; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.json.JSONException; +import org.json.JSONObject; + +/** + * An {@link Activity} that previews compositions, using {@link + * androidx.media3.transformer.CompositionPlayer}. + */ +public final class CompositionPreviewActivity extends AppCompatActivity { + private static final String TAG = "CompPreviewActivity"; + + private final ArrayList sequenceAssetTitles = new ArrayList<>(); + + @Nullable private boolean[] selectedMediaItems = null; + private String[] presetFileDescriptions = new String[0]; + @Nullable private AssetItemAdapter assetItemAdapter; + @Nullable private CompositionPlayer compositionPlayer; + @Nullable private Transformer transformer; + @Nullable private File outputFile; + private @MonotonicNonNull PlayerView playerView; + private @MonotonicNonNull AppCompatButton exportButton; + private @MonotonicNonNull AppCompatTextView exportInformationTextView; + private @MonotonicNonNull Stopwatch exportStopwatch; + + @Override + protected void onCreate(@Nullable Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.composition_preview_activity); + playerView = findViewById(R.id.composition_player_view); + + findViewById(R.id.preview_button).setOnClickListener(this::previewComposition); + findViewById(R.id.edit_sequence_button).setOnClickListener(this::selectPresetFile); + RecyclerView presetList = findViewById(R.id.composition_preset_list); + presetList.addItemDecoration(new DividerItemDecoration(this, DividerItemDecoration.VERTICAL)); + LinearLayoutManager layoutManager = + new LinearLayoutManager(this, LinearLayoutManager.VERTICAL, /* reverseLayout= */ false); + presetList.setLayoutManager(layoutManager); + + exportInformationTextView = findViewById(R.id.export_information_text); + exportButton = findViewById(R.id.composition_export_button); + exportButton.setOnClickListener(this::exportComposition); + + presetFileDescriptions = getResources().getStringArray(R.array.preset_descriptions); + // Select two media items by default. + selectedMediaItems = new boolean[presetFileDescriptions.length]; + selectedMediaItems[0] = true; + selectedMediaItems[2] = true; + for (int i = 0; i < checkNotNull(selectedMediaItems).length; i++) { + if (checkNotNull(selectedMediaItems)[i]) { + sequenceAssetTitles.add(presetFileDescriptions[i]); + } + } + assetItemAdapter = new AssetItemAdapter(sequenceAssetTitles); + presetList.setAdapter(assetItemAdapter); + + exportStopwatch = + Stopwatch.createUnstarted( + new Ticker() { + @Override + public long read() { + return android.os.SystemClock.elapsedRealtimeNanos(); + } + }); + } + + @Override + protected void onStart() { + super.onStart(); + checkStateNotNull(playerView).onResume(); + } + + @Override + protected void onStop() { + super.onStop(); + checkStateNotNull(playerView).onPause(); + releasePlayer(); + cancelExport(); + checkStateNotNull(exportStopwatch).reset(); + } + + private Composition prepareComposition() { + // Reading from resources here does not create a performance bottleneck, this + // method is called as part of more expensive operations. + String[] presetFileUris = getResources().getStringArray(/* id= */ R.array.preset_uris); + checkState( + /* expression= */ checkStateNotNull(presetFileUris).length == presetFileDescriptions.length, + /* errorMessage= */ "Unexpected array length " + + getResources().getResourceName(R.array.preset_uris)); + int[] presetDurationsUs = getResources().getIntArray(/* id= */ R.array.preset_durations); + checkState( + /* expression= */ checkStateNotNull(presetDurationsUs).length + == presetFileDescriptions.length, + /* errorMessage= */ "Unexpected array length " + + getResources().getResourceName(R.array.preset_durations)); + + List mediaItems = new ArrayList<>(); + ImmutableList effects = + ImmutableList.of( + MatrixTransformationFactory.createDizzyCropEffect(), RgbFilter.createGrayscaleFilter()); + for (int i = 0; i < checkNotNull(selectedMediaItems).length; i++) { + if (checkNotNull(selectedMediaItems)[i]) { + Uri uri = Uri.parse(presetFileUris[i]); + MediaItem.Builder mediaItemBuilder = new MediaItem.Builder().setUri(uri); + if (MimeTypes.isImage(getCommonImageMimeTypeFromExtension(uri))) { + mediaItemBuilder.setImageDurationMs(Util.usToMs(presetDurationsUs[i])); + } + MediaItem mediaItem = mediaItemBuilder.build(); + SonicAudioProcessor pitchChanger = new SonicAudioProcessor(); + pitchChanger.setPitch(mediaItems.size() % 2 == 0 ? 2f : 0.2f); + EditedMediaItem.Builder itemBuilder = + new EditedMediaItem.Builder(mediaItem) + .setEffects( + new Effects( + /* audioProcessors= */ ImmutableList.of(pitchChanger), + /* videoEffects= */ effects)) + .setDurationUs(presetDurationsUs[i]); + mediaItems.add(itemBuilder.build()); + } + } + EditedMediaItemSequence videoSequence = new EditedMediaItemSequence(mediaItems); + SonicAudioProcessor sampleRateChanger = new SonicAudioProcessor(); + sampleRateChanger.setOutputSampleRateHz(8_000); + return new Composition.Builder(/* sequences= */ ImmutableList.of(videoSequence)) + .setEffects( + new Effects( + /* audioProcessors= */ ImmutableList.of(sampleRateChanger), + /* videoEffects= */ ImmutableList.of())) + .build(); + } + + private void previewComposition(View view) { + releasePlayer(); + Composition composition = prepareComposition(); + checkStateNotNull(playerView).setPlayer(null); + + CompositionPlayer player = new CompositionPlayer.Builder(getApplicationContext()).build(); + this.compositionPlayer = player; + checkStateNotNull(playerView).setPlayer(compositionPlayer); + checkStateNotNull(playerView).setControllerAutoShow(false); + player.addListener( + new Player.Listener() { + @Override + public void onPlayerError(PlaybackException error) { + Toast.makeText(getApplicationContext(), "Preview error: " + error, Toast.LENGTH_LONG) + .show(); + Log.e(TAG, "Preview error", error); + } + }); + player.setComposition(composition); + player.prepare(); + player.play(); + } + + private void selectPresetFile(View view) { + new AlertDialog.Builder(/* context= */ this) + .setTitle(R.string.select_preset_file_title) + .setMultiChoiceItems( + presetFileDescriptions, + checkNotNull(selectedMediaItems), + this::selectPresetFileInDialog) + .setPositiveButton(android.R.string.ok, /* listener= */ null) + .setCancelable(false) + .create() + .show(); + } + + private void selectPresetFileInDialog(DialogInterface dialog, int which, boolean isChecked) { + if (selectedMediaItems == null) { + return; + } + selectedMediaItems[which] = isChecked; + // The items will be added to a the sequence in the order they were selected. + if (isChecked) { + sequenceAssetTitles.add(presetFileDescriptions[which]); + checkNotNull(assetItemAdapter).notifyItemInserted(sequenceAssetTitles.size() - 1); + } else { + int index = sequenceAssetTitles.indexOf(presetFileDescriptions[which]); + sequenceAssetTitles.remove(presetFileDescriptions[which]); + checkNotNull(assetItemAdapter).notifyItemRemoved(index); + } + } + + private void exportComposition(View view) { + // Cancel and clean up files from any ongoing export. + cancelExport(); + + Composition composition = prepareComposition(); + + try { + outputFile = + createExternalCacheFile( + "composition-preview-" + Clock.DEFAULT.elapsedRealtime() + ".mp4"); + } catch (IOException e) { + Toast.makeText( + getApplicationContext(), + "Aborting export! Unable to create output file: " + e, + Toast.LENGTH_LONG) + .show(); + Log.e(TAG, "Aborting export! Unable to create output file: " + e); + return; + } + String filePath = outputFile.getAbsolutePath(); + + transformer = + new Transformer.Builder(this) + .addListener( + new Transformer.Listener() { + @Override + public void onCompleted(Composition composition, ExportResult exportResult) { + checkStateNotNull(exportStopwatch).stop(); + long elapsedTimeMs = exportStopwatch.elapsed(TimeUnit.MILLISECONDS); + String details = + getString(R.string.export_completed, elapsedTimeMs / 1000.f, filePath); + Log.i(TAG, details); + checkStateNotNull(exportInformationTextView).setText(details); + + try { + JSONObject resultJson = + JsonUtil.exportResultAsJsonObject(exportResult) + .put("elapsedTimeMs", elapsedTimeMs) + .put("device", JsonUtil.getDeviceDetailsAsJsonObject()); + for (String line : Util.split(resultJson.toString(2), "\n")) { + Log.i(TAG, line); + } + } catch (JSONException e) { + Log.w(TAG, "Unable to convert exportResult to JSON", e); + } + } + + @Override + public void onError( + Composition composition, + ExportResult exportResult, + ExportException exportException) { + checkStateNotNull(exportStopwatch).stop(); + Toast.makeText( + getApplicationContext(), + "Export error: " + exportException, + Toast.LENGTH_LONG) + .show(); + Log.e(TAG, "Export error", exportException); + checkStateNotNull(exportInformationTextView).setText(R.string.export_error); + } + }) + .build(); + + checkStateNotNull(exportInformationTextView).setText(R.string.export_started); + checkStateNotNull(exportStopwatch).reset(); + exportStopwatch.start(); + checkStateNotNull(transformer).start(composition, filePath); + Log.i(TAG, "Export started"); + } + + private void releasePlayer() { + if (compositionPlayer != null) { + compositionPlayer.release(); + compositionPlayer = null; + } + } + + /** Cancels any ongoing export operation, and deletes output file contents. */ + private void cancelExport() { + if (transformer != null) { + transformer.cancel(); + transformer = null; + } + if (outputFile != null) { + outputFile.delete(); + outputFile = null; + } + checkStateNotNull(exportInformationTextView).setText(""); + } + + /** + * Creates a {@link File} of the {@code fileName} in the application cache directory. + * + *

If a file of that name already exists, it is overwritten. + */ + // TODO: b/320636291 - Refactor duplicate createExternalCacheFile functions. + private File createExternalCacheFile(String fileName) throws IOException { + File file = new File(getExternalCacheDir(), fileName); + if (file.exists() && !file.delete()) { + throw new IOException("Could not delete file: " + file.getAbsolutePath()); + } + if (!file.createNewFile()) { + throw new IOException("Could not create file: " + file.getAbsolutePath()); + } + return file; + } +} diff --git a/demos/composition/src/main/java/androidx/media3/demo/composition/MatrixTransformationFactory.java b/demos/composition/src/main/java/androidx/media3/demo/composition/MatrixTransformationFactory.java new file mode 100644 index 0000000000..85cc61d19f --- /dev/null +++ b/demos/composition/src/main/java/androidx/media3/demo/composition/MatrixTransformationFactory.java @@ -0,0 +1,93 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.demo.composition; + +import android.graphics.Matrix; +import androidx.media3.common.C; +import androidx.media3.common.util.Util; +import androidx.media3.effect.GlMatrixTransformation; +import androidx.media3.effect.MatrixTransformation; + +/** + * Factory for {@link GlMatrixTransformation GlMatrixTransformations} and {@link + * MatrixTransformation MatrixTransformations} that create video effects by applying transformation + * matrices to the individual video frames. + */ +/* package */ final class MatrixTransformationFactory { + /** + * Returns a {@link MatrixTransformation} that rescales the frames over the first {@link + * #ZOOM_DURATION_SECONDS} seconds, such that the rectangle filled with the input frame increases + * linearly in size from a single point to filling the full output frame. + */ + public static MatrixTransformation createZoomInTransition() { + return MatrixTransformationFactory::calculateZoomInTransitionMatrix; + } + + /** + * Returns a {@link MatrixTransformation} that crops frames to a rectangle that moves on an + * ellipse. + */ + public static MatrixTransformation createDizzyCropEffect() { + return MatrixTransformationFactory::calculateDizzyCropMatrix; + } + + /** + * Returns a {@link GlMatrixTransformation} that rotates a frame in 3D around the y-axis and + * applies perspective projection to 2D. + */ + public static GlMatrixTransformation createSpin3dEffect() { + return MatrixTransformationFactory::calculate3dSpinMatrix; + } + + private static final float ZOOM_DURATION_SECONDS = 2f; + private static final float DIZZY_CROP_ROTATION_PERIOD_US = 5_000_000f; + + private static Matrix calculateZoomInTransitionMatrix(long presentationTimeUs) { + Matrix transformationMatrix = new Matrix(); + float scale = Math.min(1, presentationTimeUs / (C.MICROS_PER_SECOND * ZOOM_DURATION_SECONDS)); + transformationMatrix.postScale(/* sx= */ scale, /* sy= */ scale); + return transformationMatrix; + } + + private static android.graphics.Matrix calculateDizzyCropMatrix(long presentationTimeUs) { + double theta = presentationTimeUs * 2 * Math.PI / DIZZY_CROP_ROTATION_PERIOD_US; + float centerX = 0.5f * (float) Math.cos(theta); + float centerY = 0.5f * (float) Math.sin(theta); + android.graphics.Matrix transformationMatrix = new android.graphics.Matrix(); + transformationMatrix.postTranslate(/* dx= */ centerX, /* dy= */ centerY); + transformationMatrix.postScale(/* sx= */ 2f, /* sy= */ 2f); + return transformationMatrix; + } + + private static float[] calculate3dSpinMatrix(long presentationTimeUs) { + float[] transformationMatrix = new float[16]; + android.opengl.Matrix.frustumM( + transformationMatrix, + /* offset= */ 0, + /* left= */ -1f, + /* right= */ 1f, + /* bottom= */ -1f, + /* top= */ 1f, + /* near= */ 3f, + /* far= */ 5f); + android.opengl.Matrix.translateM( + transformationMatrix, /* mOffset= */ 0, /* x= */ 0f, /* y= */ 0f, /* z= */ -4f); + float theta = Util.usToMs(presentationTimeUs) / 10f; + android.opengl.Matrix.rotateM( + transformationMatrix, /* mOffset= */ 0, theta, /* x= */ 0f, /* y= */ 1f, /* z= */ 0f); + return transformationMatrix; + } +} diff --git a/demos/composition/src/main/java/androidx/media3/demo/composition/package-info.java b/demos/composition/src/main/java/androidx/media3/demo/composition/package-info.java new file mode 100644 index 0000000000..068f941e6b --- /dev/null +++ b/demos/composition/src/main/java/androidx/media3/demo/composition/package-info.java @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@NonNullApi +@OptIn(markerClass = UnstableApi.class) +package androidx.media3.demo.composition; + +import androidx.annotation.OptIn; +import androidx.media3.common.util.NonNullApi; +import androidx.media3.common.util.UnstableApi; diff --git a/demos/composition/src/main/res/layout/composition_preview_activity.xml b/demos/composition/src/main/res/layout/composition_preview_activity.xml new file mode 100644 index 0000000000..83819ec920 --- /dev/null +++ b/demos/composition/src/main/res/layout/composition_preview_activity.xml @@ -0,0 +1,117 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/demos/composition/src/main/res/layout/preset_item.xml b/demos/composition/src/main/res/layout/preset_item.xml new file mode 100644 index 0000000000..41933efc0c --- /dev/null +++ b/demos/composition/src/main/res/layout/preset_item.xml @@ -0,0 +1,30 @@ + + + + + + + diff --git a/demos/composition/src/main/res/mipmap-hdpi/ic_launcher.png b/demos/composition/src/main/res/mipmap-hdpi/ic_launcher.png new file mode 100644 index 0000000000000000000000000000000000000000..adaa93220eb81c58e5c85874d1cf127a3188e419 GIT binary patch literal 3394 zcmV-I4ZZS-P)d>%xF$xW+t8d`u>^S&KV-GPLkW5 z`RC5^;-~lC!ku)^Ohlv?a=C@{LON&3(GBWZt}HiPtth81qyN7FauI`bxyAo)XVqgh zW3>@#CO*5}T%G@AK=NDHsZ^KMh4m_H1?vziij`JcTAI%)hQxiE_}?Ls_Z3mLuDVWX zS^p(KZspvdA!{OQJ1dz7PLR=Pv`VrZ>R@dXbv7*LzHZeSkZU=U@4#Bj$|wiLsE8zP zjUtt*CGx4WEBIUu40Ve(I+Sxi*XjmH{ms413S+6EC@KJ(K2_|i^|BRCO@`vH zvKRxd&J_Bf3hDvqqb-nZf%6zE79A0tLZWISqY6}PadGkEtSUjOQZP0c44yD!6&$LL zQuVDUFE6hb%j)25wdHW5UVK#tVXDp&eZ-Zrva)~XIwT%PQA|wC zQ!Fty+W@(UYYhZDmU+u5&ZUyjVE|K6yQh$naG3KcPKA`8pMU-Q`SZ0|VvIHd;(Glp z4aiFJDZnr(!;!a1$%=Mr;7(6Z58@*!eboU8^D8MSnPbIl1q%TL^9me9h6}%uu_((K zpbYG4bmPX2g)EbWH|9L8FS+t z0*O;;K|ujsym--*CB|+Zkc(LJgvwdVae#z^hLGX%63mDWSwto#Chopfz^>G_m}*;L zJ=$83e}78)aWGx+8%kFf4yEK@2Ac-8a||ihLy0$5b~^U>}5qrbp87E6|4$wR>8H{ z-0E||G#Tx+093fOKZSMxkjk~|C0{Dq-oMgv7iW9X*87K1+yF1i`T8p|BqiBpM$5>^ zNME^fWm8s}#bX8Q$;ru?Ago8gu^mwO8$$uYq@v#&Ql?DoM?)y+8To$()PE?Y4;e%m z!+j}#&1zB#@->CTlM?ZCKa(X`R0QOTK*jF0Rl1i}fDR7uF$?Hme;?$f)ISE(p6=2W zD*}?roHB_D_wLmsKFP82{w$CglsWWQKQhrBK76a_he)DxI2@lu8&9Ku?{_eZ)JXrrSQO{n8KV9UxGK;hVbaIS025Gg-0}>iKm~wym zkql{RHm!*;o9*7cyDv+(XU`rtmJT=n`Wy$USEPGs2MBI62$o?#KtdTKM^M4$O=K*p z(y@wy{n^>GXWvDrkxbO_$Ats*3S`zA5OigMmra1;`wsyGieZ|3 z)Nx4qnw~#&<`|fl9(8) z&Wd7VV~;S4+awnWNcZx~FW=oDvceq8@zk!FyRc3CIP2mM>r4fHNhF0-6De`*H|{w71NSqCG{Nv4b+kj;7)> zXUxxJoL+tP)ut>R*!&A9C@82U1IpG4P{yJ`w7-v!#W7~*W4;)OU6nCr6e)Rm=2p}> z-+%x8gRHWE0s;aaU_b@x4Qf?X7=46Jjq|3>t*i^e$J+bY9S~G>;)JPSD)Y zv&nnOB^ve1ar)BhE85s{xJ7`%9aE6|$;rv9>;>qP(LdR?)#zvW0y1l%9~ED^Sf#DTsZ*ymf`xjk0l8s$&HIqIK~ZZ! zGd~QWO)c!dSJ(m^_TBizj0*Vp%9esGu~-AoHW zzKhP&U!51;63%Zgzd=5WFIFh$v*-f7HETJoeqJB0_z#sL3=x^4F8}ljnP7;j|b|-}n27M*Mynd2Xri{jX7B z%nj6U4=KEZpuKzdZot^VfTfd?H;|mo?&6(nKNOM&_*wvxn*7y1dTst%dev_oP5kzN z$-E1;MADg?Ire-uj}0nbrlPmse%lLUm+_}U43e5Xd-m;|u@OIp-6gF^6F^pI!s@FO zcqxra6vZJgFPu1W;sB4?17m0D)RH$+ayGm7r=NcE=7)$XE3!YJVaqPlqQeQ4U7!xY zcC{Yt-syu6J{XNL|MABk?kt^J;-;*|jT?6d7ifIx(xq*f{#XWk1NtyHmSWQj9Qqob zh&_QAE5^*ym6yCx3-QPmM28O_-Wf-b!itRc05p4J1j;xLjB4iMFp@K1z<@3oE1&fBu~; zU1btC6~#&4sZ*zW`P!P_Ck|nAL<44B?Oa^0NiaT3FVQdi<0w6^82Oz62d+i?_U+rn zV{0SEsApN9V0L_|-K0sAC;82XC`*3wRLB-FW!<$(;k+FXO%W;iPW^EU!o`IP7ot5p zJf6karc9aQ&eGjV;>M)z;))uM9Xqx!@6&RK$n4aAH7Sp#sUS^TA5QxtGf8ps?=^^1 zbaZqs?-mAPOc)zWXA>Z3YBuEsxwf(#N4f5G!HhQ59K))sIyO)i}~ zcSgmnIW|q=!|iOb6Rwt!WI$%Yt#aOYR>D^yCj!x-MZD~YpYSCMh&=f zHyVvAv6V_vk7A^XO->b>OH|7jbYB1;n zFG;d~|NZyRxo?l7kD{-7mP3>N*=L`V-a`Z|I|dA!Z8J6H}*79TgZDxIQW>DhZEMgjFHQ zeVrnbk^hH3)KY(jyb8*YNT`dS0u$;+8Tp~%R{INnDEqfXQ@iqWGuDHw$NAfR!Ozcc($b|%zX}Zv-NcW1kML3u z{G>M=c{sLhTDEN2S15}*gtef~=4b=L)(+s114i6@?DP``ft zMmP%=;F_~q2sIa;KEFc)3NPzrep1o{^^JjiG{v$$j^J-@V^i z?tSUk&(8}0g3tz_h0tFK?d@&p*?!OJ!omBa1-tofC|W35I5WB*>ZWZl_4Q1F~p> z2qf?zUf=9~F@mUZ0&V3u&LYLAtrMXd>XY{gwi-c!0zW`-|1N?q762zeX3L`s`e>qO z@&95P!`0Y<0KXD}bArJ+A3(b_)|Z7zi|$wYO+xq}cDRq9bvA3(K!JCXpqc;QWHKG_ znS?MJU}s7Op4TrtNqf8)7U=EmJ%$j1`b0wLJlNTO0x#^;K2UQJm+g(JBpsan&jix3`b2=YOY?whyoAqienlse65;|} zg=H?K#52ObgK_q21hE$(&54n>SaC5F4jM>qSMDnA-SiB2NnpU4k&?=xWKz)7v0*6%)hNhmLp7Wlx z?~cG#*&N6mrEsufx!nGJ@97y(_e~6(c{2_k%t?T*6Mq0_H2UNVYPI@LfiDEv6NE;i z8IO$&dq!@GDj;!1hT>uw(rz?WA?6K}a=Z#Z9{saY_3XcPgY?hb+Gi`Drbm!(^{xEr}lc1%g3tnjY<)1@ONr~Ic+}he&w|)Ee;dTOL zWo55nqIl3(0v*Sq;f6vES)*OgeC4!QNr2s%VecN71lrr%)hQ_{VRizgrKMvrfoY`@ zFllO_YDFyMM8tX2yIL0G7XdPJQ=^Cg`l~J@BV(eSKw)9w_`VRRePDofE0my|yuc%Y z2Qvdep!q+S1X5E|Bkct8^72Mu5>rbha97<0@{}q_IB*NLCj0^?!xLQ+DF46@3C*4b zoktG4ln{n|&F%^Fi>ynd`Ul!JL198)!@|D2S&FA3cm_)Zy${7=H3#HkkJ^O`r{yZ>Vya2e1 z@fb!M8ykxe^I%~i7mTqr#TNo|j#R?YsKx#zvN^HPnVJHe!5}G-RaRF1j+jGi*jXtN zR8di}jIY1Fl@me&3w|nc3sxDq5Hi9NJQC>Gx&(63kqy*F?6X?sb<97?@o$BrHE zu>>3}ET@NJMOq-GoObqspJ!|p5%?iyr$d|7Uu8jB)Dlo9%!Tf}T({+dw{p6srlt_F zzUZ(O;QgSesAxI&sJ!(kBWD7NL*=jy!|Bnt7Nb8bfh|dUAnw-+hc@wPx8bO&2~7HS zUv^|x#AwBG2ePxXNj||2+e&_WHg)RMH*ksWdfQSu6Yyr+F5ZJitg>AiNpj~oTFzmYJyJRcG;3cY{-eyJy4@J3*s z@($!xwt~sb1?*S_wSNn-y-om6oH#MS)2ZCRUAuP4aD(5=juk~xs@1qA5SLm72UX3W z>oNu8U<;MNQRLBTwY{4+ZP`pGZ(|fXx> zj)7=nuUxru469We334imYER(3fb9hbjX|=(Xh<_Zjxg9Vt}hpRkVu4)4lzgcu;UqP zpoj>7+xlv2YwJl*Pv1#n`UPGnr^v|2=SYXRZ-}q1t}dn3pa>-(V(H^+C;pA54LJ2h zed2jf>6$fb5@{?Nv(K{eL{}#_??$T1T6|r6Xn-wpwSS^C~Hn~t= zN~Q8U?0YheL1R6UpCPaqOp_y$NYp}9s2#l3(SVF&qA2KL+F`6==^`kyys$3kdoH`0000< KMNUMnLSTZLL=Vyc literal 0 HcmV?d00001 diff --git a/demos/composition/src/main/res/mipmap-xhdpi/ic_launcher.png b/demos/composition/src/main/res/mipmap-xhdpi/ic_launcher.png new file mode 100644 index 0000000000000000000000000000000000000000..2101026c9fe1b48ab11a23fd440cf2f80af2afae GIT binary patch literal 4886 zcmV+x6Y1=UP)NZw)cXxMpfx5T2C&3}PVZZ;p-o4i2PAJ@j z+=Q8)wZ4!Axsm_BXYYOXxj|RrxHs+%BJu*>GCn21kMjaLPsdZ=m^Y{(+<2>%EO28%%wuqUDMu8OH$8%zKu7G4&}qFQk04YA1g*$9*T z-fAmCrB}H$Kq34SU>Gp7@Ek5m5M2v1I8?C_X8FB7VgIvuz5!mq7wBXR;06n?h|Qh^ zpAc-s4G;!&GQPme(+%+E*a@BkO92bdVTL>$4o@VHrSfOt+~Do02C(pgS3oS_4`l^Z zo>0t&81Rby&*~gy9`Es{e^8wD9OKI!Tp)18WVKp2jmH3M;1&be{m1d9 zjuM7eWu?2zR$)RxLBX-u*w`n5Q!~LS@l#V%#hHJEh}<7?G!99^gxuWRqXRF(=;(_=tajB`c+L4k@u+Sx9J@^Gl=@-gH zbtLBlx_R@aA5c{psY<%p+1bzG#bO+-^QBvxg}5s4BkSp3$(EDi>G?o{CL$EY9zA-r z3SiC3!~idWh;ewGB~Da4FkquP1DLI$BwbxbHcOsIw=OV5C|U`vyjeG=4PYlx_v#6Q zW1pr5FuUI*DgFSSOY+EkKo~hWIdtX9mA;@;qG0A=0pN&_+$2GE8k#f#5uZfT(GWp_6rYpFk-=oLzv*V}fW~yFWgb4PcD=P|8`jfTZkf%}Uawq@;u?Q>N4ay5-B4`v4syZWv*J zub3*RVMCPR$CU=e{Td|1QbqL0AaN|E&kCUZuju*rB^tn(-W*7IySeS{`Y?3x;K6=E zTb$e#fsPS33|Bwi=x0n)p8c*g;P&tk)dsAoZ4lKF&m7eTFy@GGD!6pfrR{2T5(xUv z7TPMSp+AJ5)~#FnB5s|~(txB1fh?Gls5gM~L)O^e$(mC3ZEx1__U+qeDpjiVAmA`m zGz4+=bNTY+`Z&M@v@!t6)1THi3UiwQjIpPIa+WV6DKD?g^FT^Uih0qZMJywMn(qh+O?~XvK&=*KE2U;{)V?3 zF$%RZAbUC68A6DLj#Q|19>{a*nYdzQV`uNEueJd~DO z+BhckhB^k?(?U-NTIp$56Fse|8Ae+h7$`L`&@ls;&|u1%GMOZ!(Ww@-NW^E)o?Qfu zeOYgyE0TaQDxw0~?hXUE`L?r*x>>NPu7NK9BZSfdiwXzf`FCZ4u}^QxUB8Y{(m2!| z#USL@z0mkUzI>&K0U|bqhK4?jHfkge(A)s?`~W%>&_k8@+Zq}u$v=oi(ggEIA5i{@ z6Atx-Pzjyy*s;Dj?}UiDUX}Q3Yw9WfbrxTH zGDnRhDLbpw1`r0WBaQy#lTWJse|NOV*5jHrYii@MNnAd;$^Z^G*MA76P0byP`NunQ za&;e>BS)wzMX|q(CdC=xy<^9YhIp{a)dnPv3#47$oU3?lzZC3N@fpkY>!mgz zIXT%papJ^QLTkGQY}vA<84k#Fl>u32w%qA@R(HOfGBDlMZItdC>mzvs$f5t~P-AVCXiQ+4&2H3^>xZ zhuaOvnlN5f5=NURB_blCd9enBhlf|eI^Y(UPOdb7=`s5}ef{ZTTGZq%b?xXH28wyb zO<^b`h_a_m6^k-eW$3M2x1zDusw1@KYlU>U;f0uYtY#JJ5>I+yG`; z_Xt%pVI3B5QQboSkdza!NJ|L_EHmLv@4Q9%`}aHAAr*3pUXW--JPEvUm4ra26F0&D z{1olzhmk1exXXavlTOgWX78}LszcKDMh4p56f>j#o!@P&YjD~GW0-+*mM$SFH`meq z0B*(+zBjNG8b2bmRyN~wVv|5LiZD%nOl%nafdzu)~bNU%tUaz0q4KNMtPkFm`lFe!@^DW7^xH!{i zpMBOlb<%JH2}wK6vtA(iO1-dz)`fadYCE$ z%%6Tt1*cDIa(yT+bR&F|{zIWH^1%H-$A}w7p7him_mC#m{pg?Cg1PNY# z`DHN+p2har68TB2T2w*51dT4W0V8^i7suFbYlMHfoE9{Hr-%W(xt={^I!R`;OGQ`I z|5g!xcJJQ37MeaMv;{gwz;_s7LPb3I@wIE$_7zXUVaU}6^u;1;kE!Rh z(|lYpRaGPn=|7p4*Xbh+po~F3lI7M-w_Ki%|0qLa$BunpXv*>5UJhV53_V!2YSr3^ zT^G17q{T{j;MhYntK(E&g7mY_QN$O^M1@0h+k7a_cpCreQ2H`rl?XAV z68_C4mudT@bh1j?imo{OZRo4fiWMuy3oV7FK&M39DB&>lXy3kl+hI+R#3iW)1~^67 zyo)sVXd-3im5^YyOTH<7etzAAmLGle(S1OtgzprAuMxt}N~cbpnrgkXGW2KzM(l{D z7^B<&mjowY8%|t89-SvNl(qEIsTsmCv2f_!yLTr{rDUE3o@T(OtFO_im`t||uRQzl zzamm`_WxaI*uQ`O`+?3L;>Iq*vCf@4KZ2iC_V3^SD|X7n3wVqH{TE%P4d+s+z^Y8J z?mzoQ<88^3C6k4AvW8wj{lL@_x!_4uL``s2*JWN8xYGdrTx{ALj3;A`R&iea#tXvs zwd&QYw-VZM?Oz;x$1cJ-Oe5i7<(V^QzMqwqMf}BWcNp-~mRocqHAibv)o#2Ku6+1H zXa)@*<+I}L?{wFJX#v?!HTPN=%$$@O+{6|>-FFE&H+fWBnZPd z#&T>M-D}(44PNMMW6jXnw!O9;INP>u1(|%heRnbt14b z%;Lo@rkE*HrZ6tZ?cTk6A5BrbgurH8$W7n8Nx<)a%hSQ~hu;gn{w};1_&F?ie(0fx zUWr5^1Hfwd@Zr|;X&s6kJa}-Lo{P^of zJw)x{Aa-#5_19krHil8cJ{~AWaV@1ohYsv(snrbMVg?%`n*RSiP4T)0OGht#?k~U} zvEl?QV@Ug|tcrm|EgWDaO&Gy75fbYi%YJ9S+CPZ$$ z@y2sCSo==oM$FWPHe2egZQ};S@251qB5eUAlC^2@Uh-&p%9~ z!k37Z{%eEZQZF00a_r^l)2APaZPUW*LBs^)q~fsU%&dzH{T4b zHc%6waF=?K>B_m@V8x0RYpbfNhGG-8brppTd`B_X4eL-*QBkDlDslAGQ%{|)t&h*> z4o#fk90C6nrmiJAp<5tASC#r_wKO{u8?eP}>R>ivYiKD}oLp8`mXn>GU7^?Y>FMeH zZ@&5FOP_e+iFfEJ;Rq29_`#g9$6`D>pbHAIh^{@H?`T@#a&$&_Y`_+5GTYh&xsAS^ z%FD}(tu{o43>h+R@#4id$PsT-%2;cL?7vtbN}^xTks(4RviXdzB|2#p>qf0$O+Z(4 zMt5xR8*==Iq~zS(+$^mS(LmRTqy75z8>>=wk*fKVv=KIvcFy<_qI5#eDR7g3Hn4x6 zmkTlOo`1TP=y2V2*InGFPoMGVhK}fp&gdTQkkf>bv$%t*bB-G4a&tsP!}z`n7cM+e zy9ZWj1JkPj!lH(&8IAzXy0kz1iGDQxl}JO{--GQ+I*w+XB3Z8>6WP+CAG)9ux}hVw zdKuYx0v2&c~z1(GLVH#Wc!ct;U{-SIo<&2pwgMf zfoA&Nga@l36&A8;0Mxk7vAwUcG!^`Y-&!8ICaE@0ri|jx?k-uv5rmFW@bA2pnr1XB_`0cAr~1`(4QCXBG>Tj>W)rr~m)}07*qo IM6N<$f{P}YasU7T literal 0 HcmV?d00001 diff --git a/demos/composition/src/main/res/mipmap-xxhdpi/ic_launcher.png b/demos/composition/src/main/res/mipmap-xxhdpi/ic_launcher.png new file mode 100644 index 0000000000000000000000000000000000000000..223ec8bd1132aab759469bd5ad2232f35f7be2c2 GIT binary patch literal 7492 zcmV-K9lPR*P)7bFjo3yR6hYmf3aeOcP}{a`bNw*d|0U;cCTESAhdF=p<#BND ze(#)nSIgQ+Boc{4BJn&=2L@r*gF)y(i#kyntnL#ek1(4lg3iwuHMt@n4EL&P$1W2%4KN`sv1kd9&jD}1l8iYZj<4mYg_}n>DmK;k!doC zK-8Ytd#%B2upQh1F)j+>AQoX0DsDZf*n~wmwTKm?d)ZF+)|%ZjwK)+eUDYNJOanW? zn;L|%l_)+zyns-GRb8`&D)ol$bt=dUuPTW^d~&;k)!;FcCKVEfWJxcs63RuGP>*R0 z-T9A11PV@^$>1O;W0l6DU%!{(q++pFS4Lx++;Sp`K)hAW0H;{BQI*EaQfwuY3XO3l zp9c}B;pl8_FcBQ(;;8nNBe+~78uKK!*3(6xI$+}T00;fvkT_nBW3h$OLC}NEkqA^r z)zLAD%kfz3Gl$w#bYMHUVww}3RU;9I_4i5OP5`YcS6bU$*4E6Z3zQEuVm(mt@2CU> z5?Q|aSL4y#5NK~Mg#jwlFZsOPqA&&6XANTp%zN^pJEs3?(0 z%mW&rxy@aHc+cM0GF3;ak!4w%G)>b%=w(ATBu}!%CsIgy2-ZR` zK7IPs8-!AcLXii98jQzpp~)i>ef#|R^Dq#&M1i&~l4lSef75~|BzZ(4RQx~h-n|uaCmfD&_xA;bbBL6fpgL3ffTX{bTL=(&!o5T$ReRGIh+p* z&-^E#W(q)kq$}0MM!=PkkpYpBks35%nVB%rjB%CERL7$UcYC{>eJ0L15>N5JQj0ins zbiIe*L6Nr7Xx+mxDP#))AkHpoK=;T4^ipoAgNS^TnVCtU?BC=&W9vgki{|X??B0Cc zqdpUw{8)5QUN`)lC(a9fDlYz29zjRfg}J?J8x zP#@)}T?^@O1Qzbzy<4P=%;35YimWD_x{u-jirH?7pg4eAbH4e1=l}_OERElh{3n7g zI!N_(q36$^r|#dszb*OP+1a@aniguRYLgZGKH%@q^XN-kNV2ttj2(_3B_^sa zjG>>Yuy_xBUXgrXSdYx0sxuav2FSY)6VHkSDD~Jd@NBQ)fWo)a-8GQuJk(%tQVtJ; zprQ8pfRzEV8b$`mBZot_rw3q)qb4y;?F~3{=FFGmdlJ*d7SXhYQCf%P?v4ELr3`?s zY3#^}M%mpdwuOG3Y`39sXOb<3J_c*S05MCMF&%QRT~$T>2$v2=Uuo21H~GG>Fji>z z+OcCtRs6&cnRr$zK-um?!LyylXn@Z2&_J%w>$AE?KU(A8Dd1x{05MBjwF0!ypDA8U zlODobJ|f1CA78t$Fji>z!rH(``N9_{6`+^DSdjyTSbPClS*)WaWo-P91AvyAwkCNKpWx-{HZ72RD#`O1v<}D;`>egoL!iPnO2AFGcS_^B6$F zO%OmFv$llv4eLRQic(ZI_UzfS4Aj6qi3KVu8eB+Z*%zjB4uFeYL#JSXsAZvAfS4uQ z+d|f{qoB*oP(o2rkL4RUckbLIVgb1*Zzw=a|CGZxxCwCSa48+2EZ3owR;wByW+@ZK zL5{y4=q2fmhA~R#gM)+j5eu)=K?Uj`TU%Soz$##i-%lj~kx|psBmiQTwqz0Hg}$-d zMo0$%%9vo%LBvEsI_S0fXXD0=b&;DFvi{Ml0icwltP*Vk05P+%f{e|ZK#Grl6U|jR zfEYb|`0%E3hn|cJ5DQ;ru&qz8yVi09v?6<9ROoYSVLAXYOa9IQvRs@&my`2P=^zr) zbaqTSc<|uAh>1OW_LM_2%Ww2Ueq#cDS=h)`5V9p!e(`Q6w-NOZ(w_&^~%pq5#p}fmaoRudnYqV!{xB$jXy|#-lqs zn1EIzK-V0^3F@;Q93b>V8%VIUEYO(=R+bPu)JiZU1{jTv?Ai&sq@)s^QpBrZjuI0F z0kmSp3erH8@B_Wn3Q+uROYrI_Mob6et`g6{QK%)48Ufi}o}kam6P}6X;pe=5{W=ZX zS0*;l%<>!k@IUt9haVC^mGO$+Ye=A~LKw~R+!|vG?ybeZbG0`MN$RlFnK(9bSSgg9G#8Qh5LF_K;|)WMDXL zku^WlG^7xKhym=0J!;snVLf7l7ZK%r4qS+YjkX)Xwe|3~02LLWt(!&cKw3ipVnxIXM~)onj__nF1)v=P zufX=uLzuF`ANG8-w8SBL{*~zfl;!4XG(gxF%~`s1Xb<^Kf(X!q{j zo$(W-vn)(005Myz#2t>kJGI0By7w>B5Ybc@LKP1p-g9YzZW$iJJtX@oDw?LyM{$B!Q;jT<+vN1*_*NyM>Z$2P+r$O|O{ z#BA)gE3jqQI&f_%HamOrg}TF7+VW+f%ghu4P-tjq^p{_L*@4)gb_SXmGqe1^U;v@j z!Ex45*2+*aK+LAE@r46}76=3Ad`Ab?L{*pB+d}5ygP_aK7S_tZ;lfAAL?dE@7(p{* zW|sdKxGY+>Y*`VXwXqw2MNI&a*&TcUOXmFsC%R7(R*_DyQV&GwKd%Dq^XEdxS|v%M zbMhg`L_L`e3p36#$DN4&QZ0NdkS81GPzylJM((@;$9qi?0+C-=dx##a?80K|tQpjW zFR{B5FI~FiOPh+!1|J~QLY47Z6VFEPR4YJa8|*g=3uj)~Ho1w@hf%KqiGH6nQ-b9i zm;>_m0zEppBqkEs(Vnws&z>MAWH!vX0L`5{w-P>UU@CsCQhk$%oRL;~tB;eD(=WtCQ2?rrV~iRAP*fq&FnI1$ z)%uN{%6RgXt@>u6IYagMP84u-beu{|7zEJFnKLO1tG0LV-hTLrI1N>00<>oAE+w4{ zQ~qfTeifQ(K)H(-FaCm<7&B%}#X({RZ3aFt4xCi4XWU?0p8f@D+8oc13)`J|G{*Q zHq(jJEt^4#i+hu|*HZ=-Of{g!jT=`cHqZ)P7FZLiiqE=FpFZ6T0IC3JjN>)jeZDgZ zfKnF=I1?`*1H%mj+}+&|5(_dD76t({c<^ANt+v0v|0n=Zg$7!<$W?LIyD0xw4!(fK zW6iK_+qPN60x@AIK;~b4^%V)}8uRAO>m^B266=X68KBjZ_Nf-2WCuIQI)04xu9RSU z0HD`lrj8gfVj!^~7ZJ@30*J&??b@}g!{w6jeaPyAc2nSL~%^s(0_+)tP`cbXk1<`R7$gJRLf8 zXfP>qRu57tK(khy6$21fl_&jr>Ho(@wqAy_!^5{_txt*0?|9SN2k#70& z<->^q3|~w@8@$Z%bSmkfnrJOCjNN6skWvBqZr>en?li_|fR4R44eSm*cd@NC)B3)=f$Wh;04vtwsa% zm?higV zY;U+8qFrk`Y8(1kV{GB-vkfy5mc4iOIE<|a)(g9*N@z+0L<%GRJA!h%; z1+aI}LfAHVHT*Q~FpR_0+}K@bk@sV`AC;jVP&mD>goFgD{Wy`Yu>)V0IOehbm~sIsd7JQC1Y8VD085yUbr`-wLt-g?X#!}S?{j#OEMTxpf$vGZfB(J?r%{KKk2`kkm|r{#0f?HfLrg-hc$;j}ghK5u#m^g9bw|EI- zHhp7yS}Fm^{_sQCcIhR=r?d7$sWDpg{)-%qMq^DrG$8X5z6+H@tKJe9K0CO&x~{`A zIFYRifR_3^fv_0i1MI4q_rbRi!uIXkw~+5-A66%y@^>HcyHG{6+BIs_Xob(t_>%hH zvCUB;S=qkeC?ViXc8UQ1$YjvzOdl3bzl&ZV7f3#neOH@&Y2fZ7UQpMqTbH)KHr={) z>x*rUS2Rx|8luzp6O009#IZ0qaU&KobEySWdf~x0Fwsei=uVwF4I-bR?-~me&PE98 z8Z4BpT)A>AruKT#dMG+Imo!VU#w>opF^i~FHMSqbpYy_L3O(MwGL3wNx~CQSPMB~u zB9v7^tAkbQwzMstIC0_^x)S6sTq3K~y8??A#uj)zf;+F$OFKBM)Baa@csQLh-A+D| zeN~5iCnj_W2xVyXt5>g1-&j|?(aIS|V13I)*@h1>BMQS+ zwe$S492@H$$F`jmV%ufMwr$(CZQHgz_`lTOsaH2nrfvrJR-M(`*Es#pf6%A6<(9na0X9wtD`8XH2P<^wY4pL^UXIuLj4I79oj;)BvePII@3j# z`3coDFKX~ibHr7QZdHsF5aI1tsfK8}exH*{rFyTq=9*iTYy?C+eTiT6-&n^A%4?7s8mBq_$Smy%R4Z-Hgl21k-3lu&q87y`U4Q=fAWBS@gtzGo z=o+5Oq}v{JY+YA}a6Jm1kqV+JC$VN}qFs8a-65O=`1$oa*@WcS3qNF05&9>;uv^n~7=PtYKvW04^-z00Tr!O&Sf7@jB;Y;3lcK-L~3cF2<86jNd0JIGk*(uC4||9pou=mFz9lsBl#1&qMtrMCW%^J#Ym1 zcI{y=jt(0d8rspRbz6wf=ZT+*m{{H$c?|;5Z1!jHi?q>q;KL6;{59EAJw#{`HcA|2 z(jZEQe0|`72Of&f(4Bt}8?4j(umDdC$;O;ylB8C5&`h5+X;KF?yylu~u71T8SKO@& zjs~#rb`k;7m<0|xfSF+(1t0gWyzJaF&X{ztqd`P-^gxV5HH^I6q;-2`sb)^yLXBoN$g3f{UXQ zbR!+jL|4|Co#kN;o*4Y1oOg+ITl3dYk=m6ye)~O#l?Tf|Hel4GXddFLH+I^XDa)a!KIW&;DK6CI!nvrf5nl!<|LMl>5u zuzhcYvZX|lWL+o~C-UTzPd@98JMMU!Y__hN0Yyk3VlWL?w&^$9^nreosBA$n7ro%` z?3GqpX>9|ii#gj|HCY!E64v==zEQ!^MKww+7K^ipJOU$0!3*!V-+p^5YI{)T?H5+k ztI7v>45nb_0M!2tn4be^I2i%k=!%?g82jFP@4Zh&Ma3e4ahM}>HE^n}gTg=ebrLoz zGgTK|2vs9#HX{Yo)-9QP?X}llSMT*a;kxUtdqO7g>2GBhD7@b2pt91t8I%0DjWx!3J=QM9&K#Ia3F8FdWE0>FJ&;gp(X?J^$FfzFKbxQ6 zbUGbai3RFyZ*N~tH?Hq=+;PX9sN>hv`M)f4_XP`!$s^(cSeO;S1*`n&V@Bcsa)6TW za=RDoqL&%1!x= zlC}V*z)Yopk}RdXVIiIOw;4|Go@`-@=;R6pm@U}L8U)81S&s=rYRrj5L{`uF?h7zCq( zUjZ=QhfbiBYxfHY2$Dq*DwC17fhw8%8fkW<>u;^}w$O%YH@%ntKG+B`gGs< z1T-TTOyj(0DLbb*Nx@7oJ^;%MQSQaBG;35xq{^RF`2S7`Kyh0|x1DoeDfc~Ylz=7y zR0LE=%FK{4OXMv1KaiXy>+hrkP$FPS05c{)6#-X7#ziFhITCGW%sqoSto+VsHI1DB O0000?3S7wcyyOM-n-02&+!#EIxIce+u1iBJ_{ZA{D_@u zcDyU!&S*>E@jvsdP}X7OsVqtzrcRKo6p#%(5zG1PuE3<= zA`eXEIYz+nMzu5Nv5#1B0mMV61gJgU$3uY&2C4%-{OZX9z2jO;A z)(PKVr@6=i#BjkurqDkA{#R>AS4A$iGyhO_NMg^dSD)p|oK4QcspI_YP>r^*N z3b&fROcF~|Hj+o*^CBOj#tYu%aH(1amHBswiR^;-Nj#3)SOY8fIc&ArMN=|p@yu{r_2BZ?Or-rRVji`=Ib%+v< z;*cH7+w_z*3JRvf)}LMwM*}~eM;M_Dq#SlTzto*^p{WCIe?7{ryVBlwfdhxa)=7nW z_XK)_SH#hvC@&0Npp{A5&lhr3C^V}$*ZTu_3%_A(ENcDLj}1bEJQJZ5#Y>0scHaj_ z_d6p(=sXaZ0mWy_xYbfvVoAkHI7DT#)#i#U8Pcn}n;RJjd9GQSVCwRm!MJa^ zA_PnC<^*qF(!}xP@UZPIW~F14JyswE)=-79U^+8sr|mSvspCQxIQKTFal^^c(I|rd zLS!qoj60ah&k)=oai)?l>@A|J2i6WRaBvmc?RGn-9?VH&oQ%=-^t@ZU+M8IznIi~t z$7){SyR*!vehy#>y|DhFJ7dLR&~gsYy_p=XFjrRgiMOnhc)vW;ApF&MeLzN4!_d&5 zrc#ihks9`JzcI%=>TrPBjSoCgB}H^bA|nlm)cL0qFzE6%^wrJCbNHjmVT^dQR9=JR zo^(OyDr%iBpkDySsCZwm1J1>CAI}rI8*M>}&SC+O2l%`cd#|u;OjzMGa9NG#dYcKx z6xwe%IXSIG%*Wtj1zLw(TKkuF1X60QRyPA)MJi!}#^r@S1$(IEain=Bd@C50GM!ah zqYHEQVAHsr8}34>{2P`#0Riz}#Y9`JjdROjf2uD%m+;Vnmx-o9KL-W|h|LFL|Dk@M zEoXy(a!X4~8(YuC@Gt^><1Pa#qeDaiYX;>o@3o)MJ6yC7d59#DM=tVD|I!1Ab9}C?WzDw zLS4nH(`DjN9BYiRjQQ+!P;*j^(bRBsRZCnhiPvy>WyO6(&ohG!Rn1S&Af)|W6)Ri1 zWk`FwK7dpMlwNM}>-I@~5%10lc@$!Dg6p_2zb}jdx`EBG|_Tz63 z2I5G&5m+~~nMJV0Q-uPlFa8cMrT9eD-F^TKO;92W5BXX0>>(c@natK}&@z{fWW2kZ zV_}}F*u3Jyp+|Mik$sgSuF(e?Ee+HfS#+H!atf0aHhPPi>=O3(_tP7NKJEl&icI`I z(YyS`ElU{_Y@+pMh*e`>{gj=KWWA9Kl|tA{T&K$5L5Bts=_dvGcE3Jqp)=;*oeLOz z0{u7~EW)&iUmUZXWl}t{4LHw0UWMW>n*LP&%kn-^ogyAm0Gua-A^cjhI01ywVT1*VjLa66 z>lhL(>KsKlqW&1q>}JXk_4o+Di3l_;UNP;{zR1-oI-0LG{@3K7Z%7?>nG5cHd?QcU zBruWY!m|t|iw*Q6!-Bu_x_S z$xV$7>3UNYKSaYs%={Nh#3vtwroS&?_XleOy}#$FAd$2yLu9UhFJVym$PJOCRNN?(%^x z_PU_MyiW6uNjo2lqyVDVyv=@t?KF!5+V`4WFt$qI3n!o7BPkd=Xu`2;%R2Q_feRWH z;d*o;BPR;mm(Pf}DzB(`45=n3fwEQ47@&nk50NAX0)F)MkznH?L?$-2o`Uy|95Q%l;Z&Shi0u6kR_+k~uFOB;k4QiZ0&2ct9P?V98Wg23Q>f|l z&oR!fW*M-9YEKZImyr5`IN{~PpS;`V%R<9sB$tQ~fi0bO;*6*xdvqtUyNV-c~TfqOcq^pV9zyLtX^G~dI(l3PDHt84ESjUJ|G zfmANRK5Z!L`UFea-m2Tqgu}}vW&zj&d3C2UOIOS$yRCSaS6BL&N-Td6@NHic6BFBD zKu39%Kto(A|D}oTt9n4gCCc%>0-EYzAy1${EnjHA;{6~&Ped{|*s z3AmAkoKsf;GETmA2^kg~RuWwYr>dpo*`Dch3o_6`BEOqTG4ara>QXIfaK{E^6~~Sx zLz!y6T+!(YKYP2*=Zv;lB*6~j0Eusx^idN3aol!oDBJ1>XoFvTW5^Enb?PRr*+7xb zwo+KT zd1y*@>p6kNH`}jdux3%5*F~^~7cKwop->)63Vj|HjRAoo<&Xd6{Z%2e4pd8bOfe1M zoPN04^Ve_uqJ^nzIZA^DtA+-?Sff0+-W<+0?_A0`fg`i#W!cYT3;<&dWMX||#N$-a z_xuwo02~K0X76L7;WjU|Y7yWuzJHBbfAmBcdR-bAcdVKw*B%{r9NA(W?yr3Sp%yBn zh5Ph?o9Ts8?!yIwJk3j094gfW)?1iNX^TLoLq-5f2-z8SHnm~7 z^70a)hEb=9q{kKr0)a>_2;0i7LAf4|KR*DmpH@A}-iR2qh^-GB(FmS5{^#Tuf9AeB zrLK25pp*r=!MnKr|JZ@0qGe+LWkdp@^+U3X2rhK~6E;f}QY*!+DrfO;++z-j&=%W-A+-HxhRhgG$H^Emm! z4zf3w3?qQSo(_a>K5>KH`ZCW0{I>UDY8QzD2}ltG#`1i0npH51ODeO(=0x+{&lmA6 z1+?Razxk?rCy8-0ts)i%X6Zpa7n6KF$^#IY2+l4F0P_i=Rsu?{`yx!*$LqbtB_%<} zQ>5@NAkx)NM!*jae_Rj28vY9M`)h;(@v5ofwI4(RevRD5>0{zy!+(%4q6OR{nfjZp z$Gq!`RJ_HddGOjtXYowu4bZ@R63N&4zEl5zX=EB;4Iv06aJ(wxk^{A=#);`su56hGGU~CC z{<2g<(R9@D7rQUh;&3hS^_iSASAzyN3ciPbu=j|)?e@MZ&GCax<)c6Z*2ZKsRGq9< zCNJi_V=tw2qVJc(I0CnnGrByA*HsMmua_Jfbn4`t_K1TLRSX-zZHP@r6sa3qAuy_W zR*8KSr`3*DWX{pK$mBVO-e3T|7tVqB!zQaQt@u5aL$Fxf5PgyMp45c?aTkSN0a@N@M0$2U6K5YX^S{pmG>7cjL z5DN-9;gjuBmiuOxdBMv!)&SY&q{&*55{EU}j&5m1P-aR>3Kvc58zQ9iUo?OO(YLa< zqiW%vuax+xH6qvmJGovjPAgxYts8dO3V8vJ9WOW&pnab@Ca4m3LgQE zK;O#+4I;dH(}W^4s1?iZc%uvyL_*mN@cUI=)~@p}=6+H)pB>Rl^sR-+hbXt^{C zXQLurlDOIJ_V^zwOxy{?rmqCBJqzT570blD<=x?LE7AddKWDIhZE`#Y%r*rCKK$eZ zp?iD+Y1mh0OI0pA<%w7kpi~PoJOenm73sc82c2EzOzkFY}IOC-PPOJJmaX zs&UFkIy7maC5jn;t;Vu%N@l@{FJGA%0Dj^J4_SbOxi?LOAH?pqcJIPrNphD%I2G8C z$QmYA^WD*`HP8qGGA5fX7x4R>3$qdTHBN{aYd0>{W^<3^-d@x~ZwQ}fE;{P<{i19d z3Iq@O9v-Qb5X*oc8Fi_EFfj1;WCK4D!*aX=5fj<`8(WvKrRRaGn--G#Oa+8PUMDaT4 zqeo-1$udL1fL393kmadMT3t3k7~}SBw`nj13e~r1P+<0sXa*nemkd_$x&Po8plg)t zOLUZW3}7FDE{>RI*YzjrWUK3*9B!mgPxWQa3Iojwwz%|!o}A4^MzL+mfaTQ6@l!X} zQN09Up+xHleD&rj=oE{Hb6v`W*i&Ek6F^inyPlXsAptDj@$>VvP@+X>wS4?7*Q>ss>|Ec^n{SFZ9LCHSbsc zV|K1?EPf6S+&AG~>8`FIj;CDSK05M!7YEvQ81d* zd>8_7adG+CFP+(m4cNF?C=hzwhkWMrH(%zS5mfB&Bu(`r5n6)0K$BxQZ*r=wYSf+J1R^UDD;mQ4Up z2~qT(14Dz$)_~=I=)?|0Yqp!mtWQnr?d=VZY-GBXho~WF+a(h@?EoKmatWpVmly`U zgLEVK@Gta8+g`|Dj3{OvC>&D{J7Zae|FpKf4ZG8uy4)El1N0dHS+JD!WdJX70(>lw z_xBAcC{%CF{}BMK+Q*U*wCc$v7`+73U!P9hUhuF!JS8DNbQ~xk8(jm`F0C{$uDUeJ zRNo@J!>W`Y9J|2!H@8Me?d3}Ru^Z#u@a(yALmIF15H$vKw9#Vzg~ ztQ5|7d;amyJNs9_4MHfqx*GlRdwqR9b)zrh6&>9bwIQ|<%yhl#dsgJ4+NGGmG1f}x z{7EzTI1FyWzBCSFiDaqBH8ND0oY`Ryb6C6PN^U2}>l*N_`}O5%9%5PrKqU?BZx2G_ zwf+0Or9q>_N?rZ0S`@&_mr8shOU_2BUU|UhnoL-@JcckyP*ikc70+WG8Uz?{6ftR$ z2tj(w*JA<3_6@opJnXQZQcML`ePmG*xMj(&Wd`=#IT zWwwGIS{b^sd5QaDHmwj#3|2!^cR()W(O`L;BTZ2#2V!&1w9f!T{5O(532H3^bKgnX zhw!LmahG6mj&xQkq-#OG{s4dcD6N%`>l-IBM}sxV3}mncEz%dn6;4UbPiIQl{W(f8 zZIuNqq-t1=(P=0tts=LCu}=Lbw8bE|i2|(Bv>+6Up+!Z*aIDMPN~qGeZ3` zh$}(@r!tC%R#Qk_9?o9&`Dc&%8lEfH(lR1PJ@)v|e*mGEJ3r1c-k%m^qU+PlmPU5 zi_cbQjmQ^UV(21~WACWu;Q+pdmdL{sp7omp=NM7sCM_@JWvNw4ufAf0Ue%YBpe0ydjsS?@(l(7+~E=ve%BACh{G{7uJR?^o9&77Puh52 z(Gx7I{9Aw0faq?T*Udio4;4gopk}{i)8!;3N~`(&7xe7pLP7NiY?O^9x?pySPFx`= zeg|^XbxWV?SX`M~kLba49f)2oCx46dceLrs#ykb{Ee=(Y2C~9Zt)aoEiaYT;Lc+~{ z&*Oy)IgYvXA6slxSYZ?jk%lM_7sbFfSM1sZSAe_v6U?fj6v#m0cqbY;6VFR>Zww;#q zWqqp#eE$hAXCO_5u8uI(4WbO63zj3~Rr|e8B+<%!YtdXoTc|W__pr3wQF&v5E%$GC zEN4C#eN#yHHB}jyhYP@oo{~=r{Y_;oUpEZO&7W$zIVDQ zz3A+GK=qd(SZ_8amas!OQ~eTd2r_o`$9@viX!AHZL|5XoU#MyO`e1I#`vcSscLZW7 z@5b|y{sTS~nL8*6uJT1T8A)sl+G3@cQjfjbqCG#~5XINNpSMN&wEuv0lNAuToF4WA zBV;+DXsKFV^%{YDN$`9}B`}eSlatf;dVi`jsQ(d;q1%)M@4NBe_YbGBMp(39 z%JJmm2{7b=tU>@yagIkEe}f{uhX48o|17+UnT;@bytYPf0=zX1Na63Ep@y&#JI+-S z&nfeNt~HZU5k<7u#pOkrbd(YBep)TvZm=evi!FOa^r)-jcVOZi%3J#Y{SG=yxU|S5 z)6uQL@96E*M4pTfi( z>|9ush4g2bAA+OEazQX|6cbZ7J-m-XQl}cS=|xxU-j9R7e-~c5%KALcr*P_i?%d{& z80VBjJnfJOiaqIjq?Hq(5JFnUhW;e zYWg3fwR&p*x_{0e;ap_}pj3-khRRtQuwtwy^8;Uwo6zL=n z&_u%s&txIhLGP;D|B3JWL2>P zlf_221v+Qk{UpHTL;XT2S)R?v(JRJ5pPj(L?pPjhKZtAfM&oWt7o)A zNqM$bg{Ez|BRS8#irbku;Fklgxo*Ty)EDZd>dCNu{JyTj;A_(x2ruI+Kngeg3rPP^ zKvYXUvSbU55=NkGrY!#eDOI=WiL%jxtfe^&KlN^5r!5)y zSF<|C{#eKIL=~7y<;nLIiP$40s_X&0gIs*@MWKAccScHE8QKp35*4q{OJJA=ra{>c zfxgnp(o?y!{riip17RmoqRgZk4Mh(BnNNONiIQaQEg!?n^D-6&xex|Plg)lTBFJ$t zBp$f})?7y{FS{KVQfHCGd}QIm*n8{oTrmg%Dv$e2hA4{ywHiUNOSx{X*?qXtZjP1q zT8m4F5(xz))E!*q$9GnOSal5Ex<5$zH?k8!_FRVz0q>{3ct=kA{|Z zfeG55aLz*|0u zaGXr@h4 zA3?u6Q_(++HEkOTM>Pt)kM}E1j_pY`0Ej*HPabsM{$VWeLspup)T=Lgfx9L?xu*Fv?ncng8 zk)Yt7(`6x#6zSe~g^~TWCaRP3xm??^^As9_u8F5#4<9~5<3){L2UmFkddc9atHq|y ztwskP=PrFXSNs1t#`V%QuhyzIWfxyT`$h0@dqhW1&qs|d$HH$z6BU=PCKrI9Ylw)W&czkeTg3QJ%wQN_t_+Gqo%CCvMg)?x1p1=SuZl zsQdO%((@GE4n3(Abu$O2T0xsy@rbumhUZ$Be}rnp481!`<{Hs!d!9I&x$^UHY$?+S z>SD9g8bB(ks+L3Ch3xAQDCExDl$Viz%_cgctr)Xh@lX@Z$Q*X|E;fwj6AraxC^|Hj z1N*z;3+zFW!>Ge60y9cdnGar7C#X*L&Nsb{~f~ zWR=BhDoLO`j$N^2BfKDZo9g&?eGG}=@X83^(UDrp_gh9_NAHvRajC`!CGorf;aS^U zagW%oaz%l^AgK>gcr(BB?~KWEgrWv}YFU?!`2Vf9%lQYYsHh-6-V0X2-n*&Cv=zmt zR{dpJ=`ZFGb4VEx2)f3U(cq!_b(9IAuXnztskQtrpY+wl=>DrVH?uT%!)mCcNVq$L!6g-0xy4R2JK-Q5{`I# zQ~xEA5$ImRBQkK`oy(BcM%5-x)6Z-@SuatpUczEB?{WcW8OE3AB5AUS0ok-f#e2A| zK{n3oX^vyUPUWXoh`e*q#Q+8aAQ1T{7c&ePW@7k}ujjCcZ8IL8Ln*B!;n|zNW z3u#Mn=cCy7eRcu+t-*m0A9@)r!RGhC_kNpx%2r5?tr#n9jm-MPnRTV3%)TWdp$d#@ z7^|}??5cEQ0j&+%5sqsyj{9AT@*JM{lp#rAT2!cW|Zno zx~W;Cp)<&_W&!~Zuc^!$>BYJMk#>gU!r{!<%o)p4r0H?_XFr5Qv+6aaUyTU_`2~Jfkm-%@yuMhsg zMMiL)iMC!4tGFjp6{p4Z+k!tf6N@Nk6p(WCy>N75ZeYCt>k|5!)&|TiwItbA*wpLz z+UlIN`wxDc5N`i-B)(y9XE|-io15ZPB3wz(euy4 zM=AonmQ$bT{d6pPc_HBrObA?l!{OYxee=!%enQJ!YGT{mGTqE19NF5GbZM~=%UKj6 zs0D$?CVChe0n0pmkv<>Vo%5SmS3u1B@^|n@KO5CsbZXX?*cFP6v?=}9@q9l{!S_Cr zxl3^AdJR72npbVKe-_I4>I?^S7uh#N7HIOoY!phn!zjWBp)~EZ^0M+p=k`eW!{fhO zKKC4tYr(fLpc!5$!83OP1|kDM%=RC1fBQ!>MiyqtcBvtAPxPlp&8?kau(N$CYksQD-YlPbh4vhd%Z z64NGbW_#Fl6DMic`pu(Q%Zx3eHZeuz6(qI=0V2o-HCJcQ$VB&EQkY#Ik|XYZ*Ws40 zvcP?JOxooV`XjoTAtgz5J;?%*1PLRwMtsQtFX9}!@hlk#yBJ)Htq?fnC!OFE=odUr z6u7NCS&i>5+JN!eT8Y=!WW7ye0(#)#368n!w1VN}?S&w~1P z+1Iq{P>SeDFu>^r9j3?mWt;NF-u}(h2F6;@*|%8HdTC#bUApgVsY>6%{rmXisru_C z%Y3xuON9`)YN2$UR+Btn3``#OD?Gj+D+j>#41)hf@f2T8;uKMY*DX*s&%zA}vm0rv zGYb7Hp-?4O&A)5x^F!*_Sdr-V`@0BN73^m5YD*l3dwn1mwI z+gQL;V2|m%-X`JHq+P&LvKXBA(Yx*pYf&H>5Hy&(PuB7pH=4g~Bao)X{`KVQ>6!8R z+LXSXn4H?yoH(jspF+&h4tikg{u8)K+l4E|=GzXu`QCI5oz7{zW)M-RzFy`@>FRB? zq|CYJ4P+lwbct3=_w-S(AmmK+*(i$Hw3X;UvhCXE{~ip)|dX+qill&JfM zx|ObMJAB`+oVn4xSHE}KE^hS==E1~C1G2=U00BkhMm7QUSbk;jk2-}0FYQ(Hej>>M zuHU)C(y~;njtvB)Z=MaGsqPKFCfX^TQaAd{I8G^Qp{i8d`gAvaPq3xD{mhknAQFNT znvd;07tPHoi^0uDZAF^w5hst2@aCEPI`SZ~SY{lFmt}_SZ-}xNeaGpwwZvo1U33o~ zyo^pI0V+!7?f^G3*PFx!uqh%z2+I$mAI4D&pw_S3${MT^C#-qr6dNJeXHD24tSH9# zgo$eXE*KxR=sqT1_Pi#N8nFK936t|kIHx*JVVB-`Bh_Cdz$0*H>K_@hEjkr)EH)~M nT_xa0rZ2p8v=`VMwgSqw1s!`SXB7Rn?en9IvUHWCN$~#w$6JiI literal 0 HcmV?d00001 diff --git a/demos/composition/src/main/res/values-night/themes.xml b/demos/composition/src/main/res/values-night/themes.xml new file mode 100644 index 0000000000..5194fe967e --- /dev/null +++ b/demos/composition/src/main/res/values-night/themes.xml @@ -0,0 +1,31 @@ + + + + + + diff --git a/demos/composition/src/main/res/values/arrays.xml b/demos/composition/src/main/res/values/arrays.xml new file mode 100644 index 0000000000..b9e79f087d --- /dev/null +++ b/demos/composition/src/main/res/values/arrays.xml @@ -0,0 +1,74 @@ + + + + + 720p H264 video and AAC audio + 1080p H265 video and AAC audio + 360p H264 video and AAC audio + 360p VP8 video and Vorbis audio + 4K H264 video and AAC audio (portrait, no B-frames) + 8k H265 video and AAC audio + Short 1080p H265 video and AAC audio + Long 180p H264 video and AAC audio + H264 video and AAC audio (portrait, H > W, 0°) + H264 video and AAC audio (portrait, H < W, 90°) + SEF slow motion with 240 fps + 480p DASH (non-square pixels) + HDR (HDR10) H265 limited range video (encoding may fail) + HDR (HLG) H265 limited range video (encoding may fail) + 720p H264 video with no audio + London JPG image (plays for 5 secs at 30 fps) + Tokyo JPG image (portrait, plays for 5 secs at 30 fps) + + + https://storage.googleapis.com/exoplayer-test-media-1/mp4/android-screens-10s.mp4 + https://storage.googleapis.com/exoplayer-test-media-0/android-block-1080-hevc.mp4 + https://html5demos.com/assets/dizzy.mp4 + https://html5demos.com/assets/dizzy.webm + https://storage.googleapis.com/exoplayer-test-media-1/mp4/portrait_4k60.mp4 + https://storage.googleapis.com/exoplayer-test-media-1/mp4/8k24fps_4s.mp4 + https://storage.googleapis.com/exoplayer-test-media-1/mp4/1920w_1080h_4s.mp4 + https://storage.googleapis.com/exoplayer-test-media-0/BigBuckBunny_320x180.mp4 + https://storage.googleapis.com/exoplayer-test-media-1/mp4/portrait_avc_aac.mp4 + https://storage.googleapis.com/exoplayer-test-media-1/mp4/portrait_rotated_avc_aac.mp4 + https://storage.googleapis.com/exoplayer-test-media-1/mp4/slow-motion/slowMotion_stopwatch_240fps_long.mp4 + https://storage.googleapis.com/exoplayer-test-media-1/gen/screens/dash-vod-single-segment/manifest-baseline.mpd + https://storage.googleapis.com/exoplayer-test-media-1/mp4/samsung-s21-hdr-hdr10.mp4 + https://storage.googleapis.com/exoplayer-test-media-1/mp4/Pixel7Pro_HLG_1080P.mp4 + https://storage.googleapis.com/exoplayer-test-media-1/mp4/sample_video_track_only.mp4 + https://storage.googleapis.com/exoplayer-test-media-1/jpg/london.jpg + https://storage.googleapis.com/exoplayer-test-media-1/jpg/tokyo.jpg + + + 10024000 + 23823000 + 25000000 + 25000000 + 3745000 + 4421000 + 3923000 + 596459000 + 3687000 + 2235000 + 47987000 + 128270000 + 4236000 + 5167000 + 1001000 + 5000000 + 5000000 + + diff --git a/demos/composition/src/main/res/values/colors.xml b/demos/composition/src/main/res/values/colors.xml new file mode 100644 index 0000000000..91d1b1023f --- /dev/null +++ b/demos/composition/src/main/res/values/colors.xml @@ -0,0 +1,24 @@ + + + + #FFBB86FC + #FF6200EE + #FF3700B3 + #FF03DAC5 + #FF018786 + #FF000000 + #FFFFFFFF + diff --git a/demos/composition/src/main/res/values/strings.xml b/demos/composition/src/main/res/values/strings.xml new file mode 100644 index 0000000000..e664b725c0 --- /dev/null +++ b/demos/composition/src/main/res/values/strings.xml @@ -0,0 +1,27 @@ + + + + Composition Demo + Edit + Preview + Single sequence preview + Single sequence items: + Choose preset file + Export + Export completed in %.3f seconds.\nOutput: %s + Export error + Export started + diff --git a/demos/composition/src/main/res/values/themes.xml b/demos/composition/src/main/res/values/themes.xml new file mode 100644 index 0000000000..29ccfdfc53 --- /dev/null +++ b/demos/composition/src/main/res/values/themes.xml @@ -0,0 +1,31 @@ + + + + + + diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerTest.java new file mode 100644 index 0000000000..da0a977d96 --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerTest.java @@ -0,0 +1,529 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.common.PlaybackException.ERROR_CODE_DECODER_INIT_FAILED; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static org.junit.Assert.assertThrows; + +import android.app.Instrumentation; +import android.content.Context; +import android.graphics.BitmapFactory; +import android.view.SurfaceHolder; +import android.view.SurfaceView; +import android.view.TextureView; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.DebugViewProvider; +import androidx.media3.common.Effect; +import androidx.media3.common.Format; +import androidx.media3.common.MediaItem; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.PlaybackException; +import androidx.media3.common.PreviewingVideoGraph; +import androidx.media3.common.SurfaceInfo; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.VideoFrameProcessor; +import androidx.media3.common.VideoGraph; +import androidx.media3.common.util.SystemClock; +import androidx.media3.common.util.Util; +import androidx.media3.datasource.AssetDataSource; +import androidx.media3.datasource.DataSourceUtil; +import androidx.media3.datasource.DataSpec; +import androidx.media3.effect.PreviewingSingleInputVideoGraph; +import androidx.media3.exoplayer.RendererCapabilities; +import androidx.media3.exoplayer.image.BitmapFactoryImageDecoder; +import androidx.media3.exoplayer.image.ImageDecoder; +import androidx.media3.exoplayer.image.ImageDecoderException; +import androidx.media3.exoplayer.source.ExternalLoader; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.rules.ActivityScenarioRule; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; +import java.io.IOException; +import java.util.List; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Instrumentation tests for {@link CompositionPlayer} */ +@RunWith(AndroidJUnit4.class) +public class CompositionPlayerTest { + + private static final long TEST_TIMEOUT_MS = 10_000; + private static final String MP4_ASSET = "asset:///media/mp4/sample.mp4"; + private static final String IMAGE_ASSET = "asset:///media/jpeg/white-1x1.jpg"; + + @Rule + public ActivityScenarioRule rule = + new ActivityScenarioRule<>(SurfaceTestActivity.class); + + private final Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); + private final Context applicationContext = instrumentation.getContext().getApplicationContext(); + + private CompositionPlayer compositionPlayer; + private SurfaceView surfaceView; + private SurfaceHolder surfaceHolder; + private TextureView textureView; + + @Before + public void setupSurfaces() { + rule.getScenario() + .onActivity( + activity -> { + surfaceView = activity.getSurfaceView(); + textureView = activity.getTextureView(); + }); + surfaceHolder = surfaceView.getHolder(); + } + + @After + public void closeActivity() { + rule.getScenario().close(); + } + + @After + public void releasePlayer() { + instrumentation.runOnMainSync( + () -> { + if (compositionPlayer != null) { + compositionPlayer.release(); + } + }); + } + + @Test + public void setVideoSurfaceView_beforeSettingComposition_surfaceViewIsPassed() throws Exception { + PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS); + instrumentation.runOnMainSync( + () -> { + compositionPlayer = new CompositionPlayer.Builder(applicationContext).build(); + compositionPlayer.setVideoSurfaceView(surfaceView); + compositionPlayer.addListener(listener); + compositionPlayer.setComposition( + new Composition.Builder( + new EditedMediaItemSequence( + new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET)) + .setDurationUs(1_000_000) + .build())) + .build()); + compositionPlayer.prepare(); + }); + + listener.waitUntilFirstFrameRendered(); + } + + @Test + public void setVideoSurfaceView_afterSettingComposition_surfaceViewIsPassed() throws Exception { + PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS); + instrumentation.runOnMainSync( + () -> { + compositionPlayer = new CompositionPlayer.Builder(applicationContext).build(); + compositionPlayer.addListener(listener); + compositionPlayer.setComposition( + new Composition.Builder( + new EditedMediaItemSequence( + new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET)) + .setDurationUs(1_000_000) + .build())) + .build()); + compositionPlayer.setVideoSurfaceView(surfaceView); + compositionPlayer.prepare(); + }); + + listener.waitUntilFirstFrameRendered(); + } + + @Test + public void setVideoSurfaceHolder_beforeSettingComposition_surfaceHolderIsPassed() + throws Exception { + PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = new CompositionPlayer.Builder(applicationContext).build(); + compositionPlayer.setVideoSurfaceHolder(surfaceHolder); + compositionPlayer.addListener(listener); + compositionPlayer.setComposition( + new Composition.Builder( + new EditedMediaItemSequence( + new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET)) + .setDurationUs(1_000_000) + .build())) + .build()); + compositionPlayer.prepare(); + }); + + listener.waitUntilFirstFrameRendered(); + } + + @Test + public void setVideoSurfaceHolder_afterSettingComposition_surfaceHolderIsPassed() + throws Exception { + PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = new CompositionPlayer.Builder(applicationContext).build(); + compositionPlayer.addListener(listener); + compositionPlayer.setComposition( + new Composition.Builder( + new EditedMediaItemSequence( + new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET)) + .setDurationUs(1_000_000) + .build())) + .build()); + compositionPlayer.setVideoSurfaceHolder(surfaceHolder); + compositionPlayer.prepare(); + }); + + listener.waitUntilFirstFrameRendered(); + } + + @Test + public void setVideoTextureView_throws() { + AtomicReference exception = new AtomicReference<>(); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = new CompositionPlayer.Builder(applicationContext).build(); + try { + compositionPlayer.setVideoTextureView(textureView); + } catch (UnsupportedOperationException e) { + exception.set(e); + } + }); + + assertThat(exception.get()).isNotNull(); + } + + @Test + public void imagePreview_imagePlaysForSetDuration() throws Exception { + PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = new CompositionPlayer.Builder(applicationContext).build(); + // Set a surface on the player even though there is no UI on this test. We need a surface + // otherwise the player will skip/drop video frames. + compositionPlayer.setVideoSurfaceView(surfaceView); + compositionPlayer.addListener(listener); + compositionPlayer.setComposition( + new Composition.Builder( + new EditedMediaItemSequence( + new EditedMediaItem.Builder( + new MediaItem.Builder() + .setUri(IMAGE_ASSET) + .setImageDurationMs(1_000) + .build()) + .setDurationUs(1_000_000) + .build())) + .build()); + compositionPlayer.prepare(); + }); + + listener.waitUntilFirstFrameRendered(); + listener.waitUntilPlayerReady(); + long playbackStartTimeMs = SystemClock.DEFAULT.elapsedRealtime(); + instrumentation.runOnMainSync(() -> compositionPlayer.play()); + listener.waitUntilPlayerEnded(); + long playbackRealTimeMs = SystemClock.DEFAULT.elapsedRealtime() - playbackStartTimeMs; + + assertThat(playbackRealTimeMs).isAtLeast(1_000); + } + + @Test + public void imagePreview_externallyLoadedImage() throws Exception { + PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS); + ExternalLoader externalImageLoader = + loadRequest -> immediateFuture(Util.getUtf8Bytes(loadRequest.uri.toString())); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = + new CompositionPlayer.Builder(applicationContext) + .setExternalImageLoader(externalImageLoader) + .setImageDecoderFactory(new TestImageDecoderFactory()) + .build(); + // Set a surface on the player even though there is no UI on this test. We need a surface + // otherwise the player will skip/drop video frames. + compositionPlayer.setVideoSurfaceView(surfaceView); + compositionPlayer.addListener(listener); + compositionPlayer.setComposition( + new Composition.Builder( + new EditedMediaItemSequence( + new EditedMediaItem.Builder( + new MediaItem.Builder() + .setUri(IMAGE_ASSET) + .setMimeType(MimeTypes.APPLICATION_EXTERNALLY_LOADED_IMAGE) + .setImageDurationMs(1_000) + .build()) + .setDurationUs(1_000_000) + .build())) + .build()); + compositionPlayer.prepare(); + }); + + listener.waitUntilFirstFrameRendered(); + } + + @Test + public void imagePreview_twoImages() throws Exception { + PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS); + EditedMediaItem image = + new EditedMediaItem.Builder( + new MediaItem.Builder().setUri(IMAGE_ASSET).setImageDurationMs(500).build()) + .setDurationUs(500_000) + .build(); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = new CompositionPlayer.Builder(applicationContext).build(); + // Set a surface on the player even though there is no UI on this test. We need a surface + // otherwise the player will skip/drop video frames. + compositionPlayer.setVideoSurfaceView(surfaceView); + compositionPlayer.addListener(listener); + compositionPlayer.setComposition( + new Composition.Builder(new EditedMediaItemSequence(image, image)).build()); + compositionPlayer.prepare(); + compositionPlayer.play(); + }); + + listener.waitUntilPlayerEnded(); + } + + @Test + public void composition_imageThenVideo() throws Exception { + PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS); + EditedMediaItem image = + new EditedMediaItem.Builder( + new MediaItem.Builder().setUri(IMAGE_ASSET).setImageDurationMs(500).build()) + .setDurationUs(500_000) + .build(); + + EditedMediaItem video = + new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET)).setDurationUs(1_000_000).build(); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = new CompositionPlayer.Builder(applicationContext).build(); + // Set a surface on the player even though there is no UI on this test. We need a surface + // otherwise the player will skip/drop video frames. + compositionPlayer.setVideoSurfaceView(surfaceView); + compositionPlayer.addListener(listener); + compositionPlayer.setComposition( + new Composition.Builder(new EditedMediaItemSequence(image, video)).build()); + compositionPlayer.prepare(); + compositionPlayer.play(); + }); + + listener.waitUntilPlayerEnded(); + } + + @Test + public void composition_videoThenImage() throws Exception { + PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS); + EditedMediaItem video = + new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET)).setDurationUs(1_000_000).build(); + EditedMediaItem image = + new EditedMediaItem.Builder( + new MediaItem.Builder().setUri(IMAGE_ASSET).setImageDurationMs(500).build()) + .setDurationUs(500_000) + .build(); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = new CompositionPlayer.Builder(applicationContext).build(); + // Set a surface on the player even though there is no UI on this test. We need a surface + // otherwise the player will skip/drop video frames. + compositionPlayer.setVideoSurfaceView(surfaceView); + compositionPlayer.addListener(listener); + compositionPlayer.setComposition( + new Composition.Builder(new EditedMediaItemSequence(video, image)).build()); + compositionPlayer.prepare(); + compositionPlayer.play(); + }); + + listener.waitUntilPlayerEnded(); + } + + @Test + public void playback_videoSinkProviderFails_playerRaisesError() { + PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS); + EditedMediaItem video = + new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET)).setDurationUs(1_000_000).build(); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = + new CompositionPlayer.Builder(applicationContext) + .setPreviewingVideoGraphFactory( + (context, + outputColorInfo, + debugViewProvider, + graphListener, + listenerExecutor, + compositionEffects, + initialTimestampOffsetUs) -> { + throw new VideoFrameProcessingException( + "Test video graph failed to initialize"); + }) + .build(); + compositionPlayer.addListener(listener); + compositionPlayer.setComposition( + new Composition.Builder(new EditedMediaItemSequence(video)).build()); + compositionPlayer.prepare(); + compositionPlayer.play(); + }); + + PlaybackException thrownException = + assertThrows(PlaybackException.class, listener::waitUntilPlayerEnded); + assertThat(thrownException.errorCode).isEqualTo(ERROR_CODE_DECODER_INIT_FAILED); + } + + @Test + public void release_videoSinkProviderFailsDuringRelease_playerDoesNotRaiseError() + throws Exception { + PlayerTestListener playerTestListener = new PlayerTestListener(TEST_TIMEOUT_MS); + EditedMediaItem video = + new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET)).setDurationUs(1_000_000).build(); + instrumentation.runOnMainSync( + () -> { + compositionPlayer = + new CompositionPlayer.Builder(applicationContext) + .setPreviewingVideoGraphFactory(FailingReleaseVideoGraph::new) + .build(); + compositionPlayer.addListener(playerTestListener); + compositionPlayer.setComposition( + new Composition.Builder(new EditedMediaItemSequence(video)).build()); + compositionPlayer.prepare(); + compositionPlayer.play(); + }); + // Wait until the player is ended to make sure the VideoGraph has been created. + playerTestListener.waitUntilPlayerEnded(); + + instrumentation.runOnMainSync(compositionPlayer::release); + + playerTestListener.waitUntilPlayerIdle(); + } + + private static final class TestImageDecoderFactory implements ImageDecoder.Factory { + + @Override + public @RendererCapabilities.Capabilities int supportsFormat(Format format) { + return format.sampleMimeType != null + && format.sampleMimeType.equals(MimeTypes.APPLICATION_EXTERNALLY_LOADED_IMAGE) + ? RendererCapabilities.create(C.FORMAT_HANDLED) + : RendererCapabilities.create(C.FORMAT_UNSUPPORTED_TYPE); + } + + @Override + public ImageDecoder createImageDecoder() { + return new BitmapFactoryImageDecoder.Factory( + /* bitmapDecoder= */ (data, length) -> { + try { + // The test serializes the image URI string to a byte array. + String assetPath = new String(data); + AssetDataSource assetDataSource = + new AssetDataSource(ApplicationProvider.getApplicationContext()); + assetDataSource.open(new DataSpec.Builder().setUri(assetPath).build()); + byte[] imageData = DataSourceUtil.readToEnd(assetDataSource); + return BitmapFactory.decodeByteArray(imageData, 0, imageData.length); + } catch (IOException e) { + throw new ImageDecoderException(e); + } + }) + .createImageDecoder(); + } + } + + private static final class FailingReleaseVideoGraph extends ForwardingVideoGraph { + public FailingReleaseVideoGraph( + Context context, + ColorInfo outputColorInfo, + DebugViewProvider debugViewProvider, + VideoGraph.Listener listener, + Executor listenerExecutor, + List compositionEffects, + long initialTimestampOffsetUs) { + super( + new PreviewingSingleInputVideoGraph.Factory() + .create( + context, + outputColorInfo, + debugViewProvider, + listener, + listenerExecutor, + compositionEffects, + initialTimestampOffsetUs)); + } + + @Override + public void release() { + throw new RuntimeException("VideoGraph release error"); + } + } + + private static class ForwardingVideoGraph implements PreviewingVideoGraph { + + private final PreviewingVideoGraph videoGraph; + + public ForwardingVideoGraph(PreviewingVideoGraph videoGraph) { + this.videoGraph = videoGraph; + } + + @Override + public void initialize() throws VideoFrameProcessingException { + videoGraph.initialize(); + } + + @Override + public void registerInput(int inputIndex) throws VideoFrameProcessingException { + videoGraph.registerInput(inputIndex); + } + + @Override + public VideoFrameProcessor getProcessor(int inputId) { + return videoGraph.getProcessor(inputId); + } + + @Override + public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { + videoGraph.setOutputSurfaceInfo(outputSurfaceInfo); + } + + @Override + public boolean hasProducedFrameWithTimestampZero() { + return videoGraph.hasProducedFrameWithTimestampZero(); + } + + @Override + public void release() { + videoGraph.release(); + } + + @Override + public void renderOutputFrame(long renderTimeNs) { + videoGraph.renderOutputFrame(renderTimeNs); + } + } +} diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlaybackTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlaybackTest.java new file mode 100644 index 0000000000..4635085905 --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlaybackTest.java @@ -0,0 +1,154 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer.mh.performance; + +import static androidx.media3.common.MimeTypes.VIDEO_H264; +import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE; +import static androidx.media3.test.utils.BitmapPixelTestUtil.createArgb8888BitmapFromRgba8888Image; +import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; +import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; +import static androidx.media3.transformer.mh.performance.PlaybackTestUtil.createTimestampOverlay; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; + +import android.app.Instrumentation; +import android.graphics.Bitmap; +import android.graphics.PixelFormat; +import android.media.Image; +import android.media.ImageReader; +import androidx.media3.common.Format; +import androidx.media3.common.MediaItem; +import androidx.media3.common.util.ConditionVariable; +import androidx.media3.common.util.Size; +import androidx.media3.common.util.Util; +import androidx.media3.transformer.Composition; +import androidx.media3.transformer.CompositionPlayer; +import androidx.media3.transformer.EditedMediaItem; +import androidx.media3.transformer.EditedMediaItemSequence; +import androidx.media3.transformer.Effects; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; +import com.google.common.collect.ImmutableList; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +/** Playback tests for {@link CompositionPlayer} */ +@RunWith(AndroidJUnit4.class) +public class CompositionPlaybackTest { + + private static final String TEST_DIRECTORY = "test-generated-goldens/ExoPlayerPlaybackTest"; + private static final String MP4_ASSET_URI_STRING = "asset:///media/mp4/sample.mp4"; + private static final Format MP4_ASSET_FORMAT = + new Format.Builder() + .setSampleMimeType(VIDEO_H264) + .setWidth(1080) + .setHeight(720) + .setFrameRate(29.97f) + .setCodecs("avc1.64001F") + .build(); + private static final Size MP4_ASSET_VIDEO_SIZE = + new Size(MP4_ASSET_FORMAT.width, MP4_ASSET_FORMAT.height); + private static final long TEST_TIMEOUT_MS = 10_000; + + @Rule public final TestName testName = new TestName(); + + private final Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); + private @MonotonicNonNull CompositionPlayer player; + private @MonotonicNonNull ImageReader outputImageReader; + private String testId; + + @Before + public void setUpTestId() { + testId = testName.getMethodName(); + } + + @After + public void tearDown() { + instrumentation.runOnMainSync( + () -> { + if (player != null) { + player.release(); + } + if (outputImageReader != null) { + outputImageReader.close(); + } + }); + } + + @Test + public void compositionPlayerPreviewTest_ensuresFirstFrameRenderedCorrectly() throws Exception { + AtomicReference renderedFirstFrameBitmap = new AtomicReference<>(); + ConditionVariable hasRenderedFirstFrameCondition = new ConditionVariable(); + outputImageReader = + ImageReader.newInstance( + MP4_ASSET_VIDEO_SIZE.getWidth(), + MP4_ASSET_VIDEO_SIZE.getHeight(), + PixelFormat.RGBA_8888, + /* maxImages= */ 1); + + instrumentation.runOnMainSync( + () -> { + player = new CompositionPlayer.Builder(instrumentation.getContext()).build(); + outputImageReader.setOnImageAvailableListener( + imageReader -> { + try (Image image = imageReader.acquireLatestImage()) { + renderedFirstFrameBitmap.set(createArgb8888BitmapFromRgba8888Image(image)); + } + hasRenderedFirstFrameCondition.open(); + }, + Util.createHandlerForCurrentOrMainLooper()); + + player.setVideoSurface(outputImageReader.getSurface(), MP4_ASSET_VIDEO_SIZE); + player.setComposition( + new Composition.Builder( + new EditedMediaItemSequence( + new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET_URI_STRING)) + .setEffects( + new Effects( + /* audioProcessors= */ ImmutableList.of(), + /* videoEffects= */ ImmutableList.of( + createTimestampOverlay()))) + .setDurationUs(1_024_000L) + .build())) + .build()); + player.prepare(); + }); + + if (!hasRenderedFirstFrameCondition.block(TEST_TIMEOUT_MS)) { + throw new TimeoutException( + Util.formatInvariant("First frame not rendered in %d ms.", TEST_TIMEOUT_MS)); + } + + assertWithMessage("First frame is not rendered.") + .that(renderedFirstFrameBitmap.get()) + .isNotNull(); + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888( + /* expected= */ readBitmap(TEST_DIRECTORY + "/first_frame.png"), + /* actual= */ renderedFirstFrameBitmap.get(), + testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + // TODO: b/315800590 - Verify onFirstFrameRendered is invoked only once. + } +} diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlayerSeekTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlayerSeekTest.java new file mode 100644 index 0000000000..8a17ac842d --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlayerSeekTest.java @@ -0,0 +1,380 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer.mh.performance; + +import static com.google.common.truth.Truth.assertThat; + +import android.app.Instrumentation; +import android.content.Context; +import android.view.SurfaceView; +import androidx.media3.common.Effect; +import androidx.media3.common.GlObjectsProvider; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.MediaItem; +import androidx.media3.effect.GlEffect; +import androidx.media3.effect.PassthroughShaderProgram; +import androidx.media3.transformer.Composition; +import androidx.media3.transformer.CompositionPlayer; +import androidx.media3.transformer.EditedMediaItem; +import androidx.media3.transformer.EditedMediaItemSequence; +import androidx.media3.transformer.Effects; +import androidx.media3.transformer.PlayerTestListener; +import androidx.media3.transformer.SurfaceTestActivity; +import androidx.test.ext.junit.rules.ActivityScenarioRule; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Instrumentation tests for {@link CompositionPlayer} {@linkplain CompositionPlayer#seekTo + * seeking}. + */ +@RunWith(AndroidJUnit4.class) +public class CompositionPlayerSeekTest { + + private static final long TEST_TIMEOUT_MS = 10_000; + private static final String MP4_ASSET = "asset:///media/mp4/sample.mp4"; + + @Rule + public ActivityScenarioRule rule = + new ActivityScenarioRule<>(SurfaceTestActivity.class); + + private final Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); + private final Context applicationContext = instrumentation.getContext().getApplicationContext(); + + private CompositionPlayer compositionPlayer; + private SurfaceView surfaceView; + + @Before + public void setupSurfaces() { + rule.getScenario().onActivity(activity -> surfaceView = activity.getSurfaceView()); + } + + @After + public void closeActivity() { + rule.getScenario().close(); + } + + @After + public void releasePlayer() { + instrumentation.runOnMainSync( + () -> { + if (compositionPlayer != null) { + compositionPlayer.release(); + } + }); + } + + // TODO: b/320244483 - Add tests that seek into the middle of the sequence. + @Test + public void seekToZero_singleSequenceOfTwoVideos() throws Exception { + PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS * 1000); + InputTimestampRecordingShaderProgram inputTimestampRecordingShaderProgram = + new InputTimestampRecordingShaderProgram(); + EditedMediaItem video = + createEditedMediaItem( + /* videoEffects= */ ImmutableList.of( + (GlEffect) (context, useHdr) -> inputTimestampRecordingShaderProgram)); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = new CompositionPlayer.Builder(applicationContext).build(); + // Set a surface on the player even though there is no UI on this test. We need a surface + // otherwise the player will skip/drop video frames. + compositionPlayer.setVideoSurfaceView(surfaceView); + compositionPlayer.addListener(listener); + compositionPlayer.setComposition( + new Composition.Builder(new EditedMediaItemSequence(video, video)).build()); + compositionPlayer.prepare(); + compositionPlayer.play(); + }); + listener.waitUntilPlayerEnded(); + listener.resetStatus(); + instrumentation.runOnMainSync(() -> compositionPlayer.seekTo(0)); + listener.waitUntilPlayerEnded(); + + ImmutableList timestampsUsOfOneSequence = + ImmutableList.of( + 1000000000000L, + 1000000033366L, + 1000000066733L, + 1000000100100L, + 1000000133466L, + 1000000166833L, + 1000000200200L, + 1000000233566L, + 1000000266933L, + 1000000300300L, + 1000000333666L, + 1000000367033L, + 1000000400400L, + 1000000433766L, + 1000000467133L, + 1000000500500L, + 1000000533866L, + 1000000567233L, + 1000000600600L, + 1000000633966L, + 1000000667333L, + 1000000700700L, + 1000000734066L, + 1000000767433L, + 1000000800800L, + 1000000834166L, + 1000000867533L, + 1000000900900L, + 1000000934266L, + 1000000967633L, + // Second video starts here. + 1000001024000L, + 1000001057366L, + 1000001090733L, + 1000001124100L, + 1000001157466L, + 1000001190833L, + 1000001224200L, + 1000001257566L, + 1000001290933L, + 1000001324300L, + 1000001357666L, + 1000001391033L, + 1000001424400L, + 1000001457766L, + 1000001491133L, + 1000001524500L, + 1000001557866L, + 1000001591233L, + 1000001624600L, + 1000001657966L, + 1000001691333L, + 1000001724700L, + 1000001758066L, + 1000001791433L, + 1000001824800L, + 1000001858166L, + 1000001891533L, + 1000001924900L, + 1000001958266L, + 1000001991633L); + + assertThat(inputTimestampRecordingShaderProgram.timestampsUs) + // Seeked after the first playback ends, so the timestamps are repeated twice. + .containsExactlyElementsIn( + new ImmutableList.Builder() + .addAll(timestampsUsOfOneSequence) + .addAll(timestampsUsOfOneSequence) + .build()) + .inOrder(); + } + + @Test + public void seekToZero_after15framesInSingleSequenceOfTwoVideos() throws Exception { + PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS * 1000); + ResettableCountDownLatch framesReceivedLatch = new ResettableCountDownLatch(15); + AtomicBoolean shaderProgramShouldBlockInput = new AtomicBoolean(); + + InputTimestampRecordingShaderProgram inputTimestampRecordingShaderProgram = + new InputTimestampRecordingShaderProgram() { + + @Override + public void queueInputFrame( + GlObjectsProvider glObjectsProvider, + GlTextureInfo inputTexture, + long presentationTimeUs) { + super.queueInputFrame(glObjectsProvider, inputTexture, presentationTimeUs); + framesReceivedLatch.countDown(); + if (framesReceivedLatch.getCount() == 0) { + shaderProgramShouldBlockInput.set(true); + } + } + + @Override + public void releaseOutputFrame(GlTextureInfo outputTexture) { + // The input listener capacity is reported in the super method, block input by skip + // reporting input capacity. + if (shaderProgramShouldBlockInput.get()) { + return; + } + super.releaseOutputFrame(outputTexture); + } + + @Override + public void flush() { + super.flush(); + shaderProgramShouldBlockInput.set(false); + framesReceivedLatch.reset(Integer.MAX_VALUE); + } + }; + EditedMediaItem video = + createEditedMediaItem( + /* videoEffects= */ ImmutableList.of( + (GlEffect) (context, useHdr) -> inputTimestampRecordingShaderProgram)); + + instrumentation.runOnMainSync( + () -> { + compositionPlayer = new CompositionPlayer.Builder(applicationContext).build(); + // Set a surface on the player even though there is no UI on this test. We need a surface + // otherwise the player will skip/drop video frames. + compositionPlayer.setVideoSurfaceView(surfaceView); + compositionPlayer.addListener(listener); + compositionPlayer.setComposition( + new Composition.Builder(new EditedMediaItemSequence(video, video)).build()); + compositionPlayer.prepare(); + compositionPlayer.play(); + }); + + // Wait until the number of frames are received, block further input on the shader program. + framesReceivedLatch.await(); + instrumentation.runOnMainSync(() -> compositionPlayer.seekTo(0)); + listener.waitUntilPlayerEnded(); + + ImmutableList expectedTimestampsUs = + ImmutableList.of( + 1000000000000L, + 1000000033366L, + 1000000066733L, + 1000000100100L, + 1000000133466L, + 1000000166833L, + 1000000200200L, + 1000000233566L, + 1000000266933L, + 1000000300300L, + 1000000333666L, + 1000000367033L, + 1000000400400L, + 1000000433766L, + 1000000467133L, + // 15 frames, seek + 1000000000000L, + 1000000033366L, + 1000000066733L, + 1000000100100L, + 1000000133466L, + 1000000166833L, + 1000000200200L, + 1000000233566L, + 1000000266933L, + 1000000300300L, + 1000000333666L, + 1000000367033L, + 1000000400400L, + 1000000433766L, + 1000000467133L, + 1000000500500L, + 1000000533866L, + 1000000567233L, + 1000000600600L, + 1000000633966L, + 1000000667333L, + 1000000700700L, + 1000000734066L, + 1000000767433L, + 1000000800800L, + 1000000834166L, + 1000000867533L, + 1000000900900L, + 1000000934266L, + 1000000967633L, + // Second video starts here. + 1000001024000L, + 1000001057366L, + 1000001090733L, + 1000001124100L, + 1000001157466L, + 1000001190833L, + 1000001224200L, + 1000001257566L, + 1000001290933L, + 1000001324300L, + 1000001357666L, + 1000001391033L, + 1000001424400L, + 1000001457766L, + 1000001491133L, + 1000001524500L, + 1000001557866L, + 1000001591233L, + 1000001624600L, + 1000001657966L, + 1000001691333L, + 1000001724700L, + 1000001758066L, + 1000001791433L, + 1000001824800L, + 1000001858166L, + 1000001891533L, + 1000001924900L, + 1000001958266L, + 1000001991633L); + + assertThat(inputTimestampRecordingShaderProgram.timestampsUs) + .containsExactlyElementsIn(expectedTimestampsUs) + .inOrder(); + } + + private static EditedMediaItem createEditedMediaItem(List videoEffects) { + return new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET)) + .setDurationUs(1_024_000) + .setEffects(new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects)) + .build(); + } + + private static class InputTimestampRecordingShaderProgram extends PassthroughShaderProgram { + public final ArrayList timestampsUs = new ArrayList<>(); + + @Override + public void queueInputFrame( + GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) { + super.queueInputFrame(glObjectsProvider, inputTexture, presentationTimeUs); + timestampsUs.add(presentationTimeUs); + } + } + + private static final class ResettableCountDownLatch { + private CountDownLatch latch; + + public ResettableCountDownLatch(int count) { + latch = new CountDownLatch(count); + } + + public void await() throws InterruptedException { + latch.await(); + } + + public void countDown() { + latch.countDown(); + } + + public long getCount() { + return latch.getCount(); + } + + public void reset(int count) { + latch = new CountDownLatch(count); + } + } +} diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/VideoCompositionPreviewPerformanceTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/VideoCompositionPreviewPerformanceTest.java new file mode 100644 index 0000000000..6919d2098a --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/VideoCompositionPreviewPerformanceTest.java @@ -0,0 +1,141 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer.mh.performance; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING; +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.common.truth.Truth.assertThat; + +import android.app.Instrumentation; +import android.os.SystemClock; +import android.view.SurfaceView; +import androidx.media3.common.Effect; +import androidx.media3.common.MediaItem; +import androidx.media3.common.PlaybackException; +import androidx.media3.common.util.Util; +import androidx.media3.effect.Contrast; +import androidx.media3.transformer.Composition; +import androidx.media3.transformer.CompositionPlayer; +import androidx.media3.transformer.EditedMediaItem; +import androidx.media3.transformer.EditedMediaItemSequence; +import androidx.media3.transformer.Effects; +import androidx.media3.transformer.PlayerTestListener; +import androidx.media3.transformer.SurfaceTestActivity; +import androidx.test.ext.junit.rules.ActivityScenarioRule; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Range; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicLong; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Performance tests for the composition previewing pipeline in {@link CompositionPlayer}. */ +@RunWith(AndroidJUnit4.class) +public class VideoCompositionPreviewPerformanceTest { + + private static final long TEST_TIMEOUT_MS = 10_000; + private static final long MEDIA_ITEM_CLIP_DURATION_MS = 500; + + @Rule + public ActivityScenarioRule rule = + new ActivityScenarioRule<>(SurfaceTestActivity.class); + + private final Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); + private SurfaceView surfaceView; + private @MonotonicNonNull CompositionPlayer player; + + @Before + public void setUpSurface() { + rule.getScenario().onActivity(activity -> surfaceView = activity.getSurfaceView()); + } + + @After + public void tearDown() { + instrumentation.runOnMainSync( + () -> { + if (player != null) { + player.release(); + } + }); + rule.getScenario().close(); + } + + /** + * This test guards against performance regressions in the effects preview pipeline that format + * switches do not cause the player to stall. + */ + @Test + public void compositionPlayerCompositionPreviewTest() throws PlaybackException, TimeoutException { + PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS); + instrumentation.runOnMainSync( + () -> { + player = new CompositionPlayer.Builder(getApplicationContext()).build(); + player.setVideoSurfaceView(surfaceView); + player.setPlayWhenReady(false); + player.addListener(listener); + player.setComposition( + new Composition.Builder( + new EditedMediaItemSequence( + getClippedEditedMediaItem(MP4_ASSET_URI_STRING, new Contrast(.2f)), + getClippedEditedMediaItem(MP4_ASSET_URI_STRING, new Contrast(-.2f)))) + .build()); + player.prepare(); + }); + + listener.waitUntilPlayerReady(); + + AtomicLong playbackStartTimeMs = new AtomicLong(); + instrumentation.runOnMainSync( + () -> { + playbackStartTimeMs.set(SystemClock.elapsedRealtime()); + checkNotNull(player).play(); + }); + + listener.waitUntilPlayerEnded(); + long compositionDurationMs = MEDIA_ITEM_CLIP_DURATION_MS * 2; + long playbackDurationMs = SystemClock.elapsedRealtime() - playbackStartTimeMs.get(); + + assertThat(playbackDurationMs) + .isIn(Range.closed(compositionDurationMs, compositionDurationMs + 250)); + } + + private static EditedMediaItem getClippedEditedMediaItem(String uri, Effect effect) { + MediaItem mediaItem = + new MediaItem.Builder() + .setUri(uri) + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setEndPositionMs(MEDIA_ITEM_CLIP_DURATION_MS) + .build()) + .build(); + + return new EditedMediaItem.Builder(mediaItem) + .setEffects( + new Effects( + /* audioProcessors= */ ImmutableList.of(), + /* videoEffects= */ ImmutableList.of(effect))) + .setDurationUs(Util.msToUs(MEDIA_ITEM_CLIP_DURATION_MS)) + .build(); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraphInputAudioSink.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraphInputAudioSink.java new file mode 100644 index 0000000000..beb0a619ab --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioGraphInputAudioSink.java @@ -0,0 +1,298 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.common.audio.AudioProcessor.EMPTY_BUFFER; +import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Assertions.checkStateNotNull; + +import androidx.annotation.Nullable; +import androidx.media3.common.AudioAttributes; +import androidx.media3.common.AuxEffectInfo; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.PlaybackParameters; +import androidx.media3.decoder.DecoderInputBuffer; +import androidx.media3.exoplayer.audio.AudioSink; +import java.nio.ByteBuffer; +import java.util.Objects; + +/** + * An {@link AudioSink} implementation that feeds an {@link AudioGraphInput}. + * + *

Should be used by {@link PreviewAudioPipeline}. + */ +/* package */ final class AudioGraphInputAudioSink implements AudioSink { + + /** + * Controller for {@link AudioGraphInputAudioSink}. + * + *

All methods will be called on the playback thread of the ExoPlayer instance writing to this + * sink. + */ + public interface Controller { + + /** + * Returns the {@link AudioGraphInput} instance associated with this {@linkplain + * AudioGraphInputAudioSink sink}. + * + *

Data {@linkplain #handleBuffer written} to the sink will be {@linkplain + * AudioGraphInput#queueInputBuffer() queued} to the {@link AudioGraphInput}. + * + * @param editedMediaItem The first {@link EditedMediaItem} queued to the {@link + * AudioGraphInput}. + * @param format The {@link Format} used to {@linkplain AudioGraphInputAudioSink#configure + * configure} the {@linkplain AudioGraphInputAudioSink sink}. + * @return The {@link AudioGraphInput}. + * @throws ExportException If there is a problem initializing the {@linkplain AudioGraphInput + * input}. + */ + AudioGraphInput getAudioGraphInput(EditedMediaItem editedMediaItem, Format format) + throws ExportException; + + /** + * Returns the position (in microseconds) that should be {@linkplain + * AudioSink#getCurrentPositionUs returned} by this sink. + */ + long getCurrentPositionUs(); + + /** Returns whether the controller is ended. */ + boolean isEnded(); + + /** See {@link #play()}. */ + default void onPlay() {} + + /** See {@link #pause()}. */ + default void onPause() {} + + /** See {@link #reset()}. */ + default void onReset() {} + } + + private final Controller controller; + + @Nullable private AudioGraphInput outputGraphInput; + @Nullable private Format currentInputFormat; + private boolean inputStreamEnded; + private boolean signalledEndOfStream; + @Nullable private EditedMediaItemInfo currentEditedMediaItemInfo; + private long offsetToCompositionTimeUs; + + public AudioGraphInputAudioSink(Controller controller) { + this.controller = controller; + } + + /** + * Informs the audio sink there is a change on the {@link EditedMediaItem} currently rendered by + * the renderer. + * + * @param editedMediaItem The {@link EditedMediaItem}. + * @param offsetToCompositionTimeUs The offset to add to the audio buffer timestamps to convert + * them to the composition time, in microseconds. + * @param isLastInSequence Whether this is the last item in the sequence. + */ + public void onMediaItemChanged( + EditedMediaItem editedMediaItem, long offsetToCompositionTimeUs, boolean isLastInSequence) { + currentEditedMediaItemInfo = new EditedMediaItemInfo(editedMediaItem, isLastInSequence); + this.offsetToCompositionTimeUs = offsetToCompositionTimeUs; + } + + // AudioSink methods + + @Override + public void configure(Format inputFormat, int specifiedBufferSize, @Nullable int[] outputChannels) + throws ConfigurationException { + checkArgument(supportsFormat(inputFormat)); + EditedMediaItem editedMediaItem = checkStateNotNull(currentEditedMediaItemInfo).editedMediaItem; + // TODO(b/303029969): Evaluate throwing vs ignoring for null outputChannels. + checkArgument(outputChannels == null); + this.currentInputFormat = inputFormat; + if (outputGraphInput == null) { + try { + outputGraphInput = controller.getAudioGraphInput(editedMediaItem, currentInputFormat); + } catch (ExportException e) { + throw new ConfigurationException(e, currentInputFormat); + } + } + + outputGraphInput.onMediaItemChanged( + editedMediaItem, editedMediaItem.durationUs, currentInputFormat, /* isLast= */ false); + } + + @Override + public boolean isEnded() { + // If we are playing the last media item in the sequence, we must also check that the controller + // is ended. + return inputStreamEnded + && (!checkStateNotNull(currentEditedMediaItemInfo).isLastInSequence + || controller.isEnded()); + } + + @Override + public boolean handleBuffer( + ByteBuffer buffer, long presentationTimeUs, int encodedAccessUnitCount) { + checkState(!inputStreamEnded); + return handleBufferInternal(buffer, presentationTimeUs, /* flags= */ 0); + } + + @Override + public void playToEndOfStream() { + inputStreamEnded = true; + // Queue end-of-stream only if playing the last media item in the sequence. + if (!signalledEndOfStream && checkStateNotNull(currentEditedMediaItemInfo).isLastInSequence) { + signalledEndOfStream = + handleBufferInternal( + EMPTY_BUFFER, C.TIME_END_OF_SOURCE, /* flags= */ C.BUFFER_FLAG_END_OF_STREAM); + } + } + + @Override + public @SinkFormatSupport int getFormatSupport(Format format) { + if (Objects.equals(format.sampleMimeType, MimeTypes.AUDIO_RAW) + && format.pcmEncoding == C.ENCODING_PCM_16BIT) { + return SINK_FORMAT_SUPPORTED_DIRECTLY; + } + + return SINK_FORMAT_UNSUPPORTED; + } + + @Override + public boolean supportsFormat(Format format) { + return getFormatSupport(format) == SINK_FORMAT_SUPPORTED_DIRECTLY; + } + + @Override + public boolean hasPendingData() { + return false; + } + + @Override + public long getCurrentPositionUs(boolean sourceEnded) { + long currentPositionUs = controller.getCurrentPositionUs(); + if (currentPositionUs != CURRENT_POSITION_NOT_SET) { + // Reset the position to the one expected by the player. + currentPositionUs -= offsetToCompositionTimeUs; + } + return currentPositionUs; + } + + @Override + public void play() { + controller.onPlay(); + } + + @Override + public void pause() { + controller.onPause(); + } + + @Override + public void flush() { + inputStreamEnded = false; + signalledEndOfStream = false; + } + + @Override + public void reset() { + flush(); + currentInputFormat = null; + currentEditedMediaItemInfo = null; + controller.onReset(); + } + + // Unsupported interface functionality. + + @Override + public void setListener(AudioSink.Listener listener) {} + + @Override + public void handleDiscontinuity() {} + + @Override + public void setAudioAttributes(AudioAttributes audioAttributes) {} + + @Nullable + @Override + public AudioAttributes getAudioAttributes() { + return null; + } + + @Override + public void setPlaybackParameters(PlaybackParameters playbackParameters) {} + + @Override + public PlaybackParameters getPlaybackParameters() { + return PlaybackParameters.DEFAULT; + } + + @Override + public void enableTunnelingV21() {} + + @Override + public void disableTunneling() {} + + @Override + public void setSkipSilenceEnabled(boolean skipSilenceEnabled) {} + + @Override + public boolean getSkipSilenceEnabled() { + return false; + } + + @Override + public void setAudioSessionId(int audioSessionId) {} + + @Override + public void setAuxEffectInfo(AuxEffectInfo auxEffectInfo) {} + + @Override + public void setVolume(float volume) {} + + // Internal methods + + private boolean handleBufferInternal(ByteBuffer buffer, long presentationTimeUs, int flags) { + checkStateNotNull(currentInputFormat); + checkState(!signalledEndOfStream); + AudioGraphInput outputGraphInput = checkNotNull(this.outputGraphInput); + + @Nullable DecoderInputBuffer outputBuffer = outputGraphInput.getInputBuffer(); + if (outputBuffer == null) { + return false; + } + outputBuffer.ensureSpaceForWrite(buffer.remaining()); + checkNotNull(outputBuffer.data).put(buffer).flip(); + outputBuffer.timeUs = + presentationTimeUs == C.TIME_END_OF_SOURCE + ? C.TIME_END_OF_SOURCE + : presentationTimeUs + offsetToCompositionTimeUs; + outputBuffer.setFlags(flags); + + return outputGraphInput.queueInputBuffer(); + } + + private static final class EditedMediaItemInfo { + public final EditedMediaItem editedMediaItem; + public final boolean isLastInSequence; + + public EditedMediaItemInfo(EditedMediaItem editedMediaItem, boolean isLastInSequence) { + this.editedMediaItem = editedMediaItem; + this.isLastInSequence = isLastInSequence; + } + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java new file mode 100644 index 0000000000..c3f7c40a49 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java @@ -0,0 +1,955 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.annotation.RestrictTo.Scope.LIBRARY_GROUP; +import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.common.util.Util.usToMs; +import static java.lang.Math.min; + +import android.content.Context; +import android.os.HandlerThread; +import android.os.Looper; +import android.os.Process; +import android.util.Pair; +import android.view.Surface; +import android.view.SurfaceHolder; +import android.view.SurfaceView; +import androidx.annotation.Nullable; +import androidx.annotation.RestrictTo; +import androidx.annotation.VisibleForTesting; +import androidx.media3.common.C; +import androidx.media3.common.MediaItem; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.PlaybackException; +import androidx.media3.common.Player; +import androidx.media3.common.PreviewingVideoGraph; +import androidx.media3.common.SimpleBasePlayer; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.VideoSize; +import androidx.media3.common.util.Clock; +import androidx.media3.common.util.HandlerWrapper; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.Size; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.common.util.Util; +import androidx.media3.effect.PreviewingSingleInputVideoGraph; +import androidx.media3.exoplayer.ExoPlaybackException; +import androidx.media3.exoplayer.ExoPlayer; +import androidx.media3.exoplayer.RendererCapabilities; +import androidx.media3.exoplayer.audio.AudioSink; +import androidx.media3.exoplayer.audio.DefaultAudioSink; +import androidx.media3.exoplayer.image.ImageDecoder; +import androidx.media3.exoplayer.source.ConcatenatingMediaSource2; +import androidx.media3.exoplayer.source.DefaultMediaSourceFactory; +import androidx.media3.exoplayer.source.ExternalLoader; +import androidx.media3.exoplayer.source.MergingMediaSource; +import androidx.media3.exoplayer.source.SilenceMediaSource; +import androidx.media3.exoplayer.source.TrackGroupArray; +import androidx.media3.exoplayer.trackselection.DefaultTrackSelector; +import androidx.media3.exoplayer.trackselection.ExoTrackSelection; +import androidx.media3.exoplayer.util.EventLogger; +import androidx.media3.exoplayer.video.CompositingVideoSinkProvider; +import androidx.media3.exoplayer.video.VideoFrameReleaseControl; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.util.ArrayList; +import java.util.List; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * A {@link Player} implementation that plays {@linkplain Composition compositions} of media assets. + * The {@link Composition} specifies how the assets should be arranged, and the audio and video + * effects to apply to them. + * + *

CompositionPlayer instances must be accessed from a single application thread. For the vast + * majority of cases this should be the application's main thread. The thread on which a + * CompositionPlayer instance must be accessed can be explicitly specified by passing a {@link + * Looper} when creating the player. If no {@link Looper} is specified, then the {@link Looper} of + * the thread that the player is created on is used, or if that thread does not have a {@link + * Looper}, the {@link Looper} of the application's main thread is used. In all cases the {@link + * Looper} of the thread from which the player must be accessed can be queried using {@link + * #getApplicationLooper()}. + */ +@UnstableApi +@RestrictTo(LIBRARY_GROUP) +public final class CompositionPlayer extends SimpleBasePlayer + implements CompositionPlayerInternal.Listener, + CompositingVideoSinkProvider.Listener, + SurfaceHolder.Callback { + + /** A builder for {@link CompositionPlayer} instances. */ + public static final class Builder { + private final Context context; + + private @MonotonicNonNull Looper looper; + private @MonotonicNonNull AudioSink audioSink; + private @MonotonicNonNull ExternalLoader externalImageLoader; + private ImageDecoder.Factory imageDecoderFactory; + private Clock clock; + private PreviewingVideoGraph.@MonotonicNonNull Factory previewingVideoGraphFactory; + private boolean built; + + /** + * Creates an instance + * + * @param context The application context. + */ + public Builder(Context context) { + this.context = context.getApplicationContext(); + imageDecoderFactory = ImageDecoder.Factory.DEFAULT; + clock = Clock.DEFAULT; + } + + /** + * Sets the {@link Looper} from which the player can be accessed and {@link Player.Listener} + * callbacks are dispatched too. + * + *

By default, the builder uses the looper of the thread that calls {@link #build()}. + * + * @param looper The {@link Looper}. + * @return This builder, for convenience. + */ + @CanIgnoreReturnValue + public Builder setLooper(Looper looper) { + this.looper = looper; + return this; + } + + /** + * Sets the {@link AudioSink} that will be used to play out audio. + * + *

By default, a {@link DefaultAudioSink} with its default configuration is used. + * + * @param audioSink The {@link AudioSink}. + * @return This builder, for convenience. + */ + @CanIgnoreReturnValue + public Builder setAudioSink(AudioSink audioSink) { + this.audioSink = audioSink; + return this; + } + + /** + * Sets the {@link ExternalLoader} for loading image media items with MIME type set to {@link + * MimeTypes#APPLICATION_EXTERNALLY_LOADED_IMAGE}. When setting an external loader, also set an + * {@link ImageDecoder.Factory} with {@link #setImageDecoderFactory(ImageDecoder.Factory)}. + * + *

By default, the player will not be able to load images with media type of {@link + * androidx.media3.common.MimeTypes#APPLICATION_EXTERNALLY_LOADED_IMAGE}. + * + * @param externalImageLoader The {@link ExternalLoader}. + * @return This builder, for convenience. + * @see DefaultMediaSourceFactory#setExternalImageLoader(ExternalLoader) + */ + @CanIgnoreReturnValue + public Builder setExternalImageLoader(ExternalLoader externalImageLoader) { + this.externalImageLoader = externalImageLoader; + return this; + } + + /** + * Sets an {@link ImageDecoder.Factory} that will create the {@link ImageDecoder} instances to + * decode images. + * + *

By default, {@link ImageDecoder.Factory#DEFAULT} is used. + * + * @param imageDecoderFactory The {@link ImageDecoder.Factory}. + * @return This builder, for convenience. + */ + @CanIgnoreReturnValue + public Builder setImageDecoderFactory(ImageDecoder.Factory imageDecoderFactory) { + this.imageDecoderFactory = imageDecoderFactory; + return this; + } + + /** + * Sets the {@link Clock} that will be used by the player. + * + *

By default, {@link Clock#DEFAULT} is used. + * + * @param clock The {@link Clock}. + * @return This builder, for convenience. + */ + @VisibleForTesting + @CanIgnoreReturnValue + public Builder setClock(Clock clock) { + this.clock = clock; + return this; + } + + /** + * Sets the {@link PreviewingVideoGraph.Factory} that will be used by the player. + * + *

By default, a {@link PreviewingSingleInputVideoGraph.Factory} is used. + * + * @param previewingVideoGraphFactory The {@link PreviewingVideoGraph.Factory}. + * @return This builder, for convenience. + */ + @VisibleForTesting + @CanIgnoreReturnValue + public Builder setPreviewingVideoGraphFactory( + PreviewingVideoGraph.Factory previewingVideoGraphFactory) { + this.previewingVideoGraphFactory = previewingVideoGraphFactory; + return this; + } + + /** + * Builds the {@link CompositionPlayer} instance. Must be called at most once. + * + *

If no {@link Looper} has been called with {@link #setLooper(Looper)}, then this method + * must be called within a {@link Looper} thread which is the thread that can access the player + * instance and where {@link Player.Listener} callbacks are dispatched. + */ + public CompositionPlayer build() { + checkState(!built); + if (looper == null) { + looper = checkStateNotNull(Looper.myLooper()); + } + if (audioSink == null) { + audioSink = new DefaultAudioSink.Builder(context).build(); + } + if (previewingVideoGraphFactory == null) { + previewingVideoGraphFactory = new PreviewingSingleInputVideoGraph.Factory(); + } + CompositionPlayer compositionPlayer = new CompositionPlayer(this); + built = true; + return compositionPlayer; + } + } + + private static final Commands AVAILABLE_COMMANDS = + new Commands.Builder() + .addAll( + COMMAND_PLAY_PAUSE, + COMMAND_PREPARE, + COMMAND_STOP, + COMMAND_SEEK_IN_CURRENT_MEDIA_ITEM, + COMMAND_SEEK_BACK, + COMMAND_SEEK_FORWARD, + COMMAND_GET_CURRENT_MEDIA_ITEM, + COMMAND_GET_TIMELINE, + COMMAND_SET_VIDEO_SURFACE, + COMMAND_RELEASE) + .build(); + + private static final @Event int[] SUPPORTED_LISTENER_EVENTS = + new int[] { + EVENT_PLAYBACK_STATE_CHANGED, + EVENT_PLAY_WHEN_READY_CHANGED, + EVENT_PLAYER_ERROR, + EVENT_POSITION_DISCONTINUITY + }; + + private static final int MAX_SUPPORTED_SEQUENCES = 2; + + private static final String TAG = "CompositionPlayer"; + + private final Context context; + private final Clock clock; + private final HandlerWrapper applicationHandler; + private final List players; + private final AudioSink finalAudioSink; + @Nullable private final ExternalLoader externalImageLoader; + private final ImageDecoder.Factory imageDecoderFactory; + private final PreviewingVideoGraph.Factory previewingVideoGraphFactory; + private final HandlerWrapper compositionInternalListenerHandler; + + private @MonotonicNonNull HandlerThread playbackThread; + private @MonotonicNonNull CompositionPlayerInternal compositionPlayerInternal; + private @MonotonicNonNull ImmutableList playlist; + private @MonotonicNonNull Composition composition; + private @MonotonicNonNull Size videoOutputSize; + private long compositionDurationUs; + private boolean playWhenReady; + private @PlayWhenReadyChangeReason int playWhenReadyChangeReason; + private boolean renderedFirstFrame; + @Nullable private Object videoOutput; + @Nullable private PlaybackException playbackException; + private @Player.State int playbackState; + @Nullable private SurfaceHolder surfaceHolder; + @Nullable private Surface displaySurface; + + private CompositionPlayer(Builder builder) { + super(checkNotNull(builder.looper), builder.clock); + context = builder.context; + clock = builder.clock; + applicationHandler = clock.createHandler(builder.looper, /* callback= */ null); + finalAudioSink = checkNotNull(builder.audioSink); + externalImageLoader = builder.externalImageLoader; + imageDecoderFactory = builder.imageDecoderFactory; + previewingVideoGraphFactory = checkNotNull(builder.previewingVideoGraphFactory); + compositionInternalListenerHandler = clock.createHandler(builder.looper, /* callback= */ null); + players = new ArrayList<>(); + compositionDurationUs = C.TIME_UNSET; + playbackState = STATE_IDLE; + } + + /** + * Sets the {@link Composition} to play. + * + *

This method should only be called once. + * + * @param composition The {@link Composition} to play. Every {@link EditedMediaItem} in the {@link + * Composition} must have its {@link EditedMediaItem#durationUs} set. + */ + public void setComposition(Composition composition) { + verifyApplicationThread(); + checkArgument( + !composition.sequences.isEmpty() + && composition.sequences.size() <= MAX_SUPPORTED_SEQUENCES); + checkState(this.composition == null); + + setCompositionInternal(composition); + if (videoOutput != null) { + if (videoOutput instanceof SurfaceHolder) { + setVideoSurfaceHolderInternal((SurfaceHolder) videoOutput); + } else if (videoOutput instanceof SurfaceView) { + SurfaceView surfaceView = (SurfaceView) videoOutput; + setVideoSurfaceHolderInternal(surfaceView.getHolder()); + } else if (videoOutput instanceof Surface) { + setVideoSurfaceInternal((Surface) videoOutput, checkNotNull(videoOutputSize)); + } else { + throw new IllegalStateException(videoOutput.getClass().toString()); + } + } + // Update the composition field at the end after everything else has been set. + this.composition = composition; + } + + /** Sets the {@link Surface} and {@link Size} to render to. */ + @VisibleForTesting + public void setVideoSurface(Surface surface, Size videoOutputSize) { + videoOutput = surface; + this.videoOutputSize = videoOutputSize; + setVideoSurfaceInternal(surface, videoOutputSize); + } + + // CompositingVideoSinkProvider.Listener methods. Called on playback thread. + + @Override + public void onFirstFrameRendered(CompositingVideoSinkProvider compositingVideoSinkProvider) { + applicationHandler.post( + () -> { + CompositionPlayer.this.renderedFirstFrame = true; + invalidateState(); + }); + } + + @Override + public void onFrameDropped(CompositingVideoSinkProvider compositingVideoSinkProvider) { + // Do not post to application thread on each dropped frame, because onFrameDropped + // may be called frequently when resources are already scarce. + } + + @Override + public void onVideoSizeChanged( + CompositingVideoSinkProvider compositingVideoSinkProvider, VideoSize videoSize) { + // TODO: b/328219481 - Report video size change to app. + } + + @Override + public void onError( + CompositingVideoSinkProvider compositingVideoSinkProvider, + VideoFrameProcessingException videoFrameProcessingException) { + // The error will also be surfaced from the underlying ExoPlayer instance via + // PlayerListener.onPlayerError, and it will arrive to the composition player twice. + applicationHandler.post( + () -> + maybeUpdatePlaybackError( + "error from video sink provider", + videoFrameProcessingException, + PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED)); + } + + // SurfaceHolder.Callback methods. Called on application thread. + + @Override + public void surfaceCreated(SurfaceHolder holder) { + videoOutputSize = new Size(holder.getSurfaceFrame().width(), holder.getSurfaceFrame().height()); + setVideoSurfaceInternal(holder.getSurface(), videoOutputSize); + } + + @Override + public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { + maybeSetOutputSurfaceInfo(width, height); + } + + @Override + public void surfaceDestroyed(SurfaceHolder holder) { + clearVideoSurfaceInternal(); + } + + // SimpleBasePlayer methods + + @Override + protected State getState() { + @Player.State int oldPlaybackState = playbackState; + updatePlaybackState(); + if (oldPlaybackState != STATE_READY && playbackState == STATE_READY && playWhenReady) { + for (int i = 0; i < players.size(); i++) { + players.get(i).setPlayWhenReady(true); + } + } else if (oldPlaybackState == STATE_READY + && playWhenReady + && playbackState == STATE_BUFFERING) { + // We were playing but a player got in buffering state, pause the players. + for (int i = 0; i < players.size(); i++) { + players.get(i).setPlayWhenReady(false); + } + } + // TODO: b/328219481 - Report video size change to app. + State.Builder state = + new State.Builder() + .setAvailableCommands(AVAILABLE_COMMANDS) + .setPlaybackState(playbackState) + .setPlayerError(playbackException) + .setPlayWhenReady(playWhenReady, playWhenReadyChangeReason) + .setContentPositionMs(this::getContentPositionMs) + .setContentBufferedPositionMs(this::getBufferedPositionMs) + .setTotalBufferedDurationMs(this::getTotalBufferedDurationMs) + .setNewlyRenderedFirstFrame(getRenderedFirstFrameAndReset()); + if (playlist != null) { + // Update the playlist only after it has been set so that SimpleBasePlayer announces a + // timeline + // change with reason TIMELINE_CHANGE_REASON_PLAYLIST_CHANGED. + state.setPlaylist(playlist); + } + return state.build(); + } + + @Override + protected ListenableFuture handlePrepare() { + checkStateNotNull(composition, "No composition set"); + + if (playbackState != Player.STATE_IDLE) { + // The player has been prepared already. + return Futures.immediateVoidFuture(); + } + for (int i = 0; i < players.size(); i++) { + players.get(i).prepare(); + } + return Futures.immediateVoidFuture(); + } + + @Override + protected ListenableFuture handleSetPlayWhenReady(boolean playWhenReady) { + this.playWhenReady = playWhenReady; + playWhenReadyChangeReason = PLAY_WHEN_READY_CHANGE_REASON_USER_REQUEST; + if (playbackState == STATE_READY) { + for (int i = 0; i < players.size(); i++) { + players.get(i).setPlayWhenReady(playWhenReady); + } + } // else, wait until all players are ready. + return Futures.immediateVoidFuture(); + } + + @Override + protected ListenableFuture handleStop() { + for (int i = 0; i < players.size(); i++) { + players.get(i).stop(); + } + return Futures.immediateVoidFuture(); + } + + @Override + protected ListenableFuture handleRelease() { + if (composition == null) { + return Futures.immediateVoidFuture(); + } + + checkState(checkStateNotNull(playbackThread).isAlive()); + // Release the players first so that they stop rendering. + for (int i = 0; i < players.size(); i++) { + players.get(i).release(); + } + checkStateNotNull(compositionPlayerInternal).release(); + removeSurfaceCallbacks(); + // Remove any queued callback from the internal player. + compositionInternalListenerHandler.removeCallbacksAndMessages(/* token= */ null); + displaySurface = null; + checkStateNotNull(playbackThread).quitSafely(); + applicationHandler.removeCallbacksAndMessages(/* token= */ null); + return Futures.immediateVoidFuture(); + } + + @Override + protected ListenableFuture handleClearVideoOutput(@Nullable Object videoOutput) { + checkArgument(Util.areEqual(videoOutput, this.videoOutput)); + + this.videoOutput = null; + if (composition == null) { + return Futures.immediateVoidFuture(); + } + removeSurfaceCallbacks(); + clearVideoSurfaceInternal(); + return Futures.immediateVoidFuture(); + } + + @Override + protected ListenableFuture handleSetVideoOutput(Object videoOutput) { + if (!(videoOutput instanceof SurfaceHolder || videoOutput instanceof SurfaceView)) { + throw new UnsupportedOperationException(videoOutput.getClass().toString()); + } + this.videoOutput = videoOutput; + if (composition == null) { + return Futures.immediateVoidFuture(); + } + if (videoOutput instanceof SurfaceHolder) { + setVideoSurfaceHolderInternal((SurfaceHolder) videoOutput); + } else { + setVideoSurfaceHolderInternal(((SurfaceView) videoOutput).getHolder()); + } + return Futures.immediateVoidFuture(); + } + + @Override + protected ListenableFuture handleSeek(int mediaItemIndex, long positionMs, int seekCommand) { + CompositionPlayerInternal compositionPlayerInternal = + checkStateNotNull(this.compositionPlayerInternal); + compositionPlayerInternal.startSeek(positionMs); + for (int i = 0; i < players.size(); i++) { + players.get(i).seekTo(positionMs); + } + compositionPlayerInternal.endSeek(); + return Futures.immediateVoidFuture(); + } + + // CompositionPlayerInternal.Listener methods + + @Override + public void onError(String message, Exception cause, int errorCode) { + maybeUpdatePlaybackError(message, cause, errorCode); + } + + // Internal methods + + private void updatePlaybackState() { + if (players.isEmpty() || playbackException != null) { + playbackState = STATE_IDLE; + return; + } + + int idleCount = 0; + int bufferingCount = 0; + int endedCount = 0; + for (int i = 0; i < players.size(); i++) { + @Player.State int playbackState = players.get(i).getPlaybackState(); + switch (playbackState) { + case STATE_IDLE: + idleCount++; + break; + case STATE_BUFFERING: + bufferingCount++; + break; + case STATE_READY: + // ignore + break; + case STATE_ENDED: + endedCount++; + break; + default: + throw new IllegalStateException(String.valueOf(playbackState)); + } + } + if (idleCount > 0) { + playbackState = STATE_IDLE; + } else if (bufferingCount > 0) { + playbackState = STATE_BUFFERING; + } else if (endedCount == players.size()) { + playbackState = STATE_ENDED; + } else { + playbackState = STATE_READY; + } + } + + @SuppressWarnings("VisibleForTests") // Calls ExoPlayer.Builder.setClock() + private void setCompositionInternal(Composition composition) { + compositionDurationUs = getCompositionDurationUs(composition); + playbackThread = new HandlerThread("CompositionPlaybackThread", Process.THREAD_PRIORITY_AUDIO); + playbackThread.start(); + // Create the audio and video composition components now in order to setup the audio and video + // pipelines. Once this method returns, further access to the audio and video pipelines must + // done on the playback thread only, to ensure related components are accessed from one thread + // only. + PreviewAudioPipeline previewAudioPipeline = + new PreviewAudioPipeline( + new DefaultAudioMixer.Factory(), + composition.effects.audioProcessors, + checkNotNull(finalAudioSink)); + CompositingVideoSinkProvider compositingVideoSinkProvider = + new CompositingVideoSinkProvider.Builder(context) + .setPreviewingVideoGraphFactory(checkNotNull(previewingVideoGraphFactory)) + .build(); + compositingVideoSinkProvider.setVideoFrameReleaseControl( + new VideoFrameReleaseControl( + context, new CompositionFrameTimingEvaluator(), /* allowedJoiningTimeMs= */ 0)); + compositingVideoSinkProvider.addListener(this); + for (int i = 0; i < composition.sequences.size(); i++) { + EditedMediaItemSequence editedMediaItemSequence = composition.sequences.get(i); + SequencePlayerRenderersWrapper playerRenderersWrapper = + i == 0 + ? SequencePlayerRenderersWrapper.create( + context, + editedMediaItemSequence, + previewAudioPipeline, + compositingVideoSinkProvider, + imageDecoderFactory) + : SequencePlayerRenderersWrapper.createForAudio( + context, editedMediaItemSequence, previewAudioPipeline); + ExoPlayer.Builder playerBuilder = + new ExoPlayer.Builder(context) + .setLooper(getApplicationLooper()) + .setPlaybackLooper(playbackThread.getLooper()) + .setRenderersFactory(playerRenderersWrapper) + .setHandleAudioBecomingNoisy(true) + .setClock(clock); + + if (i == 0) { + playerBuilder.setTrackSelector(new CompositionTrackSelector(context)); + } + + ExoPlayer player = playerBuilder.build(); + player.addListener(new PlayerListener(i)); + player.addAnalyticsListener(new EventLogger()); + setPlayerSequence(player, editedMediaItemSequence, /* shouldGenerateSilence= */ i == 0); + players.add(player); + if (i == 0) { + // Invalidate the player state before initializing the playlist to force SimpleBasePlayer + // to collect a state while the playlist is null. Consequently, once the playlist is + // initialized, SimpleBasePlayer will raise a timeline change callback with reason + // TIMELINE_CHANGE_REASON_PLAYLIST_CHANGED. + invalidateState(); + playlist = createPlaylist(); + } + } + // From here after, composition player accessed the audio and video pipelines via the internal + // player. The internal player ensures access to the components is done on the playback thread. + compositionPlayerInternal = + new CompositionPlayerInternal( + playbackThread.getLooper(), + clock, + previewAudioPipeline, + compositingVideoSinkProvider, + /* listener= */ this, + compositionInternalListenerHandler); + } + + /** Sets the {@linkplain EditedMediaItemSequence sequence} to be played by the player. */ + private void setPlayerSequence( + ExoPlayer player, EditedMediaItemSequence sequence, boolean shouldGenerateSilence) { + ConcatenatingMediaSource2.Builder mediaSourceBuilder = + new ConcatenatingMediaSource2.Builder().useDefaultMediaSourceFactory(context); + + for (int i = 0; i < sequence.editedMediaItems.size(); i++) { + EditedMediaItem editedMediaItem = sequence.editedMediaItems.get(i); + checkArgument(editedMediaItem.durationUs != C.TIME_UNSET); + long durationUs = editedMediaItem.getPresentationDurationUs(); + + if (shouldGenerateSilence) { + DefaultMediaSourceFactory defaultMediaSourceFactory = + new DefaultMediaSourceFactory(context); + if (externalImageLoader != null) { + defaultMediaSourceFactory.setExternalImageLoader(externalImageLoader); + } + mediaSourceBuilder.add( + new MergingMediaSource( + defaultMediaSourceFactory.createMediaSource(editedMediaItem.mediaItem), + // Generate silence as long as the MediaItem without clipping, because the actual + // media track starts at the clipped position. For example, if a video is 1000ms + // long and clipped 900ms from the start, its MediaSource will be enabled at 900ms + // during track selection, rather than at 0ms. + new SilenceMediaSource(editedMediaItem.durationUs)), + /* initialPlaceholderDurationMs= */ usToMs(durationUs)); + } else { + mediaSourceBuilder.add( + editedMediaItem.mediaItem, /* initialPlaceholderDurationMs= */ usToMs(durationUs)); + } + } + player.setMediaSource(mediaSourceBuilder.build()); + } + + private long getContentPositionMs() { + return players.isEmpty() ? C.TIME_UNSET : players.get(0).getContentPosition(); + } + + private long getBufferedPositionMs() { + if (players.isEmpty()) { + return 0; + } + // Return the minimum buffered position among players. + long minBufferedPositionMs = Integer.MAX_VALUE; + for (int i = 0; i < players.size(); i++) { + minBufferedPositionMs = min(minBufferedPositionMs, players.get(i).getBufferedPosition()); + } + return minBufferedPositionMs; + } + + private long getTotalBufferedDurationMs() { + if (players.isEmpty()) { + return 0; + } + // Return the minimum total buffered duration among players. + long minTotalBufferedDurationMs = Integer.MAX_VALUE; + for (int i = 0; i < players.size(); i++) { + minTotalBufferedDurationMs = + min(minTotalBufferedDurationMs, players.get(i).getTotalBufferedDuration()); + } + return minTotalBufferedDurationMs; + } + + private boolean getRenderedFirstFrameAndReset() { + boolean value = renderedFirstFrame; + renderedFirstFrame = false; + return value; + } + + private void maybeUpdatePlaybackError( + String errorMessage, Exception cause, @PlaybackException.ErrorCode int errorCode) { + if (playbackException == null) { + playbackException = new PlaybackException(errorMessage, cause, errorCode); + for (int i = 0; i < players.size(); i++) { + players.get(i).stop(); + } + invalidateState(); + } else { + Log.w(TAG, errorMessage, cause); + } + } + + private void setVideoSurfaceHolderInternal(SurfaceHolder surfaceHolder) { + removeSurfaceCallbacks(); + this.surfaceHolder = surfaceHolder; + surfaceHolder.addCallback(this); + Surface surface = surfaceHolder.getSurface(); + if (surface != null && surface.isValid()) { + videoOutputSize = + new Size( + surfaceHolder.getSurfaceFrame().width(), surfaceHolder.getSurfaceFrame().height()); + setVideoSurfaceInternal(surface, videoOutputSize); + } else { + clearVideoSurfaceInternal(); + } + } + + private void setVideoSurfaceInternal(Surface surface, Size videoOutputSize) { + displaySurface = surface; + maybeSetOutputSurfaceInfo(videoOutputSize.getWidth(), videoOutputSize.getHeight()); + } + + private void maybeSetOutputSurfaceInfo(int width, int height) { + Surface surface = displaySurface; + if (width == 0 || height == 0 || surface == null || compositionPlayerInternal == null) { + return; + } + compositionPlayerInternal.setOutputSurfaceInfo(surface, new Size(width, height)); + } + + private void clearVideoSurfaceInternal() { + displaySurface = null; + if (compositionPlayerInternal != null) { + compositionPlayerInternal.clearOutputSurface(); + } + } + + private void removeSurfaceCallbacks() { + if (surfaceHolder != null) { + surfaceHolder.removeCallback(this); + surfaceHolder = null; + } + } + + private ImmutableList createPlaylist() { + checkNotNull(compositionDurationUs != C.TIME_UNSET); + return ImmutableList.of( + new MediaItemData.Builder("CompositionTimeline") + .setMediaItem(MediaItem.EMPTY) + .setDurationUs(compositionDurationUs) + .build()); + } + + private static long getCompositionDurationUs(Composition composition) { + checkState(!composition.sequences.isEmpty()); + + long compositionDurationUs = getSequenceDurationUs(composition.sequences.get(0)); + for (int i = 0; i < composition.sequences.size(); i++) { + long sequenceDurationUs = getSequenceDurationUs(composition.sequences.get(i)); + checkArgument( + compositionDurationUs == sequenceDurationUs, + Util.formatInvariant( + "Non-matching sequence durations. First sequence duration: %d us, sequence [%d]" + + " duration: %d us", + compositionDurationUs, i, sequenceDurationUs)); + } + + return compositionDurationUs; + } + + private static long getSequenceDurationUs(EditedMediaItemSequence sequence) { + long compositionDurationUs = 0; + for (int i = 0; i < sequence.editedMediaItems.size(); i++) { + compositionDurationUs += sequence.editedMediaItems.get(i).getPresentationDurationUs(); + } + checkState(compositionDurationUs > 0, String.valueOf(compositionDurationUs)); + return compositionDurationUs; + } + + /** + * A {@link VideoFrameReleaseControl.FrameTimingEvaluator} for composition frames. + * + *

    + *
  • Signals to {@linkplain + * VideoFrameReleaseControl.FrameTimingEvaluator#shouldForceReleaseFrame(long, long) force + * release} a frame if the frame is late by more than {@link #FRAME_LATE_THRESHOLD_US} and + * the elapsed time since the previous frame release is greater than {@link + * #FRAME_RELEASE_THRESHOLD_US}. + *
  • Signals to {@linkplain + * VideoFrameReleaseControl.FrameTimingEvaluator#shouldDropFrame(long, long, boolean) drop a + * frame} if the frame is late by more than {@link #FRAME_LATE_THRESHOLD_US} and the frame + * is not marked as the last one. + *
  • Signals to never {@linkplain + * VideoFrameReleaseControl.FrameTimingEvaluator#shouldIgnoreFrame(long, long, long, + * boolean, boolean) ignore} a frame. + *
+ */ + private static final class CompositionFrameTimingEvaluator + implements VideoFrameReleaseControl.FrameTimingEvaluator { + + /** The time threshold, in microseconds, after which a frame is considered late. */ + private static final long FRAME_LATE_THRESHOLD_US = -30_000; + + /** + * The maximum elapsed time threshold, in microseconds, since last releasing a frame after which + * a frame can be force released. + */ + private static final long FRAME_RELEASE_THRESHOLD_US = 100_000; + + @Override + public boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs) { + return earlyUs < FRAME_LATE_THRESHOLD_US + && elapsedSinceLastReleaseUs > FRAME_RELEASE_THRESHOLD_US; + } + + @Override + public boolean shouldDropFrame(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) { + return earlyUs < FRAME_LATE_THRESHOLD_US && !isLastFrame; + } + + @Override + public boolean shouldIgnoreFrame( + long earlyUs, + long positionUs, + long elapsedRealtimeUs, + boolean isLastFrame, + boolean treatDroppedBuffersAsSkipped) { + // TODO: b/293873191 - Handle very late buffers and drop to key frame. + return false; + } + } + + private final class PlayerListener implements Player.Listener { + private final int playerIndex; + + public PlayerListener(int playerIndex) { + this.playerIndex = playerIndex; + } + + @Override + public void onEvents(Player player, Events events) { + if (events.containsAny(SUPPORTED_LISTENER_EVENTS)) { + invalidateState(); + } + } + + @Override + public void onPlayWhenReadyChanged(boolean playWhenReady, int reason) { + playWhenReadyChangeReason = reason; + } + + @Override + public void onPlayerError(PlaybackException error) { + maybeUpdatePlaybackError("error from player " + playerIndex, error, error.errorCode); + } + } + + /** + * A {@link DefaultTrackSelector} extension to de-select generated audio when the audio from the + * media is playable. + */ + private static final class CompositionTrackSelector extends DefaultTrackSelector { + + private static final String SILENCE_AUDIO_TRACK_GROUP_ID = "1:"; + + public CompositionTrackSelector(Context context) { + super(context); + } + + @Nullable + @Override + protected Pair selectAudioTrack( + MappedTrackInfo mappedTrackInfo, + @RendererCapabilities.Capabilities int[][][] rendererFormatSupports, + @RendererCapabilities.AdaptiveSupport int[] rendererMixedMimeTypeAdaptationSupports, + Parameters params) + throws ExoPlaybackException { + int audioRenderIndex = C.INDEX_UNSET; + for (int i = 0; i < mappedTrackInfo.getRendererCount(); i++) { + if (mappedTrackInfo.getRendererType(i) == C.TRACK_TYPE_AUDIO) { + audioRenderIndex = i; + break; + } + } + checkState(audioRenderIndex != C.INDEX_UNSET); + + TrackGroupArray audioTrackGroups = mappedTrackInfo.getTrackGroups(audioRenderIndex); + // If there's only one audio TrackGroup, it'll be silence, there's no need to override track + // selection. + if (audioTrackGroups.length > 1) { + boolean mediaAudioIsPlayable = false; + int silenceAudioTrackGroupIndex = C.INDEX_UNSET; + for (int i = 0; i < audioTrackGroups.length; i++) { + if (audioTrackGroups.get(i).id.startsWith(SILENCE_AUDIO_TRACK_GROUP_ID)) { + silenceAudioTrackGroupIndex = i; + continue; + } + // For non-silence tracks + for (int j = 0; j < audioTrackGroups.get(i).length; j++) { + mediaAudioIsPlayable |= + RendererCapabilities.getFormatSupport( + rendererFormatSupports[audioRenderIndex][i][j]) + == C.FORMAT_HANDLED; + } + } + checkState(silenceAudioTrackGroupIndex != C.INDEX_UNSET); + + if (mediaAudioIsPlayable) { + // Disable silence if the media's audio track is playable. + int silenceAudioTrackIndex = audioTrackGroups.length - 1; + rendererFormatSupports[audioRenderIndex][silenceAudioTrackIndex][0] = + RendererCapabilities.create(C.FORMAT_UNSUPPORTED_TYPE); + } + } + + return super.selectAudioTrack( + mappedTrackInfo, rendererFormatSupports, rendererMixedMimeTypeAdaptationSupports, params); + } + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayerInternal.java new file mode 100644 index 0000000000..a5e664052c --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayerInternal.java @@ -0,0 +1,237 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkState; + +import android.os.Handler; +import android.os.Looper; +import android.os.Message; +import android.view.Surface; +import androidx.media3.common.PlaybackException; +import androidx.media3.common.util.Clock; +import androidx.media3.common.util.ConditionVariable; +import androidx.media3.common.util.HandlerWrapper; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.Size; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.video.CompositingVideoSinkProvider; + +/** Provides access to the composition preview audio and video components on the playback thread. */ +/* package */ final class CompositionPlayerInternal implements Handler.Callback { + + /** A listener for events. */ + public interface Listener { + + /** + * Called when an error occurs + * + * @param message The error message. + * @param cause The error cause. + * @param errorCode The error code. + */ + void onError(String message, Exception cause, @PlaybackException.ErrorCode int errorCode); + } + + private static final String TAG = "CompPlayerInternal"; + private static final int MSG_SET_OUTPUT_SURFACE_INFO = 0; + private static final int MSG_CLEAR_OUTPUT_SURFACE = 1; + private static final int MSG_START_SEEK = 2; + private static final int MSG_END_SEEK = 3; + private static final int MSG_RELEASE = 4; + + private final Clock clock; + private final HandlerWrapper handler; + + /** Must be accessed on the playback thread only. */ + private final PreviewAudioPipeline previewAudioPipeline; + + /** Must be accessed on the playback thread only. */ + private final CompositingVideoSinkProvider compositingVideoSinkProvider; + + private final Listener listener; + private final HandlerWrapper listenerHandler; + + private boolean released; + + /** + * Creates a instance. + * + * @param playbackLooper The playback thread {@link Looper}. + * @param clock The {@link Clock} used. + * @param previewAudioPipeline The {@link PreviewAudioPipeline}. + * @param compositingVideoSinkProvider The {@link CompositingVideoSinkProvider}. + * @param listener A {@link Listener} to send callbacks back to the player. + * @param listenerHandler A {@link HandlerWrapper} to dispatch {@link Listener} callbacks. + */ + public CompositionPlayerInternal( + Looper playbackLooper, + Clock clock, + PreviewAudioPipeline previewAudioPipeline, + CompositingVideoSinkProvider compositingVideoSinkProvider, + Listener listener, + HandlerWrapper listenerHandler) { + this.clock = clock; + this.handler = clock.createHandler(playbackLooper, /* callback= */ this); + this.previewAudioPipeline = previewAudioPipeline; + this.compositingVideoSinkProvider = compositingVideoSinkProvider; + this.listener = listener; + this.listenerHandler = listenerHandler; + } + + // Public methods + + /** Sets the output surface information on the video pipeline. */ + public void setOutputSurfaceInfo(Surface surface, Size size) { + handler + .obtainMessage(MSG_SET_OUTPUT_SURFACE_INFO, new OutputSurfaceInfo(surface, size)) + .sendToTarget(); + } + + /** Clears the output surface from the video pipeline. */ + public void clearOutputSurface() { + handler.obtainMessage(MSG_CLEAR_OUTPUT_SURFACE).sendToTarget(); + } + + public void startSeek(long positionMs) { + handler.obtainMessage(MSG_START_SEEK, positionMs).sendToTarget(); + } + + public void endSeek() { + handler.obtainMessage(MSG_END_SEEK).sendToTarget(); + } + + /** + * Releases internal components on the playback thread and blocks the current thread until the + * components are released. + */ + public void release() { + checkState(!released); + // Set released to true now to silence any pending listener callback. + released = true; + ConditionVariable conditionVariable = new ConditionVariable(); + handler.obtainMessage(MSG_RELEASE, conditionVariable).sendToTarget(); + clock.onThreadBlocked(); + try { + conditionVariable.block(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + + // Handler.Callback methods + + @Override + public boolean handleMessage(Message message) { + try { + switch (message.what) { + case MSG_SET_OUTPUT_SURFACE_INFO: + setOutputSurfaceInfoOnInternalThread( + /* outputSurfaceInfo= */ (OutputSurfaceInfo) message.obj); + break; + case MSG_CLEAR_OUTPUT_SURFACE: + clearOutputSurfaceInternal(); + break; + case MSG_START_SEEK: + // Video seeking is currently handled by the video renderers, specifically in + // onPositionReset. + previewAudioPipeline.startSeek(/* positionUs= */ Util.msToUs((long) message.obj)); + break; + case MSG_END_SEEK: + previewAudioPipeline.endSeek(); + break; + case MSG_RELEASE: + releaseInternal(/* conditionVariable= */ (ConditionVariable) message.obj); + break; + default: + maybeRaiseError( + /* message= */ "Unknown message", + new IllegalStateException(String.valueOf(message.what)), + /* errorCode= */ PlaybackException.ERROR_CODE_UNSPECIFIED); + } + } catch (RuntimeException e) { + maybeRaiseError( + /* message= */ "Unknown error", + e, + /* errorCode= */ PlaybackException.ERROR_CODE_UNSPECIFIED); + } + return true; + } + + // Internal methods + + private void releaseInternal(ConditionVariable conditionVariable) { + try { + previewAudioPipeline.release(); + compositingVideoSinkProvider.clearOutputSurfaceInfo(); + compositingVideoSinkProvider.release(); + } catch (RuntimeException e) { + Log.e(TAG, "error while releasing the player", e); + } finally { + conditionVariable.open(); + } + } + + private void clearOutputSurfaceInternal() { + try { + compositingVideoSinkProvider.clearOutputSurfaceInfo(); + } catch (RuntimeException e) { + maybeRaiseError( + /* message= */ "error clearing video output", + e, + /* errorCode= */ PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED); + } + } + + private void setOutputSurfaceInfoOnInternalThread(OutputSurfaceInfo outputSurfaceInfo) { + try { + compositingVideoSinkProvider.setOutputSurfaceInfo( + outputSurfaceInfo.surface, outputSurfaceInfo.size); + } catch (RuntimeException e) { + maybeRaiseError( + /* message= */ "error setting surface view", + e, + /* errorCode= */ PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSING_FAILED); + } + } + + private void maybeRaiseError( + String message, Exception cause, @PlaybackException.ErrorCode int errorCode) { + try { + listenerHandler.post( + () -> { + // This code runs on the application thread, hence access to the `release` field does + // not need to be synchronized. + if (!released) { + listener.onError(message, cause, errorCode); + } + }); + } catch (RuntimeException e) { + Log.e(TAG, "error", e); + } + } + + private static final class OutputSurfaceInfo { + public final Surface surface; + public final Size size; + + public OutputSurfaceInfo(Surface surface, Size size) { + this.surface = surface; + this.size = size; + } + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/PreviewAudioPipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/PreviewAudioPipeline.java new file mode 100644 index 0000000000..44444d813a --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/PreviewAudioPipeline.java @@ -0,0 +1,211 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import static androidx.media3.common.util.Util.sampleCountToDurationUs; + +import androidx.media3.common.Format; +import androidx.media3.common.audio.AudioProcessor; +import androidx.media3.common.audio.AudioProcessor.AudioFormat; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.audio.AudioSink; +import com.google.common.collect.ImmutableList; +import java.nio.ByteBuffer; +import java.util.Objects; + +/** + * Processes input from {@link AudioGraphInputAudioSink} instances, plumbing the data through an + * {@link AudioGraph} and writing the output to the provided {@link AudioSink}. + * + *

Multiple streams of {@linkplain #createInput() input} are not currently supported. + */ +/* package */ final class PreviewAudioPipeline { + private final AudioSink finalAudioSink; + private final AudioGraph audioGraph; + + private int audioGraphInputsCreated; + private int inputAudioSinksCreated; + private int inputAudioSinksPlaying; + private AudioFormat outputAudioFormat; + private long outputFramesWritten; + private long seekPositionUs; + + /** + * Creates an instance. + * + * @param mixerFactory The {@linkplain AudioMixer.Factory factory} used to {@linkplain + * AudioMixer.Factory#create() create} the underlying {@link AudioMixer}. + * @param effects The composition-level audio effects that are applied after mixing. + * @param finalAudioSink The {@linkplain AudioSink sink} for processed output audio. + */ + public PreviewAudioPipeline( + AudioMixer.Factory mixerFactory, + ImmutableList effects, + AudioSink finalAudioSink) { + audioGraph = new AudioGraph(mixerFactory, effects); + this.finalAudioSink = finalAudioSink; + + outputAudioFormat = AudioFormat.NOT_SET; + } + + /** Releases any underlying resources. */ + public void release() { + audioGraph.reset(); + finalAudioSink.reset(); + finalAudioSink.release(); + audioGraphInputsCreated = 0; + inputAudioSinksCreated = 0; + inputAudioSinksPlaying = 0; + } + + /** Returns an {@link AudioSink} for a single sequence of non-overlapping raw PCM audio. */ + public AudioGraphInputAudioSink createInput() { + return new AudioGraphInputAudioSink(new SinkController()); + } + + /** + * Processes data through the underlying components. + * + * @return Whether more data can be processed by immediately calling this method again. + */ + public boolean processData() + throws ExportException, + AudioSink.WriteException, + AudioSink.InitializationException, + AudioSink.ConfigurationException { + // Do not process any data until the input audio sinks have created audio graph inputs. + if (inputAudioSinksCreated == 0 || inputAudioSinksCreated != audioGraphInputsCreated) { + return false; + } + + if (Objects.equals(outputAudioFormat, AudioFormat.NOT_SET)) { + AudioFormat audioGraphAudioFormat = audioGraph.getOutputAudioFormat(); + if (Objects.equals(audioGraphAudioFormat, AudioFormat.NOT_SET)) { + return false; + } + + finalAudioSink.configure( + Util.getPcmFormat(audioGraphAudioFormat), + /* specifiedBufferSize= */ 0, + /* outputChannels= */ null); + outputAudioFormat = audioGraphAudioFormat; + } + + if (audioGraph.isEnded()) { + if (finalAudioSink.isEnded()) { + return false; + } + finalAudioSink.playToEndOfStream(); + return false; + } + + ByteBuffer audioBuffer = audioGraph.getOutput(); + if (!audioBuffer.hasRemaining()) { + return false; + } + + int bytesToWrite = audioBuffer.remaining(); + boolean bufferHandled = + finalAudioSink.handleBuffer( + audioBuffer, getBufferPresentationTimeUs(), /* encodedAccessUnitCount= */ 1); + outputFramesWritten += + (bytesToWrite - audioBuffer.remaining()) / outputAudioFormat.bytesPerFrame; + return bufferHandled; + } + + private long getBufferPresentationTimeUs() { + return seekPositionUs + + sampleCountToDurationUs(outputFramesWritten, outputAudioFormat.sampleRate); + } + + /** + * Handles the steps that need to be executed for a seek before seeking the upstream players. + * + * @param positionUs The seek position, in microseconds. + */ + public void startSeek(long positionUs) { + finalAudioSink.pause(); + audioGraph.blockInput(); + audioGraph.setPendingStartTimeUs(positionUs); + audioGraph.flush(); + finalAudioSink.flush(); + outputFramesWritten = 0; + seekPositionUs = positionUs; + } + + /** Handles the steps that need to be executed for a seek after seeking the upstream players. */ + public void endSeek() { + audioGraph.unblockInput(); + } + + private final class SinkController implements AudioGraphInputAudioSink.Controller { + private boolean playing; + + public SinkController() { + inputAudioSinksCreated++; + } + + @Override + public AudioGraphInput getAudioGraphInput(EditedMediaItem editedMediaItem, Format format) + throws ExportException { + AudioGraphInput audioGraphInput = audioGraph.registerInput(editedMediaItem, format); + audioGraphInputsCreated++; + return audioGraphInput; + } + + @Override + public long getCurrentPositionUs() { + return finalAudioSink.getCurrentPositionUs(/* sourceEnded= */ false); + } + + @Override + public boolean isEnded() { + return finalAudioSink.isEnded(); + } + + @Override + public void onPlay() { + if (playing) { + return; + } + playing = true; + + inputAudioSinksPlaying++; + if (inputAudioSinksCreated == inputAudioSinksPlaying) { + finalAudioSink.play(); + } + } + + @Override + public void onPause() { + if (!playing) { + return; + } + playing = false; + + if (inputAudioSinksCreated == inputAudioSinksPlaying) { + finalAudioSink.pause(); + } + inputAudioSinksPlaying--; + } + + @Override + public void onReset() { + onPause(); + } + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java new file mode 100644 index 0000000000..d140477782 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java @@ -0,0 +1,433 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.exoplayer.DefaultRenderersFactory.DEFAULT_ALLOWED_VIDEO_JOINING_TIME_MS; +import static androidx.media3.exoplayer.DefaultRenderersFactory.MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY; + +import android.content.Context; +import android.graphics.Bitmap; +import android.os.Handler; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.Effect; +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.PlaybackException; +import androidx.media3.common.util.ConstantRateTimestampIterator; +import androidx.media3.exoplayer.ExoPlaybackException; +import androidx.media3.exoplayer.Renderer; +import androidx.media3.exoplayer.RenderersFactory; +import androidx.media3.exoplayer.audio.AudioRendererEventListener; +import androidx.media3.exoplayer.audio.AudioSink; +import androidx.media3.exoplayer.audio.MediaCodecAudioRenderer; +import androidx.media3.exoplayer.image.ImageDecoder; +import androidx.media3.exoplayer.image.ImageOutput; +import androidx.media3.exoplayer.image.ImageRenderer; +import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter; +import androidx.media3.exoplayer.mediacodec.MediaCodecSelector; +import androidx.media3.exoplayer.metadata.MetadataOutput; +import androidx.media3.exoplayer.source.MediaSource; +import androidx.media3.exoplayer.text.TextOutput; +import androidx.media3.exoplayer.video.CompositingVideoSinkProvider; +import androidx.media3.exoplayer.video.MediaCodecVideoRenderer; +import androidx.media3.exoplayer.video.VideoFrameReleaseControl; +import androidx.media3.exoplayer.video.VideoRendererEventListener; +import androidx.media3.exoplayer.video.VideoSink; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import java.util.ArrayList; +import java.util.List; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** Wraps {@link EditedMediaItemSequence} specific rendering logic and state. */ +/* package */ final class SequencePlayerRenderersWrapper implements RenderersFactory { + + private static final int DEFAULT_FRAME_RATE = 30; + + private final Context context; + private final EditedMediaItemSequence sequence; + private final PreviewAudioPipeline previewAudioPipeline; + @Nullable private final CompositingVideoSinkProvider compositingVideoSinkProvider; + @Nullable private final ImageDecoder.Factory imageDecoderFactory; + + /** Creates a renderers wrapper for a player that will play video, image and audio. */ + public static SequencePlayerRenderersWrapper create( + Context context, + EditedMediaItemSequence sequence, + PreviewAudioPipeline previewAudioPipeline, + CompositingVideoSinkProvider compositingVideoSinkProvider, + ImageDecoder.Factory imageDecoderFactory) { + return new SequencePlayerRenderersWrapper( + context, sequence, previewAudioPipeline, compositingVideoSinkProvider, imageDecoderFactory); + } + + /** Creates a renderers wrapper that for a player that will only play audio. */ + public static SequencePlayerRenderersWrapper createForAudio( + Context context, + EditedMediaItemSequence sequence, + PreviewAudioPipeline previewAudioPipeline) { + return new SequencePlayerRenderersWrapper( + context, + sequence, + previewAudioPipeline, + /* compositingVideoSinkProvider= */ null, + /* imageDecoderFactory= */ null); + } + + private SequencePlayerRenderersWrapper( + Context context, + EditedMediaItemSequence sequence, + PreviewAudioPipeline previewAudioPipeline, + @Nullable CompositingVideoSinkProvider compositingVideoSinkProvider, + @Nullable ImageDecoder.Factory imageDecoderFactory) { + this.context = context; + this.sequence = sequence; + this.previewAudioPipeline = previewAudioPipeline; + this.compositingVideoSinkProvider = compositingVideoSinkProvider; + this.imageDecoderFactory = imageDecoderFactory; + } + + @Override + public Renderer[] createRenderers( + Handler eventHandler, + VideoRendererEventListener videoRendererEventListener, + AudioRendererEventListener audioRendererEventListener, + TextOutput textRendererOutput, + MetadataOutput metadataRendererOutput) { + List renderers = new ArrayList<>(); + renderers.add( + new SequenceAudioRenderer( + context, + /* sequencePlayerRenderersWrapper= */ this, + eventHandler, + audioRendererEventListener, + previewAudioPipeline.createInput())); + + if (compositingVideoSinkProvider != null) { + renderers.add( + new SequenceVideoRenderer( + checkStateNotNull(context), + eventHandler, + videoRendererEventListener, + /* sequencePlayerRenderersWrapper= */ this)); + renderers.add(new SequenceImageRenderer(/* sequencePlayerRenderersWrapper= */ this)); + } + + return renderers.toArray(new Renderer[0]); + } + + private static final class SequenceAudioRenderer extends MediaCodecAudioRenderer { + private final SequencePlayerRenderersWrapper sequencePlayerRenderersWrapper; + private final AudioGraphInputAudioSink audioSink; + + @Nullable private EditedMediaItem pendingEditedMediaItem; + private long pendingOffsetToCompositionTimeUs; + + // TODO - b/320007703: Revisit the abstractions needed here (editedMediaItemProvider and + // Supplier) once we finish all the wiring to support multiple sequences. + public SequenceAudioRenderer( + Context context, + SequencePlayerRenderersWrapper sequencePlayerRenderersWrapper, + @Nullable Handler eventHandler, + @Nullable AudioRendererEventListener eventListener, + AudioGraphInputAudioSink audioSink) { + super(context, MediaCodecSelector.DEFAULT, eventHandler, eventListener, audioSink); + this.sequencePlayerRenderersWrapper = sequencePlayerRenderersWrapper; + this.audioSink = audioSink; + } + + // MediaCodecAudioRenderer methods + + @Override + public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { + super.render(positionUs, elapsedRealtimeUs); + try { + while (sequencePlayerRenderersWrapper.previewAudioPipeline.processData()) {} + } catch (ExportException + | AudioSink.WriteException + | AudioSink.InitializationException + | AudioSink.ConfigurationException e) { + throw createRendererException( + e, /* format= */ null, ExoPlaybackException.ERROR_CODE_AUDIO_TRACK_WRITE_FAILED); + } + } + + @Override + protected void onStreamChanged( + Format[] formats, + long startPositionUs, + long offsetUs, + MediaSource.MediaPeriodId mediaPeriodId) + throws ExoPlaybackException { + checkState(getTimeline().getWindowCount() == 1); + int mediaItemIndex = getTimeline().getIndexOfPeriod(mediaPeriodId.periodUid); + // We must first update the pending media item state before calling super.onStreamChanged() + // because the super method will call onProcessedStreamChange() + pendingEditedMediaItem = + sequencePlayerRenderersWrapper.sequence.editedMediaItems.get(mediaItemIndex); + // Reverse engineer how timestamps and offsets are computed with a ConcatenatingMediaSource2 + // to compute an offset converting buffer timestamps to composition timestamps. + // startPositionUs is not used because it is equal to offsetUs + clipping start time + seek + // position when seeking from any MediaItem in the playlist to the first MediaItem. + // TODO(b/331547894): remove this reverse-engineered logic by moving away from using a + // ConcatenatingMediaSource2. + // The offset to convert the sample timestamps to composition time is negative because we need + // to remove the large offset added by ExoPlayer to make sure the decoder doesn't received any + // negative timestamps. We also need to remove the clipping start position. + pendingOffsetToCompositionTimeUs = -offsetUs; + if (mediaItemIndex == 0) { + pendingOffsetToCompositionTimeUs -= + pendingEditedMediaItem.mediaItem.clippingConfiguration.startPositionUs; + } + for (int i = 0; i < mediaItemIndex; i++) { + pendingOffsetToCompositionTimeUs += + sequencePlayerRenderersWrapper + .sequence + .editedMediaItems + .get(i) + .getPresentationDurationUs(); + } + super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId); + } + + @Override + protected void onProcessedStreamChange() { + super.onProcessedStreamChange(); + onMediaItemChanged(); + } + + @Override + protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException { + super.onPositionReset(positionUs, joining); + onMediaItemChanged(); + } + + // Other methods + + private void onMediaItemChanged() { + EditedMediaItem currentEditedMediaItem = checkStateNotNull(pendingEditedMediaItem); + // Use reference equality intentionally. + boolean isLastInSequence = + currentEditedMediaItem + == Iterables.getLast(sequencePlayerRenderersWrapper.sequence.editedMediaItems); + audioSink.onMediaItemChanged( + currentEditedMediaItem, pendingOffsetToCompositionTimeUs, isLastInSequence); + } + } + + private static final class SequenceVideoRenderer extends MediaCodecVideoRenderer { + private final SequencePlayerRenderersWrapper sequencePlayerRenderersWrapper; + private final VideoSink videoSink; + @Nullable private ImmutableList pendingEffect; + + public SequenceVideoRenderer( + Context context, + Handler eventHandler, + VideoRendererEventListener videoRendererEventListener, + SequencePlayerRenderersWrapper sequencePlayerRenderersWrapper) { + super( + context, + MediaCodecAdapter.Factory.getDefault(context), + MediaCodecSelector.DEFAULT, + DEFAULT_ALLOWED_VIDEO_JOINING_TIME_MS, + /* enableDecoderFallback= */ false, + eventHandler, + videoRendererEventListener, + MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY, + /* assumedMinimumCodecOperatingRate= */ DEFAULT_FRAME_RATE, + checkStateNotNull(sequencePlayerRenderersWrapper.compositingVideoSinkProvider)); + this.sequencePlayerRenderersWrapper = sequencePlayerRenderersWrapper; + videoSink = + checkStateNotNull(sequencePlayerRenderersWrapper.compositingVideoSinkProvider).getSink(); + } + + @Override + protected void onStreamChanged( + Format[] formats, + long startPositionUs, + long offsetUs, + MediaSource.MediaPeriodId mediaPeriodId) + throws ExoPlaybackException { + checkState(getTimeline().getWindowCount() == 1); + super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId); + pendingEffect = + sequencePlayerRenderersWrapper.sequence.editedMediaItems.get( + getTimeline().getIndexOfPeriod(mediaPeriodId.periodUid)) + .effects + .videoEffects; + } + + @Override + protected void onReadyToRegisterVideoSinkInputStream() { + @Nullable ImmutableList pendingEffect = this.pendingEffect; + if (pendingEffect != null) { + videoSink.setPendingVideoEffects(pendingEffect); + this.pendingEffect = null; + } + } + } + + private static final class SequenceImageRenderer extends ImageRenderer { + private final SequencePlayerRenderersWrapper sequencePlayerRenderersWrapper; + private final CompositingVideoSinkProvider compositingVideoSinkProvider; + private final VideoSink videoSink; + private final VideoFrameReleaseControl videoFrameReleaseControl; + + private ImmutableList videoEffects; + private @MonotonicNonNull ConstantRateTimestampIterator timestampIterator; + private boolean inputStreamPendingRegistration; + @Nullable private ExoPlaybackException pendingExoPlaybackException; + private long streamOffsetUs; + private boolean mayRenderStartOfStream; + + public SequenceImageRenderer(SequencePlayerRenderersWrapper sequencePlayerRenderersWrapper) { + super( + checkStateNotNull(sequencePlayerRenderersWrapper.imageDecoderFactory), ImageOutput.NO_OP); + this.sequencePlayerRenderersWrapper = sequencePlayerRenderersWrapper; + compositingVideoSinkProvider = + checkStateNotNull(sequencePlayerRenderersWrapper.compositingVideoSinkProvider); + videoSink = compositingVideoSinkProvider.getSink(); + videoFrameReleaseControl = + checkStateNotNull(compositingVideoSinkProvider.getVideoFrameReleaseControl()); + videoEffects = ImmutableList.of(); + streamOffsetUs = C.TIME_UNSET; + } + + // ImageRenderer methods + + @Override + protected void onEnabled(boolean joining, boolean mayRenderStartOfStream) + throws ExoPlaybackException { + super.onEnabled(joining, mayRenderStartOfStream); + this.mayRenderStartOfStream = mayRenderStartOfStream; + videoFrameReleaseControl.onEnabled(mayRenderStartOfStream); + if (joining) { + videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false); + } + if (!videoSink.isInitialized()) { + Format format = new Format.Builder().build(); + try { + videoSink.initialize(format, getClock()); + } catch (VideoSink.VideoSinkException e) { + throw createRendererException( + e, format, PlaybackException.ERROR_CODE_VIDEO_FRAME_PROCESSOR_INIT_FAILED); + } + } + // TODO - b/328444280: Do not set a listener on VideoSink, but MediaCodecVideoRenderer must + // unregister itself as a listener too. + videoSink.setListener(VideoSink.Listener.NO_OP, /* executor= */ (runnable) -> {}); + } + + @Override + protected void onDisabled() { + super.onDisabled(); + videoFrameReleaseControl.onDisabled(); + } + + @Override + public boolean isReady() { + // If the renderer was enabled with mayRenderStartOfStream set to false, meaning the image + // renderer is playing after a video, we don't need to wait until the first frame is rendered. + // If the renderer was enabled with mayRenderStartOfStream, we must wait until the first frame + // is rendered, which is checked by VideoSink.isReady(). + return super.isReady() && (!mayRenderStartOfStream || videoSink.isReady()); + } + + @Override + protected void onReset() { + super.onReset(); + pendingExoPlaybackException = null; + } + + @Override + protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException { + videoSink.flush(); + super.onPositionReset(positionUs, joining); + videoFrameReleaseControl.reset(); + if (joining) { + videoFrameReleaseControl.join(/* renderNextFrameImmediately= */ false); + } + } + + @Override + protected void onStarted() throws ExoPlaybackException { + super.onStarted(); + videoFrameReleaseControl.onStarted(); + } + + @Override + protected void onStopped() { + super.onStopped(); + videoFrameReleaseControl.onStopped(); + } + + @Override + protected void onStreamChanged( + Format[] formats, + long startPositionUs, + long offsetUs, + MediaSource.MediaPeriodId mediaPeriodId) + throws ExoPlaybackException { + checkState(getTimeline().getWindowCount() == 1); + super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId); + streamOffsetUs = offsetUs; + EditedMediaItem editedMediaItem = + sequencePlayerRenderersWrapper.sequence.editedMediaItems.get( + getTimeline().getIndexOfPeriod(mediaPeriodId.periodUid)); + videoEffects = editedMediaItem.effects.videoEffects; + timestampIterator = + new ConstantRateTimestampIterator( + editedMediaItem.getPresentationDurationUs(), /* frameRate= */ DEFAULT_FRAME_RATE); + inputStreamPendingRegistration = true; + } + + @Override + public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { + if (pendingExoPlaybackException != null) { + ExoPlaybackException exoPlaybackException = pendingExoPlaybackException; + pendingExoPlaybackException = null; + throw exoPlaybackException; + } + super.render(positionUs, elapsedRealtimeUs); + compositingVideoSinkProvider.render(positionUs, elapsedRealtimeUs); + } + + @Override + protected boolean processOutputBuffer( + long positionUs, long elapsedRealtimeUs, Bitmap outputImage, long timeUs) { + if (inputStreamPendingRegistration) { + checkState(streamOffsetUs != C.TIME_UNSET); + videoSink.setPendingVideoEffects(videoEffects); + videoSink.setStreamOffsetUs(streamOffsetUs); + videoSink.registerInputStream( + VideoSink.INPUT_TYPE_BITMAP, + new Format.Builder() + .setSampleMimeType(MimeTypes.IMAGE_RAW) + .setWidth(outputImage.getWidth()) + .setHeight(outputImage.getHeight()) + .setColorInfo(ColorInfo.SRGB_BT709_FULL) + .build()); + videoFrameReleaseControl.setFrameRate(/* frameRate= */ DEFAULT_FRAME_RATE); + inputStreamPendingRegistration = false; + } + return videoSink.queueBitmap(outputImage, checkStateNotNull(timestampIterator)); + } + } +} diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerAudioPlaybackTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerAudioPlaybackTest.java new file mode 100644 index 0000000000..af2e080a90 --- /dev/null +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerAudioPlaybackTest.java @@ -0,0 +1,326 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.transformer.TestUtil.ASSET_URI_PREFIX; +import static androidx.media3.transformer.TestUtil.FILE_AUDIO_RAW; +import static androidx.media3.transformer.TestUtil.FILE_AUDIO_RAW_STEREO_48000KHZ; +import static androidx.media3.transformer.TestUtil.createAudioEffects; +import static androidx.media3.transformer.TestUtil.createVolumeScalingAudioProcessor; + +import android.content.Context; +import androidx.media3.common.MediaItem; +import androidx.media3.common.Player; +import androidx.media3.exoplayer.audio.AudioSink; +import androidx.media3.exoplayer.audio.DefaultAudioSink; +import androidx.media3.test.utils.CapturingAudioSink; +import androidx.media3.test.utils.DumpFileAsserts; +import androidx.media3.test.utils.FakeClock; +import androidx.media3.test.utils.robolectric.TestPlayerRunHelper; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Audio playback unit tests for {@link CompositionPlayer}. + * + *

These tests focus on audio because the video pipeline doesn't work in Robolectric. + */ +@RunWith(AndroidJUnit4.class) +public final class CompositionPlayerAudioPlaybackTest { + + private final Context context = ApplicationProvider.getApplicationContext(); + private CapturingAudioSink capturingAudioSink; + + @Before + public void setUp() throws Exception { + capturingAudioSink = new CapturingAudioSink(new DefaultAudioSink.Builder(context).build()); + } + + @Test + public void playback_outputsCorrectSamples() throws Exception { + CompositionPlayer player = createCompositionPlayer(context, capturingAudioSink); + EditedMediaItem editedMediaItem1 = + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW)) + .setDurationUs(1_000_000L) + .build(); + EditedMediaItem editedMediaItem2 = + new EditedMediaItem.Builder( + MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW_STEREO_48000KHZ)) + .setDurationUs(348_000L) + .build(); + EditedMediaItemSequence sequence = + new EditedMediaItemSequence(editedMediaItem1, editedMediaItem2); + Composition composition = new Composition.Builder(sequence).build(); + + player.setComposition(composition); + player.prepare(); + player.play(); + TestPlayerRunHelper.run(player).untilState(Player.STATE_ENDED); + player.release(); + + DumpFileAsserts.assertOutput( + context, capturingAudioSink, "audiosinkdumps/wav/sample.wav_then_sample_rf64.wav.dump"); + } + + @Test + public void playback_compositionWithEffects_outputsCorrectSamples() throws Exception { + CompositionPlayer player = createCompositionPlayer(context, capturingAudioSink); + EditedMediaItem editedMediaItem1 = + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW)) + .setDurationUs(1_000_000L) + .setEffects(createAudioEffects(createVolumeScalingAudioProcessor(0.5f))) + .build(); + EditedMediaItem editedMediaItem2 = + new EditedMediaItem.Builder( + MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW_STEREO_48000KHZ)) + .setDurationUs(348_000L) + .setEffects(createAudioEffects(createVolumeScalingAudioProcessor(2f))) + .build(); + EditedMediaItemSequence sequence = + new EditedMediaItemSequence(editedMediaItem1, editedMediaItem2); + Composition composition = new Composition.Builder(sequence).build(); + + player.setComposition(composition); + player.prepare(); + player.play(); + TestPlayerRunHelper.run(player).untilState(Player.STATE_ENDED); + player.release(); + + DumpFileAsserts.assertOutput( + context, + capturingAudioSink, + "audiosinkdumps/wav/sample.wav-lowVolume_then_sample_rf64.wav-highVolume.dump"); + } + + @Test + public void playback_singleAudioItemWithEffects_outputsCorrectSamples() throws Exception { + CompositionPlayer player = createCompositionPlayer(context, capturingAudioSink); + EditedMediaItem audioEditedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW)) + .setRemoveVideo(true) + .setDurationUs(1_000_000L) + .setEffects(createAudioEffects(createVolumeScalingAudioProcessor(2f))) + .build(); + Composition composition = + new Composition.Builder(new EditedMediaItemSequence(audioEditedMediaItem)).build(); + + player.setComposition(composition); + player.prepare(); + player.play(); + TestPlayerRunHelper.run(player).untilState(Player.STATE_ENDED); + player.release(); + + DumpFileAsserts.assertOutput( + context, capturingAudioSink, "audiosinkdumps/" + FILE_AUDIO_RAW + "/highVolume.dump"); + } + + @Test + public void playback_singleAudioItemWithCompositionLevelEffects_outputsCorrectSamples() + throws Exception { + CompositionPlayer player = createCompositionPlayer(context, capturingAudioSink); + EditedMediaItem audioEditedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW)) + .setRemoveVideo(true) + .setDurationUs(1_000_000L) + .build(); + Composition composition = + new Composition.Builder(new EditedMediaItemSequence(audioEditedMediaItem)) + .setEffects(createAudioEffects(createVolumeScalingAudioProcessor(2f))) + .build(); + + player.setComposition(composition); + player.prepare(); + player.play(); + TestPlayerRunHelper.run(player).untilState(Player.STATE_ENDED); + player.release(); + + DumpFileAsserts.assertOutput( + context, capturingAudioSink, "audiosinkdumps/" + FILE_AUDIO_RAW + "/highVolume.dump"); + } + + @Test + public void playback_compositionWithClipping_outputsCorrectSamples() throws Exception { + CompositionPlayer player = createCompositionPlayer(context, capturingAudioSink); + MediaItem mediaItem1 = + new MediaItem.Builder() + .setUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW) + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setStartPositionMs(300) + .setEndPositionMs(800) + .build()) + .build(); + EditedMediaItem editedMediaItem1 = + new EditedMediaItem.Builder(mediaItem1).setDurationUs(1_000_000L).build(); + MediaItem mediaItem2 = + new MediaItem.Builder() + .setUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW_STEREO_48000KHZ) + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setStartPositionMs(100) + .setEndPositionMs(300) + .build()) + .build(); + EditedMediaItem editedMediaItem2 = + new EditedMediaItem.Builder(mediaItem2).setDurationUs(348_000L).build(); + Composition composition = + new Composition.Builder(new EditedMediaItemSequence(editedMediaItem1, editedMediaItem2)) + .build(); + + player.setComposition(composition); + player.prepare(); + player.play(); + TestPlayerRunHelper.run(player).untilState(Player.STATE_ENDED); + player.release(); + + DumpFileAsserts.assertOutput( + context, + capturingAudioSink, + "audiosinkdumps/wav/sample.wav_clipped_then_sample_rf64_clipped.wav.dump"); + } + + @Test + public void seekTo_outputsCorrectSamples() throws Exception { + CompositionPlayer player = createCompositionPlayer(context, capturingAudioSink); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW)) + .setDurationUs(1_000_000L) + .build(); + EditedMediaItemSequence sequence = new EditedMediaItemSequence(editedMediaItem); + Composition composition = new Composition.Builder(sequence).build(); + player.setComposition(composition); + + player.seekTo(/* positionMs= */ 500); + player.prepare(); + player.play(); + TestPlayerRunHelper.run(player).untilState(Player.STATE_ENDED); + player.release(); + + DumpFileAsserts.assertOutput( + context, capturingAudioSink, "audiosinkdumps/" + FILE_AUDIO_RAW + "/seek_to_500_ms.dump"); + } + + @Test + public void seekToNextMediaItem_outputsCorrectSamples() throws Exception { + CompositionPlayer player = createCompositionPlayer(context, capturingAudioSink); + EditedMediaItem editedMediaItem1 = + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW)) + .setDurationUs(1_000_000L) + .build(); + EditedMediaItem editedMediaItem2 = + new EditedMediaItem.Builder( + MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW_STEREO_48000KHZ)) + .setDurationUs(348_000L) + .build(); + EditedMediaItemSequence sequence = + new EditedMediaItemSequence(editedMediaItem1, editedMediaItem2); + Composition composition = new Composition.Builder(sequence).build(); + player.setComposition(composition); + + player.seekTo(/* positionMs= */ 1200); + player.prepare(); + player.play(); + TestPlayerRunHelper.run(player).untilState(Player.STATE_ENDED); + player.release(); + + DumpFileAsserts.assertOutput( + context, + capturingAudioSink, + "audiosinkdumps/wav/sample.wav_then_sample_rf64.wav_seek_to_1200_ms.dump"); + } + + @Test + public void seekToPreviousMediaItem_outputsCorrectSamples() throws Exception { + CompositionPlayer player = createCompositionPlayer(context, capturingAudioSink); + EditedMediaItem editedMediaItem1 = + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW)) + .setDurationUs(1_000_000L) + .build(); + EditedMediaItem editedMediaItem2 = + new EditedMediaItem.Builder( + MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW_STEREO_48000KHZ)) + .setDurationUs(348_000L) + .build(); + EditedMediaItemSequence sequence = + new EditedMediaItemSequence(editedMediaItem1, editedMediaItem2); + Composition composition = new Composition.Builder(sequence).build(); + player.setComposition(composition); + + player.seekTo(/* positionMs= */ 1200); + player.seekTo(/* positionMs= */ 500); + player.prepare(); + player.play(); + TestPlayerRunHelper.run(player).untilState(Player.STATE_ENDED); + player.release(); + + DumpFileAsserts.assertOutput( + context, + capturingAudioSink, + "audiosinkdumps/wav/sample.wav_then_sample_rf64.wav_seek_to_500_ms.dump"); + } + + @Test + public void seekTo_withClipping_outputsCorrectSamples() throws Exception { + CompositionPlayer player = createCompositionPlayer(context, capturingAudioSink); + MediaItem mediaItem1 = + new MediaItem.Builder() + .setUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW) + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setStartPositionMs(200) + .setEndPositionMs(900) + .build()) + .build(); + EditedMediaItem editedMediaItem1 = + new EditedMediaItem.Builder(mediaItem1).setDurationUs(1_000_000L).build(); + MediaItem mediaItem2 = + new MediaItem.Builder() + .setUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW_STEREO_48000KHZ) + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setStartPositionMs(100) + .setEndPositionMs(300) + .build()) + .build(); + EditedMediaItem editedMediaItem2 = + new EditedMediaItem.Builder(mediaItem2).setDurationUs(348_000L).build(); + EditedMediaItemSequence sequence = + new EditedMediaItemSequence(editedMediaItem1, editedMediaItem2); + Composition composition = new Composition.Builder(sequence).build(); + player.setComposition(composition); + + player.seekTo(/* positionMs= */ 800); + player.prepare(); + player.play(); + TestPlayerRunHelper.run(player).untilState(Player.STATE_ENDED); + player.release(); + + DumpFileAsserts.assertOutput( + context, + capturingAudioSink, + "audiosinkdumps/wav/sample.wav_then_sample_rf64.wav_clipped_seek_to_800_ms.dump"); + } + + private static CompositionPlayer createCompositionPlayer(Context context, AudioSink audioSink) { + return new CompositionPlayer.Builder(context) + .setClock(new FakeClock(/* isAutoAdvancing= */ true)) + .setAudioSink(audioSink) + .build(); + } +} diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerTest.java new file mode 100644 index 0000000000..fa7d930431 --- /dev/null +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/CompositionPlayerTest.java @@ -0,0 +1,592 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.transformer.TestUtil.ASSET_URI_PREFIX; +import static androidx.media3.transformer.TestUtil.FILE_AUDIO_RAW; +import static androidx.media3.transformer.TestUtil.FILE_AUDIO_RAW_STEREO_48000KHZ; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +import android.content.Context; +import android.graphics.SurfaceTexture; +import android.os.Handler; +import android.os.HandlerThread; +import android.os.Looper; +import android.view.Surface; +import android.view.TextureView; +import androidx.media3.common.C; +import androidx.media3.common.MediaItem; +import androidx.media3.common.Player; +import androidx.media3.common.Timeline; +import androidx.media3.common.util.ConditionVariable; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.NullableType; +import androidx.media3.exoplayer.audio.AudioSink; +import androidx.media3.exoplayer.audio.DefaultAudioSink; +import androidx.media3.exoplayer.audio.ForwardingAudioSink; +import androidx.media3.test.utils.FakeClock; +import androidx.media3.test.utils.robolectric.RobolectricUtil; +import androidx.media3.test.utils.robolectric.TestPlayerRunHelper; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; +import org.mockito.Mockito; + +/** Unit tests for {@link CompositionPlayer}. */ +@RunWith(AndroidJUnit4.class) +public class CompositionPlayerTest { + private static final long TEST_TIMEOUT_MS = 1_000; + + @Test + public void builder_buildCalledTwice_throws() { + CompositionPlayer.Builder builder = + new CompositionPlayer.Builder(ApplicationProvider.getApplicationContext()); + + CompositionPlayer player = builder.build(); + + assertThrows(IllegalStateException.class, builder::build); + + player.release(); + } + + @Test + public void builder_buildCalledOnNonHandlerThread_throws() throws InterruptedException { + AtomicReference<@NullableType Exception> exception = new AtomicReference<>(); + ConditionVariable conditionVariable = new ConditionVariable(); + + Thread thread = + new Thread( + () -> { + try { + new Composition.Builder(ApplicationProvider.getApplicationContext()).build(); + } catch (Exception e) { + exception.set(e); + } finally { + conditionVariable.open(); + } + }); + thread.start(); + + conditionVariable.block(); + thread.join(); + + assertThat(exception.get()).isNotNull(); + } + + @Test + public void instance_accessedByWrongThread_throws() throws InterruptedException { + CompositionPlayer player = buildCompositionPlayer(); + AtomicReference<@NullableType RuntimeException> exception = new AtomicReference<>(); + ConditionVariable conditionVariable = new ConditionVariable(); + HandlerThread handlerThread = new HandlerThread("test"); + handlerThread.start(); + + new Handler(handlerThread.getLooper()) + .post( + () -> { + try { + player.setComposition(buildComposition()); + } catch (RuntimeException e) { + exception.set(e); + } finally { + conditionVariable.open(); + } + }); + conditionVariable.block(); + player.release(); + handlerThread.quit(); + handlerThread.join(); + + assertThat(exception.get()).isInstanceOf(IllegalStateException.class); + assertThat(exception.get()).hasMessageThat().contains("Player is accessed on the wrong thread"); + } + + @Test + public void instance_withSpecifiedApplicationLooper_callbacksDispatchedOnSpecifiedThread() + throws Exception { + HandlerThread applicationHandlerThread = new HandlerThread("app-thread"); + applicationHandlerThread.start(); + Looper applicationLooper = applicationHandlerThread.getLooper(); + Handler applicationThreadHandler = new Handler(applicationLooper); + AtomicReference callbackThread = new AtomicReference<>(); + ConditionVariable eventsArrived = new ConditionVariable(); + CompositionPlayer player = + createCompositionPlayerBuilder().setLooper(applicationLooper).build(); + // Listeners can be added by any thread. + player.addListener( + new Player.Listener() { + @Override + public void onEvents(Player player, Player.Events events) { + callbackThread.set(Thread.currentThread()); + eventsArrived.open(); + } + }); + + applicationThreadHandler.post( + () -> { + player.setComposition(buildComposition()); + player.prepare(); + }); + if (!eventsArrived.block(TEST_TIMEOUT_MS)) { + throw new TimeoutException(); + } + // Use a separate condition variable to releasing the player to avoid race conditions + // with the condition variable used for the callback. + ConditionVariable released = new ConditionVariable(); + applicationThreadHandler.post( + () -> { + player.release(); + released.open(); + }); + if (!released.block(TEST_TIMEOUT_MS)) { + throw new TimeoutException(); + } + applicationHandlerThread.quit(); + applicationHandlerThread.join(); + + assertThat(eventsArrived.isOpen()).isTrue(); + assertThat(callbackThread.get()).isEqualTo(applicationLooper.getThread()); + } + + @Test + public void release_onNewlyCreateInstance() { + CompositionPlayer player = buildCompositionPlayer(); + + player.release(); + } + + @Test + public void release_audioFailsDuringRelease_onlyLogsError() throws Exception { + Log.Logger logger = mock(Log.Logger.class); + Log.setLogger(logger); + AudioSink audioSink = + new ForwardingAudioSink( + new DefaultAudioSink.Builder(ApplicationProvider.getApplicationContext()).build()) { + @Override + public void release() { + throw new RuntimeException("AudioSink release error"); + } + }; + CompositionPlayer player = createCompositionPlayerBuilder().setAudioSink(audioSink).build(); + Player.Listener listener = mock(Player.Listener.class); + player.addListener(listener); + + player.setComposition(buildComposition()); + player.prepare(); + TestPlayerRunHelper.run(player).untilState(Player.STATE_READY); + + player.release(); + + verify(listener, never()).onPlayerError(any()); + verify(logger) + .e( + eq("CompPlayerInternal"), + eq("error while releasing the player"), + argThat( + throwable -> + throwable instanceof RuntimeException + && throwable.getMessage().contains("AudioSink release error"))); + } + + @Test + public void getAvailableCommands_returnsSpecificCommands() { + CompositionPlayer player = buildCompositionPlayer(); + + assertThat(getList(player.getAvailableCommands())) + .containsExactly( + Player.COMMAND_PLAY_PAUSE, + Player.COMMAND_PREPARE, + Player.COMMAND_STOP, + Player.COMMAND_SEEK_IN_CURRENT_MEDIA_ITEM, + Player.COMMAND_SEEK_BACK, + Player.COMMAND_SEEK_FORWARD, + Player.COMMAND_GET_CURRENT_MEDIA_ITEM, + Player.COMMAND_GET_TIMELINE, + Player.COMMAND_SET_VIDEO_SURFACE, + Player.COMMAND_RELEASE); + + player.release(); + } + + @Test + public void setComposition_calledTwice_throws() { + Composition composition = buildComposition(); + CompositionPlayer player = buildCompositionPlayer(); + + player.setComposition(composition); + + assertThrows(IllegalStateException.class, () -> player.setComposition(composition)); + + player.release(); + } + + @Test + public void setComposition_threeSequences_throws() { + CompositionPlayer player = buildCompositionPlayer(); + + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(MediaItem.EMPTY).setDurationUs(1_000).build(); + Composition composition = + new Composition.Builder( + ImmutableList.of( + new EditedMediaItemSequence(editedMediaItem), + new EditedMediaItemSequence(editedMediaItem), + new EditedMediaItemSequence(editedMediaItem))) + .build(); + + assertThrows(IllegalArgumentException.class, () -> player.setComposition(composition)); + + player.release(); + } + + @Test + public void setComposition_unmatchingDurations_throws() { + CompositionPlayer player = buildCompositionPlayer(); + + Composition composition = + new Composition.Builder( + ImmutableList.of( + new EditedMediaItemSequence( + new EditedMediaItem.Builder(MediaItem.EMPTY).setDurationUs(1).build()), + new EditedMediaItemSequence( + new EditedMediaItem.Builder(MediaItem.EMPTY).setDurationUs(2).build()))) + .build(); + + assertThrows(IllegalArgumentException.class, () -> player.setComposition(composition)); + + player.release(); + } + + @Test + public void prepare_withoutCompositionSet_throws() { + CompositionPlayer player = buildCompositionPlayer(); + + assertThrows(IllegalStateException.class, player::prepare); + + player.release(); + } + + @Test + public void playWhenReady_calledBeforePrepare_startsPlayingAfterPrepareCalled() throws Exception { + CompositionPlayer player = buildCompositionPlayer(); + + player.setPlayWhenReady(true); + player.setComposition(buildComposition()); + player.prepare(); + + TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED); + player.release(); + } + + @Test + public void playWhenReady_triggersPlayWhenReadyCallbackWithReason() throws Exception { + CompositionPlayer player = buildCompositionPlayer(); + AtomicInteger playWhenReadyReason = new AtomicInteger(-1); + player.addListener( + new Player.Listener() { + @Override + public void onPlayWhenReadyChanged( + boolean playWhenReady, @Player.PlayWhenReadyChangeReason int reason) { + playWhenReadyReason.set(reason); + } + }); + + player.setPlayWhenReady(true); + RobolectricUtil.runMainLooperUntil(() -> playWhenReadyReason.get() != -1); + + assertThat(playWhenReadyReason.get()) + .isEqualTo(Player.PLAY_WHEN_READY_CHANGE_REASON_USER_REQUEST); + } + + @Test + public void setVideoTextureView_throws() { + Context context = ApplicationProvider.getApplicationContext(); + CompositionPlayer player = buildCompositionPlayer(); + + assertThrows( + UnsupportedOperationException.class, + () -> player.setVideoTextureView(new TextureView(context))); + + player.release(); + } + + @Test + public void setVideoSurface_withNonNullSurface_throws() { + CompositionPlayer player = buildCompositionPlayer(); + Surface surface = new Surface(new SurfaceTexture(/* texName= */ 0)); + + assertThrows(UnsupportedOperationException.class, () -> player.setVideoSurface(surface)); + + player.release(); + surface.release(); + } + + @Test + public void clearVideoSurface_specifiedSurfaceNotPreviouslySet_throws() { + CompositionPlayer player = buildCompositionPlayer(); + + assertThrows( + IllegalArgumentException.class, + () -> player.clearVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 0)))); + + player.release(); + } + + @Test + public void getTotalBufferedDuration_playerStillIdle_returnsZero() { + CompositionPlayer player = buildCompositionPlayer(); + + assertThat(player.getTotalBufferedDuration()).isEqualTo(0); + + player.release(); + } + + @Test + public void getTotalBufferedDuration_setCompositionButNotPrepare_returnsZero() { + CompositionPlayer player = buildCompositionPlayer(); + + player.setComposition(buildComposition()); + + assertThat(player.getTotalBufferedDuration()).isEqualTo(0); + + player.release(); + } + + @Test + public void getTotalBufferedDuration_playerReady_returnsNonZero() throws Exception { + CompositionPlayer player = buildCompositionPlayer(); + + player.setComposition(buildComposition()); + player.prepare(); + TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_READY); + + assertThat(player.getTotalBufferedDuration()).isGreaterThan(0); + + player.release(); + } + + @Test + public void getDuration_withoutComposition_returnsTimeUnset() { + CompositionPlayer player = buildCompositionPlayer(); + + assertThat(player.getDuration()).isEqualTo(C.TIME_UNSET); + + player.release(); + } + + @Test + public void getDuration_withComposition_returnsDuration() throws Exception { + CompositionPlayer player = buildCompositionPlayer(); + Composition composition = buildComposition(); + + player.setComposition(composition); + player.prepare(); + TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_READY); + + // Refer to the durations in buildComposition(). + assertThat(player.getDuration()).isEqualTo(1_348); + + player.release(); + } + + @Test + public void addListener_callsSupportedCallbacks() throws Exception { + CompositionPlayer player = buildCompositionPlayer(); + Composition composition = buildComposition(); + List playbackStates = new ArrayList<>(); + AtomicBoolean playing = new AtomicBoolean(); + Player.Listener listener = + spy( + new Player.Listener() { + @Override + public void onPlaybackStateChanged(int playbackState) { + if (playbackStates.isEmpty() + || Iterables.getLast(playbackStates) != playbackState) { + playbackStates.add(playbackState); + } + } + + @Override + public void onIsPlayingChanged(boolean isPlaying) { + playing.set(isPlaying); + } + }); + InOrder inOrder = Mockito.inOrder(listener); + + player.setComposition(composition); + player.addListener(listener); + player.prepare(); + TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_READY); + + inOrder + .verify(listener) + .onTimelineChanged(any(), eq(Player.TIMELINE_CHANGE_REASON_PLAYLIST_CHANGED)); + inOrder.verify(listener).onPlaybackStateChanged(Player.STATE_BUFFERING); + inOrder.verify(listener).onPlaybackStateChanged(Player.STATE_READY); + + player.setPlayWhenReady(true); + + // Ensure that Player.Listener.onIsPlayingChanged(true) is called. + RobolectricUtil.runMainLooperUntil(playing::get); + inOrder + .verify(listener) + .onPlayWhenReadyChanged(true, Player.PLAY_WHEN_READY_CHANGE_REASON_USER_REQUEST); + inOrder.verify(listener).onIsPlayingChanged(true); + + TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED); + inOrder.verify(listener).onPlaybackStateChanged(Player.STATE_ENDED); + + player.release(); + TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_IDLE); + inOrder.verify(listener).onPlaybackStateChanged(Player.STATE_IDLE); + + assertThat(playbackStates) + .containsExactly( + Player.STATE_BUFFERING, Player.STATE_READY, Player.STATE_ENDED, Player.STATE_IDLE) + .inOrder(); + } + + @Test + public void addListener_callsOnEventsWithSupportedEvents() throws Exception { + CompositionPlayer player = buildCompositionPlayer(); + Composition composition = buildComposition(); + Player.Listener mockListener = mock(Player.Listener.class); + ArgumentCaptor eventsCaptor = ArgumentCaptor.forClass(Player.Events.class); + ImmutableSet supportedEvents = + ImmutableSet.of( + Player.EVENT_TIMELINE_CHANGED, + Player.EVENT_MEDIA_ITEM_TRANSITION, + Player.EVENT_PLAYBACK_STATE_CHANGED, + Player.EVENT_PLAY_WHEN_READY_CHANGED, + Player.EVENT_IS_PLAYING_CHANGED); + + player.setComposition(composition); + player.addListener(mockListener); + player.prepare(); + player.play(); + TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED); + player.release(); + + verify(mockListener, atLeastOnce()).onEvents(any(), eventsCaptor.capture()); + List eventsList = eventsCaptor.getAllValues(); + for (Player.Events events : eventsList) { + assertThat(events.size()).isNotEqualTo(0); + for (int j = 0; j < events.size(); j++) { + assertThat(supportedEvents).contains(events.get(j)); + } + } + } + + @Test + public void play_withCorrectTimelineUpdated() throws Exception { + CompositionPlayer player = buildCompositionPlayer(); + Composition composition = buildComposition(); + Player.Listener mockListener = mock(Player.Listener.class); + ArgumentCaptor timelineCaptor = ArgumentCaptor.forClass(Timeline.class); + ArgumentCaptor timelineChangeReasonCaptor = ArgumentCaptor.forClass(Integer.class); + player.setComposition(composition); + player.addListener(mockListener); + player.prepare(); + player.play(); + TestPlayerRunHelper.runUntilPlaybackState(player, Player.STATE_ENDED); + player.release(); + + verify(mockListener) + .onTimelineChanged(timelineCaptor.capture(), timelineChangeReasonCaptor.capture()); + assertThat(timelineCaptor.getAllValues()).hasSize(1); + assertThat(timelineChangeReasonCaptor.getAllValues()).hasSize(1); + Timeline timeline = timelineCaptor.getValue(); + assertThat(timeline.getWindowCount()).isEqualTo(1); + assertThat(timeline.getPeriodCount()).isEqualTo(1); + // Refer to the durations in buildComposition(). + assertThat(timeline.getWindow(/* windowIndex= */ 0, new Timeline.Window()).durationUs) + .isEqualTo(1_348_000L); + assertThat(timelineChangeReasonCaptor.getValue()) + .isEqualTo(Player.TIMELINE_CHANGE_REASON_PLAYLIST_CHANGED); + } + + @Test + public void seekPastDuration_ends() throws Exception { + CompositionPlayer player = buildCompositionPlayer(); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW)) + .setDurationUs(1_000_000L) + .build(); + EditedMediaItemSequence sequence = new EditedMediaItemSequence(editedMediaItem); + Composition composition = new Composition.Builder(sequence).build(); + player.setComposition(composition); + player.prepare(); + player.play(); + + player.seekTo(/* positionMs= */ 1100); + TestPlayerRunHelper.run(player).untilState(Player.STATE_ENDED); + player.release(); + } + + private static CompositionPlayer buildCompositionPlayer() { + return createCompositionPlayerBuilder().build(); + } + + private static CompositionPlayer.Builder createCompositionPlayerBuilder() { + return new CompositionPlayer.Builder(ApplicationProvider.getApplicationContext()) + .setClock(new FakeClock(/* isAutoAdvancing= */ true)); + } + + private static Composition buildComposition() { + // Use raw audio-only assets which can be played in robolectric tests. + EditedMediaItem editedMediaItem1 = + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW)) + .setDurationUs(1_000_000L) + .build(); + EditedMediaItem editedMediaItem2 = + new EditedMediaItem.Builder( + MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW_STEREO_48000KHZ)) + .setDurationUs(348_000L) + .build(); + EditedMediaItemSequence sequence = + new EditedMediaItemSequence(editedMediaItem1, editedMediaItem2); + return new Composition.Builder(sequence).build(); + } + + private static List getList(Player.Commands commands) { + List commandList = new ArrayList<>(); + for (int i = 0; i < commands.size(); i++) { + commandList.add(commands.get(i)); + } + return commandList; + } +} diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/PreviewAudioPipelineTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/PreviewAudioPipelineTest.java new file mode 100644 index 0000000000..b0f9565ebb --- /dev/null +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/PreviewAudioPipelineTest.java @@ -0,0 +1,194 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.atMostOnce; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import androidx.media3.exoplayer.audio.AudioSink; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit tests for {@link PreviewAudioPipeline}. */ +@RunWith(AndroidJUnit4.class) +public class PreviewAudioPipelineTest { + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + private PreviewAudioPipeline previewAudioPipeline; + @Mock AudioSink outputAudioSink; + + @Before + public void setUp() { + previewAudioPipeline = + new PreviewAudioPipeline( + new DefaultAudioMixer.Factory(), /* effects= */ ImmutableList.of(), outputAudioSink); + } + + @After + public void tearDown() { + previewAudioPipeline.release(); + } + + @Test + public void processData_noAudioSinksCreated_returnsFalse() throws Exception { + assertThat(previewAudioPipeline.processData()).isFalse(); + } + + @Test + public void processData_audioSinkHasNotConfiguredYet_returnsFalse() throws Exception { + AudioGraphInputAudioSink unused = previewAudioPipeline.createInput(); + + assertThat(previewAudioPipeline.processData()).isFalse(); + } + + @Test + public void inputPlay_withOneInput_playsOutputSink() throws Exception { + AudioGraphInputAudioSink inputAudioSink = previewAudioPipeline.createInput(); + + inputAudioSink.play(); + + verify(outputAudioSink).play(); + } + + @Test + public void inputPause_withOneInput_pausesOutputSink() throws Exception { + AudioGraphInputAudioSink inputAudioSink = previewAudioPipeline.createInput(); + + inputAudioSink.play(); + inputAudioSink.pause(); + + verify(outputAudioSink).pause(); + } + + @Test + public void inputReset_withOneInput_pausesOutputSink() { + AudioGraphInputAudioSink inputAudioSink = previewAudioPipeline.createInput(); + + inputAudioSink.play(); + inputAudioSink.reset(); + + verify(outputAudioSink).pause(); + } + + @Test + public void inputPlay_whenPlaying_doesNotPlayOutputSink() throws Exception { + AudioGraphInputAudioSink inputAudioSink = previewAudioPipeline.createInput(); + inputAudioSink.play(); + inputAudioSink.play(); + + verify(outputAudioSink, atMostOnce()).play(); + } + + @Test + public void inputPause_whenNotPlaying_doesNotPauseOutputSink() throws Exception { + AudioGraphInputAudioSink inputAudioSink = previewAudioPipeline.createInput(); + + inputAudioSink.pause(); + + verify(outputAudioSink, never()).pause(); + } + + @Test + public void someInputPlay_withMultipleInputs_doesNotPlayOutputSink() throws Exception { + AudioGraphInputAudioSink inputAudioSink1 = previewAudioPipeline.createInput(); + AudioGraphInputAudioSink inputAudioSink2 = previewAudioPipeline.createInput(); + AudioGraphInputAudioSink unused = previewAudioPipeline.createInput(); + + inputAudioSink1.play(); + inputAudioSink2.play(); + verify(outputAudioSink, never()).play(); + } + + @Test + public void allInputPlay_withMultipleInputs_playsOutputSinkOnce() throws Exception { + AudioGraphInputAudioSink inputAudioSink1 = previewAudioPipeline.createInput(); + AudioGraphInputAudioSink inputAudioSink2 = previewAudioPipeline.createInput(); + AudioGraphInputAudioSink inputAudioSink3 = previewAudioPipeline.createInput(); + + inputAudioSink1.play(); + inputAudioSink2.play(); + inputAudioSink3.play(); + + verify(outputAudioSink, atMostOnce()).play(); + } + + @Test + public void firstInputPause_withMultipleInputs_pausesOutputSink() throws Exception { + InOrder inOrder = inOrder(outputAudioSink); + AudioGraphInputAudioSink inputAudioSink1 = previewAudioPipeline.createInput(); + AudioGraphInputAudioSink inputAudioSink2 = previewAudioPipeline.createInput(); + AudioGraphInputAudioSink inputAudioSink3 = previewAudioPipeline.createInput(); + + inputAudioSink1.play(); + inputAudioSink2.play(); + inputAudioSink3.play(); + inputAudioSink2.pause(); + + inOrder.verify(outputAudioSink).pause(); + inOrder.verifyNoMoreInteractions(); + } + + @Test + public void allInputPause_withMultipleInputs_pausesOutputSinkOnce() throws Exception { + AudioGraphInputAudioSink inputAudioSink1 = previewAudioPipeline.createInput(); + AudioGraphInputAudioSink inputAudioSink2 = previewAudioPipeline.createInput(); + AudioGraphInputAudioSink inputAudioSink3 = previewAudioPipeline.createInput(); + + inputAudioSink1.play(); + inputAudioSink2.play(); + inputAudioSink3.play(); + inputAudioSink2.pause(); + inputAudioSink1.pause(); + inputAudioSink3.pause(); + + verify(outputAudioSink, atMostOnce()).pause(); + } + + @Test + public void inputPlayAfterPause_withMultipleInputs_playsOutputSink() throws Exception { + InOrder inOrder = inOrder(outputAudioSink); + AudioGraphInputAudioSink inputAudioSink1 = previewAudioPipeline.createInput(); + AudioGraphInputAudioSink inputAudioSink2 = previewAudioPipeline.createInput(); + AudioGraphInputAudioSink inputAudioSink3 = previewAudioPipeline.createInput(); + + inputAudioSink1.play(); + inputAudioSink2.play(); + inputAudioSink3.play(); + inputAudioSink2.pause(); + inputAudioSink1.pause(); + inputAudioSink2.play(); + inputAudioSink1.play(); + + inOrder.verify(outputAudioSink).play(); + inOrder.verify(outputAudioSink).pause(); + inOrder.verify(outputAudioSink).play(); + Mockito.verifyNoMoreInteractions(outputAudioSink); + } +}