diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 24343bc548..8e3cc628a3 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -8,7 +8,8 @@ * ExoPlayer: * Transformer: * Add `MediaProjectionAssetLoader`, which provides media from a - `MediaProjection` for screen recording. + `MediaProjection` for screen recording, and add support for screen + recording to the Transformer demo app. * Add `#getInputFormat()` to `Codec` interface. * Shift the responsibility to release the `GlObjectsProvider` onto the caller in `DefaultVideoFrameProcessor` and `DefaultVideoCompositor` when diff --git a/demos/transformer/build.gradle b/demos/transformer/build.gradle index b0b01c20b2..2e93ac8fcb 100644 --- a/demos/transformer/build.gradle +++ b/demos/transformer/build.gradle @@ -76,6 +76,7 @@ dependencies { implementation 'androidx.appcompat:appcompat:' + androidxAppCompatVersion implementation 'androidx.constraintlayout:constraintlayout:' + androidxConstraintLayoutVersion implementation 'androidx.recyclerview:recyclerview:' + androidxRecyclerViewVersion + implementation 'androidx.window:window:' + androidxWindowVersion implementation 'com.google.android.material:material:' + androidxMaterialVersion implementation project(modulePrefix + 'lib-effect') implementation project(modulePrefix + 'lib-exoplayer') diff --git a/demos/transformer/src/main/AndroidManifest.xml b/demos/transformer/src/main/AndroidManifest.xml index 8c58f1ad6a..907cda9c7a 100644 --- a/demos/transformer/src/main/AndroidManifest.xml +++ b/demos/transformer/src/main/AndroidManifest.xml @@ -24,6 +24,10 @@ + + + + + diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index 9b1ec9eb02..074f577b42 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -15,6 +15,7 @@ */ package androidx.media3.demo.transformer; +import static android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.exoplayer.DefaultLoadControl.DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS; @@ -22,15 +23,25 @@ import static androidx.media3.exoplayer.DefaultLoadControl.DEFAULT_BUFFER_FOR_PL import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED; import android.app.Activity; +import android.app.Notification; +import android.app.NotificationChannel; +import android.app.NotificationManager; +import android.app.Service; +import android.content.BroadcastReceiver; import android.content.Context; import android.content.Intent; +import android.content.IntentFilter; import android.content.pm.PackageManager; import android.graphics.Bitmap; import android.graphics.Color; +import android.graphics.Rect; import android.graphics.drawable.Drawable; +import android.media.projection.MediaProjection; +import android.media.projection.MediaProjectionManager; import android.net.Uri; import android.os.Bundle; import android.os.Handler; +import android.os.IBinder; import android.text.Spannable; import android.text.SpannableString; import android.text.style.ForegroundColorSpan; @@ -42,9 +53,13 @@ import android.widget.Button; import android.widget.ImageView; import android.widget.TextView; import android.widget.Toast; +import androidx.activity.result.contract.ActivityResultContracts; import androidx.annotation.Nullable; import androidx.annotation.StringRes; import androidx.appcompat.app.AppCompatActivity; +import androidx.core.app.NotificationCompat; +import androidx.core.content.ContextCompat; +import androidx.localbroadcastmanager.content.LocalBroadcastManager; import androidx.media3.common.C; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; @@ -91,10 +106,13 @@ import androidx.media3.transformer.ExportResult; import androidx.media3.transformer.InAppFragmentedMp4Muxer; import androidx.media3.transformer.InAppMp4Muxer; import androidx.media3.transformer.JsonUtil; +import androidx.media3.transformer.MediaProjectionAssetLoader; import androidx.media3.transformer.ProgressHolder; import androidx.media3.transformer.Transformer; +import androidx.media3.transformer.VideoEncoderSettings; import androidx.media3.ui.AspectRatioFrameLayout; import androidx.media3.ui.PlayerView; +import androidx.window.layout.WindowMetricsCalculator; import com.google.android.material.card.MaterialCardView; import com.google.android.material.progressindicator.LinearProgressIndicator; import com.google.common.base.Stopwatch; @@ -116,12 +134,13 @@ import org.json.JSONObject; public final class TransformerActivity extends AppCompatActivity { private static final String TAG = "TransformerActivity"; private static final int IMAGE_DURATION_MS = 5_000; - private static final int IMAGE_FRAME_RATE_FPS = 30; + private static final int DEFAULT_FRAME_RATE_FPS = 30; private static int LOAD_CONTROL_MIN_BUFFER_MS = 5_000; private static int LOAD_CONTROL_MAX_BUFFER_MS = 5_000; private Button displayInputButton; private MaterialCardView inputCardView; + private MaterialCardView outputCardView; private TextView inputTextView; private ImageView inputImageView; private PlayerView inputPlayerView; @@ -133,10 +152,13 @@ public final class TransformerActivity extends AppCompatActivity { private LinearProgressIndicator progressIndicator; private Button pauseButton; private Button resumeButton; + private Button stopCaptureButton; private Stopwatch exportStopwatch; private AspectRatioFrameLayout debugFrame; @Nullable private DebugTextViewHelper debugTextViewHelper; + @Nullable private Intent screenCaptureToken; + @Nullable private MediaProjection mediaProjection; @Nullable private ExoPlayer inputPlayer; @Nullable private ExoPlayer outputPlayer; @Nullable private Transformer transformer; @@ -149,6 +171,7 @@ public final class TransformerActivity extends AppCompatActivity { setContentView(R.layout.transformer_activity); inputCardView = findViewById(R.id.input_card_view); + outputCardView = findViewById(R.id.output_card_view); inputTextView = findViewById(R.id.input_text_view); inputImageView = findViewById(R.id.input_image_view); inputPlayerView = findViewById(R.id.input_player_view); @@ -162,6 +185,8 @@ public final class TransformerActivity extends AppCompatActivity { pauseButton.setOnClickListener(view -> pauseExport()); resumeButton = findViewById(R.id.resume_button); resumeButton.setOnClickListener(view -> startExport()); + stopCaptureButton = findViewById(R.id.stop_capture_button); + stopCaptureButton.setOnClickListener(view -> mediaProjection.stop()); debugFrame = findViewById(R.id.debug_aspect_ratio_frame_layout); displayInputButton = findViewById(R.id.display_input_button); displayInputButton.setOnClickListener(view -> toggleInputVideoDisplay()); @@ -180,7 +205,10 @@ public final class TransformerActivity extends AppCompatActivity { protected void onStart() { super.onStart(); - startExport(); + // Restart exporting, unless this is a capture session which can run in the background. + if (!isUsingMediaProjection()) { + startExport(); + } inputPlayerView.onResume(); outputPlayerView.onResume(); @@ -192,13 +220,70 @@ public final class TransformerActivity extends AppCompatActivity { inputPlayerView.onPause(); outputPlayerView.onPause(); - releasePlayers(); + + // Keep the capture session going to allow capturing other apps while backgrounded. + if (!isUsingMediaProjection()) { + releasePlayers(); + cleanUpExport(); + } + } + + @Override + protected void onDestroy() { + super.onDestroy(); + + if (isUsingMediaProjection()) { + releasePlayers(); + mediaProjection.stop(); + mediaProjection = null; + screenCaptureToken = null; + } cleanUpExport(); } private void startExport() { Intent intent = getIntent(); Uri inputUri = checkNotNull(intent.getData()); + + if (inputUri.toString().equals("transformer_surface_asset:media_projection") + && screenCaptureToken == null) { + // MediaProjection can only start once the foreground service is running. + MediaProjectionManager mediaProjectionManager = + (MediaProjectionManager) getSystemService(MEDIA_PROJECTION_SERVICE); + Context context = this; + LocalBroadcastManager.getInstance(context) + .registerReceiver( + new BroadcastReceiver() { + @Override + public void onReceive(Context context, Intent intent) { + String action = checkNotNull(intent.getAction()); + if (action.equals(DemoMediaProjectionService.ACTION_EVENT_STARTED)) { + LocalBroadcastManager.getInstance(context) + .unregisterReceiver(/* receiver= */ this); + // The service has started so media projection can start. + startExport(); + } + } + }, + new IntentFilter(DemoMediaProjectionService.ACTION_EVENT_STARTED)); + registerForActivityResult( + new ActivityResultContracts.StartActivityForResult(), + activityResult -> { + int resultCode = activityResult.getResultCode(); + if (resultCode == RESULT_OK) { + screenCaptureToken = activityResult.getData(); + Intent startServiceIntent = new Intent(context, DemoMediaProjectionService.class); + ContextCompat.startForegroundService(context, startServiceIntent); + } else if (resultCode == RESULT_CANCELED) { + finish(); + } + }) + .launch(mediaProjectionManager.createScreenCaptureIntent()); + inputCardView.setVisibility(View.GONE); + outputCardView.setVisibility(View.GONE); + return; + } + try { outputFile = createExternalCacheFile("transformer-output-" + Clock.DEFAULT.elapsedRealtime() + ".mp4"); @@ -225,10 +310,20 @@ public final class TransformerActivity extends AppCompatActivity { outputVideoTextView.setVisibility(View.GONE); debugTextView.setVisibility(View.GONE); informationTextView.setText(R.string.export_started); + outputCardView.setVisibility(View.VISIBLE); progressViewGroup.setVisibility(View.VISIBLE); pauseButton.setVisibility(View.VISIBLE); resumeButton.setVisibility(View.GONE); progressIndicator.setProgress(0); + if (isUsingMediaProjection()) { + pauseButton.setVisibility(View.GONE); + resumeButton.setVisibility(View.GONE); + stopCaptureButton.setVisibility(View.VISIBLE); + } else { + pauseButton.setVisibility(View.VISIBLE); + resumeButton.setVisibility(View.GONE); + stopCaptureButton.setVisibility(View.GONE); + } Handler mainHandler = new Handler(getMainLooper()); ProgressHolder progressHolder = new ProgressHolder(); mainHandler.post( @@ -294,11 +389,6 @@ public final class TransformerActivity extends AppCompatActivity { transformerBuilder.setVideoMimeType(videoMimeType); } - transformerBuilder.setEncoderFactory( - new DefaultEncoderFactory.Builder(this.getApplicationContext()) - .setEnableFallback(bundle.getBoolean(ConfigurationActivity.ENABLE_FALLBACK)) - .build()); - if (!bundle.getBoolean(ConfigurationActivity.ABORT_SLOW_EXPORT)) { transformerBuilder.setMaxDelayBetweenMuxerSamplesMs(C.TIME_UNSET); } @@ -321,6 +411,32 @@ public final class TransformerActivity extends AppCompatActivity { } } + VideoEncoderSettings videoEncoderSettings = VideoEncoderSettings.DEFAULT; + if (screenCaptureToken != null) { + MediaProjectionManager mediaProjectionManager = + (MediaProjectionManager) getSystemService(MEDIA_PROJECTION_SERVICE); + MediaProjection mediaProjection = + mediaProjectionManager.getMediaProjection(RESULT_OK, checkNotNull(screenCaptureToken)); + Rect bounds = + WindowMetricsCalculator.getOrCreate() + .computeCurrentWindowMetrics(/* activity= */ this) + .getBounds(); + int densityDpi = getResources().getConfiguration().densityDpi; + transformerBuilder.setAssetLoaderFactory( + new MediaProjectionAssetLoader.Factory(mediaProjection, bounds, densityDpi)); + this.mediaProjection = mediaProjection; + videoEncoderSettings = + videoEncoderSettings + .buildUpon() + .setRepeatPreviousFrameIntervalUs(C.MICROS_PER_SECOND / DEFAULT_FRAME_RATE_FPS) + .build(); + } + transformerBuilder.setEncoderFactory( + new DefaultEncoderFactory.Builder(this.getApplicationContext()) + .setEnableFallback(bundle.getBoolean(ConfigurationActivity.ENABLE_FALLBACK)) + .setRequestedVideoEncoderSettings(videoEncoderSettings) + .build()); + return transformerBuilder.build(); } @@ -339,7 +455,7 @@ public final class TransformerActivity extends AppCompatActivity { private Composition createComposition(MediaItem mediaItem, @Nullable Bundle bundle) { EditedMediaItem.Builder editedMediaItemBuilder = new EditedMediaItem.Builder(mediaItem); // For image inputs. Automatically ignored if input is audio/video. - editedMediaItemBuilder.setFrameRate(IMAGE_FRAME_RATE_FPS); + editedMediaItemBuilder.setFrameRate(DEFAULT_FRAME_RATE_FPS); if (bundle != null) { ImmutableList audioProcessors = createAudioProcessorsFromBundle(bundle); ImmutableList videoEffects = createVideoEffectsFromBundle(bundle); @@ -640,6 +756,9 @@ public final class TransformerActivity extends AppCompatActivity { informationTextView.setText(R.string.export_error); progressViewGroup.setVisibility(View.GONE); debugFrame.removeAllViews(); + if (isUsingMediaProjection()) { + mediaProjection.stop(); + } Toast.makeText(getApplicationContext(), "Export error: " + exportException, Toast.LENGTH_LONG) .show(); Log.e(TAG, "Export error", exportException); @@ -717,6 +836,12 @@ public final class TransformerActivity extends AppCompatActivity { } catch (ExecutionException | InterruptedException e) { throw new IllegalArgumentException("Failed to load bitmap.", e); } + } else if (isUsingMediaProjection()) { + inputCardView.setVisibility(View.GONE); + displayInputButton.setVisibility(View.GONE); + Intent stopIntent = new Intent(/* context= */ this, DemoMediaProjectionService.class); + stopIntent.setAction(DemoMediaProjectionService.ACTION_STOP); + ContextCompat.startForegroundService(/* context= */ this, stopIntent); } else { inputPlayerView.setVisibility(View.VISIBLE); inputImageView.setVisibility(View.GONE); @@ -829,6 +954,54 @@ public final class TransformerActivity extends AppCompatActivity { oldOutputFile = outputFile; } + private boolean isUsingMediaProjection() { + return mediaProjection != null; + } + + /** Foreground service that's required by the media projection APIs. */ + public static final class DemoMediaProjectionService extends Service { + private static final String CHANNEL_ID = "DemoMediaProjectionServiceChannel"; + private static final String CHANNEL_NAME = "Media projection"; + private static final int NOTIFICATION_ID = 1; + private static final String ACTION_EVENT_STARTED = "started"; + private static final String ACTION_STOP = "stop"; + + @Override + @Nullable + public IBinder onBind(Intent intent) { + return null; + } + + @Override + public int onStartCommand(Intent intent, int flags, int startId) { + if (ACTION_STOP.equals(intent.getAction())) { + stopSelf(); + } else { + Context context = this; + Notification notification = + new NotificationCompat.Builder(context, CHANNEL_ID) + .setOngoing(true) + .setSmallIcon(R.drawable.exo_icon_play) + .build(); + if (Util.SDK_INT >= 26) { + NotificationChannel channel = + new NotificationChannel( + CHANNEL_ID, CHANNEL_NAME, NotificationManager.IMPORTANCE_HIGH); + NotificationManager manager = getSystemService(NotificationManager.class); + manager.createNotificationChannel(channel); + } + if (Util.SDK_INT >= 29) { + startForeground(NOTIFICATION_ID, notification, FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION); + } else { + startForeground(NOTIFICATION_ID, notification); + } + // Notify that the service is started (and it's now safe to set up media projection). + LocalBroadcastManager.getInstance(context).sendBroadcast(new Intent(ACTION_EVENT_STARTED)); + } + return START_STICKY; + } + } + private final class DemoDebugViewProvider implements DebugViewProvider { @Nullable private SurfaceView surfaceView; diff --git a/demos/transformer/src/main/res/layout/transformer_activity.xml b/demos/transformer/src/main/res/layout/transformer_activity.xml index 22f6370092..9f2ce1f3fe 100644 --- a/demos/transformer/src/main/res/layout/transformer_activity.xml +++ b/demos/transformer/src/main/res/layout/transformer_activity.xml @@ -165,6 +165,12 @@ android:layout_width="match_parent" android:text="@string/resume"/> +