Add support for screen recording to the Transformer demo

Screen recording continues even if the transformer activity is backgrounded,
to support recording other apps.

PiperOrigin-RevId: 726454538

RELEASENOTES.md

@@ -8,7 +8,8 @@
 *   ExoPlayer:
 *   Transformer:
     *   Add `MediaProjectionAssetLoader`, which provides media from a
-        `MediaProjection` for screen recording.
+        `MediaProjection` for screen recording, and add support for screen
+        recording to the Transformer demo app.
     *   Add `#getInputFormat()` to `Codec` interface.
     *   Shift the responsibility to release the `GlObjectsProvider` onto the
         caller in `DefaultVideoFrameProcessor` and `DefaultVideoCompositor` when

demos/transformer/build.gradle

@@ -76,6 +76,7 @@ dependencies {
     implementation 'androidx.appcompat:appcompat:' + androidxAppCompatVersion
     implementation 'androidx.constraintlayout:constraintlayout:' + androidxConstraintLayoutVersion
     implementation 'androidx.recyclerview:recyclerview:' + androidxRecyclerViewVersion
+    implementation 'androidx.window:window:' + androidxWindowVersion
     implementation 'com.google.android.material:material:' + androidxMaterialVersion
     implementation project(modulePrefix + 'lib-effect')
     implementation project(modulePrefix + 'lib-exoplayer')

demos/transformer/src/main/AndroidManifest.xml

@@ -24,6 +24,10 @@
   <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
   <uses-permission android:name="android.permission.READ_MEDIA_VIDEO"/>
 
+  <!-- For media projection. -->
+  <uses-permission android:name="android.permission.FOREGROUND_SERVICE"/>
+  <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION"/>
+
   <application
       android:allowBackup="false"
       android:icon="@mipmap/ic_launcher"
@@ -64,5 +68,9 @@
         android:label="@string/app_name"
         android:exported="true"
         android:theme="@style/Theme.MaterialComponents.DayNight.NoActionBar"/>
+    <service
+      android:name=".TransformerActivity$DemoMediaProjectionService"
+      android:foregroundServiceType="mediaProjection"
+      android:exported="false"/>
   </application>
 </manifest>

demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java

@@ -15,6 +15,7 @@
  */
 package androidx.media3.demo.transformer;
 
+import static android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION;
 import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.common.util.Assertions.checkState;
 import static androidx.media3.exoplayer.DefaultLoadControl.DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS;
@@ -22,15 +23,25 @@ import static androidx.media3.exoplayer.DefaultLoadControl.DEFAULT_BUFFER_FOR_PL
 import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED;
 
 import android.app.Activity;
+import android.app.Notification;
+import android.app.NotificationChannel;
+import android.app.NotificationManager;
+import android.app.Service;
+import android.content.BroadcastReceiver;
 import android.content.Context;
 import android.content.Intent;
+import android.content.IntentFilter;
 import android.content.pm.PackageManager;
 import android.graphics.Bitmap;
 import android.graphics.Color;
+import android.graphics.Rect;
 import android.graphics.drawable.Drawable;
+import android.media.projection.MediaProjection;
+import android.media.projection.MediaProjectionManager;
 import android.net.Uri;
 import android.os.Bundle;
 import android.os.Handler;
+import android.os.IBinder;
 import android.text.Spannable;
 import android.text.SpannableString;
 import android.text.style.ForegroundColorSpan;
@@ -42,9 +53,13 @@ import android.widget.Button;
 import android.widget.ImageView;
 import android.widget.TextView;
 import android.widget.Toast;
+import androidx.activity.result.contract.ActivityResultContracts;
 import androidx.annotation.Nullable;
 import androidx.annotation.StringRes;
 import androidx.appcompat.app.AppCompatActivity;
+import androidx.core.app.NotificationCompat;
+import androidx.core.content.ContextCompat;
+import androidx.localbroadcastmanager.content.LocalBroadcastManager;
 import androidx.media3.common.C;
 import androidx.media3.common.DebugViewProvider;
 import androidx.media3.common.Effect;
@@ -91,10 +106,13 @@ import androidx.media3.transformer.ExportResult;
 import androidx.media3.transformer.InAppFragmentedMp4Muxer;
 import androidx.media3.transformer.InAppMp4Muxer;
 import androidx.media3.transformer.JsonUtil;
+import androidx.media3.transformer.MediaProjectionAssetLoader;
 import androidx.media3.transformer.ProgressHolder;
 import androidx.media3.transformer.Transformer;
+import androidx.media3.transformer.VideoEncoderSettings;
 import androidx.media3.ui.AspectRatioFrameLayout;
 import androidx.media3.ui.PlayerView;
+import androidx.window.layout.WindowMetricsCalculator;
 import com.google.android.material.card.MaterialCardView;
 import com.google.android.material.progressindicator.LinearProgressIndicator;
 import com.google.common.base.Stopwatch;
@@ -116,12 +134,13 @@ import org.json.JSONObject;
 public final class TransformerActivity extends AppCompatActivity {
   private static final String TAG = "TransformerActivity";
   private static final int IMAGE_DURATION_MS = 5_000;
-  private static final int IMAGE_FRAME_RATE_FPS = 30;
+  private static final int DEFAULT_FRAME_RATE_FPS = 30;
   private static int LOAD_CONTROL_MIN_BUFFER_MS = 5_000;
   private static int LOAD_CONTROL_MAX_BUFFER_MS = 5_000;
 
   private Button displayInputButton;
   private MaterialCardView inputCardView;
+  private MaterialCardView outputCardView;
   private TextView inputTextView;
   private ImageView inputImageView;
   private PlayerView inputPlayerView;
@@ -133,10 +152,13 @@ public final class TransformerActivity extends AppCompatActivity {
   private LinearProgressIndicator progressIndicator;
   private Button pauseButton;
   private Button resumeButton;
+  private Button stopCaptureButton;
   private Stopwatch exportStopwatch;
   private AspectRatioFrameLayout debugFrame;
 
   @Nullable private DebugTextViewHelper debugTextViewHelper;
+  @Nullable private Intent screenCaptureToken;
+  @Nullable private MediaProjection mediaProjection;
   @Nullable private ExoPlayer inputPlayer;
   @Nullable private ExoPlayer outputPlayer;
   @Nullable private Transformer transformer;
@@ -149,6 +171,7 @@ public final class TransformerActivity extends AppCompatActivity {
     setContentView(R.layout.transformer_activity);
 
     inputCardView = findViewById(R.id.input_card_view);
+    outputCardView = findViewById(R.id.output_card_view);
     inputTextView = findViewById(R.id.input_text_view);
     inputImageView = findViewById(R.id.input_image_view);
     inputPlayerView = findViewById(R.id.input_player_view);
@@ -162,6 +185,8 @@ public final class TransformerActivity extends AppCompatActivity {
     pauseButton.setOnClickListener(view -> pauseExport());
     resumeButton = findViewById(R.id.resume_button);
     resumeButton.setOnClickListener(view -> startExport());
+    stopCaptureButton = findViewById(R.id.stop_capture_button);
+    stopCaptureButton.setOnClickListener(view -> mediaProjection.stop());
     debugFrame = findViewById(R.id.debug_aspect_ratio_frame_layout);
     displayInputButton = findViewById(R.id.display_input_button);
     displayInputButton.setOnClickListener(view -> toggleInputVideoDisplay());
@@ -180,7 +205,10 @@ public final class TransformerActivity extends AppCompatActivity {
   protected void onStart() {
     super.onStart();
 
-    startExport();
+    // Restart exporting, unless this is a capture session which can run in the background.
+    if (!isUsingMediaProjection()) {
+      startExport();
+    }
 
     inputPlayerView.onResume();
     outputPlayerView.onResume();
@@ -192,13 +220,70 @@ public final class TransformerActivity extends AppCompatActivity {
 
     inputPlayerView.onPause();
     outputPlayerView.onPause();
-    releasePlayers();
+
+    // Keep the capture session going to allow capturing other apps while backgrounded.
+    if (!isUsingMediaProjection()) {
+      releasePlayers();
+      cleanUpExport();
+    }
+  }
+
+  @Override
+  protected void onDestroy() {
+    super.onDestroy();
+
+    if (isUsingMediaProjection()) {
+      releasePlayers();
+      mediaProjection.stop();
+      mediaProjection = null;
+      screenCaptureToken = null;
+    }
     cleanUpExport();
   }
 
   private void startExport() {
     Intent intent = getIntent();
     Uri inputUri = checkNotNull(intent.getData());
+
+    if (inputUri.toString().equals("transformer_surface_asset:media_projection")
+        && screenCaptureToken == null) {
+      // MediaProjection can only start once the foreground service is running.
+      MediaProjectionManager mediaProjectionManager =
+          (MediaProjectionManager) getSystemService(MEDIA_PROJECTION_SERVICE);
+      Context context = this;
+      LocalBroadcastManager.getInstance(context)
+          .registerReceiver(
+              new BroadcastReceiver() {
+                @Override
+                public void onReceive(Context context, Intent intent) {
+                  String action = checkNotNull(intent.getAction());
+                  if (action.equals(DemoMediaProjectionService.ACTION_EVENT_STARTED)) {
+                    LocalBroadcastManager.getInstance(context)
+                        .unregisterReceiver(/* receiver= */ this);
+                    // The service has started so media projection can start.
+                    startExport();
+                  }
+                }
+              },
+              new IntentFilter(DemoMediaProjectionService.ACTION_EVENT_STARTED));
+      registerForActivityResult(
+              new ActivityResultContracts.StartActivityForResult(),
+              activityResult -> {
+                int resultCode = activityResult.getResultCode();
+                if (resultCode == RESULT_OK) {
+                  screenCaptureToken = activityResult.getData();
+                  Intent startServiceIntent = new Intent(context, DemoMediaProjectionService.class);
+                  ContextCompat.startForegroundService(context, startServiceIntent);
+                } else if (resultCode == RESULT_CANCELED) {
+                  finish();
+                }
+              })
+          .launch(mediaProjectionManager.createScreenCaptureIntent());
+      inputCardView.setVisibility(View.GONE);
+      outputCardView.setVisibility(View.GONE);
+      return;
+    }
+
     try {
       outputFile =
           createExternalCacheFile("transformer-output-" + Clock.DEFAULT.elapsedRealtime() + ".mp4");
@@ -225,10 +310,20 @@ public final class TransformerActivity extends AppCompatActivity {
     outputVideoTextView.setVisibility(View.GONE);
     debugTextView.setVisibility(View.GONE);
     informationTextView.setText(R.string.export_started);
+    outputCardView.setVisibility(View.VISIBLE);
     progressViewGroup.setVisibility(View.VISIBLE);
     pauseButton.setVisibility(View.VISIBLE);
     resumeButton.setVisibility(View.GONE);
     progressIndicator.setProgress(0);
+    if (isUsingMediaProjection()) {
+      pauseButton.setVisibility(View.GONE);
+      resumeButton.setVisibility(View.GONE);
+      stopCaptureButton.setVisibility(View.VISIBLE);
+    } else {
+      pauseButton.setVisibility(View.VISIBLE);
+      resumeButton.setVisibility(View.GONE);
+      stopCaptureButton.setVisibility(View.GONE);
+    }
     Handler mainHandler = new Handler(getMainLooper());
     ProgressHolder progressHolder = new ProgressHolder();
     mainHandler.post(
@@ -294,11 +389,6 @@ public final class TransformerActivity extends AppCompatActivity {
         transformerBuilder.setVideoMimeType(videoMimeType);
       }
 
-      transformerBuilder.setEncoderFactory(
-          new DefaultEncoderFactory.Builder(this.getApplicationContext())
-              .setEnableFallback(bundle.getBoolean(ConfigurationActivity.ENABLE_FALLBACK))
-              .build());
-
       if (!bundle.getBoolean(ConfigurationActivity.ABORT_SLOW_EXPORT)) {
         transformerBuilder.setMaxDelayBetweenMuxerSamplesMs(C.TIME_UNSET);
       }
@@ -321,6 +411,32 @@ public final class TransformerActivity extends AppCompatActivity {
       }
     }
 
+    VideoEncoderSettings videoEncoderSettings = VideoEncoderSettings.DEFAULT;
+    if (screenCaptureToken != null) {
+      MediaProjectionManager mediaProjectionManager =
+          (MediaProjectionManager) getSystemService(MEDIA_PROJECTION_SERVICE);
+      MediaProjection mediaProjection =
+          mediaProjectionManager.getMediaProjection(RESULT_OK, checkNotNull(screenCaptureToken));
+      Rect bounds =
+          WindowMetricsCalculator.getOrCreate()
+              .computeCurrentWindowMetrics(/* activity= */ this)
+              .getBounds();
+      int densityDpi = getResources().getConfiguration().densityDpi;
+      transformerBuilder.setAssetLoaderFactory(
+          new MediaProjectionAssetLoader.Factory(mediaProjection, bounds, densityDpi));
+      this.mediaProjection = mediaProjection;
+      videoEncoderSettings =
+          videoEncoderSettings
+              .buildUpon()
+              .setRepeatPreviousFrameIntervalUs(C.MICROS_PER_SECOND / DEFAULT_FRAME_RATE_FPS)
+              .build();
+    }
+    transformerBuilder.setEncoderFactory(
+        new DefaultEncoderFactory.Builder(this.getApplicationContext())
+            .setEnableFallback(bundle.getBoolean(ConfigurationActivity.ENABLE_FALLBACK))
+            .setRequestedVideoEncoderSettings(videoEncoderSettings)
+            .build());
+
     return transformerBuilder.build();
   }
 
@@ -339,7 +455,7 @@ public final class TransformerActivity extends AppCompatActivity {
   private Composition createComposition(MediaItem mediaItem, @Nullable Bundle bundle) {
     EditedMediaItem.Builder editedMediaItemBuilder = new EditedMediaItem.Builder(mediaItem);
     // For image inputs. Automatically ignored if input is audio/video.
-    editedMediaItemBuilder.setFrameRate(IMAGE_FRAME_RATE_FPS);
+    editedMediaItemBuilder.setFrameRate(DEFAULT_FRAME_RATE_FPS);
     if (bundle != null) {
       ImmutableList<AudioProcessor> audioProcessors = createAudioProcessorsFromBundle(bundle);
       ImmutableList<Effect> videoEffects = createVideoEffectsFromBundle(bundle);
@@ -640,6 +756,9 @@ public final class TransformerActivity extends AppCompatActivity {
     informationTextView.setText(R.string.export_error);
     progressViewGroup.setVisibility(View.GONE);
     debugFrame.removeAllViews();
+    if (isUsingMediaProjection()) {
+      mediaProjection.stop();
+    }
     Toast.makeText(getApplicationContext(), "Export error: " + exportException, Toast.LENGTH_LONG)
         .show();
     Log.e(TAG, "Export error", exportException);
@@ -717,6 +836,12 @@ public final class TransformerActivity extends AppCompatActivity {
       } catch (ExecutionException | InterruptedException e) {
         throw new IllegalArgumentException("Failed to load bitmap.", e);
       }
+    } else if (isUsingMediaProjection()) {
+      inputCardView.setVisibility(View.GONE);
+      displayInputButton.setVisibility(View.GONE);
+      Intent stopIntent = new Intent(/* context= */ this, DemoMediaProjectionService.class);
+      stopIntent.setAction(DemoMediaProjectionService.ACTION_STOP);
+      ContextCompat.startForegroundService(/* context= */ this, stopIntent);
     } else {
       inputPlayerView.setVisibility(View.VISIBLE);
       inputImageView.setVisibility(View.GONE);
@@ -829,6 +954,54 @@ public final class TransformerActivity extends AppCompatActivity {
     oldOutputFile = outputFile;
   }
 
+  private boolean isUsingMediaProjection() {
+    return mediaProjection != null;
+  }
+
+  /** Foreground service that's required by the media projection APIs. */
+  public static final class DemoMediaProjectionService extends Service {
+    private static final String CHANNEL_ID = "DemoMediaProjectionServiceChannel";
+    private static final String CHANNEL_NAME = "Media projection";
+    private static final int NOTIFICATION_ID = 1;
+    private static final String ACTION_EVENT_STARTED = "started";
+    private static final String ACTION_STOP = "stop";
+
+    @Override
+    @Nullable
+    public IBinder onBind(Intent intent) {
+      return null;
+    }
+
+    @Override
+    public int onStartCommand(Intent intent, int flags, int startId) {
+      if (ACTION_STOP.equals(intent.getAction())) {
+        stopSelf();
+      } else {
+        Context context = this;
+        Notification notification =
+            new NotificationCompat.Builder(context, CHANNEL_ID)
+                .setOngoing(true)
+                .setSmallIcon(R.drawable.exo_icon_play)
+                .build();
+        if (Util.SDK_INT >= 26) {
+          NotificationChannel channel =
+              new NotificationChannel(
+                  CHANNEL_ID, CHANNEL_NAME, NotificationManager.IMPORTANCE_HIGH);
+          NotificationManager manager = getSystemService(NotificationManager.class);
+          manager.createNotificationChannel(channel);
+        }
+        if (Util.SDK_INT >= 29) {
+          startForeground(NOTIFICATION_ID, notification, FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION);
+        } else {
+          startForeground(NOTIFICATION_ID, notification);
+        }
+        // Notify that the service is started (and it's now safe to set up media projection).
+        LocalBroadcastManager.getInstance(context).sendBroadcast(new Intent(ACTION_EVENT_STARTED));
+      }
+      return START_STICKY;
+    }
+  }
+
   private final class DemoDebugViewProvider implements DebugViewProvider {
 
     @Nullable private SurfaceView surfaceView;

demos/transformer/src/main/res/layout/transformer_activity.xml

@@ -165,6 +165,12 @@
             android:layout_width="match_parent"
             android:text="@string/resume"/>
 
+          <Button
+            android:id="@+id/stop_capture_button"
+            android:layout_height="wrap_content"
+            android:layout_width="match_parent"
+            android:text="@string/stop_capture"/>
+
           <androidx.media3.ui.AspectRatioFrameLayout
             android:id="@+id/debug_aspect_ratio_frame_layout"
             android:layout_width="match_parent"

demos/transformer/src/main/res/values/arrays.xml

@@ -58,6 +58,7 @@
     <item>HDR (HDR10+) H265 limited range video (encoding may fail)</item>
     <item>HDR (HLG) H265 limited range video (encoding may fail)</item>
     <item>720p H264 video with no audio (B-frames)</item>
+    <item>Record screen</item>
   </string-array>
   <string-array name="preset_uris">
     <item>https://storage.googleapis.com/exoplayer-test-media-1/mp4/android-screens-10s.mp4</item>
@@ -77,5 +78,6 @@
     <item>https://storage.googleapis.com/exoplayer-test-media-1/mp4/samsung-s21-hdr-hdr10.mp4</item>
     <item>https://storage.googleapis.com/exoplayer-test-media-1/mp4/Pixel7Pro_HLG_1080P.mp4</item>
     <item>https://storage.googleapis.com/exoplayer-test-media-1/mp4/sample_video_track_only.mp4</item>
+    <item>transformer_surface_asset:media_projection</item>
   </string-array>
 </resources>

demos/transformer/src/main/res/values/strings.xml

@@ -44,6 +44,7 @@
   <string name="debug_preview" translatable="false">Debug preview:</string>
   <string name="pause" translatable="false">Pause</string>
   <string name="resume" translatable="false">Resume</string>
+  <string name="stop_capture" translatable="false">Stop capture</string>
   <string name="debug_preview_not_available" translatable="false">No debug preview available.</string>
   <string name="export_started" translatable="false">Export started</string>
   <string name="export_timer" translatable="false">Export started %d seconds ago.</string>