Add test for per-MediaItem effect

The test transcodes four media items with distinct effects, keeping one frame
each; extracts the four frames in the produced video, and compares them against
the expected results.

PiperOrigin-RevId: 539697344

libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java

@@ -25,6 +25,7 @@ import android.content.Context;
 import android.graphics.Bitmap;
 import android.graphics.BitmapFactory;
 import android.graphics.Color;
+import android.graphics.ImageFormat;
 import android.graphics.Matrix;
 import android.graphics.PixelFormat;
 import android.media.Image;
@@ -145,6 +146,34 @@ public class BitmapPixelTestUtil {
     return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888);
   }
 
+  /**
+   * Returns a grayscale bitmap from the Luma channel in the {@link ImageFormat#YUV_420_888} image.
+   */
+  @RequiresApi(19)
+  public static Bitmap createGrayscaleArgb8888BitmapFromYuv420888Image(Image image) {
+    int width = image.getWidth();
+    int height = image.getHeight();
+    assertThat(image.getPlanes()).hasLength(3);
+    assertThat(image.getFormat()).isEqualTo(ImageFormat.YUV_420_888);
+    Image.Plane lumaPlane = image.getPlanes()[0];
+    ByteBuffer lumaBuffer = lumaPlane.getBuffer();
+    int[] colors = new int[width * height];
+
+    for (int y = 0; y < height; y++) {
+      for (int x = 0; x < width; x++) {
+        int offset = y * lumaPlane.getRowStride() + x * lumaPlane.getPixelStride();
+        int lumaValue = lumaBuffer.get(offset) & 0xFF;
+        colors[y * width + x] =
+            Color.argb(
+                /* alpha= */ 255,
+                /* red= */ lumaValue,
+                /* green= */ lumaValue,
+                /* blue= */ lumaValue);
+      }
+    }
+    return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888);
+  }
+
   /**
    * Returns a solid {@link Bitmap} with every pixel having the same color.
    *

libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java

@@ -74,6 +74,8 @@ public final class AndroidTestUtil {
   public static final String PNG_ASSET_URI_STRING =
       "asset:///media/bitmap/input_images/media3test.png";
   public static final String JPG_ASSET_URI_STRING = "asset:///media/bitmap/input_images/london.jpg";
+  public static final String JPG_PORTRAIT_ASSET_URI_STRING =
+      "asset:///media/bitmap/input_images/tokyo.jpg";
 
   public static final String MP4_ASSET_URI_STRING = "asset:///media/mp4/sample.mp4";
   public static final Format MP4_ASSET_FORMAT =
@@ -85,6 +87,17 @@ public final class AndroidTestUtil {
           .setCodecs("avc1.64001F")
           .build();
 
+  public static final String MP4_PORTRAIT_ASSET_URI_STRING =
+      "asset:///media/mp4/sample_portrait.mp4";
+  public static final Format MP4_PORTRAIT_ASSET_FORMAT =
+      new Format.Builder()
+          .setSampleMimeType(VIDEO_H264)
+          .setWidth(720)
+          .setHeight(1080)
+          .setFrameRate(29.97f)
+          .setCodecs("avc1.64001F")
+          .build();
+
   public static final String MP4_ASSET_AV1_VIDEO_URI_STRING = "asset:///media/mp4/sample_av1.mp4";
   public static final Format MP4_ASSET_AV1_VIDEO_FORMAT =
       new Format.Builder()

libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerSequenceEffectTest.java

@@ -0,0 +1,264 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package androidx.media3.transformer;
+
+import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.maybeSaveTestBitmap;
+import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
+import static androidx.media3.transformer.AndroidTestUtil.JPG_ASSET_URI_STRING;
+import static androidx.media3.transformer.AndroidTestUtil.JPG_PORTRAIT_ASSET_URI_STRING;
+import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING;
+import static androidx.media3.transformer.AndroidTestUtil.MP4_PORTRAIT_ASSET_URI_STRING;
+import static com.google.common.truth.Truth.assertThat;
+
+import android.content.Context;
+import android.graphics.Bitmap;
+import android.media.Image;
+import androidx.annotation.Nullable;
+import androidx.media3.common.Effect;
+import androidx.media3.common.MediaItem;
+import androidx.media3.common.util.Util;
+import androidx.media3.effect.BitmapOverlay;
+import androidx.media3.effect.OverlayEffect;
+import androidx.media3.effect.Presentation;
+import androidx.media3.effect.RgbFilter;
+import androidx.media3.effect.ScaleAndRotateTransformation;
+import androidx.media3.test.utils.BitmapPixelTestUtil;
+import androidx.test.core.app.ApplicationProvider;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import com.google.common.collect.ImmutableList;
+import java.io.IOException;
+import java.util.List;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/**
+ * Tests for using different {@linkplain Effect effects} for {@link MediaItem MediaItems} in one
+ * {@link EditedMediaItemSequence} .
+ */
+@RunWith(AndroidJUnit4.class)
+public final class TransformerSequenceEffectTest {
+
+  private static final ImmutableList<Effect> NO_EFFECT = ImmutableList.of();
+  private static final String PNG_ASSET_BASE_PATH = "media/bitmap/transformer_sequence_effect_test";
+  private static final String OVERLAY_PNG_ASSET_PATH = "media/bitmap/input_images/media3test.png";
+  private static final int EXPORT_WIDTH = 360;
+  private static final int EXPORT_HEIGHT = 240;
+
+  /**
+   * Allowing more difference than {@link
+   * BitmapPixelTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} for YUV to RGB conversion on
+   * emulators.
+   */
+  private static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE =
+      8 * BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
+
+  private final Context context = ApplicationProvider.getApplicationContext();
+
+  @Test
+  public void export_withNoCompositionPresentationAndWithPerMediaItemEffects() throws Exception {
+    String testId = "export_withNoCompositionPresentationAndWithPerMediaItemEffects";
+    OverlayEffect overlayEffect = createOverlayEffect();
+    Composition composition =
+        createComposition(
+            /* presentation= */ null,
+            oneFrameFromVideo(
+                MP4_ASSET_URI_STRING,
+                ImmutableList.of(
+                    Presentation.createForWidthAndHeight(
+                        EXPORT_WIDTH, EXPORT_HEIGHT, Presentation.LAYOUT_SCALE_TO_FIT))),
+            oneFrameFromImage(
+                JPG_ASSET_URI_STRING,
+                ImmutableList.of(
+                    new ScaleAndRotateTransformation.Builder().setRotationDegrees(72).build(),
+                    overlayEffect)),
+            oneFrameFromImage(JPG_ASSET_URI_STRING, NO_EFFECT),
+            // Transition to a different aspect ratio.
+            oneFrameFromImage(
+                JPG_ASSET_URI_STRING,
+                ImmutableList.of(
+                    Presentation.createForWidthAndHeight(
+                        EXPORT_WIDTH / 2, EXPORT_HEIGHT, Presentation.LAYOUT_SCALE_TO_FIT),
+                    new ScaleAndRotateTransformation.Builder().setRotationDegrees(90).build(),
+                    overlayEffect)));
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+            .build()
+            .run(testId, composition);
+
+    assertThat(result.filePath).isNotNull();
+    assertBitmapsMatchExpected(
+        extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId);
+  }
+
+  @Test
+  public void export_withCompositionPresentationAndWithPerMediaItemEffects() throws Exception {
+    String testId = "export_withCompositionPresentationAndWithPerMediaItemEffects";
+    Composition composition =
+        createComposition(
+            Presentation.createForWidthAndHeight(
+                EXPORT_WIDTH, /* height= */ EXPORT_WIDTH, Presentation.LAYOUT_SCALE_TO_FIT),
+            oneFrameFromImage(
+                JPG_ASSET_URI_STRING,
+                ImmutableList.of(
+                    new ScaleAndRotateTransformation.Builder().setRotationDegrees(90).build(),
+                    Presentation.createForWidthAndHeight(
+                        EXPORT_WIDTH, EXPORT_HEIGHT, Presentation.LAYOUT_SCALE_TO_FIT))),
+            oneFrameFromImage(JPG_ASSET_URI_STRING, NO_EFFECT),
+            oneFrameFromVideo(
+                MP4_ASSET_URI_STRING, ImmutableList.of(RgbFilter.createInvertedFilter())),
+            oneFrameFromVideo(
+                MP4_ASSET_URI_STRING,
+                ImmutableList.of(
+                    Presentation.createForWidthAndHeight(
+                        EXPORT_WIDTH / 2, EXPORT_HEIGHT, Presentation.LAYOUT_SCALE_TO_FIT),
+                    createOverlayEffect())));
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+            .build()
+            .run(testId, composition);
+
+    assertThat(result.filePath).isNotNull();
+    assertBitmapsMatchExpected(
+        extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId);
+  }
+
+  @Test
+  public void export_withCompositionPresentationAndNoVideoEffects() throws Exception {
+    String testId = "export_withCompositionPresentationAndNoVideoEffects";
+    Composition composition =
+        createComposition(
+            Presentation.createForHeight(EXPORT_HEIGHT),
+            oneFrameFromImage(JPG_ASSET_URI_STRING, NO_EFFECT),
+            oneFrameFromVideo(MP4_PORTRAIT_ASSET_URI_STRING, NO_EFFECT),
+            oneFrameFromVideo(MP4_ASSET_URI_STRING, NO_EFFECT),
+            oneFrameFromImage(JPG_PORTRAIT_ASSET_URI_STRING, NO_EFFECT));
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+            .build()
+            .run(testId, composition);
+
+    assertThat(result.filePath).isNotNull();
+    assertBitmapsMatchExpected(
+        extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId);
+  }
+
+  @Test
+  public void export_withCompositionPresentationAndNoVideoEffectsForFirstMediaItem()
+      throws Exception {
+    String testId = "export_withCompositionPresentationAndNoVideoEffectsForFirstMediaItem";
+    Composition composition =
+        createComposition(
+            Presentation.createForHeight(EXPORT_HEIGHT),
+            oneFrameFromVideo(MP4_ASSET_URI_STRING, NO_EFFECT),
+            oneFrameFromVideo(
+                MP4_PORTRAIT_ASSET_URI_STRING, ImmutableList.of(RgbFilter.createInvertedFilter())));
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
+            .build()
+            .run(testId, composition);
+
+    assertThat(result.filePath).isNotNull();
+    assertBitmapsMatchExpected(
+        extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId);
+  }
+
+  private static OverlayEffect createOverlayEffect() throws IOException {
+    return new OverlayEffect(
+        ImmutableList.of(
+            BitmapOverlay.createStaticBitmapOverlay(readBitmap(OVERLAY_PNG_ASSET_PATH))));
+  }
+
+  private static Composition createComposition(
+      @Nullable Presentation presentation, EditedMediaItem... editedMediaItems) {
+    Composition.Builder builder =
+        new Composition.Builder(
+            ImmutableList.of(new EditedMediaItemSequence(ImmutableList.copyOf(editedMediaItems))));
+    if (presentation != null) {
+      builder.setEffects(
+          new Effects(/* audioProcessors= */ ImmutableList.of(), ImmutableList.of(presentation)));
+    }
+    return builder.build();
+  }
+
+  private static EditedMediaItem oneFrameFromVideo(String uri, List<Effect> effects) {
+    return new EditedMediaItem.Builder(
+            MediaItem.fromUri(uri)
+                .buildUpon()
+                .setClippingConfiguration(
+                    new MediaItem.ClippingConfiguration.Builder()
+                        // Clip to only the first frame.
+                        .setEndPositionMs(50)
+                        .build())
+                .build())
+        .setRemoveAudio(true)
+        .setEffects(
+            new Effects(/* audioProcessors= */ ImmutableList.of(), ImmutableList.copyOf(effects)))
+        .build();
+  }
+
+  private static EditedMediaItem oneFrameFromImage(String uri, List<Effect> effects) {
+    return new EditedMediaItem.Builder(MediaItem.fromUri(uri))
+        // 50ms for a 20-fps video is one frame.
+        .setFrameRate(20)
+        .setDurationUs(50_000)
+        .setEffects(
+            new Effects(/* audioProcessors= */ ImmutableList.of(), ImmutableList.copyOf(effects)))
+        .build();
+  }
+
+  private static ImmutableList<Bitmap> extractBitmapsFromVideo(Context context, String filePath)
+      throws IOException, InterruptedException {
+    ImmutableList.Builder<Bitmap> bitmaps = new ImmutableList.Builder<>();
+    try (VideoDecodingWrapper decodingWrapper =
+        new VideoDecodingWrapper(
+            context, filePath, /* comparisonInterval= */ 1, /* maxImagesAllowed= */ 1)) {
+      while (true) {
+        @Nullable Image image = decodingWrapper.runUntilComparisonFrameOrEnded();
+        if (image == null) {
+          break;
+        }
+        bitmaps.add(BitmapPixelTestUtil.createGrayscaleArgb8888BitmapFromYuv420888Image(image));
+        image.close();
+      }
+    }
+    return bitmaps.build();
+  }
+
+  private static void assertBitmapsMatchExpected(List<Bitmap> actualBitmaps, String testId)
+      throws IOException {
+    for (int i = 0; i < actualBitmaps.size(); i++) {
+      Bitmap actualBitmap = actualBitmaps.get(i);
+      String subTestId = testId + "_" + i;
+      Bitmap expectedBitmap =
+          readBitmap(Util.formatInvariant("%s/%s.png", PNG_ASSET_BASE_PATH, subTestId));
+
+      maybeSaveTestBitmap(
+          testId, /* bitmapLabel= */ String.valueOf(i), actualBitmap, /* path= */ null);
+      float averagePixelAbsoluteDifference =
+          getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, subTestId);
+      assertThat(averagePixelAbsoluteDifference)
+          .isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
+    }
+  }
+}