Add motion photo support to Transformer

PiperOrigin-RevId: 683540867

libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java

@@ -223,6 +223,18 @@ public final class AndroidTestUtil {
                   .setHeight(4080)
                   .build())
           .build();
+  public static final AssetInfo JPG_PIXEL_MOTION_PHOTO_ASSET =
+      new AssetInfo.Builder("asset:///media/jpeg/pixel-motion-photo-2-hevc-tracks.jpg")
+          .setVideoFormat(
+              new Format.Builder()
+                  .setSampleMimeType(VIDEO_H265)
+                  .setWidth(1024)
+                  .setHeight(768)
+                  .setFrameRate(27.61f)
+                  .setCodecs("hvc1.1.6.L153")
+                  .build())
+          .setVideoFrameCount(58)
+          .build();
 
   public static final AssetInfo WEBP_LARGE =
       new AssetInfo.Builder("asset:///media/webp/black_large.webp")

libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java

@@ -22,6 +22,7 @@ import static androidx.media3.common.util.MediaFormatUtil.createFormatFromMediaF
 import static androidx.media3.common.util.Util.isRunningOnEmulator;
 import static androidx.media3.test.utils.TestUtil.retrieveTrackFormat;
 import static androidx.media3.transformer.AndroidTestUtil.JPG_ASSET;
+import static androidx.media3.transformer.AndroidTestUtil.JPG_PIXEL_MOTION_PHOTO_ASSET;
 import static androidx.media3.transformer.AndroidTestUtil.MP3_ASSET;
 import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET;
 import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_PHOTOS_TRIM_OPTIMIZATION_VIDEO;
@@ -1543,6 +1544,55 @@ public class TransformerEndToEndTest {
     assertThat(new File(result.filePath).length()).isGreaterThan(0);
   }
 
+  @Test
+  public void motionPhoto_withNoDurationSet_exportsVideo() throws Exception {
+    Transformer transformer = new Transformer.Builder(context).build();
+    assumeFormatsSupported(
+        context,
+        testId,
+        /* inputFormat= */ null,
+        /* outputFormat= */ JPG_PIXEL_MOTION_PHOTO_ASSET.videoFormat);
+    EditedMediaItem motionPhotoItem =
+        new EditedMediaItem.Builder(MediaItem.fromUri(JPG_PIXEL_MOTION_PHOTO_ASSET.uri)).build();
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, transformer)
+            .build()
+            .run(testId, motionPhotoItem);
+
+    assertThat(result.exportResult.videoFrameCount)
+        .isEqualTo(JPG_PIXEL_MOTION_PHOTO_ASSET.videoFrameCount);
+  }
+
+  @Test
+  public void motionPhoto_withDurationSet_exportsImage() throws Exception {
+    Transformer transformer = new Transformer.Builder(context).build();
+    MediaItem motionPhotoItem =
+        new MediaItem.Builder()
+            .setUri(JPG_PIXEL_MOTION_PHOTO_ASSET.uri)
+            .setImageDurationMs(500)
+            .build();
+    // Downscale to make sure the resolution is supported by the encoder.
+    Effect downscalingEffect =
+        Presentation.createForWidthAndHeight(
+            /* width= */ 480, /* height= */ 360, Presentation.LAYOUT_SCALE_TO_FIT);
+    EditedMediaItem motionPhotoEditedItem =
+        new EditedMediaItem.Builder(motionPhotoItem)
+            .setFrameRate(30)
+            .setEffects(
+                new Effects(
+                    /* audioProcessors= */ ImmutableList.of(),
+                    /* videoEffects= */ ImmutableList.of(downscalingEffect)))
+            .build();
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, transformer)
+            .build()
+            .run(testId, motionPhotoEditedItem);
+
+    assertThat(result.exportResult.videoFrameCount).isEqualTo(15); // 0.5 sec at 30 fps
+  }
+
   @Test
   public void audioTranscode_processesInInt16Pcm() throws Exception {
     FormatTrackingAudioBufferSink audioFormatTracker = new FormatTrackingAudioBufferSink();

libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java

@@ -17,6 +17,7 @@
 package androidx.media3.transformer;
 
 import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.transformer.TransformerUtil.isImage;
 
 import android.content.Context;
 import android.graphics.BitmapFactory;
@@ -25,7 +26,6 @@ import android.os.Looper;
 import androidx.annotation.Nullable;
 import androidx.media3.common.C;
 import androidx.media3.common.MediaItem;
-import androidx.media3.common.MimeTypes;
 import androidx.media3.common.util.BitmapLoader;
 import androidx.media3.common.util.Clock;
 import androidx.media3.common.util.Log;
@@ -141,7 +141,11 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
       AssetLoader.Listener listener,
       CompositionSettings compositionSettings) {
     MediaItem mediaItem = editedMediaItem.mediaItem;
-    if (isImage(mediaItem)) {
+    boolean isImage = isImage(context, mediaItem);
+    // TODO: b/350499931 - use the MediaItem's imageDurationMs instead of the EditedMediaItem's
+    //  durationUs to export motion photos as video
+    boolean exportVideoFromMotionPhoto = isImage && editedMediaItem.durationUs == C.TIME_UNSET;
+    if (isImage && !exportVideoFromMotionPhoto) {
       if (checkNotNull(mediaItem.localConfiguration).imageDurationMs == C.TIME_UNSET) {
         Log.w(TAG, "The imageDurationMs field must be set on image MediaItems.");
       }
@@ -160,9 +164,4 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
     return exoPlayerAssetLoaderFactory.createAssetLoader(
         editedMediaItem, looper, listener, compositionSettings);
   }
-
-  private boolean isImage(MediaItem mediaItem) {
-    @Nullable String mimeType = ImageAssetLoader.getImageMimeType(context, mediaItem);
-    return mimeType != null && MimeTypes.isImage(mimeType);
-  }
 }

libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java

@@ -28,6 +28,7 @@ import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE;
 import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED;
 import static androidx.media3.transformer.Transformer.PROGRESS_STATE_UNAVAILABLE;
 import static androidx.media3.transformer.Transformer.PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
+import static androidx.media3.transformer.TransformerUtil.isImage;
 import static java.lang.Math.min;
 
 import android.content.Context;
@@ -139,6 +140,7 @@ public final class ExoPlayerAssetLoader implements AssetLoader {
    */
   private static final long EMULATOR_RELEASE_TIMEOUT_MS = 5_000;
 
+  private final Context context;
   private final EditedMediaItem editedMediaItem;
   private final CapturingDecoderFactory decoderFactory;
   private final ExoPlayer player;
@@ -154,6 +156,7 @@ public final class ExoPlayerAssetLoader implements AssetLoader {
       Looper looper,
       Listener listener,
       Clock clock) {
+    this.context = context;
     this.editedMediaItem = editedMediaItem;
     this.decoderFactory = new CapturingDecoderFactory(decoderFactory);
 
@@ -339,10 +342,13 @@ public final class ExoPlayerAssetLoader implements AssetLoader {
           // listener callbacks are called in the right order.
           player.play();
         } else {
+          String errorMessage = "The asset loader has no audio or video track to output.";
+          if (isImage(context, editedMediaItem.mediaItem)) {
+            errorMessage += " Try setting an image duration on input image MediaItems.";
+          }
           assetLoaderListener.onError(
               ExportException.createForAssetLoader(
-                  new IllegalStateException("The asset loader has no track to output."),
+                  new IllegalStateException(errorMessage), ERROR_CODE_FAILED_RUNTIME_CHECK));
-                  ERROR_CODE_FAILED_RUNTIME_CHECK));
         }
       } catch (RuntimeException e) {
         assetLoaderListener.onError(

libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java

@@ -28,7 +28,6 @@ import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED
 import static com.google.common.util.concurrent.Futures.immediateFailedFuture;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
-import android.content.ContentResolver;
 import android.content.Context;
 import android.graphics.Bitmap;
 import android.os.Looper;
@@ -36,7 +35,6 @@ import androidx.annotation.Nullable;
 import androidx.media3.common.C;
 import androidx.media3.common.ColorInfo;
 import androidx.media3.common.Format;
-import androidx.media3.common.MediaItem;
 import androidx.media3.common.MimeTypes;
 import androidx.media3.common.ParserException;
 import androidx.media3.common.util.BitmapLoader;
@@ -44,12 +42,10 @@ import androidx.media3.common.util.ConstantRateTimestampIterator;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
 import androidx.media3.transformer.SampleConsumer.InputResult;
-import com.google.common.base.Ascii;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.FutureCallback;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
-import java.util.Objects;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 
@@ -126,41 +122,6 @@ public final class ImageAssetLoader implements AssetLoader {
     progressState = PROGRESS_STATE_NOT_STARTED;
   }
 
-  /**
-   * Returns the image MIME type corresponding to a {@link MediaItem}.
-   *
-   * <p>This method only supports some common image MIME types.
-   *
-   * @param context The {@link Context}.
-   * @param mediaItem The {@link MediaItem} to inspect.
-   * @return The MIME type.
-   */
-  @Nullable
-  public static String getImageMimeType(Context context, MediaItem mediaItem) {
-    if (mediaItem.localConfiguration == null) {
-      return null;
-    }
-    MediaItem.LocalConfiguration localConfiguration = mediaItem.localConfiguration;
-    @Nullable String mimeType = localConfiguration.mimeType;
-    if (mimeType == null) {
-      if (Objects.equals(localConfiguration.uri.getScheme(), ContentResolver.SCHEME_CONTENT)) {
-        ContentResolver cr = context.getContentResolver();
-        mimeType = cr.getType(localConfiguration.uri);
-      } else {
-        @Nullable String uriPath = localConfiguration.uri.getPath();
-        if (uriPath == null) {
-          return null;
-        }
-        int fileExtensionStart = uriPath.lastIndexOf(".");
-        if (fileExtensionStart >= 0 && fileExtensionStart < uriPath.length() - 1) {
-          String extension = Ascii.toLowerCase(uriPath.substring(fileExtensionStart + 1));
-          mimeType = getCommonImageMimeTypeFromExtension(extension);
-        }
-      }
-    }
-    return mimeType;
-  }
-
   @Override
   // Ignore Future returned by scheduledExecutorService because failures are already handled in the
   // runnable.
@@ -172,7 +133,7 @@ public final class ImageAssetLoader implements AssetLoader {
     ListenableFuture<Bitmap> future;
 
     @Nullable
-    String mimeType = ImageAssetLoader.getImageMimeType(context, editedMediaItem.mediaItem);
+    String mimeType = TransformerUtil.getImageMimeType(context, editedMediaItem.mediaItem);
     if (mimeType == null || !bitmapLoader.supportsMimeType(mimeType)) {
       future =
           immediateFailedFuture(
@@ -276,47 +237,4 @@ public final class ImageAssetLoader implements AssetLoader {
       listener.onError(ExportException.createForAssetLoader(e, ERROR_CODE_UNSPECIFIED));
     }
   }
-
-  @Nullable
-  private static String getCommonImageMimeTypeFromExtension(String extension) {
-    switch (extension) {
-      case "bmp":
-      case "dib":
-        return MimeTypes.IMAGE_BMP;
-      case "heif":
-        return MimeTypes.IMAGE_HEIF;
-      case "heic":
-        return MimeTypes.IMAGE_HEIC;
-      case "jpg":
-      case "jpeg":
-      case "jpe":
-      case "jif":
-      case "jfif":
-      case "jfi":
-        return MimeTypes.IMAGE_JPEG;
-      case "png":
-        return MimeTypes.IMAGE_PNG;
-      case "webp":
-        return MimeTypes.IMAGE_WEBP;
-      case "gif":
-        return "image/gif";
-      case "tiff":
-      case "tif":
-        return "image/tiff";
-      case "raw":
-      case "arw":
-      case "cr2":
-      case "k25":
-        return "image/raw";
-      case "svg":
-      case "svgz":
-        return "image/svg+xml";
-      case "ico":
-        return "image/x-icon";
-      case "avif":
-        return MimeTypes.IMAGE_AVIF;
-      default:
-        return null;
-    }
-  }
 }

libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java

@@ -24,6 +24,8 @@ import static androidx.media3.transformer.Composition.HDR_MODE_TONE_MAP_HDR_TO_S
 import static androidx.media3.transformer.EncoderUtil.getSupportedEncodersForHdrEditing;
 import static java.lang.Math.round;
 
+import android.content.ContentResolver;
+import android.content.Context;
 import android.media.MediaCodec;
 import android.media.MediaCodecInfo;
 import android.util.Pair;
@@ -32,6 +34,7 @@ import androidx.media3.common.C;
 import androidx.media3.common.ColorInfo;
 import androidx.media3.common.Effect;
 import androidx.media3.common.Format;
+import androidx.media3.common.MediaItem;
 import androidx.media3.common.Metadata;
 import androidx.media3.common.MimeTypes;
 import androidx.media3.common.util.UnstableApi;
@@ -39,7 +42,9 @@ import androidx.media3.effect.GlEffect;
 import androidx.media3.effect.ScaleAndRotateTransformation;
 import androidx.media3.extractor.metadata.mp4.SlowMotionData;
 import androidx.media3.transformer.Composition.HdrMode;
+import com.google.common.base.Ascii;
 import com.google.common.collect.ImmutableList;
+import java.util.Objects;
 
 /** Utility methods for Transformer. */
 @UnstableApi
@@ -279,4 +284,88 @@ public final class TransformerUtil {
     }
     return Pair.create(requestedOutputMimeType, hdrMode);
   }
+
+  /** Returns whether the provided {@link MediaItem} corresponds to an image. */
+  public static boolean isImage(Context context, MediaItem mediaItem) {
+    @Nullable String mimeType = getImageMimeType(context, mediaItem);
+    return mimeType != null && MimeTypes.isImage(mimeType);
+  }
+
+  /**
+   * Returns the image MIME type corresponding to a {@link MediaItem}.
+   *
+   * <p>This method only supports some common image MIME types.
+   *
+   * @param context The {@link Context}.
+   * @param mediaItem The {@link MediaItem} to inspect.
+   * @return The MIME type.
+   */
+  @Nullable
+  public static String getImageMimeType(Context context, MediaItem mediaItem) {
+    if (mediaItem.localConfiguration == null) {
+      return null;
+    }
+    MediaItem.LocalConfiguration localConfiguration = mediaItem.localConfiguration;
+    @Nullable String mimeType = localConfiguration.mimeType;
+    if (mimeType == null) {
+      if (Objects.equals(localConfiguration.uri.getScheme(), ContentResolver.SCHEME_CONTENT)) {
+        ContentResolver cr = context.getContentResolver();
+        mimeType = cr.getType(localConfiguration.uri);
+      } else {
+        @Nullable String uriPath = localConfiguration.uri.getPath();
+        if (uriPath == null) {
+          return null;
+        }
+        int fileExtensionStart = uriPath.lastIndexOf(".");
+        if (fileExtensionStart >= 0 && fileExtensionStart < uriPath.length() - 1) {
+          String extension = Ascii.toLowerCase(uriPath.substring(fileExtensionStart + 1));
+          mimeType = getCommonImageMimeTypeFromExtension(extension);
+        }
+      }
+    }
+    return mimeType;
+  }
+
+  @Nullable
+  private static String getCommonImageMimeTypeFromExtension(String extension) {
+    switch (extension) {
+      case "bmp":
+      case "dib":
+        return MimeTypes.IMAGE_BMP;
+      case "heif":
+        return MimeTypes.IMAGE_HEIF;
+      case "heic":
+        return MimeTypes.IMAGE_HEIC;
+      case "jpg":
+      case "jpeg":
+      case "jpe":
+      case "jif":
+      case "jfif":
+      case "jfi":
+        return MimeTypes.IMAGE_JPEG;
+      case "png":
+        return MimeTypes.IMAGE_PNG;
+      case "webp":
+        return MimeTypes.IMAGE_WEBP;
+      case "gif":
+        return "image/gif";
+      case "tiff":
+      case "tif":
+        return "image/tiff";
+      case "raw":
+      case "arw":
+      case "cr2":
+      case "k25":
+        return "image/raw";
+      case "svg":
+      case "svgz":
+        return "image/svg+xml";
+      case "ico":
+        return "image/x-icon";
+      case "avif":
+        return MimeTypes.IMAGE_AVIF;
+      default:
+        return null;
+    }
+  }
 }