Transformer: add api to cutomize image loading

PiperOrigin-RevId: 598793134

libraries/common/src/main/java/androidx/media3/common/FileTypes.java

@@ -19,6 +19,7 @@ import static androidx.media3.common.MimeTypes.normalizeMimeType;
 import static java.lang.annotation.ElementType.TYPE_USE;
 
 import android.net.Uri;
+import android.text.TextUtils;
 import androidx.annotation.IntDef;
 import androidx.annotation.Nullable;
 import androidx.annotation.VisibleForTesting;
@@ -29,6 +30,7 @@ import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;
 
 /** Defines common file type constants and helper methods. */
 @UnstableApi
@@ -327,4 +329,30 @@ public final class FileTypes {
       return FileTypes.UNKNOWN;
     }
   }
+
+  /**
+   * Returns the file extension of the given {@link Uri} or an empty string if there is no
+   * extension.
+   *
+   * <p>This method is a convenience method for obtaining the extension of a url and has undefined
+   * results for other Strings.
+   */
+  public static String getFileExtensionFromUri(Uri uri) {
+    String path = uri.getPath();
+    if (TextUtils.isEmpty(path)) {
+      return "";
+    }
+    int filenamePos = path.lastIndexOf('/');
+    String filename = 0 <= filenamePos ? path.substring(filenamePos + 1) : path;
+
+    // If the filename contains special characters, we don't consider it valid for our matching
+    // purposes.
+    if (!filename.isEmpty() && Pattern.matches("[a-zA-Z_0-9\\.\\-\\(\\)\\%]+", filename)) {
+      int dotPos = filename.lastIndexOf('.');
+      if (0 <= dotPos) {
+        return filename.substring(dotPos + 1);
+      }
+    }
+    return "";
+  }
 }

libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java

@@ -16,18 +16,28 @@
 
 package androidx.media3.transformer;
 
-import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.common.util.Assertions.checkState;
 
+import android.content.ContentResolver;
 import android.content.Context;
+import android.graphics.BitmapFactory;
+import android.graphics.ColorSpace;
 import android.os.Looper;
 import androidx.annotation.Nullable;
+import androidx.media3.common.FileTypes;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.MimeTypes;
+import androidx.media3.common.util.BitmapLoader;
 import androidx.media3.common.util.Clock;
 import androidx.media3.common.util.UnstableApi;
+import androidx.media3.common.util.Util;
+import androidx.media3.datasource.DataSourceBitmapLoader;
+import androidx.media3.datasource.DefaultDataSource;
 import androidx.media3.exoplayer.source.MediaSource;
 import com.google.common.base.Ascii;
-import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.MoreExecutors;
+import java.util.Objects;
+import java.util.concurrent.Executors;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 /** The default {@link AssetLoader.Factory} implementation. */
@@ -39,6 +49,7 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
   private final boolean forceInterpretHdrAsSdr;
   private final Clock clock;
   private final MediaSource.@MonotonicNonNull Factory mediaSourceFactory;
+  private final BitmapLoader bitmapLoader;
 
   private AssetLoader.@MonotonicNonNull Factory imageAssetLoaderFactory;
   private AssetLoader.@MonotonicNonNull Factory exoPlayerAssetLoaderFactory;
@@ -46,6 +57,10 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
   /**
    * Creates an instance.
    *
+   * <p>Uses {@link DataSourceBitmapLoader} to load images, setting the {@link
+   * android.graphics.BitmapFactory.Options#inPreferredColorSpace} to {@link
+   * android.graphics.ColorSpace.Named#SRGB} when possible.
+   *
    * @param context The {@link Context}.
    * @param decoderFactory The {@link Codec.DecoderFactory} to use to decode the samples (if
    *     necessary).
@@ -64,6 +79,37 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
     this.forceInterpretHdrAsSdr = forceInterpretHdrAsSdr;
     this.clock = clock;
     this.mediaSourceFactory = null;
+    @Nullable BitmapFactory.Options options = null;
+    if (Util.SDK_INT >= 26) {
+      options = new BitmapFactory.Options();
+      options.inPreferredColorSpace = ColorSpace.get(ColorSpace.Named.SRGB);
+    }
+    this.bitmapLoader =
+        new DataSourceBitmapLoader(
+            MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor()),
+            new DefaultDataSource.Factory(context),
+            options);
+  }
+
+  /**
+   * Creates an instance with the default {@link Clock} and {@link Codec.DecoderFactory}.
+   *
+   * <p>For multi-picture formats (e.g. gifs), a single image frame from the container is loaded.
+   * The frame loaded is determined by the {@link BitmapLoader} implementation.
+   *
+   * @param context The {@link Context}.
+   * @param forceInterpretHdrAsSdr Whether to apply {@link
+   *     Composition#HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR}.
+   * @param bitmapLoader The {@link BitmapLoader} to use to load and decode images.
+   */
+  public DefaultAssetLoaderFactory(
+      Context context, boolean forceInterpretHdrAsSdr, BitmapLoader bitmapLoader) {
+    this.context = context.getApplicationContext();
+    this.decoderFactory = new DefaultDecoderFactory(context);
+    this.forceInterpretHdrAsSdr = forceInterpretHdrAsSdr;
+    this.clock = Clock.DEFAULT;
+    this.mediaSourceFactory = null;
+    this.bitmapLoader = bitmapLoader;
   }
 
   /**
@@ -78,18 +124,21 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
    *     testing.
    * @param mediaSourceFactory The {@link MediaSource.Factory} to use to retrieve the samples to
    *     transform when an {@link ExoPlayerAssetLoader} is used.
+   * @param bitmapLoader The {@link BitmapLoader} to use to load and decode images.
    */
   public DefaultAssetLoaderFactory(
       Context context,
       Codec.DecoderFactory decoderFactory,
       boolean forceInterpretHdrAsSdr,
       Clock clock,
-      MediaSource.Factory mediaSourceFactory) {
+      MediaSource.Factory mediaSourceFactory,
+      BitmapLoader bitmapLoader) {
     this.context = context.getApplicationContext();
     this.decoderFactory = decoderFactory;
     this.forceInterpretHdrAsSdr = forceInterpretHdrAsSdr;
     this.clock = clock;
     this.mediaSourceFactory = mediaSourceFactory;
+    this.bitmapLoader = bitmapLoader;
   }
 
   @Override
@@ -98,7 +147,7 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
     MediaItem mediaItem = editedMediaItem.mediaItem;
     if (isImage(mediaItem.localConfiguration)) {
       if (imageAssetLoaderFactory == null) {
-        imageAssetLoaderFactory = new ImageAssetLoader.Factory(context);
+        imageAssetLoaderFactory = new ImageAssetLoader.Factory(bitmapLoader);
       }
       return imageAssetLoaderFactory.createAssetLoader(editedMediaItem, looper, listener);
     }
@@ -113,21 +162,71 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
     return exoPlayerAssetLoaderFactory.createAssetLoader(editedMediaItem, looper, listener);
   }
 
-  private static boolean isImage(@Nullable MediaItem.LocalConfiguration localConfiguration) {
+  private boolean isImage(@Nullable MediaItem.LocalConfiguration localConfiguration) {
     if (localConfiguration == null) {
       return false;
     }
-    if (localConfiguration.mimeType != null) {
+    @Nullable String mimeType = localConfiguration.mimeType;
-      return MimeTypes.isImage(localConfiguration.mimeType);
+    if (mimeType == null) {
+      if (Objects.equals(localConfiguration.uri.getScheme(), ContentResolver.SCHEME_CONTENT)) {
+        ContentResolver cr = context.getContentResolver();
+        mimeType = cr.getType(localConfiguration.uri);
+      } else {
+        String fileExtension = FileTypes.getFileExtensionFromUri(localConfiguration.uri);
+        mimeType = getCommonImageMimeTypeFromExtension(Ascii.toLowerCase(fileExtension));
+      }
     }
-    ImmutableList<String> supportedImageTypes =
+    if (mimeType == null) {
-        ImmutableList.of(".png", ".webp", ".jpg", ".jpeg", ".heic", ".heif", ".bmp");
-    String uriPath = checkNotNull(localConfiguration.uri.getPath());
-    int fileExtensionStart = uriPath.lastIndexOf(".");
-    if (fileExtensionStart < 0) {
       return false;
     }
-    String extension = Ascii.toLowerCase(uriPath.substring(fileExtensionStart));
+    if (!MimeTypes.isImage(mimeType)) {
-    return supportedImageTypes.contains(extension);
+      return false;
+    }
+    checkState(
+        bitmapLoader.supportsMimeType(mimeType),
+        "Image format not supported by given bitmapLoader");
+    return true;
+  }
+
+  @Nullable
+  private static String getCommonImageMimeTypeFromExtension(String extension) {
+    switch (extension) {
+      case "bmp":
+      case "dib":
+        return MimeTypes.IMAGE_BMP;
+      case "heif":
+      case "heic":
+        return MimeTypes.IMAGE_HEIF;
+      case "jpg":
+      case "jpeg":
+      case "jpe":
+      case "jif":
+      case "jfif":
+      case "jfi":
+        return MimeTypes.IMAGE_JPEG;
+      case "png":
+        return MimeTypes.IMAGE_PNG;
+      case "webp":
+        return MimeTypes.IMAGE_WEBP;
+      case "gif":
+        return "image/gif";
+      case "tiff":
+      case "tif":
+        return "image/tiff";
+      case "raw":
+      case "arw":
+      case "cr2":
+      case "k25":
+        return "image/raw";
+      case "svg":
+      case "svgz":
+        return "image/svg+xml";
+      case "ico":
+        return "image/x-icon";
+      case "avif":
+        return "image/avif";
+      default:
+        return null;
+    }
   }
 }

libraries/transformer/src/main/java/androidx/media3/transformer/EditedMediaItem.java

@@ -45,7 +45,9 @@ public final class EditedMediaItem {
      * Creates an instance.
      *
      * <p>For image inputs, the values passed into {@link #setRemoveAudio}, {@link #setRemoveVideo}
-     * and {@link #setFlattenForSlowMotion} will be ignored.
+     * and {@link #setFlattenForSlowMotion} will be ignored. For multi-picture formats (e.g. gifs),
+     * a single image frame from the container is displayed if the {@link DefaultAssetLoaderFactory}
+     * is used.
      *
      * @param mediaItem The {@link MediaItem} on which transformations are applied.
      */

libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java

@@ -27,10 +27,7 @@ import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE;
 import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
-import android.content.Context;
 import android.graphics.Bitmap;
-import android.graphics.BitmapFactory;
-import android.graphics.ColorSpace;
 import android.os.Looper;
 import androidx.annotation.Nullable;
 import androidx.media3.common.C;
@@ -41,16 +38,11 @@ import androidx.media3.common.MimeTypes;
 import androidx.media3.common.util.BitmapLoader;
 import androidx.media3.common.util.ConstantRateTimestampIterator;
 import androidx.media3.common.util.UnstableApi;
-import androidx.media3.common.util.Util;
-import androidx.media3.datasource.DataSource;
-import androidx.media3.datasource.DataSourceBitmapLoader;
-import androidx.media3.datasource.DefaultDataSource;
 import androidx.media3.transformer.SampleConsumer.InputResult;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.FutureCallback;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.MoreExecutors;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 
@@ -67,16 +59,21 @@ public final class ImageAssetLoader implements AssetLoader {
   /** An {@link AssetLoader.Factory} for {@link ImageAssetLoader} instances. */
   public static final class Factory implements AssetLoader.Factory {
 
-    private final Context context;
+    private final BitmapLoader bitmapLoader;
 
-    public Factory(Context context) {
+    /**
-      this.context = context.getApplicationContext();
+     * Creates an instance.
+     *
+     * @param bitmapLoader The {@link BitmapLoader} to use to load and decode images.
+     */
+    public Factory(BitmapLoader bitmapLoader) {
+      this.bitmapLoader = bitmapLoader;
     }
 
     @Override
     public AssetLoader createAssetLoader(
         EditedMediaItem editedMediaItem, Looper looper, Listener listener) {
-      return new ImageAssetLoader(context, editedMediaItem, listener);
+      return new ImageAssetLoader(editedMediaItem, listener, bitmapLoader);
     }
   }
 
@@ -85,7 +82,7 @@ public final class ImageAssetLoader implements AssetLoader {
   private static final int QUEUE_BITMAP_INTERVAL_MS = 10;
 
   private final EditedMediaItem editedMediaItem;
-  private final DataSource.Factory dataSourceFactory;
+  private final BitmapLoader bitmapLoader;
   private final Listener listener;
   private final ScheduledExecutorService scheduledExecutorService;
 
@@ -94,12 +91,13 @@ public final class ImageAssetLoader implements AssetLoader {
 
   private volatile int progress;
 
-  private ImageAssetLoader(Context context, EditedMediaItem editedMediaItem, Listener listener) {
+  private ImageAssetLoader(
+      EditedMediaItem editedMediaItem, Listener listener, BitmapLoader bitmapLoader) {
     checkState(editedMediaItem.durationUs != C.TIME_UNSET);
     checkState(editedMediaItem.frameRate != C.RATE_UNSET_INT);
     this.editedMediaItem = editedMediaItem;
-    dataSourceFactory = new DefaultDataSource.Factory(context);
     this.listener = listener;
+    this.bitmapLoader = bitmapLoader;
     scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
     progressState = PROGRESS_STATE_NOT_STARTED;
   }
@@ -112,18 +110,11 @@ public final class ImageAssetLoader implements AssetLoader {
     progressState = PROGRESS_STATE_AVAILABLE;
     listener.onDurationUs(editedMediaItem.durationUs);
     listener.onTrackCount(1);
-    @Nullable BitmapFactory.Options options = null;
-    if (Util.SDK_INT >= 26) {
-      options = new BitmapFactory.Options();
-      options.inPreferredColorSpace = ColorSpace.get(ColorSpace.Named.SRGB);
-    }
-    BitmapLoader bitmapLoader =
-        new DataSourceBitmapLoader(
-            MoreExecutors.listeningDecorator(scheduledExecutorService), dataSourceFactory, options);
     MediaItem.LocalConfiguration localConfiguration =
         checkNotNull(editedMediaItem.mediaItem.localConfiguration);
 
     ListenableFuture<Bitmap> future = bitmapLoader.loadBitmap(localConfiguration.uri);
+
     Futures.addCallback(
         future,
         new FutureCallback<Bitmap>() {

libraries/transformer/src/test/java/androidx/media3/transformer/ImageAssetLoaderTest.java

@@ -23,6 +23,7 @@ import android.os.Looper;
 import androidx.media3.common.Format;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.util.TimestampIterator;
+import androidx.media3.datasource.DataSourceBitmapLoader;
 import androidx.test.core.app.ApplicationProvider;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import java.time.Duration;
@@ -120,7 +121,8 @@ public class ImageAssetLoaderTest {
             .setDurationUs(1_000_000)
             .setFrameRate(30)
             .build();
-    return new ImageAssetLoader.Factory(ApplicationProvider.getApplicationContext())
+    return new ImageAssetLoader.Factory(
+            new DataSourceBitmapLoader(ApplicationProvider.getApplicationContext()))
         .createAssetLoader(editedMediaItem, Looper.myLooper(), listener);
   }