loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add motion photo support to Transformer

Browse Source 
PiperOrigin-RevId: 683540867
This commit is contained in:
kimvde 2024-10-08 02:55:41 -07:00 committed by Copybara-Service
parent bd192c17ca
commit 52f08d46c2
6 changed files with 166 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View File

@ -223,6 +223,18 @@ public final class AndroidTestUtil {
.setHeight(4080)
.build())
.build();
public static final AssetInfo JPG_PIXEL_MOTION_PHOTO_ASSET =
new AssetInfo.Builder("asset:///media/jpeg/pixel-motion-photo-2-hevc-tracks.jpg")
.setVideoFormat(
new Format.Builder()
.setSampleMimeType(VIDEO_H265)
.setWidth(1024)
.setHeight(768)
.setFrameRate(27.61f)
.setCodecs("hvc1.1.6.L153")
.build())
.setVideoFrameCount(58)
.build();
public static final AssetInfo WEBP_LARGE =
new AssetInfo.Builder("asset:///media/webp/black_large.webp")

50
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -22,6 +22,7 @@ import static androidx.media3.common.util.MediaFormatUtil.createFormatFromMediaF
import static androidx.media3.common.util.Util.isRunningOnEmulator;
import static androidx.media3.test.utils.TestUtil.retrieveTrackFormat;
import static androidx.media3.transformer.AndroidTestUtil.JPG_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.JPG_PIXEL_MOTION_PHOTO_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.MP3_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_PHOTOS_TRIM_OPTIMIZATION_VIDEO;
@ -1543,6 +1544,55 @@ public class TransformerEndToEndTest {
assertThat(new File(result.filePath).length()).isGreaterThan(0);
}
@Test
public void motionPhoto_withNoDurationSet_exportsVideo() throws Exception {
Transformer transformer = new Transformer.Builder(context).build();
assumeFormatsSupported(
context,
testId,
/* inputFormat= */ null,
/* outputFormat= */ JPG_PIXEL_MOTION_PHOTO_ASSET.videoFormat);
EditedMediaItem motionPhotoItem =
new EditedMediaItem.Builder(MediaItem.fromUri(JPG_PIXEL_MOTION_PHOTO_ASSET.uri)).build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, motionPhotoItem);
assertThat(result.exportResult.videoFrameCount)
.isEqualTo(JPG_PIXEL_MOTION_PHOTO_ASSET.videoFrameCount);
}
@Test
public void motionPhoto_withDurationSet_exportsImage() throws Exception {
Transformer transformer = new Transformer.Builder(context).build();
MediaItem motionPhotoItem =
new MediaItem.Builder()
.setUri(JPG_PIXEL_MOTION_PHOTO_ASSET.uri)
.setImageDurationMs(500)
.build();
// Downscale to make sure the resolution is supported by the encoder.
Effect downscalingEffect =
Presentation.createForWidthAndHeight(
/* width= */ 480, /* height= */ 360, Presentation.LAYOUT_SCALE_TO_FIT);
EditedMediaItem motionPhotoEditedItem =
new EditedMediaItem.Builder(motionPhotoItem)
.setFrameRate(30)
.setEffects(
new Effects(
/* audioProcessors= */ ImmutableList.of(),
/* videoEffects= */ ImmutableList.of(downscalingEffect)))
.build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, motionPhotoEditedItem);
assertThat(result.exportResult.videoFrameCount).isEqualTo(15); // 0.5 sec at 30 fps
}
@Test
public void audioTranscode_processesInInt16Pcm() throws Exception {
FormatTrackingAudioBufferSink audioFormatTracker = new FormatTrackingAudioBufferSink();

13
libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java
View File

@ -17,6 +17,7 @@
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.transformer.TransformerUtil.isImage;
import android.content.Context;
import android.graphics.BitmapFactory;
@ -25,7 +26,6 @@ import android.os.Looper;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.MediaItem;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.BitmapLoader;
import androidx.media3.common.util.Clock;
import androidx.media3.common.util.Log;
@ -141,7 +141,11 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
AssetLoader.Listener listener,
CompositionSettings compositionSettings) {
MediaItem mediaItem = editedMediaItem.mediaItem;
if (isImage(mediaItem)) {
boolean isImage = isImage(context, mediaItem);
// TODO: b/350499931 - use the MediaItem's imageDurationMs instead of the EditedMediaItem's
// durationUs to export motion photos as video
boolean exportVideoFromMotionPhoto = isImage && editedMediaItem.durationUs == C.TIME_UNSET;
if (isImage && !exportVideoFromMotionPhoto) {
if (checkNotNull(mediaItem.localConfiguration).imageDurationMs == C.TIME_UNSET) {
Log.w(TAG, "The imageDurationMs field must be set on image MediaItems.");
}
@ -160,9 +164,4 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
return exoPlayerAssetLoaderFactory.createAssetLoader(
editedMediaItem, looper, listener, compositionSettings);
}
private boolean isImage(MediaItem mediaItem) {
@Nullable String mimeType = ImageAssetLoader.getImageMimeType(context, mediaItem);
return mimeType != null && MimeTypes.isImage(mimeType);
}
}

10
libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java
View File

@ -28,6 +28,7 @@ import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_UNAVAILABLE;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
import static androidx.media3.transformer.TransformerUtil.isImage;
import static java.lang.Math.min;
import android.content.Context;
@ -139,6 +140,7 @@ public final class ExoPlayerAssetLoader implements AssetLoader {
*/
private static final long EMULATOR_RELEASE_TIMEOUT_MS = 5_000;
private final Context context;
private final EditedMediaItem editedMediaItem;
private final CapturingDecoderFactory decoderFactory;
private final ExoPlayer player;
@ -154,6 +156,7 @@ public final class ExoPlayerAssetLoader implements AssetLoader {
Looper looper,
Listener listener,
Clock clock) {
this.context = context;
this.editedMediaItem = editedMediaItem;
this.decoderFactory = new CapturingDecoderFactory(decoderFactory);
@ -339,10 +342,13 @@ public final class ExoPlayerAssetLoader implements AssetLoader {
// listener callbacks are called in the right order.
player.play();
} else {
String errorMessage = "The asset loader has no audio or video track to output.";
if (isImage(context, editedMediaItem.mediaItem)) {
errorMessage += " Try setting an image duration on input image MediaItems.";
}
assetLoaderListener.onError(
ExportException.createForAssetLoader(
new IllegalStateException("The asset loader has no track to output."),
ERROR_CODE_FAILED_RUNTIME_CHECK));
new IllegalStateException(errorMessage), ERROR_CODE_FAILED_RUNTIME_CHECK));
}
} catch (RuntimeException e) {
assetLoaderListener.onError(

84
libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java
View File

@ -28,7 +28,6 @@ import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED
import static com.google.common.util.concurrent.Futures.immediateFailedFuture;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import android.content.ContentResolver;
import android.content.Context;
import android.graphics.Bitmap;
import android.os.Looper;
@ -36,7 +35,6 @@ import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
import androidx.media3.common.MediaItem;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.ParserException;
import androidx.media3.common.util.BitmapLoader;
@ -44,12 +42,10 @@ import androidx.media3.common.util.ConstantRateTimestampIterator;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import androidx.media3.transformer.SampleConsumer.InputResult;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import java.util.Objects;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@ -126,41 +122,6 @@ public final class ImageAssetLoader implements AssetLoader {
progressState = PROGRESS_STATE_NOT_STARTED;
}
/**
* Returns the image MIME type corresponding to a {@link MediaItem}.
*
* <p>This method only supports some common image MIME types.
*
* @param context The {@link Context}.
* @param mediaItem The {@link MediaItem} to inspect.
* @return The MIME type.
*/
@Nullable
public static String getImageMimeType(Context context, MediaItem mediaItem) {
if (mediaItem.localConfiguration == null) {
return null;
}
MediaItem.LocalConfiguration localConfiguration = mediaItem.localConfiguration;
@Nullable String mimeType = localConfiguration.mimeType;
if (mimeType == null) {
if (Objects.equals(localConfiguration.uri.getScheme(), ContentResolver.SCHEME_CONTENT)) {
ContentResolver cr = context.getContentResolver();
mimeType = cr.getType(localConfiguration.uri);
} else {
@Nullable String uriPath = localConfiguration.uri.getPath();
if (uriPath == null) {
return null;
}
int fileExtensionStart = uriPath.lastIndexOf(".");
if (fileExtensionStart >= 0 && fileExtensionStart < uriPath.length() - 1) {
String extension = Ascii.toLowerCase(uriPath.substring(fileExtensionStart + 1));
mimeType = getCommonImageMimeTypeFromExtension(extension);
}
}
}
return mimeType;
}
@Override
// Ignore Future returned by scheduledExecutorService because failures are already handled in the
// runnable.
@ -172,7 +133,7 @@ public final class ImageAssetLoader implements AssetLoader {
ListenableFuture<Bitmap> future;
@Nullable
String mimeType = ImageAssetLoader.getImageMimeType(context, editedMediaItem.mediaItem);
String mimeType = TransformerUtil.getImageMimeType(context, editedMediaItem.mediaItem);
if (mimeType == null || !bitmapLoader.supportsMimeType(mimeType)) {
future =
immediateFailedFuture(
@ -276,47 +237,4 @@ public final class ImageAssetLoader implements AssetLoader {
listener.onError(ExportException.createForAssetLoader(e, ERROR_CODE_UNSPECIFIED));
}
}
@Nullable
private static String getCommonImageMimeTypeFromExtension(String extension) {
switch (extension) {
case "bmp":
case "dib":
return MimeTypes.IMAGE_BMP;
case "heif":
return MimeTypes.IMAGE_HEIF;
case "heic":
return MimeTypes.IMAGE_HEIC;
case "jpg":
case "jpeg":
case "jpe":
case "jif":
case "jfif":
case "jfi":
return MimeTypes.IMAGE_JPEG;
case "png":
return MimeTypes.IMAGE_PNG;
case "webp":
return MimeTypes.IMAGE_WEBP;
case "gif":
return "image/gif";
case "tiff":
case "tif":
return "image/tiff";
case "raw":
case "arw":
case "cr2":
case "k25":
return "image/raw";
case "svg":
case "svgz":
return "image/svg+xml";
case "ico":
return "image/x-icon";
case "avif":
return MimeTypes.IMAGE_AVIF;
default:
return null;
}
}
}

89
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java
View File

@ -24,6 +24,8 @@ import static androidx.media3.transformer.Composition.HDR_MODE_TONE_MAP_HDR_TO_S
import static androidx.media3.transformer.EncoderUtil.getSupportedEncodersForHdrEditing;
import static java.lang.Math.round;
import android.content.ContentResolver;
import android.content.Context;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.util.Pair;
@ -32,6 +34,7 @@ import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.Effect;
import androidx.media3.common.Format;
import androidx.media3.common.MediaItem;
import androidx.media3.common.Metadata;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.UnstableApi;
@ -39,7 +42,9 @@ import androidx.media3.effect.GlEffect;
import androidx.media3.effect.ScaleAndRotateTransformation;
import androidx.media3.extractor.metadata.mp4.SlowMotionData;
import androidx.media3.transformer.Composition.HdrMode;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableList;
import java.util.Objects;
/** Utility methods for Transformer. */
@UnstableApi
@ -279,4 +284,88 @@ public final class TransformerUtil {
}
return Pair.create(requestedOutputMimeType, hdrMode);
}
/** Returns whether the provided {@link MediaItem} corresponds to an image. */
public static boolean isImage(Context context, MediaItem mediaItem) {
@Nullable String mimeType = getImageMimeType(context, mediaItem);
return mimeType != null && MimeTypes.isImage(mimeType);
}
/**
* Returns the image MIME type corresponding to a {@link MediaItem}.
*
* <p>This method only supports some common image MIME types.
*
* @param context The {@link Context}.
* @param mediaItem The {@link MediaItem} to inspect.
* @return The MIME type.
*/
@Nullable
public static String getImageMimeType(Context context, MediaItem mediaItem) {
if (mediaItem.localConfiguration == null) {
return null;
}
MediaItem.LocalConfiguration localConfiguration = mediaItem.localConfiguration;
@Nullable String mimeType = localConfiguration.mimeType;
if (mimeType == null) {
if (Objects.equals(localConfiguration.uri.getScheme(), ContentResolver.SCHEME_CONTENT)) {
ContentResolver cr = context.getContentResolver();
mimeType = cr.getType(localConfiguration.uri);
} else {
@Nullable String uriPath = localConfiguration.uri.getPath();
if (uriPath == null) {
return null;
}
int fileExtensionStart = uriPath.lastIndexOf(".");
if (fileExtensionStart >= 0 && fileExtensionStart < uriPath.length() - 1) {
String extension = Ascii.toLowerCase(uriPath.substring(fileExtensionStart + 1));
mimeType = getCommonImageMimeTypeFromExtension(extension);
}
}
}
return mimeType;
}
@Nullable
private static String getCommonImageMimeTypeFromExtension(String extension) {
switch (extension) {
case "bmp":
case "dib":
return MimeTypes.IMAGE_BMP;
case "heif":
return MimeTypes.IMAGE_HEIF;
case "heic":
return MimeTypes.IMAGE_HEIC;
case "jpg":
case "jpeg":
case "jpe":
case "jif":
case "jfif":
case "jfi":
return MimeTypes.IMAGE_JPEG;
case "png":
return MimeTypes.IMAGE_PNG;
case "webp":
return MimeTypes.IMAGE_WEBP;
case "gif":
return "image/gif";
case "tiff":
case "tif":
return "image/tiff";
case "raw":
case "arw":
case "cr2":
case "k25":
return "image/raw";
case "svg":
case "svgz":
return "image/svg+xml";
case "ico":
return "image/x-icon";
case "avif":
return MimeTypes.IMAGE_AVIF;
default:
return null;
}
}
}