loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compositor: Add VideoCompositorSettings to Composition.

Browse Source 
This allows apps using Transformer to customize how a Composition is used.

PiperOrigin-RevId: 567633129
This commit is contained in:
huangdarwin 2023-09-22 08:48:48 -07:00 committed by Copybara-Service
parent 42d9879d47
commit ba8c85a277
9 changed files with 141 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
libraries/test_data/src/test/assets/media/bitmap/transformer_multi_sequence_composition_test/export_withTwoSequencesWithVideoCompositorSettings_succeeds_0.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 62 KiB

83
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerMultiSequenceCompositionTest.java
View File

@ -32,11 +32,14 @@ import android.content.Context;
import android.graphics.Bitmap;
import androidx.media3.common.Effect;
import androidx.media3.common.MediaItem;
import androidx.media3.common.util.Size;
import androidx.media3.common.util.Util;
import androidx.media3.effect.AlphaScale;
import androidx.media3.effect.Contrast;
import androidx.media3.effect.OverlaySettings;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.ScaleAndRotateTransformation;
import androidx.media3.effect.VideoCompositorSettings;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
@ -49,6 +52,9 @@ import org.junit.runner.RunWith;
@RunWith(AndroidJUnit4.class)
public final class TransformerMultiSequenceCompositionTest {
// Bitmaps are generated on a Pixel 6 or 7 Pro instead of an emulator, due to an emulator bug.
// TODO: b/301242589 - Fix this test on the crow emulator, and re-generate bitmaps using the crow
// emulator, for consistency with other pixel tests.
private static final String PNG_ASSET_BASE_PATH =
"media/bitmap/transformer_multi_sequence_composition_test";
@ -86,7 +92,8 @@ public final class TransformerMultiSequenceCompositionTest {
.build()))),
/* secondSequenceMediaItems= */ ImmutableList.of(
editedMediaItemByClippingVideo(
MP4_ASSET_URI_STRING, /* effects= */ ImmutableList.of())));
MP4_ASSET_URI_STRING, /* effects= */ ImmutableList.of())),
VideoCompositorSettings.DEFAULT);
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
@ -125,7 +132,70 @@ public final class TransformerMultiSequenceCompositionTest {
.build()))),
/* secondSequenceMediaItems= */ ImmutableList.of(
editedMediaItemOfOneFrameImage(
JPG_ASSET_URI_STRING, /* effects= */ ImmutableList.of())));
JPG_ASSET_URI_STRING, /* effects= */ ImmutableList.of())),
VideoCompositorSettings.DEFAULT);
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
.build()
.run(testId, composition);
assertThat(result.filePath).isNotNull();
assertBitmapsMatchExpected(
extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId);
}
@Test
public void export_withTwoSequencesWithVideoCompositorSettings_succeeds() throws Exception {
String testId = "export_withTwoSequencesWithVideoCompositorSettings_succeeds";
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
context,
testId,
/* inputFormat= */ MP4_ASSET_FORMAT,
/* outputFormat= */ MP4_ASSET_FORMAT)) {
return;
}
VideoCompositorSettings pictureInPictureVideoCompositorSettings =
new VideoCompositorSettings() {
@Override
public Size getOutputSize(List<Size> inputSizes) {
return inputSizes.get(0);
}
@Override
public OverlaySettings getOverlaySettings(int inputId, long presentationTimeUs) {
if (inputId == 0) {
// This tests all OverlaySettings builder variables.
return new OverlaySettings.Builder()
.setScale(.25f, .25f)
.setOverlayFrameAnchor(1, -1)
.setBackgroundFrameAnchor(.9f, -.7f)
.build();
} else {
return new OverlaySettings.Builder().build();
}
}
};
Composition composition =
createComposition(
/* compositionEffects= */ ImmutableList.of(
new Contrast(0.1f),
Presentation.createForWidthAndHeight(
EXPORT_WIDTH, EXPORT_HEIGHT, Presentation.LAYOUT_SCALE_TO_FIT)),
/* firstSequenceMediaItems= */ ImmutableList.of(
editedMediaItemByClippingVideo(
MP4_ASSET_URI_STRING,
/* effects= */ ImmutableList.of(
new AlphaScale(0.5f),
new ScaleAndRotateTransformation.Builder()
.setRotationDegrees(180)
.build()))),
/* secondSequenceMediaItems= */ ImmutableList.of(
editedMediaItemByClippingVideo(
MP4_ASSET_URI_STRING, /* effects= */ ImmutableList.of())),
pictureInPictureVideoCompositorSettings);
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build())
@ -165,7 +235,8 @@ public final class TransformerMultiSequenceCompositionTest {
private static Composition createComposition(
List<Effect> compositionEffects,
List<EditedMediaItem> firstSequenceMediaItems,
List<EditedMediaItem> secondSequenceMediaItems) {
List<EditedMediaItem> secondSequenceMediaItems,
VideoCompositorSettings videoCompositorSettings) {
return new Composition.Builder(
ImmutableList.of(
@ -174,6 +245,7 @@ public final class TransformerMultiSequenceCompositionTest {
.setEffects(
new Effects(
/* audioProcessors= */ ImmutableList.of(), /* videoEffects= */ compositionEffects))
.setVideoCompositorSettings(videoCompositorSettings)
.build();
}
@ -181,12 +253,11 @@ public final class TransformerMultiSequenceCompositionTest {
throws IOException {
for (int i = 0; i < actualBitmaps.size(); i++) {
Bitmap actualBitmap = actualBitmaps.get(i);
maybeSaveTestBitmap(
testId, /* bitmapLabel= */ String.valueOf(i), actualBitmap, /* path= */ null);
String subTestId = testId + "_" + i;
Bitmap expectedBitmap =
readBitmap(Util.formatInvariant("%s/%s.png", PNG_ASSET_BASE_PATH, subTestId));
maybeSaveTestBitmap(
testId, /* bitmapLabel= */ String.valueOf(i), actualBitmap, /* path= */ null);
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, subTestId);
assertThat(averagePixelAbsoluteDifference)

31
libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java
View File

@ -23,6 +23,7 @@ import androidx.annotation.IntDef;
import androidx.media3.common.MediaItem;
import androidx.media3.common.audio.AudioProcessor;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.effect.VideoCompositorSettings;
import com.google.common.collect.ImmutableList;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.lang.annotation.Documented;
@ -43,6 +44,7 @@ public final class Composition {
public static final class Builder {
private ImmutableList<EditedMediaItemSequence> sequences;
private VideoCompositorSettings videoCompositorSettings;
private Effects effects;
private boolean forceAudioTrack;
private boolean transmuxAudio;
@ -69,12 +71,14 @@ public final class Composition {
!sequences.isEmpty(),
"The composition must contain at least one EditedMediaItemSequence.");
this.sequences = ImmutableList.copyOf(sequences);
videoCompositorSettings = VideoCompositorSettings.DEFAULT;
effects = Effects.EMPTY;
}
/** Creates a new instance to build upon the provided {@link Composition}. */
private Builder(Composition composition) {
sequences = composition.sequences;
videoCompositorSettings = composition.videoCompositorSettings;
effects = composition.effects;
forceAudioTrack = composition.forceAudioTrack;
transmuxAudio = composition.transmuxAudio;
@ -82,6 +86,20 @@ public final class Composition {
hdrMode = composition.hdrMode;
}
/**
* Sets the {@link VideoCompositorSettings} to apply to the {@link Composition}.
*
* <p>The default value is {@link VideoCompositorSettings#DEFAULT}.
*
* @param videoCompositorSettings The {@link VideoCompositorSettings}.
* @return This builder.
*/
@CanIgnoreReturnValue
public Builder setVideoCompositorSettings(VideoCompositorSettings videoCompositorSettings) {
this.videoCompositorSettings = videoCompositorSettings;
return this;
}
/**
* Sets the {@link Effects} to apply to the {@link Composition}.
*
@ -207,7 +225,13 @@ public final class Composition {
/** Builds a {@link Composition} instance. */
public Composition build() {
return new Composition(
sequences, effects, forceAudioTrack, transmuxAudio, transmuxVideo, hdrMode);
sequences,
videoCompositorSettings,
effects,
forceAudioTrack,
transmuxAudio,
transmuxVideo,
hdrMode);
}
/**
@ -309,6 +333,9 @@ public final class Composition {
*/
public final ImmutableList<EditedMediaItemSequence> sequences;
/** The {@link VideoCompositorSettings} to apply to the composition. */
public final VideoCompositorSettings videoCompositorSettings;
/** The {@link Effects} to apply to the composition. */
public final Effects effects;
@ -347,6 +374,7 @@ public final class Composition {
private Composition(
List<EditedMediaItemSequence> sequences,
VideoCompositorSettings videoCompositorSettings,
Effects effects,
boolean forceAudioTrack,
boolean transmuxAudio,
@ -356,6 +384,7 @@ public final class Composition {
!transmuxAudio || !forceAudioTrack,
"Audio transmuxing and audio track forcing are not allowed together.");
this.sequences = ImmutableList.copyOf(sequences);
this.videoCompositorSettings = videoCompositorSettings;
this.effects = effects;
this.transmuxAudio = transmuxAudio;
this.transmuxVideo = transmuxVideo;

7
libraries/transformer/src/main/java/androidx/media3/transformer/MultipleInputVideoGraph.java
View File

@ -72,6 +72,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
DebugViewProvider debugViewProvider,
Listener listener,
Executor listenerExecutor,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects,
long initialTimestampOffsetUs) {
return new MultipleInputVideoGraph(
@ -81,6 +82,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
debugViewProvider,
listener,
listenerExecutor,
videoCompositorSettings,
compositionEffects,
initialTimestampOffsetUs);
}
@ -99,6 +101,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final DebugViewProvider debugViewProvider;
private final Listener listener;
private final Executor listenerExecutor;
private final VideoCompositorSettings videoCompositorSettings;
private final List<Effect> compositionEffects;
private final List<VideoFrameProcessingWrapper> preProcessingWrappers;
@ -128,6 +131,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
DebugViewProvider debugViewProvider,
Listener listener,
Executor listenerExecutor,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects,
long initialTimestampOffsetUs) {
this.context = context;
@ -136,6 +140,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.debugViewProvider = debugViewProvider;
this.listener = listener;
this.listenerExecutor = listenerExecutor;
this.videoCompositorSettings = videoCompositorSettings;
this.compositionEffects = new ArrayList<>(compositionEffects);
this.initialTimestampOffsetUs = initialTimestampOffsetUs;
lastRenderedPresentationTimeUs = C.TIME_UNSET;
@ -219,7 +224,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
new DefaultVideoCompositor(
context,
glObjectsProvider,
VideoCompositorSettings.DEFAULT,
videoCompositorSettings,
sharedExecutorService,
new VideoCompositor.Listener() {
// All of this listener's methods are called on the sharedExecutorService.

12
libraries/transformer/src/main/java/androidx/media3/transformer/SingleInputVideoGraph.java
View File

@ -17,6 +17,7 @@
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.content.Context;
@ -30,6 +31,7 @@ import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.VideoGraph;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.VideoCompositorSettings;
import java.util.List;
import java.util.concurrent.Executor;
@ -52,6 +54,11 @@ import java.util.concurrent.Executor;
private boolean released;
private volatile boolean hasProducedFrameWithTimestampZero;
/**
* Creates an instance.
*
* <p>{@code videoCompositorSettings} must be {@link VideoCompositorSettings#DEFAULT}.
*/
public SingleInputVideoGraph(
Context context,
VideoFrameProcessor.Factory videoFrameProcessorFactory,
@ -60,9 +67,14 @@ import java.util.concurrent.Executor;
Listener listener,
DebugViewProvider debugViewProvider,
Executor listenerExecutor,
VideoCompositorSettings videoCompositorSettings,
boolean renderFramesAutomatically,
@Nullable Presentation presentation,
long initialTimestampOffsetUs) {
checkState(
VideoCompositorSettings.DEFAULT.equals(videoCompositorSettings),
"SingleInputVideoGraph does not use VideoCompositor, and therefore cannot apply"
+ " VideoCompositorSettings");
this.context = context;
this.videoFrameProcessorFactory = videoFrameProcessorFactory;
this.inputColorInfo = inputColorInfo;

1
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
View File

@ -582,6 +582,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
context,
firstAssetLoaderInputFormat,
transformationRequest,
composition.videoCompositorSettings,
composition.effects.videoEffects,
videoFrameProcessorFactory,
encoderFactory,

5
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java
View File

@ -23,6 +23,7 @@ import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.Effect;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.VideoCompositorSettings;
import java.util.List;
import java.util.concurrent.Executor;
@ -46,6 +47,7 @@ import java.util.concurrent.Executor;
DebugViewProvider debugViewProvider,
Listener listener,
Executor listenerExecutor,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects,
long initialTimestampOffsetUs) {
@Nullable Presentation presentation = null;
@ -63,6 +65,7 @@ import java.util.concurrent.Executor;
listener,
debugViewProvider,
listenerExecutor,
videoCompositorSettings,
/* renderFramesAutomatically= */ true,
presentation,
initialTimestampOffsetUs);
@ -77,6 +80,7 @@ import java.util.concurrent.Executor;
Listener listener,
DebugViewProvider debugViewProvider,
Executor listenerExecutor,
VideoCompositorSettings videoCompositorSettings,
boolean renderFramesAutomatically,
@Nullable Presentation presentation,
long initialTimestampOffsetUs) {
@ -88,6 +92,7 @@ import java.util.concurrent.Executor;
listener,
debugViewProvider,
listenerExecutor,
videoCompositorSettings,
renderFramesAutomatically,
presentation,
initialTimestampOffsetUs);

5
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java
View File

@ -23,6 +23,7 @@ import androidx.media3.common.Effect;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.VideoGraph;
import androidx.media3.effect.VideoCompositorSettings;
import java.util.List;
import java.util.concurrent.Executor;
@ -40,7 +41,10 @@ import java.util.concurrent.Executor;
* @param debugViewProvider A {@link DebugViewProvider}.
* @param listener A {@link Listener}.
* @param listenerExecutor The {@link Executor} on which the {@code listener} is invoked.
* @param videoCompositorSettings The {@link VideoCompositorSettings} to apply to the
* composition.
* @param compositionEffects A list of {@linkplain Effect effects} to apply to the composition.
* @param initialTimestampOffsetUs The timestamp offset for the first frame, in microseconds.
* @return A new instance.
* @throws VideoFrameProcessingException If a problem occurs while creating the {@link
* VideoFrameProcessor}.
@ -52,6 +56,7 @@ import java.util.concurrent.Executor;
DebugViewProvider debugViewProvider,
Listener listener,
Executor listenerExecutor,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects,
long initialTimestampOffsetUs)
throws VideoFrameProcessingException;

5
libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java
View File

@ -48,6 +48,7 @@ import androidx.media3.common.util.Log;
import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer;
import androidx.media3.effect.DebugTraceUtil;
import androidx.media3.effect.VideoCompositorSettings;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.MoreExecutors;
@ -78,6 +79,7 @@ import org.checkerframework.dataflow.qual.Pure;
Context context,
Format firstInputFormat,
TransformationRequest transformationRequest,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects,
VideoFrameProcessor.Factory videoFrameProcessorFactory,
Codec.EncoderFactory encoderFactory,
@ -151,6 +153,7 @@ import org.checkerframework.dataflow.qual.Pure;
videoGraphOutputColor,
errorConsumer,
debugViewProvider,
videoCompositorSettings,
compositionEffects);
videoGraph.initialize();
} catch (VideoFrameProcessingException e) {
@ -477,6 +480,7 @@ import org.checkerframework.dataflow.qual.Pure;
ColorInfo videoFrameProcessorOutputColor,
Consumer<ExportException> errorConsumer,
DebugViewProvider debugViewProvider,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects)
throws VideoFrameProcessingException {
this.errorConsumer = errorConsumer;
@ -493,6 +497,7 @@ import org.checkerframework.dataflow.qual.Pure;
debugViewProvider,
/* listener= */ thisRef,
/* listenerExecutor= */ MoreExecutors.directExecutor(),
videoCompositorSettings,
compositionEffects,
initialTimestampOffsetUs);
}