loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handle multiple sequences in a Composition

Browse Source 
This lays the groundwork for full multi-asset, and more particularly for
adding looping background audio.

PiperOrigin-RevId: 512887888
This commit is contained in:
kimvde 2023-02-28 11:53:43 +00:00 committed by tonihei
parent 408b4449ff
commit d2d5174f09
5 changed files with 187 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
libraries/transformer/src/main/java/androidx/media3/transformer/CompositeAssetLoader.java
View File

@ -73,6 +73,7 @@ import java.util.concurrent.atomic.AtomicInteger;
private final AtomicInteger nonEndedTracks;
private AssetLoader currentAssetLoader;
private boolean trackCountReported;
private int processedInputsSize;
private volatile long currentDurationUs;
@ -197,8 +198,11 @@ import java.util.concurrent.atomic.AtomicInteger;
if (currentMediaItemIndex.get() == 0) {
boolean addForcedAudioTrack =
forceAudioTrack && nonEndedTracks.get() == 1 && trackType == C.TRACK_TYPE_VIDEO;
if (!trackCountReported) {
int trackCount = nonEndedTracks.get() + (addForcedAudioTrack ? 1 : 0);
compositeAssetLoaderListener.onTrackCount(trackCount);
trackCountReported = true;
}
sampleConsumer =
new SampleConsumerWrapper(
compositeAssetLoaderListener.onTrackAdded(

15
libraries/transformer/src/main/java/androidx/media3/transformer/FallbackListener.java
View File

@ -23,6 +23,7 @@ import androidx.media3.common.C;
import androidx.media3.common.util.HandlerWrapper;
import androidx.media3.common.util.ListenerSet;
import androidx.media3.common.util.Util;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Listener for fallback {@link TransformationRequest TransformationRequests} from the audio and
@ -31,12 +32,12 @@ import androidx.media3.common.util.Util;
/* package */ final class FallbackListener {
private final Composition composition;
private final TransformationRequest originalTransformationRequest;
private final ListenerSet<Transformer.Listener> transformerListeners;
private final HandlerWrapper transformerListenerHandler;
private final TransformationRequest originalTransformationRequest;
private final AtomicInteger trackCount;
private TransformationRequest fallbackTransformationRequest;
private int trackCount;
/**
* Creates a new instance.
@ -58,6 +59,7 @@ import androidx.media3.common.util.Util;
this.transformerListenerHandler = transformerListenerHandler;
this.originalTransformationRequest = originalTransformationRequest;
this.fallbackTransformationRequest = originalTransformationRequest;
trackCount = new AtomicInteger();
}
/**
@ -65,9 +67,11 @@ import androidx.media3.common.util.Util;
*
* <p>The track count must be set before a transformation request is {@linkplain
* #onTransformationRequestFinalized(TransformationRequest) finalized}.
*
* <p>Can be called from any thread.
*/
public void setTrackCount(@IntRange(from = 1) int trackCount) {
this.trackCount = trackCount;
this.trackCount.set(trackCount);
}
/**
@ -86,7 +90,7 @@ import androidx.media3.common.util.Util;
* #setTrackCount(int)}.
*/
public void onTransformationRequestFinalized(TransformationRequest transformationRequest) {
checkState(trackCount-- > 0);
checkState(trackCount.getAndDecrement() > 0);
TransformationRequest.Builder fallbackRequestBuilder =
fallbackTransformationRequest.buildUpon();
@ -107,7 +111,8 @@ import androidx.media3.common.util.Util;
TransformationRequest newFallbackTransformationRequest = fallbackRequestBuilder.build();
fallbackTransformationRequest = newFallbackTransformationRequest;
if (trackCount == 0 && !originalTransformationRequest.equals(fallbackTransformationRequest)) {
if (trackCount.get() == 0
&& !originalTransformationRequest.equals(fallbackTransformationRequest)) {
transformerListenerHandler.post(
() ->
transformerListeners.sendEvent(

6
libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java
View File

@ -69,7 +69,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private final SparseArray<TrackInfo> trackTypeToInfo;
private final ScheduledExecutorService abortScheduledExecutorService;
private int trackCount;
private boolean isReady;
private boolean isEnded;
private @C.TrackType int previousTrackType;
@ -79,6 +78,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private boolean isAborted;
private @MonotonicNonNull Muxer muxer;
private volatile int trackCount;
public MuxerWrapper(String outputPath, Muxer.Factory muxerFactory, Listener listener) {
this.outputPath = outputPath;
this.muxerFactory = muxerFactory;
@ -95,6 +96,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* <p>The track count must be set before any track format is {@linkplain #addTrackFormat(Format)
* added}.
*
* <p>Can be called from any thread.
*
* @throws IllegalStateException If a track format was {@linkplain #addTrackFormat(Format) added}
* before calling this method.
*/
@ -135,6 +138,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* the track.
*/
public void addTrackFormat(Format format) throws Muxer.MuxerException {
int trackCount = this.trackCount;
checkState(trackCount > 0, "The track count should be set before the formats are added.");
checkState(trackTypeToInfo.size() < trackCount, "All track formats have already been added.");
@Nullable String sampleMimeType = format.sampleMimeType;

5
libraries/transformer/src/main/java/androidx/media3/transformer/SamplePipeline.java
View File

@ -39,8 +39,9 @@ import java.util.List;
*
* <p>The {@link SampleConsumer} and {@link OnMediaItemChangedListener} methods must be called from
* the same thread. This thread can change when the {@link
* OnMediaItemChangedListener#onMediaItemChanged(EditedMediaItem, Format, long) MediaItem} changes,
* and can be different from the thread used to call the other {@code SamplePipeline} methods.
* OnMediaItemChangedListener#onMediaItemChanged(EditedMediaItem, long, Format, boolean) MediaItem}
* changes, and can be different from the thread used to call the other {@code SamplePipeline}
* methods.
*/
/* package */ abstract class SamplePipeline implements SampleConsumer, OnMediaItemChangedListener {

281
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
View File

@ -22,6 +22,7 @@ import static androidx.media3.transformer.AssetLoader.SUPPORTED_OUTPUT_TYPE_DECO
import static androidx.media3.transformer.AssetLoader.SUPPORTED_OUTPUT_TYPE_ENCODED;
import static androidx.media3.transformer.ExportException.ERROR_CODE_FAILED_RUNTIME_CHECK;
import static androidx.media3.transformer.ExportException.ERROR_CODE_MUXING_FAILED;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED;
import static java.lang.annotation.ElementType.TYPE_USE;
@ -53,7 +54,7 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/* package */ final class TransformerInternal {
/* package */ final class TransformerInternal implements MuxerWrapper.Listener {
public interface Listener {
@ -88,15 +89,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private static final int DRAIN_PIPELINES_DELAY_MS = 10;
private final Context context;
private final TransformationRequest transformationRequest;
private final CapturingEncoderFactory encoderFactory;
private final Listener listener;
private final HandlerWrapper applicationHandler;
private final DebugViewProvider debugViewProvider;
private final Clock clock;
private final HandlerThread internalHandlerThread;
private final HandlerWrapper internalHandler;
private final CompositeAssetLoader compositeAssetLoader;
private final List<CompositeAssetLoader> compositeAssetLoaders;
private final AtomicInteger totalInputTrackCount;
private final AtomicInteger unreportedInputTrackCounts;
private final List<SamplePipeline> samplePipelines;
private final MuxerWrapper muxerWrapper;
private final ConditionVariable transformerConditionVariable;
@ -108,6 +109,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private volatile boolean released;
// Warning suppression is needed to assign the MuxerWrapper with "this" as listener.
@SuppressWarnings("assignment.type.incompatible")
public TransformerInternal(
Context context,
Composition composition,
@ -122,37 +125,34 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
DebugViewProvider debugViewProvider,
Clock clock) {
this.context = context;
this.transformationRequest = transformationRequest;
this.encoderFactory = new CapturingEncoderFactory(encoderFactory);
this.listener = listener;
this.applicationHandler = applicationHandler;
this.debugViewProvider = debugViewProvider;
this.clock = clock;
internalHandlerThread = new HandlerThread("Transformer:Internal");
internalHandlerThread.start();
compositeAssetLoaders = new ArrayList<>();
Looper internalLooper = internalHandlerThread.getLooper();
EditedMediaItemSequence sequence = composition.sequences.get(0);
ImmutableList<Effect> compositionVideoEffects = composition.effects.videoEffects;
@Nullable
Presentation presentation =
compositionVideoEffects.isEmpty() ? null : (Presentation) compositionVideoEffects.get(0);
ComponentListener componentListener =
new ComponentListener(
sequence,
presentation,
composition.transmuxAudio,
composition.transmuxVideo,
fallbackListener);
compositeAssetLoader =
for (int i = 0; i < composition.sequences.size(); i++) {
CompositeAssetLoaderListener compositeAssetLoaderListener =
new CompositeAssetLoaderListener(
/* sequenceIndex= */ i,
composition,
transformationRequest,
fallbackListener,
debugViewProvider);
compositeAssetLoaders.add(
new CompositeAssetLoader(
sequence,
composition.sequences.get(i),
composition.forceAudioTrack,
assetLoaderFactory,
internalLooper,
componentListener,
clock);
compositeAssetLoaderListener,
clock));
}
totalInputTrackCount = new AtomicInteger();
unreportedInputTrackCounts = new AtomicInteger(composition.sequences.size());
samplePipelines = new ArrayList<>();
muxerWrapper = new MuxerWrapper(outputPath, muxerFactory, componentListener);
transformerConditionVariable = new ConditionVariable();
exportResultBuilder = new ExportResult.Builder();
// It's safe to use "this" because we don't send a message before exiting the constructor.
@ -160,6 +160,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
HandlerWrapper internalHandler =
clock.createHandler(internalLooper, /* callback= */ this::handleMessage);
this.internalHandler = internalHandler;
// It's safe to use "this" because we don't mux any data before exiting the constructor.
@SuppressWarnings("nullness:argument.type.incompatible")
MuxerWrapper muxerWrapper = new MuxerWrapper(outputPath, muxerFactory, /* listener= */ this);
this.muxerWrapper = muxerWrapper;
}
public void start() {
@ -192,6 +196,51 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
}
// MuxerWrapper.Listener implementation
@Override
public void onTrackEnded(
@C.TrackType int trackType, Format format, int averageBitrate, int sampleCount) {
if (trackType == C.TRACK_TYPE_AUDIO) {
exportResultBuilder.setAverageAudioBitrate(averageBitrate).setPcmEncoding(format.pcmEncoding);
if (format.channelCount != Format.NO_VALUE) {
exportResultBuilder.setChannelCount(format.channelCount);
}
if (format.sampleRate != Format.NO_VALUE) {
exportResultBuilder.setSampleRate(format.sampleRate);
}
} else if (trackType == C.TRACK_TYPE_VIDEO) {
exportResultBuilder
.setAverageVideoBitrate(averageBitrate)
.setColorInfo(format.colorInfo)
.setVideoFrameCount(sampleCount);
if (format.height != Format.NO_VALUE) {
exportResultBuilder.setHeight(format.height);
}
if (format.width != Format.NO_VALUE) {
exportResultBuilder.setWidth(format.width);
}
}
}
@Override
public void onEnded(long durationMs, long fileSizeBytes) {
exportResultBuilder.setDurationMs(durationMs).setFileSizeBytes(fileSizeBytes);
internalHandler
.obtainMessage(MSG_END, END_REASON_COMPLETED, /* unused */ 0, /* exportException */ null)
.sendToTarget();
}
@Override
public void onError(ExportException exportException) {
internalHandler
.obtainMessage(MSG_END, END_REASON_ERROR, /* unused */ 0, exportException)
.sendToTarget();
}
// Private methods.
private boolean handleMessage(Message msg) {
// Some messages cannot be ignored when resources have been released. End messages must be
// handled to report release timeouts and to unblock the transformer condition variable in case
@ -229,7 +278,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
private void startInternal() {
compositeAssetLoader.start();
for (int i = 0; i < compositeAssetLoaders.size(); i++) {
compositeAssetLoaders.get(i).start();
}
}
private void registerSamplePipelineInternal(SamplePipeline samplePipeline) {
@ -251,10 +302,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
private void endInternal(@EndReason int endReason, @Nullable ExportException exportException) {
ImmutableList<ExportResult.ProcessedInput> processedInputs =
compositeAssetLoader.getProcessedInputs();
ImmutableList.Builder<ExportResult.ProcessedInput> processedInputsBuilder =
new ImmutableList.Builder<>();
for (int i = 0; i < compositeAssetLoaders.size(); i++) {
processedInputsBuilder.addAll(compositeAssetLoaders.get(i).getProcessedInputs());
}
exportResultBuilder
.setProcessedInputs(processedInputs)
.setProcessedInputs(processedInputsBuilder.build())
.setAudioEncoderName(encoderFactory.getAudioEncoderName())
.setVideoEncoderName(encoderFactory.getVideoEncoderName());
@ -262,27 +316,40 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@Nullable ExportException releaseExportException = null;
if (!released) {
released = true;
for (int i = 0; i < compositeAssetLoaders.size(); i++) {
try {
try {
compositeAssetLoader.release();
} finally {
try {
for (int i = 0; i < samplePipelines.size(); i++) {
samplePipelines.get(i).release();
}
} finally {
muxerWrapper.release(forCancellation);
}
}
} catch (Muxer.MuxerException e) {
releaseExportException = ExportException.createForMuxer(e, ERROR_CODE_MUXING_FAILED);
compositeAssetLoaders.get(i).release();
} catch (RuntimeException e) {
if (releaseExportException == null) {
releaseExportException = ExportException.createForUnexpected(e);
// cancelException is not reported through a listener. It is thrown in cancel(), as this
// method is blocking.
cancelException = e;
}
}
}
for (int i = 0; i < samplePipelines.size(); i++) {
try {
samplePipelines.get(i).release();
} catch (RuntimeException e) {
if (releaseExportException == null) {
releaseExportException = ExportException.createForUnexpected(e);
cancelException = e;
}
}
}
try {
muxerWrapper.release(forCancellation);
} catch (Muxer.MuxerException e) {
if (releaseExportException == null) {
releaseExportException = ExportException.createForMuxer(e, ERROR_CODE_MUXING_FAILED);
}
} catch (RuntimeException e) {
if (releaseExportException == null) {
releaseExportException = ExportException.createForUnexpected(e);
cancelException = e;
}
}
// Quit thread lazily so that all events that got triggered when releasing the AssetLoader are
// still delivered.
internalHandler.post(internalHandlerThread::quitSafely);
@ -314,49 +381,43 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
private void updateProgressInternal(ProgressHolder progressHolder) {
progressState = compositeAssetLoader.getProgress(progressHolder);
int progressSum = 0;
ProgressHolder individualProgressHolder = new ProgressHolder();
for (int i = 0; i < compositeAssetLoaders.size(); i++) {
progressState = compositeAssetLoaders.get(i).getProgress(individualProgressHolder);
if (progressState != PROGRESS_STATE_AVAILABLE) {
transformerConditionVariable.open();
return;
}
progressSum += individualProgressHolder.progress;
}
progressHolder.progress = progressSum / compositeAssetLoaders.size();
transformerConditionVariable.open();
}
private class ComponentListener implements AssetLoader.Listener, MuxerWrapper.Listener {
private final class CompositeAssetLoaderListener implements AssetLoader.Listener {
// The first EditedMediaItem in the sequence determines which SamplePipeline to use.
private final EditedMediaItem firstEditedMediaItem;
@Nullable private final Presentation compositionPresentation;
private final int mediaItemCount;
private final boolean transmuxAudio;
private final boolean transmuxVideo;
private final int sequenceIndex;
private final ImmutableList<EditedMediaItem> editedMediaItems;
private final Composition composition;
private final TransformationRequest transformationRequest;
private final FallbackListener fallbackListener;
private final AtomicInteger trackCount;
private final DebugViewProvider debugViewProvider;
private boolean trackAdded;
public ComponentListener(
EditedMediaItemSequence sequence,
@Nullable Presentation compositionPresentation,
boolean transmuxAudio,
boolean transmuxVideo,
FallbackListener fallbackListener) {
firstEditedMediaItem = sequence.editedMediaItems.get(0);
this.compositionPresentation = compositionPresentation;
mediaItemCount = sequence.editedMediaItems.size();
this.transmuxAudio = transmuxAudio;
this.transmuxVideo = transmuxVideo;
public CompositeAssetLoaderListener(
int sequenceIndex,
Composition composition,
TransformationRequest transformationRequest,
FallbackListener fallbackListener,
DebugViewProvider debugViewProvider) {
this.sequenceIndex = sequenceIndex;
editedMediaItems = composition.sequences.get(sequenceIndex).editedMediaItems;
this.composition = composition;
this.transformationRequest = transformationRequest;
this.fallbackListener = fallbackListener;
trackCount = new AtomicInteger();
this.debugViewProvider = debugViewProvider;
}
// AssetLoader.Listener and MuxerWrapper.Listener implementation.
@Override
public void onError(ExportException exportException) {
internalHandler
.obtainMessage(MSG_END, END_REASON_ERROR, /* unused */ 0, exportException)
.sendToTarget();
}
// AssetLoader.Listener implementation.
@Override
public void onDurationUs(long durationUs) {}
@ -369,7 +430,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
ERROR_CODE_FAILED_RUNTIME_CHECK));
return;
}
this.trackCount.set(trackCount);
totalInputTrackCount.addAndGet(trackCount);
unreportedInputTrackCounts.decrementAndGet();
if (unreportedInputTrackCounts.get() == 0) {
muxerWrapper.setTrackCount(totalInputTrackCount.get());
fallbackListener.setTrackCount(totalInputTrackCount.get());
}
}
@Override
@ -379,14 +445,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long streamStartPositionUs,
long streamOffsetUs)
throws ExportException {
if (!trackAdded) {
// Call setTrackCount() methods here so that they are called from the same thread as the
// MuxerWrapper and FallbackListener methods called when building the sample pipelines.
muxerWrapper.setTrackCount(trackCount.get());
fallbackListener.setTrackCount(trackCount.get());
trackAdded = true;
}
SamplePipeline samplePipeline =
getSamplePipeline(
firstInputFormat,
@ -399,48 +457,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
MimeTypes.isAudio(firstInputFormat.sampleMimeType)
? C.TRACK_TYPE_AUDIO
: C.TRACK_TYPE_VIDEO;
compositeAssetLoader.addOnMediaItemChangedListener(samplePipeline, trackType);
compositeAssetLoaders
.get(sequenceIndex)
.addOnMediaItemChangedListener(samplePipeline, trackType);
internalHandler.obtainMessage(MSG_REGISTER_SAMPLE_PIPELINE, samplePipeline).sendToTarget();
return samplePipeline;
}
// MuxerWrapper.Listener implementation.
@Override
public void onTrackEnded(
@C.TrackType int trackType, Format format, int averageBitrate, int sampleCount) {
if (trackType == C.TRACK_TYPE_AUDIO) {
exportResultBuilder
.setAverageAudioBitrate(averageBitrate)
.setPcmEncoding(format.pcmEncoding);
if (format.channelCount != Format.NO_VALUE) {
exportResultBuilder.setChannelCount(format.channelCount);
}
if (format.sampleRate != Format.NO_VALUE) {
exportResultBuilder.setSampleRate(format.sampleRate);
}
} else if (trackType == C.TRACK_TYPE_VIDEO) {
exportResultBuilder
.setAverageVideoBitrate(averageBitrate)
.setColorInfo(format.colorInfo)
.setVideoFrameCount(sampleCount);
if (format.height != Format.NO_VALUE) {
exportResultBuilder.setHeight(format.height);
}
if (format.width != Format.NO_VALUE) {
exportResultBuilder.setWidth(format.width);
}
}
}
@Override
public void onEnded(long durationMs, long fileSizeBytes) {
exportResultBuilder.setDurationMs(durationMs).setFileSizeBytes(fileSizeBytes);
internalHandler
.obtainMessage(MSG_END, END_REASON_COMPLETED, /* unused */ 0, /* exportException */ null)
.sendToTarget();
public void onError(ExportException exportException) {
TransformerInternal.this.onError(exportException);
}
// Private methods.
@ -452,6 +478,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
long streamOffsetUs)
throws ExportException {
if (shouldTranscode) {
EditedMediaItem firstEditedMediaItem = editedMediaItems.get(0);
if (MimeTypes.isAudio(firstInputFormat.sampleMimeType)) {
return new AudioSamplePipeline(
firstInputFormat,
@ -463,6 +490,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
muxerWrapper,
fallbackListener);
} else { // MIME type is video or image.
ImmutableList<Effect> compositionVideoEffects = composition.effects.videoEffects;
@Nullable
Presentation compositionPresentation =
compositionVideoEffects.isEmpty()
? null
: (Presentation) compositionVideoEffects.get(0);
return new VideoSamplePipeline(
context,
firstInputFormat,
@ -513,7 +546,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
private boolean shouldTranscodeAudio(Format inputFormat) {
if (mediaItemCount > 1 && !transmuxAudio) {
if (editedMediaItems.size() > 1 && !composition.transmuxAudio) {
return true;
}
if (encoderFactory.audioNeedsEncoding()) {
@ -527,6 +560,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
&& !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) {
return true;
}
EditedMediaItem firstEditedMediaItem = editedMediaItems.get(0);
if (firstEditedMediaItem.flattenForSlowMotion && isSlowMotion(inputFormat)) {
return true;
}
@ -552,9 +586,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private boolean shouldTranscodeVideo(
Format inputFormat, long streamStartPositionUs, long streamOffsetUs) {
if (mediaItemCount > 1 && !transmuxVideo) {
if (editedMediaItems.size() > 1 && !composition.transmuxVideo) {
return true;
}
EditedMediaItem firstEditedMediaItem = editedMediaItems.get(0);
if ((streamStartPositionUs - streamOffsetUs) != 0
&& !firstEditedMediaItem.mediaItem.clippingConfiguration.startsAtKeyFrame) {
return true;