loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move slow mo video flattening to AssetLoader

Browse Source 
PiperOrigin-RevId: 493300556
This commit is contained in:
kimvde 2022-12-06 15:31:32 +00:00 committed by Ian Baker
parent 339205f428
commit 0d12de8134
7 changed files with 83 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java
View File

@ -66,12 +66,7 @@ import org.checkerframework.dataflow.qual.Pure;
MuxerWrapper muxerWrapper,
FallbackListener fallbackListener)
throws TransformationException {
super(
inputFormat,
streamStartPositionUs,
streamOffsetUs,
transformationRequest.flattenForSlowMotion,
muxerWrapper);
super(inputFormat, streamStartPositionUs, muxerWrapper);
if (forceSilentAudioDurationUs != C.TIME_UNSET) {
silentAudioGenerator =
@ -153,23 +148,23 @@ import org.checkerframework.dataflow.qual.Pure;
return true;
}
@Override
public void release() {
audioProcessingPipeline.reset();
encoder.release();
}
@Override
@Nullable
protected DecoderInputBuffer dequeueInputBufferInternal() {
public DecoderInputBuffer dequeueInputBuffer() {
return hasPendingInputBuffer ? null : inputBuffer;
}
@Override
protected void queueInputBufferInternal() {
public void queueInputBuffer() {
hasPendingInputBuffer = true;
}
@Override
public void release() {
audioProcessingPipeline.reset();
encoder.release();
}
@Override
protected boolean processDataUpToMuxer() throws TransformationException {
if (!audioProcessingPipeline.isOperational()) {

63
libraries/transformer/src/main/java/androidx/media3/transformer/BaseSamplePipeline.java
View File

@ -16,7 +16,6 @@
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import androidx.annotation.Nullable;
@ -24,35 +23,20 @@ import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.decoder.DecoderInputBuffer;
import java.nio.ByteBuffer;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/* package */ abstract class BaseSamplePipeline implements SamplePipeline {
private final long streamStartPositionUs;
private final long streamOffsetUs;
private final MuxerWrapper muxerWrapper;
private final @C.TrackType int trackType;
private final @MonotonicNonNull SefSlowMotionFlattener sefVideoSlowMotionFlattener;
@Nullable private DecoderInputBuffer inputBuffer;
private boolean muxerWrapperTrackAdded;
public BaseSamplePipeline(
Format inputFormat,
long streamStartPositionUs,
long streamOffsetUs,
boolean flattenForSlowMotion,
MuxerWrapper muxerWrapper) {
Format inputFormat, long streamStartPositionUs, MuxerWrapper muxerWrapper) {
this.streamStartPositionUs = streamStartPositionUs;
this.streamOffsetUs = streamOffsetUs;
this.muxerWrapper = muxerWrapper;
trackType = MimeTypes.getTrackType(inputFormat.sampleMimeType);
sefVideoSlowMotionFlattener =
flattenForSlowMotion && trackType == C.TRACK_TYPE_VIDEO
? new SefSlowMotionFlattener(inputFormat)
: null;
}
protected static TransformationException createNoSupportedMimeTypeException(
@ -66,32 +50,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
}
@Nullable
@Override
public DecoderInputBuffer dequeueInputBuffer() throws TransformationException {
inputBuffer = dequeueInputBufferInternal();
return inputBuffer;
}
@Override
public void queueInputBuffer() throws TransformationException {
DecoderInputBuffer inputBuffer = checkNotNull(this.inputBuffer);
checkNotNull(inputBuffer.data);
if (!shouldDropInputBuffer(inputBuffer)) {
queueInputBufferInternal();
}
}
@Override
public boolean processData() throws TransformationException {
return feedMuxer() || processDataUpToMuxer();
}
@Nullable
protected abstract DecoderInputBuffer dequeueInputBufferInternal() throws TransformationException;
protected abstract void queueInputBufferInternal() throws TransformationException;
protected abstract boolean processDataUpToMuxer() throws TransformationException;
@Nullable
@ -104,30 +67,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
protected abstract boolean isMuxerInputEnded();
/**
* Preprocesses an {@linkplain DecoderInputBuffer input buffer} queued to the pipeline and returns
* whether it should be dropped.
*/
@RequiresNonNull("#1.data")
private boolean shouldDropInputBuffer(DecoderInputBuffer inputBuffer) {
ByteBuffer inputBytes = inputBuffer.data;
if (sefVideoSlowMotionFlattener == null || inputBuffer.isEndOfStream()) {
return false;
}
long presentationTimeUs = inputBuffer.timeUs - streamOffsetUs;
boolean shouldDropInputBuffer =
sefVideoSlowMotionFlattener.dropOrTransformSample(inputBytes, presentationTimeUs);
if (shouldDropInputBuffer) {
inputBytes.clear();
} else {
inputBuffer.timeUs =
streamOffsetUs + sefVideoSlowMotionFlattener.getSamplePresentationTimeUs();
}
return shouldDropInputBuffer;
}
/**
* Attempts to pass encoded data to the muxer, and returns whether it may be possible to pass more
* data immediately by calling this method again.

19
libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java
View File

@ -75,6 +75,7 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener;
MediaItem mediaItem,
boolean removeAudio,
boolean removeVideo,
boolean flattenForSlowMotion,
MediaSource.Factory mediaSourceFactory,
Codec.DecoderFactory decoderFactory,
Looper looper,
@ -99,7 +100,8 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener;
ExoPlayer.Builder playerBuilder =
new ExoPlayer.Builder(
context,
new RenderersFactoryImpl(removeAudio, removeVideo, decoderFactory, listener))
new RenderersFactoryImpl(
removeAudio, removeVideo, flattenForSlowMotion, decoderFactory, listener))
.setMediaSourceFactory(mediaSourceFactory)
.setTrackSelector(trackSelector)
.setLoadControl(loadControl)
@ -143,16 +145,19 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener;
private final TransformerMediaClock mediaClock;
private final boolean removeAudio;
private final boolean removeVideo;
private final boolean flattenForSlowMotion;
private final Codec.DecoderFactory decoderFactory;
private final ExoPlayerAssetLoader.Listener assetLoaderListener;
public RenderersFactoryImpl(
boolean removeAudio,
boolean removeVideo,
boolean flattenForSlowMotion,
Codec.DecoderFactory decoderFactory,
ExoPlayerAssetLoader.Listener assetLoaderListener) {
this.removeAudio = removeAudio;
this.removeVideo = removeVideo;
this.flattenForSlowMotion = flattenForSlowMotion;
this.decoderFactory = decoderFactory;
this.assetLoaderListener = assetLoaderListener;
mediaClock = new TransformerMediaClock();
@ -171,13 +176,21 @@ import androidx.media3.exoplayer.video.VideoRendererEventListener;
if (!removeAudio) {
renderers[index] =
new ExoPlayerAssetLoaderRenderer(
C.TRACK_TYPE_AUDIO, decoderFactory, mediaClock, assetLoaderListener);
C.TRACK_TYPE_AUDIO,
/* flattenForSlowMotion= */ false,
decoderFactory,
mediaClock,
assetLoaderListener);
index++;
}
if (!removeVideo) {
renderers[index] =
new ExoPlayerAssetLoaderRenderer(
C.TRACK_TYPE_VIDEO, decoderFactory, mediaClock, assetLoaderListener);
C.TRACK_TYPE_VIDEO,
flattenForSlowMotion,
decoderFactory,
mediaClock,
assetLoaderListener);
index++;
}
return renderers;

40
libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java
View File

@ -40,6 +40,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private static final String TAG = "ExoPlayerAssetLoaderRenderer";
private final boolean flattenForSlowMotion;
private final Codec.DecoderFactory decoderFactory;
private final TransformerMediaClock mediaClock;
private final ExoPlayerAssetLoader.Listener assetLoaderListener;
@ -48,6 +49,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private boolean isTransformationRunning;
private long streamStartPositionUs;
private long streamOffsetUs;
private @MonotonicNonNull SefSlowMotionFlattener sefVideoSlowMotionFlattener;
private @MonotonicNonNull Codec decoder;
@Nullable private ByteBuffer pendingDecoderOutputBuffer;
private SamplePipeline.@MonotonicNonNull Input samplePipelineInput;
@ -55,10 +57,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
public ExoPlayerAssetLoaderRenderer(
int trackType,
boolean flattenForSlowMotion,
Codec.DecoderFactory decoderFactory,
TransformerMediaClock mediaClock,
ExoPlayerAssetLoader.Listener assetLoaderListener) {
super(trackType);
this.flattenForSlowMotion = flattenForSlowMotion;
this.decoderFactory = decoderFactory;
this.mediaClock = mediaClock;
this.assetLoaderListener = assetLoaderListener;
@ -161,6 +165,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
Format inputFormat = checkNotNull(formatHolder.format);
samplePipelineInput =
assetLoaderListener.onTrackAdded(inputFormat, streamStartPositionUs, streamOffsetUs);
if (getTrackType() == C.TRACK_TYPE_VIDEO && flattenForSlowMotion) {
sefVideoSlowMotionFlattener = new SefSlowMotionFlattener(inputFormat);
}
if (samplePipelineInput.expectsDecodedData()) {
decoder = decoderFactory.createForAudioDecoding(inputFormat);
}
@ -228,6 +235,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return false;
}
if (shouldDropInputBuffer(decoderInputBuffer)) {
return true;
}
decoder.queueInputBuffer(decoderInputBuffer);
return true;
}
@ -249,6 +260,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return false;
}
if (shouldDropInputBuffer(samplePipelineInputBuffer)) {
return true;
}
samplePipelineInput.queueInputBuffer();
if (samplePipelineInputBuffer.isEndOfStream()) {
isEnded = true;
@ -279,4 +294,29 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return false;
}
}
/**
* Preprocesses an {@linkplain DecoderInputBuffer input buffer} queued to the pipeline and returns
* whether it should be dropped.
*
* <p>The input buffer is cleared if it should be dropped.
*/
private boolean shouldDropInputBuffer(DecoderInputBuffer inputBuffer) {
ByteBuffer inputBytes = checkNotNull(inputBuffer.data);
if (sefVideoSlowMotionFlattener == null || inputBuffer.isEndOfStream()) {
return false;
}
long presentationTimeUs = inputBuffer.timeUs - streamOffsetUs;
boolean shouldDropInputBuffer =
sefVideoSlowMotionFlattener.dropOrTransformSample(inputBytes, presentationTimeUs);
if (shouldDropInputBuffer) {
inputBytes.clear();
} else {
inputBuffer.timeUs =
streamOffsetUs + sefVideoSlowMotionFlattener.getSamplePresentationTimeUs();
}
return shouldDropInputBuffer;
}
}

18
libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java
View File

@ -31,16 +31,10 @@ import androidx.media3.decoder.DecoderInputBuffer;
public PassthroughSamplePipeline(
Format format,
long streamStartPositionUs,
long streamOffsetUs,
TransformationRequest transformationRequest,
MuxerWrapper muxerWrapper,
FallbackListener fallbackListener) {
super(
format,
streamStartPositionUs,
streamOffsetUs,
transformationRequest.flattenForSlowMotion,
muxerWrapper);
super(format, streamStartPositionUs, muxerWrapper);
this.format = format;
buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT);
fallbackListener.onTransformationRequestFinalized(transformationRequest);
@ -51,22 +45,22 @@ import androidx.media3.decoder.DecoderInputBuffer;
return false;
}
@Override
public void release() {}
@Override
@Nullable
protected DecoderInputBuffer dequeueInputBufferInternal() {
public DecoderInputBuffer dequeueInputBuffer() {
return hasPendingBuffer ? null : buffer;
}
@Override
protected void queueInputBufferInternal() {
public void queueInputBuffer() {
if (buffer.data != null && buffer.data.hasRemaining()) {
hasPendingBuffer = true;
}
}
@Override
public void release() {}
@Override
protected boolean processDataUpToMuxer() {
return false;

2
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
View File

@ -157,6 +157,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
mediaItem,
removeAudio,
removeVideo,
transformationRequest.flattenForSlowMotion,
mediaSourceFactory,
decoderFactory,
internalLooper,
@ -497,7 +498,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
return new PassthroughSamplePipeline(
inputFormat,
streamStartPositionUs,
streamOffsetUs,
transformationRequest,
muxerWrapper,
fallbackListener);

26
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -87,13 +87,7 @@ import org.checkerframework.dataflow.qual.Pure;
FallbackListener fallbackListener,
DebugViewProvider debugViewProvider)
throws TransformationException {
super(
inputFormat,
streamStartPositionUs,
streamOffsetUs,
transformationRequest.flattenForSlowMotion,
muxerWrapper);
super(inputFormat, streamStartPositionUs, muxerWrapper);
if (ColorInfo.isTransferHdr(inputFormat.colorInfo)) {
if (transformationRequest.hdrMode
== TransformationRequest.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR) {
@ -229,27 +223,27 @@ import org.checkerframework.dataflow.qual.Pure;
return false;
}
@Override
public void release() {
frameProcessor.release();
decoder.release();
encoderWrapper.release();
}
@Override
@Nullable
protected DecoderInputBuffer dequeueInputBufferInternal() throws TransformationException {
public DecoderInputBuffer dequeueInputBuffer() throws TransformationException {
return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null;
}
@Override
protected void queueInputBufferInternal() throws TransformationException {
public void queueInputBuffer() throws TransformationException {
if (decoderInputBuffer.isDecodeOnly()) {
decodeOnlyPresentationTimestamps.add(decoderInputBuffer.timeUs);
}
decoder.queueInputBuffer(decoderInputBuffer);
}
@Override
public void release() {
frameProcessor.release();
decoder.release();
encoderWrapper.release();
}
@Override
protected boolean processDataUpToMuxer() throws TransformationException {
if (decoder.isEnded()) {