From 1d888d20dc7c1cd66f76420847f7274b313131cf Mon Sep 17 00:00:00 2001 From: kimvde Date: Thu, 10 Nov 2022 10:32:35 +0000 Subject: [PATCH] Add TransformerInternal The player is still driving the transformation at this point. The transformer thread will be added in another CL. PiperOrigin-RevId: 487479148 --- .../transformer/ExoPlayerAssetLoader.java | 104 +------ ...java => ExoPlayerAssetLoaderRenderer.java} | 86 +++--- .../media3/transformer/MuxerWrapper.java | 2 +- .../media3/transformer/Transformer.java | 30 +- .../transformer/TransformerAudioRenderer.java | 132 --------- .../transformer/TransformerInternal.java | 263 ++++++++++++++++++ .../transformer/TransformerVideoRenderer.java | 169 ----------- .../VideoTranscodingSamplePipeline.java | 6 +- 8 files changed, 345 insertions(+), 447 deletions(-) rename libraries/transformer/src/main/java/androidx/media3/transformer/{TransformerBaseRenderer.java => ExoPlayerAssetLoaderRenderer.java} (63%) delete mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java delete mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java index 93d173c453..d7f29fefde 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java @@ -32,9 +32,7 @@ import android.os.Handler; import android.os.Looper; import androidx.annotation.Nullable; import androidx.media3.common.C; -import androidx.media3.common.DebugViewProvider; -import androidx.media3.common.Effect; -import androidx.media3.common.FrameProcessor; +import androidx.media3.common.Format; import androidx.media3.common.MediaItem; import androidx.media3.common.PlaybackException; import androidx.media3.common.Player; @@ -51,29 +49,27 @@ import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.text.TextOutput; import androidx.media3.exoplayer.trackselection.DefaultTrackSelector; import androidx.media3.exoplayer.video.VideoRendererEventListener; -import com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* package */ final class ExoPlayerAssetLoader { public interface Listener { + void onTrackRegistered(); + + SamplePipeline onTrackAdded(Format format, long streamStartPositionUs, long streamOffsetUs) + throws TransformationException; + void onEnded(); void onError(Exception e); } private final Context context; - private final TransformationRequest transformationRequest; - private final ImmutableList videoEffects; private final boolean removeAudio; private final boolean removeVideo; private final MediaSource.Factory mediaSourceFactory; - private final Codec.DecoderFactory decoderFactory; - private final Codec.EncoderFactory encoderFactory; - private final FrameProcessor.Factory frameProcessorFactory; private final Looper looper; - private final DebugViewProvider debugViewProvider; private final Clock clock; private @MonotonicNonNull MuxerWrapper muxerWrapper; @@ -82,28 +78,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public ExoPlayerAssetLoader( Context context, - TransformationRequest transformationRequest, - ImmutableList videoEffects, boolean removeAudio, boolean removeVideo, MediaSource.Factory mediaSourceFactory, - Codec.DecoderFactory decoderFactory, - Codec.EncoderFactory encoderFactory, - FrameProcessor.Factory frameProcessorFactory, Looper looper, - DebugViewProvider debugViewProvider, Clock clock) { this.context = context; - this.transformationRequest = transformationRequest; - this.videoEffects = videoEffects; this.removeAudio = removeAudio; this.removeVideo = removeVideo; this.mediaSourceFactory = mediaSourceFactory; - this.decoderFactory = decoderFactory; - this.encoderFactory = encoderFactory; - this.frameProcessorFactory = frameProcessorFactory; this.looper = looper; - this.debugViewProvider = debugViewProvider; this.clock = clock; progressState = PROGRESS_STATE_NO_TRANSFORMATION; } @@ -112,7 +96,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; MediaItem mediaItem, MuxerWrapper muxerWrapper, Listener listener, - FallbackListener fallbackListener, Transformer.AsyncErrorListener asyncErrorListener) { this.muxerWrapper = muxerWrapper; @@ -134,20 +117,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; ExoPlayer.Builder playerBuilder = new ExoPlayer.Builder( context, - new RenderersFactoryImpl( - context, - muxerWrapper, - removeAudio, - removeVideo, - transformationRequest, - mediaItem.clippingConfiguration.startsAtKeyFrame, - videoEffects, - frameProcessorFactory, - encoderFactory, - decoderFactory, - fallbackListener, - asyncErrorListener, - debugViewProvider)) + new RenderersFactoryImpl(removeAudio, removeVideo, listener, asyncErrorListener)) .setMediaSourceFactory(mediaSourceFactory) .setTrackSelector(trackSelector) .setLoadControl(loadControl) @@ -187,48 +157,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final class RenderersFactoryImpl implements RenderersFactory { - private final Context context; - private final MuxerWrapper muxerWrapper; private final TransformerMediaClock mediaClock; private final boolean removeAudio; private final boolean removeVideo; - private final TransformationRequest transformationRequest; - private final boolean clippingStartsAtKeyFrame; - private final ImmutableList videoEffects; - private final FrameProcessor.Factory frameProcessorFactory; - private final Codec.EncoderFactory encoderFactory; - private final Codec.DecoderFactory decoderFactory; - private final FallbackListener fallbackListener; + private final ExoPlayerAssetLoader.Listener assetLoaderListener; private final Transformer.AsyncErrorListener asyncErrorListener; - private final DebugViewProvider debugViewProvider; public RenderersFactoryImpl( - Context context, - MuxerWrapper muxerWrapper, boolean removeAudio, boolean removeVideo, - TransformationRequest transformationRequest, - boolean clippingStartsAtKeyFrame, - ImmutableList videoEffects, - FrameProcessor.Factory frameProcessorFactory, - Codec.EncoderFactory encoderFactory, - Codec.DecoderFactory decoderFactory, - FallbackListener fallbackListener, - Transformer.AsyncErrorListener asyncErrorListener, - DebugViewProvider debugViewProvider) { - this.context = context; - this.muxerWrapper = muxerWrapper; + ExoPlayerAssetLoader.Listener assetLoaderListener, + Transformer.AsyncErrorListener asyncErrorListener) { this.removeAudio = removeAudio; this.removeVideo = removeVideo; - this.transformationRequest = transformationRequest; - this.clippingStartsAtKeyFrame = clippingStartsAtKeyFrame; - this.videoEffects = videoEffects; - this.frameProcessorFactory = frameProcessorFactory; - this.encoderFactory = encoderFactory; - this.decoderFactory = decoderFactory; - this.fallbackListener = fallbackListener; + this.assetLoaderListener = assetLoaderListener; this.asyncErrorListener = asyncErrorListener; - this.debugViewProvider = debugViewProvider; mediaClock = new TransformerMediaClock(); } @@ -244,31 +187,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; int index = 0; if (!removeAudio) { renderers[index] = - new TransformerAudioRenderer( - muxerWrapper, - mediaClock, - transformationRequest, - encoderFactory, - decoderFactory, - asyncErrorListener, - fallbackListener); + new ExoPlayerAssetLoaderRenderer( + C.TRACK_TYPE_AUDIO, mediaClock, assetLoaderListener, asyncErrorListener); index++; } if (!removeVideo) { renderers[index] = - new TransformerVideoRenderer( - context, - muxerWrapper, - mediaClock, - transformationRequest, - clippingStartsAtKeyFrame, - videoEffects, - frameProcessorFactory, - encoderFactory, - decoderFactory, - asyncErrorListener, - fallbackListener, - debugViewProvider); + new ExoPlayerAssetLoaderRenderer( + C.TRACK_TYPE_VIDEO, mediaClock, assetLoaderListener, asyncErrorListener); index++; } return renderers; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java similarity index 63% rename from libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java rename to libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java index 0a54b43d77..0efc831da4 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoaderRenderer.java @@ -16,46 +16,53 @@ package androidx.media3.transformer; +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.decoder.DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED; +import static androidx.media3.exoplayer.source.SampleStream.FLAG_REQUIRE_FORMAT; + import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.exoplayer.BaseRenderer; +import androidx.media3.exoplayer.FormatHolder; import androidx.media3.exoplayer.MediaClock; import androidx.media3.exoplayer.RendererCapabilities; import androidx.media3.exoplayer.source.SampleStream.ReadDataResult; -import com.google.errorprone.annotations.ForOverride; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; -/* package */ abstract class TransformerBaseRenderer extends BaseRenderer { +/* package */ final class ExoPlayerAssetLoaderRenderer extends BaseRenderer { - protected final MuxerWrapper muxerWrapper; - protected final TransformerMediaClock mediaClock; - protected final TransformationRequest transformationRequest; - protected final Transformer.AsyncErrorListener asyncErrorListener; - protected final FallbackListener fallbackListener; + private static final String TAG = "ExoPlayerAssetLoaderRenderer"; + + private final TransformerMediaClock mediaClock; + private final ExoPlayerAssetLoader.Listener assetLoaderListener; + private final Transformer.AsyncErrorListener asyncErrorListener; + private final DecoderInputBuffer decoderInputBuffer; private boolean isTransformationRunning; - protected long streamOffsetUs; - protected long streamStartPositionUs; - protected @MonotonicNonNull SamplePipeline samplePipeline; + private long streamOffsetUs; + private long streamStartPositionUs; + private @MonotonicNonNull SamplePipeline samplePipeline; - public TransformerBaseRenderer( + public ExoPlayerAssetLoaderRenderer( int trackType, - MuxerWrapper muxerWrapper, TransformerMediaClock mediaClock, - TransformationRequest transformationRequest, - Transformer.AsyncErrorListener asyncErrorListener, - FallbackListener fallbackListener) { + ExoPlayerAssetLoader.Listener assetLoaderListener, + Transformer.AsyncErrorListener asyncErrorListener) { super(trackType); - this.muxerWrapper = muxerWrapper; this.mediaClock = mediaClock; - this.transformationRequest = transformationRequest; + this.assetLoaderListener = assetLoaderListener; this.asyncErrorListener = asyncErrorListener; - this.fallbackListener = fallbackListener; + decoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED); + } + + @Override + public String getName() { + return TAG; } /** @@ -65,7 +72,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * @return The {@link Capabilities} for this format. */ @Override - public final @Capabilities int supportsFormat(Format format) { + public @Capabilities int supportsFormat(Format format) { return RendererCapabilities.create( MimeTypes.getTrackType(format.sampleMimeType) == getTrackType() ? C.FORMAT_HANDLED @@ -73,22 +80,22 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } @Override - public final MediaClock getMediaClock() { + public MediaClock getMediaClock() { return mediaClock; } @Override - public final boolean isReady() { + public boolean isReady() { return isSourceReady(); } @Override - public final boolean isEnded() { + public boolean isEnded() { return samplePipeline != null && samplePipeline.isEnded(); } @Override - public final void render(long positionUs, long elapsedRealtimeUs) { + public void render(long positionUs, long elapsedRealtimeUs) { try { if (!isTransformationRunning || isEnded() || !ensureConfigured()) { return; @@ -97,43 +104,56 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; while (samplePipeline.processData() || feedPipelineFromInput()) {} } catch (TransformationException e) { isTransformationRunning = false; - asyncErrorListener.onTransformationException(e); + asyncErrorListener.onTransformationError(e); } } @Override - protected final void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) { + protected void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) { this.streamOffsetUs = offsetUs; this.streamStartPositionUs = startPositionUs; } @Override - protected final void onEnabled(boolean joining, boolean mayRenderStartOfStream) { - muxerWrapper.registerTrack(); - fallbackListener.registerTrack(); + protected void onEnabled(boolean joining, boolean mayRenderStartOfStream) { + assetLoaderListener.onTrackRegistered(); mediaClock.updateTimeForTrackType(getTrackType(), 0L); } @Override - protected final void onStarted() { + protected void onStarted() { isTransformationRunning = true; } @Override - protected final void onStopped() { + protected void onStopped() { isTransformationRunning = false; } @Override - protected final void onReset() { + protected void onReset() { if (samplePipeline != null) { samplePipeline.release(); } } - @ForOverride @EnsuresNonNullIf(expression = "samplePipeline", result = true) - protected abstract boolean ensureConfigured() throws TransformationException; + private boolean ensureConfigured() throws TransformationException { + if (samplePipeline != null) { + return true; + } + + FormatHolder formatHolder = getFormatHolder(); + @ReadDataResult + int result = readSource(formatHolder, decoderInputBuffer, /* readFlags= */ FLAG_REQUIRE_FORMAT); + if (result != C.RESULT_FORMAT_READ) { + return false; + } + Format inputFormat = checkNotNull(formatHolder.format); + samplePipeline = + assetLoaderListener.onTrackAdded(inputFormat, streamStartPositionUs, streamOffsetUs); + return true; + } /** * Attempts to read input data and pass the input data to the sample pipeline. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java index f22b8668fa..ea7bc24953 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java @@ -311,7 +311,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return; } isAborted = true; - asyncErrorListener.onTransformationException( + asyncErrorListener.onTransformationError( TransformationException.createForMuxer( new IllegalStateException( "No output sample written in the last " diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index de50b10f56..6d1dfd75f8 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -35,7 +35,6 @@ import androidx.media3.common.FrameProcessor; import androidx.media3.common.MediaItem; import androidx.media3.common.MediaLibraryInfo; import androidx.media3.common.MimeTypes; -import androidx.media3.common.PlaybackException; import androidx.media3.common.util.Clock; import androidx.media3.common.util.ListenerSet; import androidx.media3.common.util.UnstableApi; @@ -552,7 +551,7 @@ public final class Transformer { private final Looper looper; private final DebugViewProvider debugViewProvider; private final Clock clock; - private final ExoPlayerAssetLoader exoPlayerAssetLoader; + private final TransformerInternal transformerInternal; @Nullable private MuxerWrapper muxerWrapper; @Nullable private String outputPath; @@ -590,8 +589,8 @@ public final class Transformer { this.looper = looper; this.debugViewProvider = debugViewProvider; this.clock = clock; - exoPlayerAssetLoader = - new ExoPlayerAssetLoader( + transformerInternal = + new TransformerInternal( context, transformationRequest, videoEffects, @@ -730,7 +729,7 @@ public final class Transformer { this.muxerWrapper = muxerWrapper; FallbackListener fallbackListener = new FallbackListener(mediaItem, listeners, transformationRequest); - exoPlayerAssetLoader.start( + transformerInternal.start( mediaItem, muxerWrapper, /* listener= */ componentListener, @@ -761,7 +760,7 @@ public final class Transformer { */ public @ProgressState int getProgress(ProgressHolder progressHolder) { verifyApplicationThread(); - return exoPlayerAssetLoader.getProgress(progressHolder); + return transformerInternal.getProgress(progressHolder); } /** @@ -791,7 +790,7 @@ public final class Transformer { */ private void releaseResources(boolean forCancellation) throws TransformationException { transformationInProgress = false; - exoPlayerAssetLoader.release(); + transformerInternal.release(); if (muxerWrapper != null) { try { muxerWrapper.release(forCancellation); @@ -836,11 +835,11 @@ public final class Transformer { * *

Can be called from any thread. */ - void onTransformationException(TransformationException exception); + void onTransformationError(TransformationException exception); } private final class ComponentListener - implements ExoPlayerAssetLoader.Listener, AsyncErrorListener { + implements TransformerInternal.Listener, AsyncErrorListener { private final MediaItem mediaItem; private final Handler handler; @@ -851,21 +850,12 @@ public final class Transformer { } @Override - public void onError(Exception e) { - TransformationException transformationException = - e instanceof PlaybackException - ? TransformationException.createForPlaybackException((PlaybackException) e) - : TransformationException.createForUnexpected(e); - handleTransformationException(transformationException); - } - - @Override - public void onEnded() { + public void onTransformationCompleted() { handleTransformationEnded(/* exception= */ null); } @Override - public void onTransformationException(TransformationException exception) { + public void onTransformationError(TransformationException exception) { if (Looper.myLooper() == looper) { handleTransformationException(exception); } else { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java deleted file mode 100644 index f998f4a269..0000000000 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package androidx.media3.transformer; - -import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.exoplayer.source.SampleStream.FLAG_REQUIRE_FORMAT; - -import androidx.annotation.Nullable; -import androidx.media3.common.C; -import androidx.media3.common.Format; -import androidx.media3.common.Metadata; -import androidx.media3.decoder.DecoderInputBuffer; -import androidx.media3.exoplayer.FormatHolder; -import androidx.media3.exoplayer.source.SampleStream.ReadDataResult; -import androidx.media3.extractor.metadata.mp4.SlowMotionData; - -/* package */ final class TransformerAudioRenderer extends TransformerBaseRenderer { - - private static final String TAG = "TAudioRenderer"; - - private final Codec.EncoderFactory encoderFactory; - private final Codec.DecoderFactory decoderFactory; - private final DecoderInputBuffer decoderInputBuffer; - - public TransformerAudioRenderer( - MuxerWrapper muxerWrapper, - TransformerMediaClock mediaClock, - TransformationRequest transformationRequest, - Codec.EncoderFactory encoderFactory, - Codec.DecoderFactory decoderFactory, - Transformer.AsyncErrorListener asyncErrorListener, - FallbackListener fallbackListener) { - super( - C.TRACK_TYPE_AUDIO, - muxerWrapper, - mediaClock, - transformationRequest, - asyncErrorListener, - fallbackListener); - this.encoderFactory = encoderFactory; - this.decoderFactory = decoderFactory; - decoderInputBuffer = - new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); - } - - @Override - public String getName() { - return TAG; - } - - /** Attempts to read the input format and to initialize the {@link SamplePipeline}. */ - @Override - protected boolean ensureConfigured() throws TransformationException { - if (samplePipeline != null) { - return true; - } - FormatHolder formatHolder = getFormatHolder(); - @ReadDataResult - int result = readSource(formatHolder, decoderInputBuffer, /* readFlags= */ FLAG_REQUIRE_FORMAT); - if (result != C.RESULT_FORMAT_READ) { - return false; - } - Format inputFormat = checkNotNull(formatHolder.format); - if (shouldPassthrough(inputFormat)) { - samplePipeline = - new PassthroughSamplePipeline( - inputFormat, - streamOffsetUs, - streamStartPositionUs, - transformationRequest, - muxerWrapper, - fallbackListener); - } else { - samplePipeline = - new AudioTranscodingSamplePipeline( - inputFormat, - streamOffsetUs, - streamStartPositionUs, - transformationRequest, - decoderFactory, - encoderFactory, - muxerWrapper, - fallbackListener); - } - return true; - } - - private boolean shouldPassthrough(Format inputFormat) { - if (encoderFactory.audioNeedsEncoding()) { - return false; - } - if (transformationRequest.audioMimeType != null - && !transformationRequest.audioMimeType.equals(inputFormat.sampleMimeType)) { - return false; - } - if (transformationRequest.audioMimeType == null - && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) { - return false; - } - if (transformationRequest.flattenForSlowMotion && isSlowMotion(inputFormat)) { - return false; - } - return true; - } - - private static boolean isSlowMotion(Format format) { - @Nullable Metadata metadata = format.metadata; - if (metadata == null) { - return false; - } - for (int i = 0; i < metadata.length(); i++) { - if (metadata.get(i) instanceof SlowMotionData) { - return true; - } - } - return false; - } -} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java new file mode 100644 index 0000000000..7d47459461 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -0,0 +1,263 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import android.content.Context; +import android.os.Looper; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.DebugViewProvider; +import androidx.media3.common.Effect; +import androidx.media3.common.Format; +import androidx.media3.common.FrameProcessor; +import androidx.media3.common.MediaItem; +import androidx.media3.common.Metadata; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.PlaybackException; +import androidx.media3.common.util.Clock; +import androidx.media3.exoplayer.source.MediaSource; +import androidx.media3.extractor.metadata.mp4.SlowMotionData; +import com.google.common.collect.ImmutableList; + +/* package */ final class TransformerInternal { + + public interface Listener { + + void onTransformationCompleted(); + + void onTransformationError(TransformationException exception); + } + + private final Context context; + private final TransformationRequest transformationRequest; + private final ImmutableList videoEffects; + private final Codec.DecoderFactory decoderFactory; + private final Codec.EncoderFactory encoderFactory; + private final FrameProcessor.Factory frameProcessorFactory; + private final DebugViewProvider debugViewProvider; + private final ExoPlayerAssetLoader exoPlayerAssetLoader; + + public TransformerInternal( + Context context, + TransformationRequest transformationRequest, + ImmutableList videoEffects, + boolean removeAudio, + boolean removeVideo, + MediaSource.Factory mediaSourceFactory, + Codec.DecoderFactory decoderFactory, + Codec.EncoderFactory encoderFactory, + FrameProcessor.Factory frameProcessorFactory, + Looper looper, + DebugViewProvider debugViewProvider, + Clock clock) { + this.context = context; + this.transformationRequest = transformationRequest; + this.videoEffects = videoEffects; + this.decoderFactory = decoderFactory; + this.encoderFactory = encoderFactory; + this.frameProcessorFactory = frameProcessorFactory; + this.debugViewProvider = debugViewProvider; + exoPlayerAssetLoader = + new ExoPlayerAssetLoader( + context, removeAudio, removeVideo, mediaSourceFactory, looper, clock); + } + + public void start( + MediaItem mediaItem, + MuxerWrapper muxerWrapper, + Listener listener, + FallbackListener fallbackListener, + Transformer.AsyncErrorListener asyncErrorListener) { + ComponentListener componentListener = + new ComponentListener( + mediaItem, muxerWrapper, listener, fallbackListener, asyncErrorListener); + exoPlayerAssetLoader.start(mediaItem, muxerWrapper, componentListener, asyncErrorListener); + } + + public @Transformer.ProgressState int getProgress(ProgressHolder progressHolder) { + return exoPlayerAssetLoader.getProgress(progressHolder); + } + + public void release() { + exoPlayerAssetLoader.release(); + } + + private class ComponentListener implements ExoPlayerAssetLoader.Listener { + + private final MediaItem mediaItem; + private final MuxerWrapper muxerWrapper; + private final TransformerInternal.Listener listener; + private final FallbackListener fallbackListener; + private final Transformer.AsyncErrorListener asyncErrorListener; + + public ComponentListener( + MediaItem mediaItem, + MuxerWrapper muxerWrapper, + Listener listener, + FallbackListener fallbackListener, + Transformer.AsyncErrorListener asyncErrorListener) { + this.mediaItem = mediaItem; + this.muxerWrapper = muxerWrapper; + this.listener = listener; + this.fallbackListener = fallbackListener; + this.asyncErrorListener = asyncErrorListener; + } + + @Override + public void onTrackRegistered() { + muxerWrapper.registerTrack(); + fallbackListener.registerTrack(); + } + + @Override + public SamplePipeline onTrackAdded( + Format format, long streamStartPositionUs, long streamOffsetUs) + throws TransformationException { + return getSamplePipeline(format, streamStartPositionUs, streamOffsetUs); + } + + @Override + public void onError(Exception e) { + TransformationException transformationException = + e instanceof PlaybackException + ? TransformationException.createForPlaybackException((PlaybackException) e) + : TransformationException.createForUnexpected(e); + listener.onTransformationError(transformationException); + } + + @Override + public void onEnded() { + listener.onTransformationCompleted(); + } + + private SamplePipeline getSamplePipeline( + Format inputFormat, long streamStartPositionUs, long streamOffsetUs) + throws TransformationException { + if (MimeTypes.isAudio(inputFormat.sampleMimeType) && shouldTranscodeAudio(inputFormat)) { + return new AudioTranscodingSamplePipeline( + inputFormat, + streamOffsetUs, + streamOffsetUs, + transformationRequest, + decoderFactory, + encoderFactory, + muxerWrapper, + fallbackListener); + } else if (MimeTypes.isVideo(inputFormat.sampleMimeType) + && shouldTranscodeVideo(inputFormat, streamStartPositionUs, streamOffsetUs)) { + return new VideoTranscodingSamplePipeline( + context, + inputFormat, + streamOffsetUs, + streamStartPositionUs, + transformationRequest, + videoEffects, + frameProcessorFactory, + decoderFactory, + encoderFactory, + muxerWrapper, + fallbackListener, + asyncErrorListener, + debugViewProvider); + } else { + return new PassthroughSamplePipeline( + inputFormat, + streamOffsetUs, + streamStartPositionUs, + transformationRequest, + muxerWrapper, + fallbackListener); + } + } + + private boolean shouldTranscodeAudio(Format inputFormat) { + if (encoderFactory.audioNeedsEncoding()) { + return true; + } + if (transformationRequest.audioMimeType != null + && !transformationRequest.audioMimeType.equals(inputFormat.sampleMimeType)) { + return true; + } + if (transformationRequest.audioMimeType == null + && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) { + return true; + } + if (transformationRequest.flattenForSlowMotion && isSlowMotion(inputFormat)) { + return true; + } + return false; + } + + private boolean isSlowMotion(Format format) { + @Nullable Metadata metadata = format.metadata; + if (metadata == null) { + return false; + } + for (int i = 0; i < metadata.length(); i++) { + if (metadata.get(i) instanceof SlowMotionData) { + return true; + } + } + return false; + } + + private boolean shouldTranscodeVideo( + Format inputFormat, long streamStartPositionUs, long streamOffsetUs) { + if ((streamStartPositionUs - streamOffsetUs) != 0 + && !mediaItem.clippingConfiguration.startsAtKeyFrame) { + return true; + } + if (encoderFactory.videoNeedsEncoding()) { + return true; + } + if (transformationRequest.enableRequestSdrToneMapping) { + return true; + } + if (transformationRequest.videoMimeType != null + && !transformationRequest.videoMimeType.equals(inputFormat.sampleMimeType)) { + return true; + } + if (transformationRequest.videoMimeType == null + && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) { + return true; + } + if (inputFormat.pixelWidthHeightRatio != 1f) { + return true; + } + if (transformationRequest.rotationDegrees != 0f) { + return true; + } + if (transformationRequest.scaleX != 1f) { + return true; + } + if (transformationRequest.scaleY != 1f) { + return true; + } + // The decoder rotates encoded frames for display by inputFormat.rotationDegrees. + int decodedHeight = + (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; + if (transformationRequest.outputHeight != C.LENGTH_UNSET + && transformationRequest.outputHeight != decodedHeight) { + return true; + } + if (!videoEffects.isEmpty()) { + return true; + } + return false; + } + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java deleted file mode 100644 index 56aeba52f5..0000000000 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2021 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package androidx.media3.transformer; - -import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.exoplayer.source.SampleStream.FLAG_REQUIRE_FORMAT; - -import android.content.Context; -import androidx.media3.common.C; -import androidx.media3.common.DebugViewProvider; -import androidx.media3.common.Effect; -import androidx.media3.common.Format; -import androidx.media3.common.FrameProcessor; -import androidx.media3.decoder.DecoderInputBuffer; -import androidx.media3.exoplayer.FormatHolder; -import androidx.media3.exoplayer.source.SampleStream.ReadDataResult; -import com.google.common.collect.ImmutableList; - -/* package */ final class TransformerVideoRenderer extends TransformerBaseRenderer { - - private static final String TAG = "TVideoRenderer"; - - private final Context context; - private final boolean clippingStartsAtKeyFrame; - private final ImmutableList effects; - private final FrameProcessor.Factory frameProcessorFactory; - private final Codec.EncoderFactory encoderFactory; - private final Codec.DecoderFactory decoderFactory; - private final DebugViewProvider debugViewProvider; - private final DecoderInputBuffer decoderInputBuffer; - - public TransformerVideoRenderer( - Context context, - MuxerWrapper muxerWrapper, - TransformerMediaClock mediaClock, - TransformationRequest transformationRequest, - boolean clippingStartsAtKeyFrame, - ImmutableList effects, - FrameProcessor.Factory frameProcessorFactory, - Codec.EncoderFactory encoderFactory, - Codec.DecoderFactory decoderFactory, - Transformer.AsyncErrorListener asyncErrorListener, - FallbackListener fallbackListener, - DebugViewProvider debugViewProvider) { - super( - C.TRACK_TYPE_VIDEO, - muxerWrapper, - mediaClock, - transformationRequest, - asyncErrorListener, - fallbackListener); - this.context = context; - this.clippingStartsAtKeyFrame = clippingStartsAtKeyFrame; - this.effects = effects; - this.frameProcessorFactory = frameProcessorFactory; - this.encoderFactory = encoderFactory; - this.decoderFactory = decoderFactory; - this.debugViewProvider = debugViewProvider; - decoderInputBuffer = - new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); - } - - @Override - public String getName() { - return TAG; - } - - /** Attempts to read the input format and to initialize the {@link SamplePipeline}. */ - @Override - protected boolean ensureConfigured() throws TransformationException { - if (samplePipeline != null) { - return true; - } - FormatHolder formatHolder = getFormatHolder(); - @ReadDataResult - int result = readSource(formatHolder, decoderInputBuffer, /* readFlags= */ FLAG_REQUIRE_FORMAT); - if (result != C.RESULT_FORMAT_READ) { - return false; - } - Format inputFormat = checkNotNull(formatHolder.format); - if (shouldTranscode(inputFormat)) { - samplePipeline = - new VideoTranscodingSamplePipeline( - context, - inputFormat, - streamOffsetUs, - streamStartPositionUs, - transformationRequest, - effects, - frameProcessorFactory, - decoderFactory, - encoderFactory, - muxerWrapper, - fallbackListener, - asyncErrorListener, - debugViewProvider); - } else { - samplePipeline = - new PassthroughSamplePipeline( - inputFormat, - streamOffsetUs, - streamStartPositionUs, - transformationRequest, - muxerWrapper, - fallbackListener); - } - return true; - } - - private boolean shouldTranscode(Format inputFormat) { - if ((streamStartPositionUs - streamOffsetUs) != 0 && !clippingStartsAtKeyFrame) { - return true; - } - if (encoderFactory.videoNeedsEncoding()) { - return true; - } - if (transformationRequest.enableRequestSdrToneMapping) { - return true; - } - if (transformationRequest.forceInterpretHdrVideoAsSdr) { - return true; - } - if (transformationRequest.videoMimeType != null - && !transformationRequest.videoMimeType.equals(inputFormat.sampleMimeType)) { - return true; - } - if (transformationRequest.videoMimeType == null - && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) { - return true; - } - if (inputFormat.pixelWidthHeightRatio != 1f) { - return true; - } - if (transformationRequest.rotationDegrees != 0f) { - return true; - } - if (transformationRequest.scaleX != 1f) { - return true; - } - if (transformationRequest.scaleY != 1f) { - return true; - } - // The decoder rotates encoded frames for display by inputFormat.rotationDegrees. - int decodedHeight = - (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; - if (transformationRequest.outputHeight != C.LENGTH_UNSET - && transformationRequest.outputHeight != decodedHeight) { - return true; - } - if (!effects.isEmpty()) { - return true; - } - return false; - } -} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 0255e606fb..bd57fca978 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -154,7 +154,7 @@ import org.checkerframework.dataflow.qual.Pure; checkNotNull(frameProcessor) .setOutputSurfaceInfo(encoderWrapper.getSurfaceInfo(width, height)); } catch (TransformationException exception) { - asyncErrorListener.onTransformationException(exception); + asyncErrorListener.onTransformationError(exception); } } @@ -165,7 +165,7 @@ import org.checkerframework.dataflow.qual.Pure; @Override public void onFrameProcessingError(FrameProcessingException exception) { - asyncErrorListener.onTransformationException( + asyncErrorListener.onTransformationError( TransformationException.createForFrameProcessingException( exception, TransformationException.ERROR_CODE_FRAME_PROCESSING_FAILED)); } @@ -175,7 +175,7 @@ import org.checkerframework.dataflow.qual.Pure; try { encoderWrapper.signalEndOfInputStream(); } catch (TransformationException exception) { - asyncErrorListener.onTransformationException(exception); + asyncErrorListener.onTransformationError(exception); } } },