From 551a47e1fc9f2b0b68b60112f6071816189d6b27 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Tue, 23 Nov 2021 10:38:52 +0000 Subject: [PATCH] Deduplicate transformer audio and video renderer implementations. This change moves methods that are the same in `TransformerAudioRenderer` and `TransformerVideoRenderer` to `TransformerBaseRenderer`. PiperOrigin-RevId: 411758928 --- .../transformer/AudioSamplePipeline.java | 1 + .../transformer/TransformerAudioRenderer.java | 105 +-------------- .../transformer/TransformerBaseRenderer.java | 127 +++++++++++++++++- .../transformer/TransformerVideoRenderer.java | 126 ++--------------- 4 files changed, 138 insertions(+), 221 deletions(-) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java index 9360162e04..d4510d4df6 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/AudioSamplePipeline.java @@ -150,6 +150,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; encoderOutputBuffer.data = encoder.getOutputBuffer(); if (encoderOutputBuffer.data != null) { encoderOutputBuffer.timeUs = checkNotNull(encoder.getOutputBufferInfo()).presentationTimeUs; + encoderOutputBuffer.setFlags(C.BUFFER_FLAG_KEY_FRAME); return encoderOutputBuffer; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java index a8f125fcf2..74110ed1be 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java @@ -17,10 +17,8 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.exoplayer.source.SampleStream.FLAG_REQUIRE_FORMAT; -import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import androidx.media3.common.C; import androidx.media3.common.Format; @@ -28,9 +26,6 @@ import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.exoplayer.ExoPlaybackException; import androidx.media3.exoplayer.FormatHolder; import androidx.media3.exoplayer.source.SampleStream.ReadDataResult; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; @RequiresApi(18) /* package */ final class TransformerAudioRenderer extends TransformerBaseRenderer { @@ -39,10 +34,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private final DecoderInputBuffer decoderInputBuffer; - private @MonotonicNonNull SamplePipeline samplePipeline; - private boolean muxerWrapperTrackAdded; - private boolean muxerWrapperTrackEnded; - public TransformerAudioRenderer( MuxerWrapper muxerWrapper, TransformerMediaClock mediaClock, Transformation transformation) { super(C.TRACK_TYPE_AUDIO, muxerWrapper, mediaClock, transformation); @@ -55,32 +46,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return TAG; } + /** Attempts to read the input format and to initialize the sample or passthrough pipeline. */ @Override - public boolean isEnded() { - return muxerWrapperTrackEnded; - } - - @Override - protected void onReset() { - if (samplePipeline != null) { - samplePipeline.release(); - } - muxerWrapperTrackAdded = false; - muxerWrapperTrackEnded = false; - } - - @Override - public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { - if (!isRendererStarted || isEnded() || !ensureRendererConfigured()) { - return; - } - - while (feedMuxerFromPipeline() || samplePipeline.processData() || feedPipelineFromInput()) {} - } - - /** Attempts to read the input format and to initialize the sample pipeline. */ - @EnsuresNonNullIf(expression = "samplePipeline", result = true) - private boolean ensureRendererConfigured() throws ExoPlaybackException { + protected boolean ensureConfigured() throws ExoPlaybackException { if (samplePipeline != null) { return true; } @@ -100,73 +68,4 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } return true; } - - /** - * Attempts to write sample pipeline output data to the muxer. - * - * @return Whether it may be possible to write more data immediately by calling this method again. - */ - @RequiresNonNull("samplePipeline") - private boolean feedMuxerFromPipeline() { - if (!muxerWrapperTrackAdded) { - @Nullable Format samplePipelineOutputFormat = samplePipeline.getOutputFormat(); - if (samplePipelineOutputFormat == null) { - return false; - } - muxerWrapperTrackAdded = true; - muxerWrapper.addTrackFormat(samplePipelineOutputFormat); - } - - if (samplePipeline.isEnded()) { - muxerWrapper.endTrack(getTrackType()); - muxerWrapperTrackEnded = true; - return false; - } - @Nullable DecoderInputBuffer samplePipelineOutputBuffer = samplePipeline.getOutputBuffer(); - if (samplePipelineOutputBuffer == null) { - return false; - } - if (!muxerWrapper.writeSample( - getTrackType(), - checkStateNotNull(samplePipelineOutputBuffer.data), - /* isKeyFrame= */ true, - samplePipelineOutputBuffer.timeUs)) { - return false; - } - samplePipeline.releaseOutputBuffer(); - return true; - } - - /** - * Attempts to pass input data to the sample pipeline. - * - * @return Whether it may be possible to pass more data immediately by calling this method again. - */ - @RequiresNonNull("samplePipeline") - private boolean feedPipelineFromInput() { - @Nullable DecoderInputBuffer samplePipelineInputBuffer = samplePipeline.dequeueInputBuffer(); - if (samplePipelineInputBuffer == null) { - return false; - } - - @ReadDataResult - int result = readSource(getFormatHolder(), samplePipelineInputBuffer, /* readFlags= */ 0); - switch (result) { - case C.RESULT_BUFFER_READ: - if (samplePipelineInputBuffer.isEndOfStream()) { - samplePipeline.queueInputBuffer(); - return false; - } - mediaClock.updateTimeForTrackType(getTrackType(), samplePipelineInputBuffer.timeUs); - samplePipelineInputBuffer.timeUs -= streamOffsetUs; - samplePipelineInputBuffer.flip(); - samplePipeline.queueInputBuffer(); - return true; - case C.RESULT_FORMAT_READ: - throw new IllegalStateException("Format changes are not supported."); - case C.RESULT_NOTHING_READ: - default: - return false; - } - } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java index 8d6a368c3e..9387812f57 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java @@ -16,15 +16,23 @@ package androidx.media3.transformer; +import static androidx.media3.common.util.Assertions.checkStateNotNull; + import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; +import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.exoplayer.BaseRenderer; import androidx.media3.exoplayer.ExoPlaybackException; import androidx.media3.exoplayer.MediaClock; import androidx.media3.exoplayer.RendererCapabilities; +import androidx.media3.exoplayer.source.SampleStream.ReadDataResult; +import com.google.errorprone.annotations.ForOverride; +import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; @RequiresApi(18) /* package */ abstract class TransformerBaseRenderer extends BaseRenderer { @@ -34,7 +42,10 @@ import androidx.media3.exoplayer.RendererCapabilities; protected final Transformation transformation; protected boolean isRendererStarted; + protected boolean muxerWrapperTrackAdded; + protected boolean muxerWrapperTrackEnded; protected long streamOffsetUs; + protected @MonotonicNonNull SamplePipeline samplePipeline; public TransformerBaseRenderer( int trackType, @@ -47,11 +58,6 @@ import androidx.media3.exoplayer.RendererCapabilities; this.transformation = transformation; } - @Override - protected void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) { - this.streamOffsetUs = offsetUs; - } - @Override @C.FormatSupport public final int supportsFormat(Format format) { @@ -84,6 +90,34 @@ import androidx.media3.exoplayer.RendererCapabilities; return mediaClock; } + @Override + public final boolean isEnded() { + return muxerWrapperTrackEnded; + } + + @Override + public final void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { + if (!isRendererStarted || isEnded() || !ensureConfigured()) { + return; + } + + while (feedMuxerFromPipeline() || samplePipeline.processData() || feedPipelineFromInput()) {} + } + + @Override + protected final void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) { + this.streamOffsetUs = offsetUs; + } + + @Override + protected final void onReset() { + if (samplePipeline != null) { + samplePipeline.release(); + } + muxerWrapperTrackAdded = false; + muxerWrapperTrackEnded = false; + } + @Override protected final void onEnabled(boolean joining, boolean mayRenderStartOfStream) { muxerWrapper.registerTrack(); @@ -91,7 +125,7 @@ import androidx.media3.exoplayer.RendererCapabilities; } @Override - protected void onStarted() throws ExoPlaybackException { + protected final void onStarted() { isRendererStarted = true; } @@ -99,4 +133,85 @@ import androidx.media3.exoplayer.RendererCapabilities; protected final void onStopped() { isRendererStarted = false; } + + @ForOverride + @EnsuresNonNullIf(expression = "samplePipeline", result = true) + protected abstract boolean ensureConfigured() throws ExoPlaybackException; + + @RequiresNonNull({"samplePipeline", "#1.data"}) + protected void maybeQueueSampleToPipeline(DecoderInputBuffer inputBuffer) { + samplePipeline.queueInputBuffer(); + } + + /** + * Attempts to write sample pipeline output data to the muxer. + * + * @return Whether it may be possible to write more data immediately by calling this method again. + */ + @RequiresNonNull("samplePipeline") + private boolean feedMuxerFromPipeline() { + if (!muxerWrapperTrackAdded) { + @Nullable Format samplePipelineOutputFormat = samplePipeline.getOutputFormat(); + if (samplePipelineOutputFormat == null) { + return false; + } + muxerWrapperTrackAdded = true; + muxerWrapper.addTrackFormat(samplePipelineOutputFormat); + } + + if (samplePipeline.isEnded()) { + muxerWrapper.endTrack(getTrackType()); + muxerWrapperTrackEnded = true; + return false; + } + + @Nullable DecoderInputBuffer samplePipelineOutputBuffer = samplePipeline.getOutputBuffer(); + if (samplePipelineOutputBuffer == null) { + return false; + } + + if (!muxerWrapper.writeSample( + getTrackType(), + checkStateNotNull(samplePipelineOutputBuffer.data), + samplePipelineOutputBuffer.isKeyFrame(), + samplePipelineOutputBuffer.timeUs)) { + return false; + } + samplePipeline.releaseOutputBuffer(); + return true; + } + + /** + * Attempts to read input data and pass the input data to the sample pipeline. + * + * @return Whether it may be possible to read more data immediately by calling this method again. + */ + @RequiresNonNull("samplePipeline") + private boolean feedPipelineFromInput() { + @Nullable DecoderInputBuffer samplePipelineInputBuffer = samplePipeline.dequeueInputBuffer(); + if (samplePipelineInputBuffer == null) { + return false; + } + + @ReadDataResult + int result = readSource(getFormatHolder(), samplePipelineInputBuffer, /* readFlags= */ 0); + switch (result) { + case C.RESULT_BUFFER_READ: + if (samplePipelineInputBuffer.isEndOfStream()) { + samplePipeline.queueInputBuffer(); + return false; + } + mediaClock.updateTimeForTrackType(getTrackType(), samplePipelineInputBuffer.timeUs); + samplePipelineInputBuffer.timeUs -= streamOffsetUs; + samplePipelineInputBuffer.flip(); + checkStateNotNull(samplePipelineInputBuffer.data); + maybeQueueSampleToPipeline(samplePipelineInputBuffer); + return true; + case C.RESULT_FORMAT_READ: + throw new IllegalStateException("Format changes are not supported."); + case C.RESULT_NOTHING_READ: + default: + return false; + } + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java index 8a1d6aa057..8f215c23ad 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java @@ -17,11 +17,9 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkNotNull; -import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.exoplayer.source.SampleStream.FLAG_REQUIRE_FORMAT; import android.content.Context; -import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import androidx.media3.common.C; import androidx.media3.common.Format; @@ -30,7 +28,6 @@ import androidx.media3.exoplayer.ExoPlaybackException; import androidx.media3.exoplayer.FormatHolder; import androidx.media3.exoplayer.source.SampleStream.ReadDataResult; import java.nio.ByteBuffer; -import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -43,9 +40,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; private final DecoderInputBuffer decoderInputBuffer; private @MonotonicNonNull SefSlowMotionFlattener sefSlowMotionFlattener; - private @MonotonicNonNull SamplePipeline samplePipeline; - private boolean muxerWrapperTrackAdded; - private boolean muxerWrapperTrackEnded; public TransformerVideoRenderer( Context context, @@ -63,32 +57,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return TAG; } - @Override - public boolean isEnded() { - return muxerWrapperTrackEnded; - } - - @Override - protected void onReset() { - if (samplePipeline != null) { - samplePipeline.release(); - } - muxerWrapperTrackAdded = false; - muxerWrapperTrackEnded = false; - } - - @Override - public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { - if (!isRendererStarted || isEnded() || !ensureRendererConfigured()) { - return; - } - - while (feedMuxerFromPipeline() || samplePipeline.processData() || feedPipelineFromInput()) {} - } - /** Attempts to read the input format and to initialize the sample or passthrough pipeline. */ - @EnsuresNonNullIf(expression = "samplePipeline", result = true) - private boolean ensureRendererConfigured() throws ExoPlaybackException { + @Override + protected boolean ensureConfigured() throws ExoPlaybackException { if (samplePipeline != null) { return true; } @@ -115,88 +86,19 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } /** - * Attempts to write sample pipeline output data to the muxer. - * - * @return Whether it may be possible to write more data immediately by calling this method again. + * Queues the input buffer to the sample pipeline unless it should be dropped because of slow + * motion flattening. */ - @RequiresNonNull("samplePipeline") - private boolean feedMuxerFromPipeline() { - if (!muxerWrapperTrackAdded) { - @Nullable Format samplePipelineOutputFormat = samplePipeline.getOutputFormat(); - if (samplePipelineOutputFormat == null) { - return false; - } - muxerWrapperTrackAdded = true; - muxerWrapper.addTrackFormat(samplePipelineOutputFormat); - } - - if (samplePipeline.isEnded()) { - muxerWrapper.endTrack(getTrackType()); - muxerWrapperTrackEnded = true; - return false; - } - - @Nullable DecoderInputBuffer samplePipelineOutputBuffer = samplePipeline.getOutputBuffer(); - if (samplePipelineOutputBuffer == null) { - return false; - } - - if (!muxerWrapper.writeSample( - getTrackType(), - checkStateNotNull(samplePipelineOutputBuffer.data), - samplePipelineOutputBuffer.isKeyFrame(), - samplePipelineOutputBuffer.timeUs)) { - return false; - } - samplePipeline.releaseOutputBuffer(); - return true; - } - - /** - * Attempts to: - * - *
    - *
  1. read input data, - *
  2. optionally, apply slow motion flattening, and - *
  3. pass input data to the sample pipeline. - *
- * - * @return Whether it may be possible to read more data immediately by calling this method again. - */ - @RequiresNonNull("samplePipeline") - private boolean feedPipelineFromInput() { - @Nullable DecoderInputBuffer samplePipelineInputBuffer = samplePipeline.dequeueInputBuffer(); - if (samplePipelineInputBuffer == null) { - return false; - } - - @ReadDataResult - int result = readSource(getFormatHolder(), samplePipelineInputBuffer, /* readFlags= */ 0); - switch (result) { - case C.RESULT_BUFFER_READ: - if (samplePipelineInputBuffer.isEndOfStream()) { - samplePipeline.queueInputBuffer(); - return false; - } - mediaClock.updateTimeForTrackType(getTrackType(), samplePipelineInputBuffer.timeUs); - samplePipelineInputBuffer.timeUs -= streamOffsetUs; - samplePipelineInputBuffer.flip(); - if (sefSlowMotionFlattener != null) { - ByteBuffer data = checkStateNotNull(samplePipelineInputBuffer.data); - boolean shouldDropSample = - sefSlowMotionFlattener.dropOrTransformSample(samplePipelineInputBuffer); - if (shouldDropSample) { - data.clear(); - return true; - } - } - samplePipeline.queueInputBuffer(); - return true; - case C.RESULT_FORMAT_READ: - throw new IllegalStateException("Format changes are not supported."); - case C.RESULT_NOTHING_READ: - default: - return false; + @Override + @RequiresNonNull({"samplePipeline", "#1.data"}) + protected void maybeQueueSampleToPipeline(DecoderInputBuffer inputBuffer) { + ByteBuffer data = inputBuffer.data; + boolean shouldDropSample = + sefSlowMotionFlattener != null && sefSlowMotionFlattener.dropOrTransformSample(inputBuffer); + if (shouldDropSample) { + data.clear(); + } else { + samplePipeline.queueInputBuffer(); } } }