From ef1788dcea76b7350c93f6c5b512d0c2a366dbfd Mon Sep 17 00:00:00 2001 From: hschlueter Date: Fri, 3 Dec 2021 10:34:55 +0000 Subject: [PATCH] Only use a FrameEditor if editing is needed. When no editing is needed, the OpenGL steps can be skipped. PiperOrigin-RevId: 413884305 --- .../RepeatedTranscodeTransformationTest.java | 7 +++ .../transformer/VideoSamplePipeline.java | 58 +++++++++++-------- 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java index f4999b4803..4c056ff803 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java @@ -19,6 +19,7 @@ import static androidx.media3.transformer.mh.AndroidTestUtil.runTransformer; import static com.google.common.truth.Truth.assertWithMessage; import android.content.Context; +import android.graphics.Matrix; import androidx.media3.common.MimeTypes; import androidx.media3.transformer.Transformer; import androidx.test.core.app.ApplicationProvider; @@ -38,9 +39,12 @@ public final class RepeatedTranscodeTransformationTest { @Test public void repeatedTranscode_givesConsistentLengthOutput() throws Exception { Context context = ApplicationProvider.getApplicationContext(); + Matrix transformationMatrix = new Matrix(); + transformationMatrix.postTranslate((float) 0.1, (float) 0.1); Transformer transformer = new Transformer.Builder(context) .setVideoMimeType(MimeTypes.VIDEO_H265) + .setTransformationMatrix(transformationMatrix) .setAudioMimeType(MimeTypes.AUDIO_AMR_NB) .build(); @@ -66,9 +70,12 @@ public final class RepeatedTranscodeTransformationTest { @Test public void repeatedTranscodeNoAudio_givesConsistentLengthOutput() throws Exception { Context context = ApplicationProvider.getApplicationContext(); + Matrix transformationMatrix = new Matrix(); + transformationMatrix.postTranslate((float) 0.1, (float) 0.1); Transformer transformer = new Transformer.Builder(context) .setVideoMimeType(MimeTypes.VIDEO_H265) + .setTransformationMatrix(transformationMatrix) .setRemoveAudio(true) .build(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java index 0949ed7ef3..8c7f7c28ce 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java @@ -28,6 +28,7 @@ import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.exoplayer.ExoPlaybackException; import com.google.common.collect.ImmutableMap; import java.io.IOException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * Pipeline to decode video samples, apply transformations on the raw samples, and re-encode them. @@ -39,12 +40,12 @@ import java.io.IOException; private final DecoderInputBuffer decoderInputBuffer; private final MediaCodecAdapterWrapper decoder; - private final FrameEditor frameEditor; - private final MediaCodecAdapterWrapper encoder; private final DecoderInputBuffer encoderOutputBuffer; - private boolean waitingForPopulatedDecoderSurface; + private @MonotonicNonNull FrameEditor frameEditor; + + private boolean waitingForFrameEditorInput; public VideoSamplePipeline( Context context, Format inputFormat, Transformation transformation, int rendererIndex) @@ -79,17 +80,23 @@ import java.io.IOException; throw createRendererException( e, rendererIndex, inputFormat, PlaybackException.ERROR_CODE_UNSPECIFIED); } - frameEditor = - FrameEditor.create( - context, - outputWidth, - outputHeight, - transformation.transformationMatrix, - /* outputSurface= */ checkNotNull(encoder.getInputSurface())); + if (inputFormat.height != transformation.outputHeight + || !transformation.transformationMatrix.isIdentity()) { + frameEditor = + FrameEditor.create( + context, + outputWidth, + outputHeight, + transformation.transformationMatrix, + /* outputSurface= */ checkNotNull(encoder.getInputSurface())); + } try { decoder = MediaCodecAdapterWrapper.createForVideoDecoding( - inputFormat, frameEditor.getInputSurface()); + inputFormat, + frameEditor == null + ? checkNotNull(encoder.getInputSurface()) + : frameEditor.getInputSurface()); } catch (IOException e) { throw createRendererException( e, rendererIndex, inputFormat, PlaybackException.ERROR_CODE_DECODER_INIT_FAILED); @@ -102,24 +109,27 @@ import java.io.IOException; return false; } - if (frameEditor.hasInputData()) { - waitingForPopulatedDecoderSurface = false; - frameEditor.processData(); - return true; + if (frameEditor != null) { + if (frameEditor.hasInputData()) { + waitingForFrameEditorInput = false; + frameEditor.processData(); + return true; + } + if (waitingForFrameEditorInput) { + return false; + } } - if (waitingForPopulatedDecoderSurface) { - return false; - } - - if (decoder.getOutputBufferInfo() != null) { + boolean decoderHasOutputBuffer = decoder.getOutputBufferInfo() != null; + if (decoderHasOutputBuffer) { decoder.releaseOutputBuffer(/* render= */ true); - waitingForPopulatedDecoderSurface = true; + waitingForFrameEditorInput = frameEditor != null; } if (decoder.isEnded()) { encoder.signalEndOfInputStream(); + return false; } - return false; + return decoderHasOutputBuffer && !waitingForFrameEditorInput; } @Override @@ -164,7 +174,9 @@ import java.io.IOException; @Override public void release() { - frameEditor.release(); + if (frameEditor != null) { + frameEditor.release(); + } decoder.release(); encoder.release(); }