From f3d76e9e2fc902757a7b7c39096eaa3e0fa9f549 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Thu, 9 Dec 2021 18:41:30 +0000 Subject: [PATCH] Keep orientation information during the transformation. The input rotation is used to rotate the video during decoding, the video is rotated so that it is in landscape orientation before encoding and a rotation is added to the output format where necessary so that the output video has the same orientation as the input. PiperOrigin-RevId: 415301328 --- .../transformer/MediaCodecAdapterWrapper.java | 9 +++++- .../transformer/VideoSamplePipeline.java | 28 +++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java index 0ca566351d..82d7dc675a 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java @@ -20,6 +20,7 @@ import static com.google.android.exoplayer2.util.Assertions.checkArgument; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Assertions.checkState; +import android.annotation.SuppressLint; import android.media.MediaCodec; import android.media.MediaCodec.BufferInfo; import android.media.MediaCodecInfo.CodecCapabilities; @@ -133,6 +134,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @return A configured and started decoder wrapper. * @throws IOException If the underlying codec cannot be created. */ + @SuppressLint("InlinedApi") public static MediaCodecAdapterWrapper createForVideoDecoding(Format format, Surface surface) throws IOException { @Nullable MediaCodecAdapter adapter = null; @@ -140,6 +142,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; MediaFormat mediaFormat = MediaFormat.createVideoFormat( checkNotNull(format.sampleMimeType), format.width, format.height); + MediaFormatUtil.maybeSetInteger( + mediaFormat, MediaFormat.KEY_ROTATION, format.rotationDegrees); MediaFormatUtil.maybeSetInteger( mediaFormat, MediaFormat.KEY_MAX_INPUT_SIZE, format.maxInputSize); MediaFormatUtil.setCsdBuffers(mediaFormat, format.initializationData); @@ -201,7 +205,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; * @param format The {@link Format} (of the output data) used to determine the underlying {@link * MediaCodec} and its configuration values. {@link Format#sampleMimeType}, {@link * Format#width} and {@link Format#height} must be set to those of the desired output video - * format. + * format. {@link Format#rotationDegrees} should be 0. The video should always be in landscape + * orientation. * @param additionalEncoderConfig A map of {@link MediaFormat}'s integer settings, where the keys * are from {@code MediaFormat.KEY_*} constants. Its values will override those in {@code * format}. @@ -212,6 +217,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Format format, Map additionalEncoderConfig) throws IOException { checkArgument(format.width != Format.NO_VALUE); checkArgument(format.height != Format.NO_VALUE); + checkArgument(format.height < format.width); + checkArgument(format.rotationDegrees == 0); @Nullable MediaCodecAdapter adapter = null; try { diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java index 0af218829c..369cdb3b40 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java @@ -37,6 +37,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final String TAG = "VideoSamplePipeline"; + private final int outputRotationDegrees; private final DecoderInputBuffer decoderInputBuffer; private final MediaCodecAdapterWrapper decoder; @@ -59,6 +60,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; encoderOutputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); + // TODO(internal b/209781577): Think about which edge length should be set for portrait videos. int outputWidth = inputFormat.width; int outputHeight = inputFormat.height; if (transformation.outputHeight != Format.NO_VALUE @@ -67,12 +69,29 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; outputHeight = transformation.outputHeight; } + if (inputFormat.height > inputFormat.width) { + // The encoder may not support encoding in portrait orientation, so the decoded video is + // rotated to landscape orientation and a rotation is added back later to the output format. + outputRotationDegrees = (inputFormat.rotationDegrees + 90) % 360; + int temp = outputWidth; + outputWidth = outputHeight; + outputHeight = temp; + } else { + outputRotationDegrees = inputFormat.rotationDegrees; + } + // The decoder rotates videos to their intended display orientation. The frameEditor rotates + // them back for improved encoder compatibility. + // TODO(internal b/201293185): After fragment shader transformations are implemented, put + // postrotation in a later vertex shader. + transformation.transformationMatrix.postRotate(outputRotationDegrees); + try { encoder = MediaCodecAdapterWrapper.createForVideoEncoding( new Format.Builder() .setWidth(outputWidth) .setHeight(outputHeight) + .setRotationDegrees(0) .setSampleMimeType( transformation.videoMimeType != null ? transformation.videoMimeType @@ -84,7 +103,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throw createRendererException( e, rendererIndex, inputFormat, PlaybackException.ERROR_CODE_UNSPECIFIED); } - if (inputFormat.height != outputHeight || !transformation.transformationMatrix.isIdentity()) { + if (inputFormat.height != outputHeight + || inputFormat.width != outputWidth + || !transformation.transformationMatrix.isIdentity()) { frameEditor = FrameEditor.create( context, @@ -150,7 +171,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override @Nullable public Format getOutputFormat() { - return encoder.getOutputFormat(); + Format format = encoder.getOutputFormat(); + return format == null + ? null + : format.buildUpon().setRotationDegrees(outputRotationDegrees).build(); } @Override