From 31b247ea1a8771c1bfe21c0c3f1b7c8c486c327a Mon Sep 17 00:00:00 2001 From: claincly Date: Wed, 2 Feb 2022 11:13:15 +0000 Subject: [PATCH] Split the implementation of Encoder/Decoder Factory PiperOrigin-RevId: 425838647 --- .../androidx/media3/transformer/Codec.java | 4 +- .../media3/transformer/CodecFactoryUtil.java | 119 +++++++++++ .../transformer/DefaultDecoderFactory.java | 72 +++++++ ...actory.java => DefaultEncoderFactory.java} | 192 +++--------------- 4 files changed, 217 insertions(+), 170 deletions(-) create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/CodecFactoryUtil.java create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java rename libraries/transformer/src/main/java/androidx/media3/transformer/{DefaultCodecFactory.java => DefaultEncoderFactory.java} (61%) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java index 600318ab0e..53ed48bf18 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java @@ -49,7 +49,7 @@ public final class Codec { public interface DecoderFactory { /** A default {@code DecoderFactory} implementation. */ - DecoderFactory DEFAULT = new DefaultCodecFactory(/* videoEncoderSelector= */ null); + DecoderFactory DEFAULT = new DefaultDecoderFactory(); /** * Returns a {@link Codec} for audio decoding. @@ -78,7 +78,7 @@ public final class Codec { public interface EncoderFactory { /** A default {@code EncoderFactory} implementation. */ - EncoderFactory DEFAULT = new DefaultCodecFactory(EncoderSelector.DEFAULT); + EncoderFactory DEFAULT = new DefaultEncoderFactory(); /** * Returns a {@link Codec} for audio encoding. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/CodecFactoryUtil.java b/libraries/transformer/src/main/java/androidx/media3/transformer/CodecFactoryUtil.java new file mode 100644 index 0000000000..87fffb66d6 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/CodecFactoryUtil.java @@ -0,0 +1,119 @@ +/* + * Copyright 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import android.media.MediaCodec; +import android.media.MediaFormat; +import android.view.Surface; +import androidx.annotation.Nullable; +import androidx.media3.common.Format; +import androidx.media3.common.util.TraceUtil; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; + +/** Utility methods for {@link Codec}'s factory methods. */ +/* package */ final class CodecFactoryUtil { + /** Creates a {@link Codec}. */ + @RequiresNonNull("#1.sampleMimeType") + public static Codec createCodec( + Format format, + MediaFormat mediaFormat, + @Nullable String mediaCodecName, + boolean isVideo, + boolean isDecoder, + @Nullable Surface outputSurface) + throws TransformationException { + @Nullable MediaCodec mediaCodec = null; + @Nullable Surface inputSurface = null; + try { + mediaCodec = + mediaCodecName != null + ? MediaCodec.createByCodecName(mediaCodecName) + : isDecoder + ? MediaCodec.createDecoderByType(format.sampleMimeType) + : MediaCodec.createEncoderByType(format.sampleMimeType); + configureCodec(mediaCodec, mediaFormat, isDecoder, outputSurface); + if (isVideo && !isDecoder) { + inputSurface = mediaCodec.createInputSurface(); + } + startCodec(mediaCodec); + } catch (Exception e) { + if (inputSurface != null) { + inputSurface.release(); + } + if (mediaCodec != null) { + mediaCodecName = mediaCodec.getName(); + mediaCodec.release(); + } + throw createTransformationException(e, format, isVideo, isDecoder, mediaCodecName); + } + return new Codec(mediaCodec, format, inputSurface); + } + + /** Creates a {@link TransformationException}. */ + public static TransformationException createTransformationException( + Exception cause, + Format format, + boolean isVideo, + boolean isDecoder, + @Nullable String mediaCodecName) { + String componentName = (isVideo ? "Video" : "Audio") + (isDecoder ? "Decoder" : "Encoder"); + if (cause instanceof IOException || cause instanceof MediaCodec.CodecException) { + return TransformationException.createForCodec( + cause, + componentName, + format, + mediaCodecName, + isDecoder + ? TransformationException.ERROR_CODE_DECODER_INIT_FAILED + : TransformationException.ERROR_CODE_ENCODER_INIT_FAILED); + } + if (cause instanceof IllegalArgumentException) { + return TransformationException.createForCodec( + cause, + componentName, + format, + mediaCodecName, + isDecoder + ? TransformationException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED + : TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); + } + return TransformationException.createForUnexpected(cause); + } + + private static void configureCodec( + MediaCodec codec, + MediaFormat mediaFormat, + boolean isDecoder, + @Nullable Surface outputSurface) { + TraceUtil.beginSection("configureCodec"); + codec.configure( + mediaFormat, + outputSurface, + /* crypto= */ null, + isDecoder ? 0 : MediaCodec.CONFIGURE_FLAG_ENCODE); + TraceUtil.endSection(); + } + + private static void startCodec(MediaCodec codec) { + TraceUtil.beginSection("startCodec"); + codec.start(); + TraceUtil.endSection(); + } + + private CodecFactoryUtil() {} +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java new file mode 100644 index 0000000000..b47a18d97c --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java @@ -0,0 +1,72 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Util.SDK_INT; +import static androidx.media3.transformer.CodecFactoryUtil.createCodec; + +import android.media.MediaFormat; +import android.view.Surface; +import androidx.media3.common.Format; +import androidx.media3.common.util.MediaFormatUtil; + +/** A default implementation of {@link Codec.DecoderFactory}. */ +/* package */ final class DefaultDecoderFactory implements Codec.DecoderFactory { + @Override + public Codec createForAudioDecoding(Format format) throws TransformationException { + MediaFormat mediaFormat = + MediaFormat.createAudioFormat( + checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount); + MediaFormatUtil.maybeSetInteger( + mediaFormat, MediaFormat.KEY_MAX_INPUT_SIZE, format.maxInputSize); + MediaFormatUtil.setCsdBuffers(mediaFormat, format.initializationData); + + return createCodec( + format, + mediaFormat, + /* mediaCodecName= */ null, + /* isVideo= */ false, + /* isDecoder= */ true, + /* outputSurface= */ null); + } + + @Override + public Codec createForVideoDecoding(Format format, Surface outputSurface) + throws TransformationException { + MediaFormat mediaFormat = + MediaFormat.createVideoFormat( + checkNotNull(format.sampleMimeType), format.width, format.height); + MediaFormatUtil.maybeSetInteger(mediaFormat, MediaFormat.KEY_ROTATION, format.rotationDegrees); + MediaFormatUtil.maybeSetInteger( + mediaFormat, MediaFormat.KEY_MAX_INPUT_SIZE, format.maxInputSize); + MediaFormatUtil.setCsdBuffers(mediaFormat, format.initializationData); + if (SDK_INT >= 29) { + // On API levels over 29, Transformer decodes as many frames as possible in one render + // cycle. This key ensures no frame dropping when the decoder's output surface is full. + mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0); + } + + return createCodec( + format, + mediaFormat, + /* mediaCodecName= */ null, + /* isVideo= */ true, + /* isDecoder= */ true, + outputSurface); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodecFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java similarity index 61% rename from libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodecFactory.java rename to libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java index 6f2ce53126..a9dd48d472 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodecFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java @@ -1,5 +1,5 @@ /* - * Copyright 2021 The Android Open Source Project + * Copyright 2022 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,89 +20,46 @@ import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.common.util.Util.SDK_INT; +import static androidx.media3.transformer.CodecFactoryUtil.createCodec; +import static androidx.media3.transformer.CodecFactoryUtil.createTransformationException; import static java.lang.Math.abs; -import android.annotation.SuppressLint; -import android.media.MediaCodec; import android.media.MediaCodecInfo; -import android.media.MediaCodecInfo.CodecCapabilities; import android.media.MediaFormat; import android.util.Pair; -import android.view.Surface; import androidx.annotation.Nullable; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; -import androidx.media3.common.util.MediaFormatUtil; -import androidx.media3.common.util.TraceUtil; +import androidx.media3.common.util.UnstableApi; import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import com.google.common.collect.ImmutableList; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.checkerframework.checker.nullness.qual.RequiresNonNull; -/** A default {@link Codec.DecoderFactory} and {@link Codec.EncoderFactory}. */ -/* package */ final class DefaultCodecFactory - implements Codec.DecoderFactory, Codec.EncoderFactory { +/** A default implementation of {@link Codec.EncoderFactory}. */ +@UnstableApi +public final class DefaultEncoderFactory implements Codec.EncoderFactory { // TODO(b/214973843): Add option to disable fallback. - // TODO(b/210591626): Fall back adaptively to H265 if possible. private static final String DEFAULT_FALLBACK_MIME_TYPE = MimeTypes.VIDEO_H264; - private static final int DEFAULT_COLOR_FORMAT = CodecCapabilities.COLOR_FormatSurface; + private static final int DEFAULT_COLOR_FORMAT = + MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; private static final int DEFAULT_FRAME_RATE = 60; private static final int DEFAULT_I_FRAME_INTERVAL_SECS = 1; @Nullable private final EncoderSelector videoEncoderSelector; + /** Creates a new instance using the {@link EncoderSelector#DEFAULT default encoder selector}. */ + public DefaultEncoderFactory() { + this.videoEncoderSelector = EncoderSelector.DEFAULT; + } + /** Creates a new instance. */ - public DefaultCodecFactory(@Nullable EncoderSelector videoEncoderSelector) { + public DefaultEncoderFactory(EncoderSelector videoEncoderSelector) { this.videoEncoderSelector = videoEncoderSelector; } - @Override - public Codec createForAudioDecoding(Format format) throws TransformationException { - MediaFormat mediaFormat = - MediaFormat.createAudioFormat( - checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount); - MediaFormatUtil.maybeSetInteger( - mediaFormat, MediaFormat.KEY_MAX_INPUT_SIZE, format.maxInputSize); - MediaFormatUtil.setCsdBuffers(mediaFormat, format.initializationData); - - return createCodec( - format, - mediaFormat, - /* mediaCodecName= */ null, - /* isVideo= */ false, - /* isDecoder= */ true, - /* outputSurface= */ null); - } - - @Override - @SuppressLint("InlinedApi") - public Codec createForVideoDecoding(Format format, Surface outputSurface) - throws TransformationException { - MediaFormat mediaFormat = - MediaFormat.createVideoFormat( - checkNotNull(format.sampleMimeType), format.width, format.height); - MediaFormatUtil.maybeSetInteger(mediaFormat, MediaFormat.KEY_ROTATION, format.rotationDegrees); - MediaFormatUtil.maybeSetInteger( - mediaFormat, MediaFormat.KEY_MAX_INPUT_SIZE, format.maxInputSize); - MediaFormatUtil.setCsdBuffers(mediaFormat, format.initializationData); - if (SDK_INT >= 29) { - // On API levels over 29, Transformer decodes as many frames as possible in one render - // cycle. This key ensures no frame dropping when the decoder's output surface is full. - mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0); - } - - return createCodec( - format, - mediaFormat, - /* mediaCodecName= */ null, - /* isVideo= */ true, - /* isDecoder= */ true, - outputSurface); - } - @Override public Codec createForAudioEncoding(Format format, List allowedMimeTypes) throws TransformationException { @@ -141,9 +98,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; checkStateNotNull(videoEncoderSelector); @Nullable - EncoderAndSupportedFormat encoderAndSupportedFormat = + Pair encoderAndClosestFormatSupport = findEncoderWithClosestFormatSupport(format, videoEncoderSelector, allowedMimeTypes); - if (encoderAndSupportedFormat == null) { + if (encoderAndClosestFormatSupport == null) { throw createTransformationException( new IllegalArgumentException( "No encoder available that supports the requested output format."), @@ -153,7 +110,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; /* mediaCodecName= */ null); } - format = encoderAndSupportedFormat.supportedFormat; + format = encoderAndClosestFormatSupport.second; MediaFormat mediaFormat = MediaFormat.createVideoFormat( checkNotNull(format.sampleMimeType), format.width, format.height); @@ -175,75 +132,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return createCodec( format, mediaFormat, - encoderAndSupportedFormat.encoderInfo.getName(), + encoderAndClosestFormatSupport.first.getName(), /* isVideo= */ true, /* isDecoder= */ false, /* outputSurface= */ null); } - @RequiresNonNull("#1.sampleMimeType") - private static Codec createCodec( - Format format, - MediaFormat mediaFormat, - @Nullable String mediaCodecName, - boolean isVideo, - boolean isDecoder, - @Nullable Surface outputSurface) - throws TransformationException { - @Nullable MediaCodec mediaCodec = null; - @Nullable Surface inputSurface = null; - try { - mediaCodec = - mediaCodecName != null - ? MediaCodec.createByCodecName(mediaCodecName) - : isDecoder - ? MediaCodec.createDecoderByType(format.sampleMimeType) - : MediaCodec.createEncoderByType(format.sampleMimeType); - configureCodec(mediaCodec, mediaFormat, isDecoder, outputSurface); - if (isVideo && !isDecoder) { - inputSurface = mediaCodec.createInputSurface(); - } - startCodec(mediaCodec); - } catch (Exception e) { - if (inputSurface != null) { - inputSurface.release(); - } - if (mediaCodec != null) { - mediaCodecName = mediaCodec.getName(); - mediaCodec.release(); - } - throw createTransformationException(e, format, isVideo, isDecoder, mediaCodecName); - } - return new Codec(mediaCodec, format, inputSurface); - } - - private static void configureCodec( - MediaCodec codec, - MediaFormat mediaFormat, - boolean isDecoder, - @Nullable Surface outputSurface) { - TraceUtil.beginSection("configureCodec"); - codec.configure( - mediaFormat, - outputSurface, - /* crypto= */ null, - isDecoder ? 0 : MediaCodec.CONFIGURE_FLAG_ENCODE); - TraceUtil.endSection(); - } - - private static void startCodec(MediaCodec codec) { - TraceUtil.beginSection("startCodec"); - codec.start(); - TraceUtil.endSection(); - } - /** - * Finds the {@link EncoderAndSupportedFormat} whose {@link EncoderAndSupportedFormat#encoderInfo - * encoder} supports the {@code requestedFormat} most closely; {@code null} if none is found. + * Finds a {@link MediaCodecInfo encoder} that supports the requested format most closely. Returns + * the {@link MediaCodecInfo encoder} and the supported {@link Format} in a {@link Pair}, or + * {@code null} if none is found. */ @RequiresNonNull("#1.sampleMimeType") @Nullable - private static EncoderAndSupportedFormat findEncoderWithClosestFormatSupport( + private static Pair findEncoderWithClosestFormatSupport( Format requestedFormat, EncoderSelector encoderSelector, List allowedMimeTypes) { String mimeType = requestedFormat.sampleMimeType; @@ -340,7 +242,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; EncoderUtil.getClosestSupportedBitrate( pickedEncoder, finalMimeType, requestedBitrate)) .build(); - return new EncoderAndSupportedFormat(pickedEncoder, encoderSupportedFormat); + return Pair.create(pickedEncoder, encoderSupportedFormat); } private interface EncoderFallbackCost { @@ -383,50 +285,4 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; // 1080p30 -> 6.2Mbps, 720p30 -> 2.7Mbps. return (int) (width * height * frameRate * 0.1); } - - private static TransformationException createTransformationException( - Exception cause, - Format format, - boolean isVideo, - boolean isDecoder, - @Nullable String mediaCodecName) { - String componentName = (isVideo ? "Video" : "Audio") + (isDecoder ? "Decoder" : "Encoder"); - if (cause instanceof IOException || cause instanceof MediaCodec.CodecException) { - return TransformationException.createForCodec( - cause, - componentName, - format, - mediaCodecName, - isDecoder - ? TransformationException.ERROR_CODE_DECODER_INIT_FAILED - : TransformationException.ERROR_CODE_ENCODER_INIT_FAILED); - } - if (cause instanceof IllegalArgumentException) { - return TransformationException.createForCodec( - cause, - componentName, - format, - mediaCodecName, - isDecoder - ? TransformationException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED - : TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED); - } - return TransformationException.createForUnexpected(cause); - } - - /** - * A class wrapping a selected {@link MediaCodecInfo encoder} and its supported {@link Format}. - */ - private static class EncoderAndSupportedFormat { - /** The {@link MediaCodecInfo} that describes the encoder. */ - public final MediaCodecInfo encoderInfo; - /** The {@link Format} that this encoder supports. */ - public final Format supportedFormat; - - /** Creates a new instance. */ - public EncoderAndSupportedFormat(MediaCodecInfo encoderInfo, Format supportedFormat) { - this.encoderInfo = encoderInfo; - this.supportedFormat = supportedFormat; - } - } }