From d91afa063a0f7fcafd358c4491ab3dbdf2f7e796 Mon Sep 17 00:00:00 2001 From: samrobinson Date: Tue, 14 Feb 2023 19:47:06 +0000 Subject: [PATCH] Align ASP and VSP on findSupportedMimeTypeForEncoder/Muxer logic. * Moved the logic to SamplePipeline. * Pass the requested values via Format. * Moved exception throwing inside the methods. * Build up the mimeTypesToCheck as a set - removing possible duplicate checks. * Simplified logic that calls the findSupportedMimeType method. * Improved javadoc. PiperOrigin-RevId: 509594062 --- .../transformer/AudioSamplePipeline.java | 54 ++++------- .../transformer/SamplePipeline.java | 92 +++++++++++++++---- .../transformer/VideoSamplePipeline.java | 73 ++------------- 3 files changed, 101 insertions(+), 118 deletions(-) diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java index 722308092c..c2631df89f 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java @@ -33,7 +33,6 @@ import com.google.android.exoplayer2.util.Util; import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.List; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedDeque; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; @@ -119,36 +118,32 @@ import org.checkerframework.dataflow.qual.Pure; audioProcessingPipeline.flush(); - String requestedMimeType = - transformationRequest.audioMimeType != null - ? transformationRequest.audioMimeType - : checkNotNull(firstInputFormat.sampleMimeType); - Format requestedOutputFormat = + Format requestedEncoderFormat = new Format.Builder() - .setSampleMimeType(requestedMimeType) + .setSampleMimeType( + transformationRequest.audioMimeType != null + ? transformationRequest.audioMimeType + : checkNotNull(firstInputFormat.sampleMimeType)) .setSampleRate(encoderInputAudioFormat.sampleRate) .setChannelCount(encoderInputAudioFormat.channelCount) .setAverageBitrate(DEFAULT_ENCODER_BITRATE) .build(); - ImmutableList muxerSupportedMimeTypes = - muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO); - - // TODO(b/259570024): investigate overhauling fallback. - @Nullable - String supportedMimeType = - findSupportedMimeTypeForEncoderAndMuxer(requestedMimeType, muxerSupportedMimeTypes); - if (supportedMimeType == null) { - throw createNoSupportedMimeTypeException(requestedOutputFormat); - } - encoder = encoderFactory.createForAudioEncoding( - requestedOutputFormat.buildUpon().setSampleMimeType(supportedMimeType).build()); - checkState(supportedMimeType.equals(encoder.getConfigurationFormat().sampleMimeType)); + requestedEncoderFormat + .buildUpon() + .setSampleMimeType( + findSupportedMimeTypeForEncoderAndMuxer( + requestedEncoderFormat, + muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO))) + .build()); + fallbackListener.onTransformationRequestFinalized( createFallbackTransformationRequest( - transformationRequest, requestedOutputFormat, encoder.getConfigurationFormat())); + transformationRequest, + requestedEncoderFormat, + /* actualFormat= */ encoder.getConfigurationFormat())); // Use the same stream offset as the input stream for encoder input buffers. nextEncoderInputBufferTimeUs = streamOffsetUs; @@ -350,23 +345,6 @@ import org.checkerframework.dataflow.qual.Pure; encoder.queueInputBuffer(encoderInputBuffer); } - @Nullable - private static String findSupportedMimeTypeForEncoderAndMuxer( - String preferredMimeType, List muxerSupportedMimeTypes) { - if (!EncoderUtil.getSupportedEncoders(preferredMimeType).isEmpty()) { - return preferredMimeType; - } else { - // No encoder supports the preferred MIME type. - for (int i = 0; i < muxerSupportedMimeTypes.size(); i++) { - String mimeType = muxerSupportedMimeTypes.get(i); - if (!EncoderUtil.getSupportedEncoders(mimeType).isEmpty()) { - return mimeType; - } - } - } - return null; - } - @Pure private static TransformationRequest createFallbackTransformationRequest( TransformationRequest transformationRequest, Format requestedFormat, Format actualFormat) { diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java index edb0ca9bb3..1c8f883a2e 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java @@ -16,7 +16,11 @@ package com.google.android.exoplayer2.transformer; +import static com.google.android.exoplayer2.transformer.EncoderUtil.getSupportedEncoders; +import static com.google.android.exoplayer2.transformer.EncoderUtil.getSupportedEncodersForHdrEditing; +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; +import static com.google.android.exoplayer2.video.ColorInfo.isTransferHdr; import androidx.annotation.Nullable; import com.google.android.exoplayer2.C; @@ -24,6 +28,9 @@ import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.video.ColorInfo; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import java.util.List; /** * Pipeline for processing media data. @@ -48,23 +55,6 @@ import com.google.android.exoplayer2.video.ColorInfo; : MimeTypes.getTrackType(firstInputFormat.sampleMimeType); } - protected static TransformationException createNoSupportedMimeTypeException(Format format) { - String errorMessage = "No MIME type is supported by both encoder and muxer."; - int errorCode = TransformationException.ERROR_CODE_ENCODING_FORMAT_UNSUPPORTED; - boolean isVideo = MimeTypes.isVideo(format.sampleMimeType); - - if (isVideo && ColorInfo.isTransferHdr(format.colorInfo)) { - errorMessage += " Requested HDR colorInfo: " + format.colorInfo; - } - - return TransformationException.createForCodec( - new IllegalArgumentException(errorMessage), - errorCode, - isVideo, - /* isDecoder= */ false, - format); - } - @Override public boolean expectsDecodedData() { return true; @@ -143,4 +133,72 @@ import com.google.android.exoplayer2.video.ColorInfo; releaseMuxerInputBuffer(); return true; } + + /** + * Finds a {@linkplain MimeTypes MIME type} that is supported by the encoder and the muxer. + * + *

The {@linkplain Format requestedFormat} determines what support is checked. + * + *

    + *
  • The {@link Format#sampleMimeType} determines whether audio or video mime types are + * considered. See {@link MimeTypes#isAudio} and {@link MimeTypes#isVideo} for more details. + *
  • The {@link Format#sampleMimeType} must be populated with the preferred {@linkplain + * MimeTypes MIME type}. This mime type will be the first checked. + *
  • When checking video support, if the HDR {@link Format#colorInfo} is set, only encoders + * that support that {@link ColorInfo} will be considered. + *
+ * + * @param requestedFormat The {@link Format} requested. + * @param muxerSupportedMimeTypes The list of sample {@linkplain MimeTypes MIME types} that the + * muxer supports. + * @return A supported {@linkplain MimeTypes MIME type}. + * @throws TransformationException If there are no supported {@linkplain MimeTypes MIME types}. + */ + protected static String findSupportedMimeTypeForEncoderAndMuxer( + Format requestedFormat, List muxerSupportedMimeTypes) throws TransformationException { + boolean isVideo = MimeTypes.isVideo(checkNotNull(requestedFormat.sampleMimeType)); + + ImmutableSet.Builder mimeTypesToCheckSetBuilder = + new ImmutableSet.Builder().add(requestedFormat.sampleMimeType); + if (isVideo) { + mimeTypesToCheckSetBuilder.add(MimeTypes.VIDEO_H265).add(MimeTypes.VIDEO_H264); + } + mimeTypesToCheckSetBuilder.addAll(muxerSupportedMimeTypes); + ImmutableList mimeTypesToCheck = mimeTypesToCheckSetBuilder.build().asList(); + + for (int i = 0; i < mimeTypesToCheck.size(); i++) { + String mimeType = mimeTypesToCheck.get(i); + + if (!muxerSupportedMimeTypes.contains(mimeType)) { + continue; + } + + if (isVideo && isTransferHdr(requestedFormat.colorInfo)) { + if (!getSupportedEncodersForHdrEditing(mimeType, requestedFormat.colorInfo).isEmpty()) { + return mimeType; + } + } else if (!getSupportedEncoders(mimeType).isEmpty()) { + return mimeType; + } + } + + throw createNoSupportedMimeTypeException(requestedFormat); + } + + private static TransformationException createNoSupportedMimeTypeException(Format format) { + String errorMessage = "No MIME type is supported by both encoder and muxer."; + int errorCode = TransformationException.ERROR_CODE_ENCODING_FORMAT_UNSUPPORTED; + boolean isVideo = MimeTypes.isVideo(format.sampleMimeType); + + if (isVideo && isTransferHdr(format.colorInfo)) { + errorMessage += " Requested HDR colorInfo: " + format.colorInfo; + } + + return TransformationException.createForCodec( + new IllegalArgumentException(errorMessage), + errorCode, + isVideo, + /* isDecoder= */ false, + format); + } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java index b28a788db2..19c4a05324 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java @@ -16,14 +16,12 @@ package com.google.android.exoplayer2.transformer; -import static com.google.android.exoplayer2.transformer.EncoderUtil.getSupportedEncoders; import static com.google.android.exoplayer2.transformer.EncoderUtil.getSupportedEncodersForHdrEditing; import static com.google.android.exoplayer2.transformer.TransformationRequest.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR; import static com.google.android.exoplayer2.transformer.TransformationRequest.HDR_MODE_KEEP_HDR; import static com.google.android.exoplayer2.transformer.TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_MEDIACODEC; import static com.google.android.exoplayer2.transformer.TransformationRequest.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL; import static com.google.android.exoplayer2.util.Assertions.checkNotNull; -import static com.google.android.exoplayer2.util.Assertions.checkState; import static com.google.android.exoplayer2.util.Util.SDK_INT; import static com.google.android.exoplayer2.video.ColorInfo.isTransferHdr; @@ -425,20 +423,16 @@ import org.checkerframework.dataflow.qual.Pure; .setColorInfo(getSupportedInputColor()) .build(); - @Nullable - String supportedMimeType = - findSupportedMimeTypeForEncoderAndMuxer( - requestedOutputMimeType, muxerSupportedMimeTypes, requestedEncoderFormat.colorInfo); - if (supportedMimeType == null) { - throw createNoSupportedMimeTypeException(requestedEncoderFormat); - } - encoder = encoderFactory.createForVideoEncoding( - requestedEncoderFormat.buildUpon().setSampleMimeType(supportedMimeType).build()); + requestedEncoderFormat + .buildUpon() + .setSampleMimeType( + findSupportedMimeTypeForEncoderAndMuxer( + requestedEncoderFormat, muxerSupportedMimeTypes)) + .build()); - Format encoderSupportedFormat = encoder.getConfigurationFormat(); - checkState(supportedMimeType.equals(encoderSupportedFormat.sampleMimeType)); + Format actualEncoderFormat = encoder.getConfigurationFormat(); boolean isInputToneMapped = isTransferHdr(inputFormat.colorInfo) && !isTransferHdr(requestedEncoderFormat.colorInfo); @@ -456,14 +450,14 @@ import org.checkerframework.dataflow.qual.Pure; transformationRequest, /* hasOutputFormatRotation= */ outputRotationDegrees != 0, requestedEncoderFormat, - encoderSupportedFormat, + actualEncoderFormat, supportedFallbackHdrMode)); encoderSurfaceInfo = new SurfaceInfo( encoder.getInputSurface(), - encoderSupportedFormat.width, - encoderSupportedFormat.height, + actualEncoderFormat.width, + actualEncoderFormat.height, outputRotationDegrees); if (releaseEncoder) { @@ -516,52 +510,5 @@ import org.checkerframework.dataflow.qual.Pure; } releaseEncoder = true; } - - /** - * Finds a {@linkplain MimeTypes MIME type} that is supported by the encoder and the muxer. - * - *

HDR editing support is checked if the {@link ColorInfo} is HDR. - * - * @param preferredMimeType The preferred {@linkplain MimeTypes MIME type}, returned if - * supported. - * @param muxerSupportedMimeTypes The list of sample {@linkplain MimeTypes MIME types} that the - * muxer supports. - * @param colorInfo The optional encoding {@link ColorInfo}. If a HDR color info is provided, - * only encoders that support it will be considered. - * @return A {@linkplain MimeTypes MIME type} that is supported by an encoder and the muxer, or - * {@code null} if no such {@linkplain MimeTypes MIME type} exists. - */ - @Nullable - private static String findSupportedMimeTypeForEncoderAndMuxer( - String preferredMimeType, - List muxerSupportedMimeTypes, - @Nullable ColorInfo colorInfo) { - ImmutableList mimeTypesToCheck = - new ImmutableList.Builder() - .add(preferredMimeType) - .add(MimeTypes.VIDEO_H265) - .add(MimeTypes.VIDEO_H264) - .addAll(muxerSupportedMimeTypes) - .build(); - - for (int i = 0; i < mimeTypesToCheck.size(); i++) { - String mimeType = mimeTypesToCheck.get(i); - if (mimeTypeAndColorAreSupported(mimeType, muxerSupportedMimeTypes, colorInfo)) { - return mimeType; - } - } - return null; - } - - private static boolean mimeTypeAndColorAreSupported( - String mimeType, List muxerSupportedMimeTypes, @Nullable ColorInfo colorInfo) { - if (!muxerSupportedMimeTypes.contains(mimeType)) { - return false; - } - - return isTransferHdr(colorInfo) - ? !getSupportedEncodersForHdrEditing(mimeType, colorInfo).isEmpty() - : !getSupportedEncoders(mimeType).isEmpty(); - } } }