diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java index 1295644308..1b201aa57a 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java @@ -29,11 +29,12 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.mediacodec.MediaCodecAdapter; import com.google.android.exoplayer2.mediacodec.MediaFormatUtil; import com.google.android.exoplayer2.mediacodec.SynchronousMediaCodecAdapter; +import com.google.android.exoplayer2.util.MimeTypes; import com.google.common.collect.ImmutableList; import java.io.IOException; import java.nio.ByteBuffer; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; -import org.checkerframework.checker.nullness.qual.RequiresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * A wrapper around {@link MediaCodecAdapter}. @@ -44,17 +45,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; */ /* package */ final class MediaCodecAdapterWrapper { + // MediaCodec decoders always output 16 bit PCM, unless configured to output PCM float. + // https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers. + private static final int MEDIA_CODEC_PCM_ENCODING = C.ENCODING_PCM_16BIT; + private final BufferInfo outputBufferInfo; private final MediaCodecAdapter codec; - private final Format format; + private @MonotonicNonNull Format outputFormat; @Nullable private ByteBuffer outputBuffer; private int inputBufferIndex; private int outputBufferIndex; private boolean inputStreamEnded; private boolean outputStreamEnded; - private boolean hasOutputFormat; /** * Returns a {@link MediaCodecAdapterWrapper} for a configured and started {@link @@ -65,12 +69,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * @return A configured and started decoder wrapper. * @throws IOException If the underlying codec cannot be created. */ - @RequiresNonNull("#1.sampleMimeType") public static MediaCodecAdapterWrapper createForAudioDecoding(Format format) throws IOException { @Nullable MediaCodec decoder = null; @Nullable MediaCodecAdapter adapter = null; try { - decoder = MediaCodec.createDecoderByType(format.sampleMimeType); + decoder = MediaCodec.createDecoderByType(checkNotNull(format.sampleMimeType)); MediaFormat mediaFormat = MediaFormat.createAudioFormat( format.sampleMimeType, format.sampleRate, format.channelCount); @@ -78,7 +81,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; adapter = new SynchronousMediaCodecAdapter.Factory().createAdapter(decoder); adapter.configure(mediaFormat, /* surface= */ null, /* crypto= */ null, /* flags= */ 0); adapter.start(); - return new MediaCodecAdapterWrapper(adapter, format); + return new MediaCodecAdapterWrapper(adapter); } catch (Exception e) { if (adapter != null) { adapter.release(); @@ -98,12 +101,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; * @return A configured and started encoder wrapper. * @throws IOException If the underlying codec cannot be created. */ - @RequiresNonNull("#1.sampleMimeType") public static MediaCodecAdapterWrapper createForAudioEncoding(Format format) throws IOException { @Nullable MediaCodec encoder = null; @Nullable MediaCodecAdapter adapter = null; try { - encoder = MediaCodec.createEncoderByType(format.sampleMimeType); + encoder = MediaCodec.createEncoderByType(checkNotNull(format.sampleMimeType)); MediaFormat mediaFormat = MediaFormat.createAudioFormat( format.sampleMimeType, format.sampleRate, format.channelCount); @@ -115,7 +117,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; /* crypto= */ null, /* flags= */ MediaCodec.CONFIGURE_FLAG_ENCODE); adapter.start(); - return new MediaCodecAdapterWrapper(adapter, format); + return new MediaCodecAdapterWrapper(adapter); } catch (Exception e) { if (adapter != null) { adapter.release(); @@ -126,9 +128,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } } - private MediaCodecAdapterWrapper(MediaCodecAdapter codec, Format format) { + private MediaCodecAdapterWrapper(MediaCodecAdapter codec) { this.codec = codec; - this.format = format; outputBufferInfo = new BufferInfo(); inputBufferIndex = C.INDEX_UNSET; outputBufferIndex = C.INDEX_UNSET; @@ -202,8 +203,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; outputBufferIndex = codec.dequeueOutputBufferIndex(outputBufferInfo); if (outputBufferIndex < 0) { - if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED && !hasOutputFormat) { - hasOutputFormat = true; + if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + outputFormat = getFormat(codec.getOutputFormat()); } return false; } @@ -228,41 +229,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return true; } - /** - * Returns a {@link Format} based on the {@link MediaCodecAdapter#getOutputFormat() mediaFormat}, - * if available. - */ + /** Returns the current output format, if available. */ @Nullable public Format getOutputFormat() { - @Nullable MediaFormat mediaFormat = hasOutputFormat ? codec.getOutputFormat() : null; - if (mediaFormat == null) { - return null; - } - - ImmutableList.Builder csdBuffers = new ImmutableList.Builder<>(); - int csdIndex = 0; - while (true) { - @Nullable ByteBuffer csdByteBuffer = mediaFormat.getByteBuffer("csd-" + csdIndex); - if (csdByteBuffer == null) { - break; - } - byte[] csdBufferData = new byte[csdByteBuffer.remaining()]; - csdByteBuffer.get(csdBufferData); - csdBuffers.add(csdBufferData); - csdIndex++; - } - - return new Format.Builder() - .setSampleMimeType(mediaFormat.getString(MediaFormat.KEY_MIME)) - .setChannelCount(mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT)) - .setSampleRate(mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)) - .setInitializationData(csdBuffers.build()) - .build(); - } - - /** Returns the {@link Format} used to create and configure the underlying {@link MediaCodec}. */ - public Format getConfigFormat() { - return format; + return outputFormat; } /** Returns the current output {@link ByteBuffer}, if available. */ @@ -299,4 +269,37 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; outputBuffer = null; codec.release(); } + + private static Format getFormat(MediaFormat mediaFormat) { + ImmutableList.Builder csdBuffers = new ImmutableList.Builder<>(); + int csdIndex = 0; + while (true) { + @Nullable ByteBuffer csdByteBuffer = mediaFormat.getByteBuffer("csd-" + csdIndex); + if (csdByteBuffer == null) { + break; + } + byte[] csdBufferData = new byte[csdByteBuffer.remaining()]; + csdByteBuffer.get(csdBufferData); + csdBuffers.add(csdBufferData); + csdIndex++; + } + String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME); + Format.Builder formatBuilder = + new Format.Builder() + .setSampleMimeType(mediaFormat.getString(MediaFormat.KEY_MIME)) + .setInitializationData(csdBuffers.build()); + if (MimeTypes.isVideo(mimeType)) { + formatBuilder + .setWidth(mediaFormat.getInteger(MediaFormat.KEY_WIDTH)) + .setHeight(mediaFormat.getInteger(MediaFormat.KEY_HEIGHT)); + } else if (MimeTypes.isAudio(mimeType)) { + // TODO(internal b/178685617): Only set the PCM encoding for audio/raw, once we have a way to + // simulate more realistic codec input/output formats in tests. + formatBuilder + .setChannelCount(mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT)) + .setSampleRate(mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)) + .setPcmEncoding(MEDIA_CODEC_PCM_ENCODING); + } + return formatBuilder.build(); + } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java index 73fc3af60e..d0fd83d4b1 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java @@ -32,7 +32,6 @@ import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat; import com.google.android.exoplayer2.audio.SonicAudioProcessor; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.source.SampleStream; -import com.google.android.exoplayer2.util.Util; import java.io.IOException; import java.nio.ByteBuffer; @@ -40,9 +39,6 @@ import java.nio.ByteBuffer; /* package */ final class TransformerAudioRenderer extends TransformerBaseRenderer { private static final String TAG = "TransformerAudioRenderer"; - // MediaCodec decoders always output 16 bit PCM, unless configured to output PCM float. - // https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers. - private static final int MEDIA_CODEC_PCM_ENCODING = C.ENCODING_PCM_16BIT; private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024; private static final float SPEED_UNSET = -1f; @@ -53,6 +49,8 @@ import java.nio.ByteBuffer; @Nullable private MediaCodecAdapterWrapper decoder; @Nullable private MediaCodecAdapterWrapper encoder; @Nullable private SpeedProvider speedProvider; + @Nullable private Format inputFormat; + @Nullable private AudioFormat encoderInputAudioFormat; private ByteBuffer sonicOutputBuffer; private long nextEncoderInputBufferTimeUs; @@ -100,6 +98,8 @@ import java.nio.ByteBuffer; encoder = null; } speedProvider = null; + inputFormat = null; + encoderInputAudioFormat = null; sonicOutputBuffer = AudioProcessor.EMPTY_BUFFER; nextEncoderInputBufferTimeUs = 0; currentSpeed = SPEED_UNSET; @@ -307,6 +307,7 @@ import java.nio.ByteBuffer; * returns whether it may be possible to write more data. */ private boolean feedEncoder(ByteBuffer inputBuffer) { + AudioFormat encoderInputAudioFormat = checkNotNull(this.encoderInputAudioFormat); MediaCodecAdapterWrapper encoder = checkNotNull(this.encoder); ByteBuffer encoderInputBufferData = checkNotNull(encoderInputBuffer.data); int bufferLimit = inputBuffer.limit(); @@ -317,9 +318,8 @@ import java.nio.ByteBuffer; nextEncoderInputBufferTimeUs += getBufferDurationUs( /* bytesWritten= */ encoderInputBufferData.position(), - /* bytesPerFrame= */ Util.getPcmFrameSize( - MEDIA_CODEC_PCM_ENCODING, encoder.getConfigFormat().channelCount), - encoder.getConfigFormat().sampleRate); + encoderInputAudioFormat.bytesPerFrame, + encoderInputAudioFormat.sampleRate); encoderInputBuffer.setFlags(0); encoderInputBuffer.flip(); @@ -342,30 +342,35 @@ import java.nio.ByteBuffer; * yet. */ private void setupEncoderAndMaybeSonic() throws ExoPlaybackException { - MediaCodecAdapterWrapper decoder = checkNotNull(this.decoder); - if (encoder != null) { return; } - - Format decoderFormat = decoder.getConfigFormat(); + // TODO(b/161127201): Use the decoder output format once the decoder is fed before setting up + // the encoder. + AudioFormat outputAudioFormat = + new AudioFormat( + checkNotNull(inputFormat).sampleRate, inputFormat.channelCount, C.ENCODING_PCM_16BIT); if (transformation.flattenForSlowMotion) { try { - configureSonic(decoderFormat); + outputAudioFormat = sonicAudioProcessor.configure(outputAudioFormat); + flushSonicAndSetSpeed(currentSpeed); } catch (AudioProcessor.UnhandledAudioFormatException e) { - throw ExoPlaybackException.createForRenderer( - e, TAG, getIndex(), /* rendererFormat= */ null, C.FORMAT_HANDLED); + throw createRendererException(e); } } - Format encoderFormat = - decoderFormat.buildUpon().setAverageBitrate(DEFAULT_ENCODER_BITRATE).build(); - checkNotNull(encoderFormat.sampleMimeType); try { - encoder = MediaCodecAdapterWrapper.createForAudioEncoding(encoderFormat); + encoder = + MediaCodecAdapterWrapper.createForAudioEncoding( + new Format.Builder() + .setSampleMimeType(checkNotNull(inputFormat).sampleMimeType) + .setSampleRate(outputAudioFormat.sampleRate) + .setChannelCount(outputAudioFormat.channelCount) + .setAverageBitrate(DEFAULT_ENCODER_BITRATE) + .build()); } catch (IOException e) { - throw ExoPlaybackException.createForRenderer( - e, TAG, getIndex(), encoderFormat, /* rendererFormatSupport= */ C.FORMAT_HANDLED); + throw createRendererException(e); } + encoderInputAudioFormat = outputAudioFormat; } /** @@ -383,15 +388,13 @@ import java.nio.ByteBuffer; if (result != C.RESULT_FORMAT_READ) { return false; } - Format decoderFormat = checkNotNull(formatHolder.format); - checkNotNull(decoderFormat.sampleMimeType); + inputFormat = checkNotNull(formatHolder.format); try { - decoder = MediaCodecAdapterWrapper.createForAudioDecoding(decoderFormat); + decoder = MediaCodecAdapterWrapper.createForAudioDecoding(inputFormat); } catch (IOException e) { - throw ExoPlaybackException.createForRenderer( - e, TAG, getIndex(), decoderFormat, /* rendererFormatSupport= */ C.FORMAT_HANDLED); + throw createRendererException(e); } - speedProvider = new SegmentSpeedProvider(decoderFormat); + speedProvider = new SegmentSpeedProvider(inputFormat); currentSpeed = speedProvider.getSpeed(0); return true; } @@ -406,18 +409,17 @@ import java.nio.ByteBuffer; return speedChanging; } - private void configureSonic(Format format) throws AudioProcessor.UnhandledAudioFormatException { - sonicAudioProcessor.configure( - new AudioFormat(format.sampleRate, format.channelCount, MEDIA_CODEC_PCM_ENCODING)); - flushSonicAndSetSpeed(currentSpeed); - } - private void flushSonicAndSetSpeed(float speed) { sonicAudioProcessor.setSpeed(speed); sonicAudioProcessor.setPitch(speed); sonicAudioProcessor.flush(); } + private ExoPlaybackException createRendererException(Throwable cause) { + return ExoPlaybackException.createForRenderer( + cause, TAG, getIndex(), inputFormat, /* rendererFormatSupport= */ C.FORMAT_HANDLED); + } + private static long getBufferDurationUs(long bytesWritten, int bytesPerFrame, int sampleRate) { long framesWritten = bytesWritten / bytesPerFrame; return framesWritten * C.MICROS_PER_SECOND / sampleRate; diff --git a/testdata/src/test/assets/transformerdumps/amr/sample_nb.amr.dump b/testdata/src/test/assets/transformerdumps/amr/sample_nb.amr.dump index f48431a0f7..c18193c16f 100644 --- a/testdata/src/test/assets/transformerdumps/amr/sample_nb.amr.dump +++ b/testdata/src/test/assets/transformerdumps/amr/sample_nb.amr.dump @@ -3,6 +3,7 @@ format 0: sampleMimeType = audio/3gpp channelCount = 1 sampleRate = 8000 + pcmEncoding = 2 sample: trackIndex = 0 dataHashCode = 924517484 diff --git a/testdata/src/test/assets/transformerdumps/mp4/sample.mp4.dump b/testdata/src/test/assets/transformerdumps/mp4/sample.mp4.dump index 4ccbeae3d7..3d74318819 100644 --- a/testdata/src/test/assets/transformerdumps/mp4/sample.mp4.dump +++ b/testdata/src/test/assets/transformerdumps/mp4/sample.mp4.dump @@ -3,6 +3,7 @@ format 0: sampleMimeType = audio/mp4a-latm channelCount = 1 sampleRate = 44100 + pcmEncoding = 2 format 1: id = 1 sampleMimeType = video/avc diff --git a/testdata/src/test/assets/transformerdumps/mp4/sample.mp4.novideo.dump b/testdata/src/test/assets/transformerdumps/mp4/sample.mp4.novideo.dump index 25e3aa791b..2e520ebb02 100644 --- a/testdata/src/test/assets/transformerdumps/mp4/sample.mp4.novideo.dump +++ b/testdata/src/test/assets/transformerdumps/mp4/sample.mp4.novideo.dump @@ -3,6 +3,7 @@ format 0: sampleMimeType = audio/mp4a-latm channelCount = 1 sampleRate = 44100 + pcmEncoding = 2 sample: trackIndex = 0 dataHashCode = 1205768497 diff --git a/testdata/src/test/assets/transformerdumps/mp4/sample_sef_slow_motion.mp4.dump b/testdata/src/test/assets/transformerdumps/mp4/sample_sef_slow_motion.mp4.dump index 5262f11ea1..672582b703 100644 --- a/testdata/src/test/assets/transformerdumps/mp4/sample_sef_slow_motion.mp4.dump +++ b/testdata/src/test/assets/transformerdumps/mp4/sample_sef_slow_motion.mp4.dump @@ -3,6 +3,7 @@ format 0: sampleMimeType = audio/mp4a-latm channelCount = 2 sampleRate = 12000 + pcmEncoding = 2 format 1: id = 2 sampleMimeType = video/avc