loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-05-06 23:20:42 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Improve format propagation in transformer

Browse Source 
- Store output format in `MediaCodecAdapterWrapper` when we get a format from
  the codec, instead of creating it on demand.
- Make format building code not audio-specific.
- Remove `MediaCodecAdapterWrapper.getConfigFormat` and instead keep track of
  the input/output formats in the renderer. This will mean that the code still
  works if an audio processor changes the audio format in future.
- Make exceptions thrown during audio rendering use the same (input) renderer
  format.
- Misc other minor cleanup.

#minor-release

PiperOrigin-RevId: 354556619
This commit is contained in:
andrewlewis 2021-01-29 18:11:23 +00:00 committed by Oliver Woodman
parent d5499ee36f
commit 84b96fdff7
6 changed files with 87 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java
View File

@ -29,11 +29,12 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.mediacodec.MediaCodecAdapter;
import com.google.android.exoplayer2.mediacodec.MediaFormatUtil;
import com.google.android.exoplayer2.mediacodec.SynchronousMediaCodecAdapter;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* A wrapper around {@link MediaCodecAdapter}.
@ -44,17 +45,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
*/
/* package */ final class MediaCodecAdapterWrapper {
// MediaCodec decoders always output 16 bit PCM, unless configured to output PCM float.
// https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers.
private static final int MEDIA_CODEC_PCM_ENCODING = C.ENCODING_PCM_16BIT;
private final BufferInfo outputBufferInfo;
private final MediaCodecAdapter codec;
private final Format format;
private @MonotonicNonNull Format outputFormat;
@Nullable private ByteBuffer outputBuffer;
private int inputBufferIndex;
private int outputBufferIndex;
private boolean inputStreamEnded;
private boolean outputStreamEnded;
private boolean hasOutputFormat;
/**
* Returns a {@link MediaCodecAdapterWrapper} for a configured and started {@link
@ -65,12 +69,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* @return A configured and started decoder wrapper.
* @throws IOException If the underlying codec cannot be created.
*/
@RequiresNonNull("#1.sampleMimeType")
public static MediaCodecAdapterWrapper createForAudioDecoding(Format format) throws IOException {
@Nullable MediaCodec decoder = null;
@Nullable MediaCodecAdapter adapter = null;
try {
decoder = MediaCodec.createDecoderByType(format.sampleMimeType);
decoder = MediaCodec.createDecoderByType(checkNotNull(format.sampleMimeType));
MediaFormat mediaFormat =
MediaFormat.createAudioFormat(
format.sampleMimeType, format.sampleRate, format.channelCount);
@ -78,7 +81,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
adapter = new SynchronousMediaCodecAdapter.Factory().createAdapter(decoder);
adapter.configure(mediaFormat, /* surface= */ null, /* crypto= */ null, /* flags= */ 0);
adapter.start();
return new MediaCodecAdapterWrapper(adapter, format);
return new MediaCodecAdapterWrapper(adapter);
} catch (Exception e) {
if (adapter != null) {
adapter.release();
@ -98,12 +101,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* @return A configured and started encoder wrapper.
* @throws IOException If the underlying codec cannot be created.
*/
@RequiresNonNull("#1.sampleMimeType")
public static MediaCodecAdapterWrapper createForAudioEncoding(Format format) throws IOException {
@Nullable MediaCodec encoder = null;
@Nullable MediaCodecAdapter adapter = null;
try {
encoder = MediaCodec.createEncoderByType(format.sampleMimeType);
encoder = MediaCodec.createEncoderByType(checkNotNull(format.sampleMimeType));
MediaFormat mediaFormat =
MediaFormat.createAudioFormat(
format.sampleMimeType, format.sampleRate, format.channelCount);
@ -115,7 +117,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/* crypto= */ null,
/* flags= */ MediaCodec.CONFIGURE_FLAG_ENCODE);
adapter.start();
return new MediaCodecAdapterWrapper(adapter, format);
return new MediaCodecAdapterWrapper(adapter);
} catch (Exception e) {
if (adapter != null) {
adapter.release();
@ -126,9 +128,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
}
}
private MediaCodecAdapterWrapper(MediaCodecAdapter codec, Format format) {
private MediaCodecAdapterWrapper(MediaCodecAdapter codec) {
this.codec = codec;
this.format = format;
outputBufferInfo = new BufferInfo();
inputBufferIndex = C.INDEX_UNSET;
outputBufferIndex = C.INDEX_UNSET;
@ -202,8 +203,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
outputBufferIndex = codec.dequeueOutputBufferIndex(outputBufferInfo);
if (outputBufferIndex < 0) {
if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED && !hasOutputFormat) {
hasOutputFormat = true;
if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
outputFormat = getFormat(codec.getOutputFormat());
}
return false;
}
@ -228,41 +229,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
return true;
}
/**
* Returns a {@link Format} based on the {@link MediaCodecAdapter#getOutputFormat() mediaFormat},
* if available.
*/
/** Returns the current output format, if available. */
@Nullable
public Format getOutputFormat() {
@Nullable MediaFormat mediaFormat = hasOutputFormat ? codec.getOutputFormat() : null;
if (mediaFormat == null) {
return null;
}
ImmutableList.Builder<byte[]> csdBuffers = new ImmutableList.Builder<>();
int csdIndex = 0;
while (true) {
@Nullable ByteBuffer csdByteBuffer = mediaFormat.getByteBuffer("csd-" + csdIndex);
if (csdByteBuffer == null) {
break;
}
byte[] csdBufferData = new byte[csdByteBuffer.remaining()];
csdByteBuffer.get(csdBufferData);
csdBuffers.add(csdBufferData);
csdIndex++;
}
return new Format.Builder()
.setSampleMimeType(mediaFormat.getString(MediaFormat.KEY_MIME))
.setChannelCount(mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT))
.setSampleRate(mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE))
.setInitializationData(csdBuffers.build())
.build();
}
/** Returns the {@link Format} used to create and configure the underlying {@link MediaCodec}. */
public Format getConfigFormat() {
return format;
return outputFormat;
}
/** Returns the current output {@link ByteBuffer}, if available. */
@ -299,4 +269,37 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
outputBuffer = null;
codec.release();
}
private static Format getFormat(MediaFormat mediaFormat) {
ImmutableList.Builder<byte[]> csdBuffers = new ImmutableList.Builder<>();
int csdIndex = 0;
while (true) {
@Nullable ByteBuffer csdByteBuffer = mediaFormat.getByteBuffer("csd-" + csdIndex);
if (csdByteBuffer == null) {
break;
}
byte[] csdBufferData = new byte[csdByteBuffer.remaining()];
csdByteBuffer.get(csdBufferData);
csdBuffers.add(csdBufferData);
csdIndex++;
}
String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
Format.Builder formatBuilder =
new Format.Builder()
.setSampleMimeType(mediaFormat.getString(MediaFormat.KEY_MIME))
.setInitializationData(csdBuffers.build());
if (MimeTypes.isVideo(mimeType)) {
formatBuilder
.setWidth(mediaFormat.getInteger(MediaFormat.KEY_WIDTH))
.setHeight(mediaFormat.getInteger(MediaFormat.KEY_HEIGHT));
} else if (MimeTypes.isAudio(mimeType)) {
// TODO(internal b/178685617): Only set the PCM encoding for audio/raw, once we have a way to
// simulate more realistic codec input/output formats in tests.
formatBuilder
.setChannelCount(mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT))
.setSampleRate(mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE))
.setPcmEncoding(MEDIA_CODEC_PCM_ENCODING);
}
return formatBuilder.build();
}
}

66
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java
View File

@ -32,7 +32,6 @@ import com.google.android.exoplayer2.audio.AudioProcessor.AudioFormat;
import com.google.android.exoplayer2.audio.SonicAudioProcessor;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.source.SampleStream;
import com.google.android.exoplayer2.util.Util;
import java.io.IOException;
import java.nio.ByteBuffer;
@ -40,9 +39,6 @@ import java.nio.ByteBuffer;
/* package */ final class TransformerAudioRenderer extends TransformerBaseRenderer {
private static final String TAG = "TransformerAudioRenderer";
// MediaCodec decoders always output 16 bit PCM, unless configured to output PCM float.
// https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers.
private static final int MEDIA_CODEC_PCM_ENCODING = C.ENCODING_PCM_16BIT;
private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024;
private static final float SPEED_UNSET = -1f;
@ -53,6 +49,8 @@ import java.nio.ByteBuffer;
@Nullable private MediaCodecAdapterWrapper decoder;
@Nullable private MediaCodecAdapterWrapper encoder;
@Nullable private SpeedProvider speedProvider;
@Nullable private Format inputFormat;
@Nullable private AudioFormat encoderInputAudioFormat;
private ByteBuffer sonicOutputBuffer;
private long nextEncoderInputBufferTimeUs;
@ -100,6 +98,8 @@ import java.nio.ByteBuffer;
encoder = null;
}
speedProvider = null;
inputFormat = null;
encoderInputAudioFormat = null;
sonicOutputBuffer = AudioProcessor.EMPTY_BUFFER;
nextEncoderInputBufferTimeUs = 0;
currentSpeed = SPEED_UNSET;
@ -307,6 +307,7 @@ import java.nio.ByteBuffer;
* returns whether it may be possible to write more data.
*/
private boolean feedEncoder(ByteBuffer inputBuffer) {
AudioFormat encoderInputAudioFormat = checkNotNull(this.encoderInputAudioFormat);
MediaCodecAdapterWrapper encoder = checkNotNull(this.encoder);
ByteBuffer encoderInputBufferData = checkNotNull(encoderInputBuffer.data);
int bufferLimit = inputBuffer.limit();
@ -317,9 +318,8 @@ import java.nio.ByteBuffer;
nextEncoderInputBufferTimeUs +=
getBufferDurationUs(
/* bytesWritten= */ encoderInputBufferData.position(),
/* bytesPerFrame= */ Util.getPcmFrameSize(
MEDIA_CODEC_PCM_ENCODING, encoder.getConfigFormat().channelCount),
encoder.getConfigFormat().sampleRate);
encoderInputAudioFormat.bytesPerFrame,
encoderInputAudioFormat.sampleRate);
encoderInputBuffer.setFlags(0);
encoderInputBuffer.flip();
@ -342,30 +342,35 @@ import java.nio.ByteBuffer;
* yet.
*/
private void setupEncoderAndMaybeSonic() throws ExoPlaybackException {
MediaCodecAdapterWrapper decoder = checkNotNull(this.decoder);
if (encoder != null) {
return;
}
Format decoderFormat = decoder.getConfigFormat();
// TODO(b/161127201): Use the decoder output format once the decoder is fed before setting up
// the encoder.
AudioFormat outputAudioFormat =
new AudioFormat(
checkNotNull(inputFormat).sampleRate, inputFormat.channelCount, C.ENCODING_PCM_16BIT);
if (transformation.flattenForSlowMotion) {
try {
configureSonic(decoderFormat);
outputAudioFormat = sonicAudioProcessor.configure(outputAudioFormat);
flushSonicAndSetSpeed(currentSpeed);
} catch (AudioProcessor.UnhandledAudioFormatException e) {
throw ExoPlaybackException.createForRenderer(
e, TAG, getIndex(), /* rendererFormat= */ null, C.FORMAT_HANDLED);
throw createRendererException(e);
}
}
Format encoderFormat =
decoderFormat.buildUpon().setAverageBitrate(DEFAULT_ENCODER_BITRATE).build();
checkNotNull(encoderFormat.sampleMimeType);
try {
encoder = MediaCodecAdapterWrapper.createForAudioEncoding(encoderFormat);
encoder =
MediaCodecAdapterWrapper.createForAudioEncoding(
new Format.Builder()
.setSampleMimeType(checkNotNull(inputFormat).sampleMimeType)
.setSampleRate(outputAudioFormat.sampleRate)
.setChannelCount(outputAudioFormat.channelCount)
.setAverageBitrate(DEFAULT_ENCODER_BITRATE)
.build());
} catch (IOException e) {
throw ExoPlaybackException.createForRenderer(
e, TAG, getIndex(), encoderFormat, /* rendererFormatSupport= */ C.FORMAT_HANDLED);
throw createRendererException(e);
}
encoderInputAudioFormat = outputAudioFormat;
}
/**
@ -383,15 +388,13 @@ import java.nio.ByteBuffer;
if (result != C.RESULT_FORMAT_READ) {
return false;
}
Format decoderFormat = checkNotNull(formatHolder.format);
checkNotNull(decoderFormat.sampleMimeType);
inputFormat = checkNotNull(formatHolder.format);
try {
decoder = MediaCodecAdapterWrapper.createForAudioDecoding(decoderFormat);
decoder = MediaCodecAdapterWrapper.createForAudioDecoding(inputFormat);
} catch (IOException e) {
throw ExoPlaybackException.createForRenderer(
e, TAG, getIndex(), decoderFormat, /* rendererFormatSupport= */ C.FORMAT_HANDLED);
throw createRendererException(e);
}
speedProvider = new SegmentSpeedProvider(decoderFormat);
speedProvider = new SegmentSpeedProvider(inputFormat);
currentSpeed = speedProvider.getSpeed(0);
return true;
}
@ -406,18 +409,17 @@ import java.nio.ByteBuffer;
return speedChanging;
}
private void configureSonic(Format format) throws AudioProcessor.UnhandledAudioFormatException {
sonicAudioProcessor.configure(
new AudioFormat(format.sampleRate, format.channelCount, MEDIA_CODEC_PCM_ENCODING));
flushSonicAndSetSpeed(currentSpeed);
}
private void flushSonicAndSetSpeed(float speed) {
sonicAudioProcessor.setSpeed(speed);
sonicAudioProcessor.setPitch(speed);
sonicAudioProcessor.flush();
}
private ExoPlaybackException createRendererException(Throwable cause) {
return ExoPlaybackException.createForRenderer(
cause, TAG, getIndex(), inputFormat, /* rendererFormatSupport= */ C.FORMAT_HANDLED);
}
private static long getBufferDurationUs(long bytesWritten, int bytesPerFrame, int sampleRate) {
long framesWritten = bytesWritten / bytesPerFrame;
return framesWritten * C.MICROS_PER_SECOND / sampleRate;

1
testdata/src/test/assets/transformerdumps/amr/sample_nb.amr.dump vendored
View File

@ -3,6 +3,7 @@ format 0:
sampleMimeType = audio/3gpp
channelCount = 1
sampleRate = 8000
pcmEncoding = 2
sample:
trackIndex = 0
dataHashCode = 924517484

1
testdata/src/test/assets/transformerdumps/mp4/sample.mp4.dump vendored
View File

@ -3,6 +3,7 @@ format 0:
sampleMimeType = audio/mp4a-latm
channelCount = 1
sampleRate = 44100
pcmEncoding = 2
format 1:
id = 1
sampleMimeType = video/avc

1
testdata/src/test/assets/transformerdumps/mp4/sample.mp4.novideo.dump vendored
View File

@ -3,6 +3,7 @@ format 0:
sampleMimeType = audio/mp4a-latm
channelCount = 1
sampleRate = 44100
pcmEncoding = 2
sample:
trackIndex = 0
dataHashCode = 1205768497

1
testdata/src/test/assets/transformerdumps/mp4/sample_sef_slow_motion.mp4.dump vendored
View File

@ -3,6 +3,7 @@ format 0:
sampleMimeType = audio/mp4a-latm
channelCount = 2
sampleRate = 12000
pcmEncoding = 2
format 1:
id = 2
sampleMimeType = video/avc