Rename MediaCodecAdapterWrapper to Codec.
Move static factories into a separate class and make it implement an interface that will let tests customize encoder/decoder creation. PiperOrigin-RevId: 417610825
This commit is contained in:
claincly
tonihei
parent
89271eb722
commit
4706d3a59f
@ -44,8 +44,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
|
||||
private final Format inputFormat;
|
||||
private final Transformation transformation;
|
||||
private final Codec.EncoderFactory encoderFactory;
|
||||
|
||||
private final MediaCodecAdapterWrapper decoder;
|
||||
private final Codec decoder;
|
||||
private final DecoderInputBuffer decoderInputBuffer;
|
||||
|
||||
private final SonicAudioProcessor sonicAudioProcessor;
|
||||
@ -55,7 +56,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
private final DecoderInputBuffer encoderOutputBuffer;
|
||||
|
||||
private @MonotonicNonNull AudioFormat encoderInputAudioFormat;
|
||||
private @MonotonicNonNull MediaCodecAdapterWrapper encoder;
|
||||
private @MonotonicNonNull Codec encoder;
|
||||
private long nextEncoderInputBufferTimeUs;
|
||||
private long encoderBufferDurationRemainder;
|
||||
|
||||
@ -63,10 +64,15 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
private boolean drainingSonicForSpeedChange;
|
||||
private float currentSpeed;
|
||||
|
||||
public AudioSamplePipeline(Format inputFormat, Transformation transformation)
|
||||
public AudioSamplePipeline(
|
||||
Format inputFormat,
|
||||
Transformation transformation,
|
||||
Codec.EncoderFactory encoderFactory,
|
||||
Codec.DecoderFactory decoderFactory)
|
||||
throws TransformationException {
|
||||
this.inputFormat = inputFormat;
|
||||
this.transformation = transformation;
|
||||
this.encoderFactory = encoderFactory;
|
||||
decoderInputBuffer =
|
||||
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
|
||||
encoderInputBuffer =
|
||||
@ -77,7 +83,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
sonicOutputBuffer = AudioProcessor.EMPTY_BUFFER;
|
||||
speedProvider = new SegmentSpeedProvider(inputFormat);
|
||||
currentSpeed = speedProvider.getSpeed(0);
|
||||
this.decoder = MediaCodecAdapterWrapper.createForAudioDecoding(inputFormat);
|
||||
this.decoder = decoderFactory.createForAudioDecoding(inputFormat);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -301,7 +307,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
}
|
||||
}
|
||||
encoder =
|
||||
MediaCodecAdapterWrapper.createForAudioEncoding(
|
||||
encoderFactory.createForAudioEncoding(
|
||||
new Format.Builder()
|
||||
.setSampleMimeType(
|
||||
transformation.audioMimeType == null
|
||||
|
||||
@ -0,0 +1,324 @@
|
||||
/*
|
||||
* Copyright 2021 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package androidx.media3.transformer;
|
||||
|
||||
import static androidx.media3.common.util.Assertions.checkNotNull;
|
||||
import static androidx.media3.common.util.Assertions.checkState;
|
||||
|
||||
import android.media.MediaCodec;
|
||||
import android.media.MediaCodec.BufferInfo;
|
||||
import android.media.MediaFormat;
|
||||
import android.view.Surface;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.Format;
|
||||
import androidx.media3.common.MimeTypes;
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
import androidx.media3.decoder.DecoderInputBuffer;
|
||||
import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import java.nio.ByteBuffer;
|
||||
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
/**
|
||||
* A wrapper around {@link MediaCodecAdapter}.
|
||||
*
|
||||
* <p>Provides a layer of abstraction for callers that need to interact with {@link MediaCodec}
|
||||
* through {@link MediaCodecAdapter}. This is done by simplifying the calls needed to queue and
|
||||
* dequeue buffers, removing the need to track buffer indices and codec events.
|
||||
*/
|
||||
@UnstableApi
|
||||
public final class Codec {
|
||||
|
||||
/** A factory for {@link Codec decoder} instances. */
|
||||
public interface DecoderFactory {
|
||||
|
||||
/** A default {@code DecoderFactory} implementation. */
|
||||
DecoderFactory DEFAULT = new DefaultCodecFactory();
|
||||
|
||||
/**
|
||||
* Returns a {@link Codec} for audio decoding.
|
||||
*
|
||||
* @param format The {@link Format} (of the input data) used to determine the underlying {@link
|
||||
* MediaCodec} and its configuration values.
|
||||
* @return A configured and started decoder wrapper.
|
||||
* @throws TransformationException If the underlying codec cannot be created.
|
||||
*/
|
||||
Codec createForAudioDecoding(Format format) throws TransformationException;
|
||||
|
||||
/**
|
||||
* Returns a {@link Codec} for video decoding.
|
||||
*
|
||||
* @param format The {@link Format} (of the input data) used to determine the underlying {@link
|
||||
* MediaCodec} and its configuration values.
|
||||
* @param surface The {@link Surface} to which the decoder output is rendered.
|
||||
* @return A configured and started decoder wrapper.
|
||||
* @throws TransformationException If the underlying codec cannot be created.
|
||||
*/
|
||||
Codec createForVideoDecoding(Format format, Surface surface) throws TransformationException;
|
||||
}
|
||||
|
||||
/** A factory for {@link Codec encoder} instances. */
|
||||
public interface EncoderFactory {
|
||||
|
||||
/** A default {@code EncoderFactory} implementation. */
|
||||
EncoderFactory DEFAULT = new DefaultCodecFactory();
|
||||
|
||||
/**
|
||||
* Returns a {@link Codec} for audio encoding.
|
||||
*
|
||||
* @param format The {@link Format} (of the output data) used to determine the underlying {@link
|
||||
* MediaCodec} and its configuration values.
|
||||
* @return A configured and started encoder wrapper.
|
||||
* @throws TransformationException If the underlying codec cannot be created.
|
||||
*/
|
||||
Codec createForAudioEncoding(Format format) throws TransformationException;
|
||||
|
||||
/**
|
||||
* Returns a {@link Codec} for video encoding.
|
||||
*
|
||||
* @param format The {@link Format} (of the output data) used to determine the underlying {@link
|
||||
* MediaCodec} and its configuration values. {@link Format#sampleMimeType}, {@link
|
||||
* Format#width} and {@link Format#height} must be set to those of the desired output video
|
||||
* format. {@link Format#rotationDegrees} should be 0. The video should always be in
|
||||
* landscape orientation.
|
||||
* @return A configured and started encoder wrapper.
|
||||
* @throws TransformationException If the underlying codec cannot be created.
|
||||
*/
|
||||
Codec createForVideoEncoding(Format format) throws TransformationException;
|
||||
}
|
||||
|
||||
// MediaCodec decoders always output 16 bit PCM, unless configured to output PCM float.
|
||||
// https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers.
|
||||
private static final int MEDIA_CODEC_PCM_ENCODING = C.ENCODING_PCM_16BIT;
|
||||
|
||||
private final BufferInfo outputBufferInfo;
|
||||
private final MediaCodecAdapter mediaCodecAdapter;
|
||||
|
||||
private @MonotonicNonNull Format outputFormat;
|
||||
@Nullable private ByteBuffer outputBuffer;
|
||||
|
||||
private int inputBufferIndex;
|
||||
private int outputBufferIndex;
|
||||
private boolean inputStreamEnded;
|
||||
private boolean outputStreamEnded;
|
||||
|
||||
/** Creates a {@code Codec} from a configured and started {@link MediaCodecAdapter}. */
|
||||
public Codec(MediaCodecAdapter mediaCodecAdapter) {
|
||||
this.mediaCodecAdapter = mediaCodecAdapter;
|
||||
outputBufferInfo = new BufferInfo();
|
||||
inputBufferIndex = C.INDEX_UNSET;
|
||||
outputBufferIndex = C.INDEX_UNSET;
|
||||
}
|
||||
|
||||
/** Returns the input {@link Surface}, or null if the input is not a surface. */
|
||||
@Nullable
|
||||
public Surface getInputSurface() {
|
||||
return mediaCodecAdapter.getInputSurface();
|
||||
}
|
||||
|
||||
/**
|
||||
* Dequeues a writable input buffer, if available.
|
||||
*
|
||||
* @param inputBuffer The buffer where the dequeued buffer data is stored.
|
||||
* @return Whether an input buffer is ready to be used.
|
||||
*/
|
||||
@EnsuresNonNullIf(expression = "#1.data", result = true)
|
||||
public boolean maybeDequeueInputBuffer(DecoderInputBuffer inputBuffer) {
|
||||
if (inputStreamEnded) {
|
||||
return false;
|
||||
}
|
||||
if (inputBufferIndex < 0) {
|
||||
inputBufferIndex = mediaCodecAdapter.dequeueInputBufferIndex();
|
||||
if (inputBufferIndex < 0) {
|
||||
return false;
|
||||
}
|
||||
inputBuffer.data = mediaCodecAdapter.getInputBuffer(inputBufferIndex);
|
||||
inputBuffer.clear();
|
||||
}
|
||||
checkNotNull(inputBuffer.data);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Queues an input buffer to the decoder. No buffers may be queued after an {@link
|
||||
* DecoderInputBuffer#isEndOfStream() end of stream} buffer has been queued.
|
||||
*/
|
||||
public void queueInputBuffer(DecoderInputBuffer inputBuffer) {
|
||||
checkState(
|
||||
!inputStreamEnded, "Input buffer can not be queued after the input stream has ended.");
|
||||
|
||||
int offset = 0;
|
||||
int size = 0;
|
||||
if (inputBuffer.data != null && inputBuffer.data.hasRemaining()) {
|
||||
offset = inputBuffer.data.position();
|
||||
size = inputBuffer.data.remaining();
|
||||
}
|
||||
int flags = 0;
|
||||
if (inputBuffer.isEndOfStream()) {
|
||||
inputStreamEnded = true;
|
||||
flags = MediaCodec.BUFFER_FLAG_END_OF_STREAM;
|
||||
}
|
||||
mediaCodecAdapter.queueInputBuffer(inputBufferIndex, offset, size, inputBuffer.timeUs, flags);
|
||||
inputBufferIndex = C.INDEX_UNSET;
|
||||
inputBuffer.data = null;
|
||||
}
|
||||
|
||||
public void signalEndOfInputStream() {
|
||||
mediaCodecAdapter.signalEndOfInputStream();
|
||||
}
|
||||
|
||||
/** Returns the current output format, if available. */
|
||||
@Nullable
|
||||
public Format getOutputFormat() {
|
||||
// The format is updated when dequeueing a 'special' buffer index, so attempt to dequeue now.
|
||||
maybeDequeueOutputBuffer();
|
||||
return outputFormat;
|
||||
}
|
||||
|
||||
/** Returns the current output {@link ByteBuffer}, if available. */
|
||||
@Nullable
|
||||
public ByteBuffer getOutputBuffer() {
|
||||
return maybeDequeueAndSetOutputBuffer() ? outputBuffer : null;
|
||||
}
|
||||
|
||||
/** Returns the {@link BufferInfo} associated with the current output buffer, if available. */
|
||||
@Nullable
|
||||
public BufferInfo getOutputBufferInfo() {
|
||||
return maybeDequeueOutputBuffer() ? outputBufferInfo : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases the current output buffer.
|
||||
*
|
||||
* <p>This should be called after the buffer has been processed. The next output buffer will not
|
||||
* be available until the previous has been released.
|
||||
*/
|
||||
public void releaseOutputBuffer() {
|
||||
releaseOutputBuffer(/* render= */ false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases the current output buffer. If the {@link MediaCodec} was configured with an output
|
||||
* surface, setting {@code render} to {@code true} will first send the buffer to the output
|
||||
* surface. The surface will release the buffer back to the codec once it is no longer
|
||||
* used/displayed.
|
||||
*
|
||||
* <p>This should be called after the buffer has been processed. The next output buffer will not
|
||||
* be available until the previous has been released.
|
||||
*/
|
||||
public void releaseOutputBuffer(boolean render) {
|
||||
outputBuffer = null;
|
||||
mediaCodecAdapter.releaseOutputBuffer(outputBufferIndex, render);
|
||||
outputBufferIndex = C.INDEX_UNSET;
|
||||
}
|
||||
|
||||
/** Returns whether the codec output stream has ended, and no more data can be dequeued. */
|
||||
public boolean isEnded() {
|
||||
return outputStreamEnded && outputBufferIndex == C.INDEX_UNSET;
|
||||
}
|
||||
|
||||
/** Releases the underlying codec. */
|
||||
public void release() {
|
||||
outputBuffer = null;
|
||||
mediaCodecAdapter.release();
|
||||
}
|
||||
|
||||
/**
|
||||
* Tries obtaining an output buffer and sets {@link #outputBuffer} to the obtained output buffer.
|
||||
*
|
||||
* @return {@code true} if a buffer is successfully obtained, {@code false} otherwise.
|
||||
*/
|
||||
private boolean maybeDequeueAndSetOutputBuffer() {
|
||||
if (!maybeDequeueOutputBuffer()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
outputBuffer = checkNotNull(mediaCodecAdapter.getOutputBuffer(outputBufferIndex));
|
||||
outputBuffer.position(outputBufferInfo.offset);
|
||||
outputBuffer.limit(outputBufferInfo.offset + outputBufferInfo.size);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if there is already an output buffer pending. Otherwise attempts to dequeue an
|
||||
* output buffer and returns whether there is a new output buffer.
|
||||
*/
|
||||
private boolean maybeDequeueOutputBuffer() {
|
||||
if (outputBufferIndex >= 0) {
|
||||
return true;
|
||||
}
|
||||
if (outputStreamEnded) {
|
||||
return false;
|
||||
}
|
||||
|
||||
outputBufferIndex = mediaCodecAdapter.dequeueOutputBufferIndex(outputBufferInfo);
|
||||
if (outputBufferIndex < 0) {
|
||||
if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
|
||||
outputFormat = getFormat(mediaCodecAdapter.getOutputFormat());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if ((outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
|
||||
outputStreamEnded = true;
|
||||
if (outputBufferInfo.size == 0) {
|
||||
releaseOutputBuffer();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if ((outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
|
||||
// Encountered a CSD buffer, skip it.
|
||||
releaseOutputBuffer();
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static Format getFormat(MediaFormat mediaFormat) {
|
||||
ImmutableList.Builder<byte[]> csdBuffers = new ImmutableList.Builder<>();
|
||||
int csdIndex = 0;
|
||||
while (true) {
|
||||
@Nullable ByteBuffer csdByteBuffer = mediaFormat.getByteBuffer("csd-" + csdIndex);
|
||||
if (csdByteBuffer == null) {
|
||||
break;
|
||||
}
|
||||
byte[] csdBufferData = new byte[csdByteBuffer.remaining()];
|
||||
csdByteBuffer.get(csdBufferData);
|
||||
csdBuffers.add(csdBufferData);
|
||||
csdIndex++;
|
||||
}
|
||||
String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
|
||||
Format.Builder formatBuilder =
|
||||
new Format.Builder()
|
||||
.setSampleMimeType(mediaFormat.getString(MediaFormat.KEY_MIME))
|
||||
.setInitializationData(csdBuffers.build());
|
||||
if (MimeTypes.isVideo(mimeType)) {
|
||||
formatBuilder
|
||||
.setWidth(mediaFormat.getInteger(MediaFormat.KEY_WIDTH))
|
||||
.setHeight(mediaFormat.getInteger(MediaFormat.KEY_HEIGHT));
|
||||
} else if (MimeTypes.isAudio(mimeType)) {
|
||||
// TODO(internal b/178685617): Only set the PCM encoding for audio/raw, once we have a way to
|
||||
// simulate more realistic codec input/output formats in tests.
|
||||
formatBuilder
|
||||
.setChannelCount(mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT))
|
||||
.setSampleRate(mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE))
|
||||
.setPcmEncoding(MEDIA_CODEC_PCM_ENCODING);
|
||||
}
|
||||
return formatBuilder.build();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,194 @@
|
||||
/*
|
||||
* Copyright 2021 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package androidx.media3.transformer;
|
||||
|
||||
import static androidx.media3.common.util.Assertions.checkArgument;
|
||||
import static androidx.media3.common.util.Assertions.checkNotNull;
|
||||
import static androidx.media3.common.util.Util.SDK_INT;
|
||||
|
||||
import android.annotation.SuppressLint;
|
||||
import android.media.MediaCodec;
|
||||
import android.media.MediaCodecInfo.CodecCapabilities;
|
||||
import android.media.MediaFormat;
|
||||
import android.view.Surface;
|
||||
import androidx.media3.common.Format;
|
||||
import androidx.media3.common.util.MediaFormatUtil;
|
||||
import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter;
|
||||
import androidx.media3.exoplayer.mediacodec.MediaCodecInfo;
|
||||
import androidx.media3.exoplayer.mediacodec.SynchronousMediaCodecAdapter;
|
||||
import java.io.IOException;
|
||||
|
||||
/** A default {@link Codec.DecoderFactory} and {@link Codec.EncoderFactory}. */
|
||||
/* package */ final class DefaultCodecFactory
|
||||
implements Codec.DecoderFactory, Codec.EncoderFactory {
|
||||
|
||||
@Override
|
||||
public Codec createForAudioDecoding(Format format) throws TransformationException {
|
||||
MediaFormat mediaFormat =
|
||||
MediaFormat.createAudioFormat(
|
||||
checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount);
|
||||
MediaFormatUtil.maybeSetInteger(
|
||||
mediaFormat, MediaFormat.KEY_MAX_INPUT_SIZE, format.maxInputSize);
|
||||
MediaFormatUtil.setCsdBuffers(mediaFormat, format.initializationData);
|
||||
|
||||
MediaCodecAdapter adapter;
|
||||
try {
|
||||
adapter =
|
||||
new MediaCodecFactory()
|
||||
.createAdapter(
|
||||
MediaCodecAdapter.Configuration.createForAudioDecoding(
|
||||
createPlaceholderMediaCodecInfo(), mediaFormat, format, /* crypto= */ null));
|
||||
} catch (Exception e) {
|
||||
throw createTransformationException(e, format, /* isVideo= */ false, /* isDecoder= */ true);
|
||||
}
|
||||
return new Codec(adapter);
|
||||
}
|
||||
|
||||
@Override
|
||||
@SuppressLint("InlinedApi")
|
||||
public Codec createForVideoDecoding(Format format, Surface surface)
|
||||
throws TransformationException {
|
||||
MediaFormat mediaFormat =
|
||||
MediaFormat.createVideoFormat(
|
||||
checkNotNull(format.sampleMimeType), format.width, format.height);
|
||||
MediaFormatUtil.maybeSetInteger(mediaFormat, MediaFormat.KEY_ROTATION, format.rotationDegrees);
|
||||
MediaFormatUtil.maybeSetInteger(
|
||||
mediaFormat, MediaFormat.KEY_MAX_INPUT_SIZE, format.maxInputSize);
|
||||
MediaFormatUtil.setCsdBuffers(mediaFormat, format.initializationData);
|
||||
if (SDK_INT >= 29) {
|
||||
// On API levels over 29, Transformer decodes as many frames as possible in one render
|
||||
// cycle. This key ensures no frame dropping when the decoder's output surface is full.
|
||||
mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0);
|
||||
}
|
||||
|
||||
MediaCodecAdapter adapter;
|
||||
try {
|
||||
adapter =
|
||||
new MediaCodecFactory()
|
||||
.createAdapter(
|
||||
MediaCodecAdapter.Configuration.createForVideoDecoding(
|
||||
createPlaceholderMediaCodecInfo(),
|
||||
mediaFormat,
|
||||
format,
|
||||
surface,
|
||||
/* crypto= */ null));
|
||||
} catch (Exception e) {
|
||||
throw createTransformationException(e, format, /* isVideo= */ true, /* isDecoder= */ true);
|
||||
}
|
||||
return new Codec(adapter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Codec createForAudioEncoding(Format format) throws TransformationException {
|
||||
MediaFormat mediaFormat =
|
||||
MediaFormat.createAudioFormat(
|
||||
checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.bitrate);
|
||||
|
||||
MediaCodecAdapter adapter;
|
||||
try {
|
||||
adapter =
|
||||
new MediaCodecFactory()
|
||||
.createAdapter(
|
||||
MediaCodecAdapter.Configuration.createForAudioEncoding(
|
||||
createPlaceholderMediaCodecInfo(), mediaFormat, format));
|
||||
} catch (Exception e) {
|
||||
throw createTransformationException(e, format, /* isVideo= */ false, /* isDecoder= */ false);
|
||||
}
|
||||
return new Codec(adapter);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Codec createForVideoEncoding(Format format) throws TransformationException {
|
||||
checkArgument(format.width != Format.NO_VALUE);
|
||||
checkArgument(format.height != Format.NO_VALUE);
|
||||
// According to interface Javadoc, format.rotationDegrees should be 0. The video should always
|
||||
// be in landscape orientation.
|
||||
checkArgument(format.height < format.width);
|
||||
checkArgument(format.rotationDegrees == 0);
|
||||
|
||||
MediaFormat mediaFormat =
|
||||
MediaFormat.createVideoFormat(
|
||||
checkNotNull(format.sampleMimeType), format.width, format.height);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, CodecCapabilities.COLOR_FormatSurface);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, 413_000);
|
||||
|
||||
MediaCodecAdapter adapter;
|
||||
try {
|
||||
adapter =
|
||||
new MediaCodecFactory()
|
||||
.createAdapter(
|
||||
MediaCodecAdapter.Configuration.createForVideoEncoding(
|
||||
createPlaceholderMediaCodecInfo(), mediaFormat, format));
|
||||
} catch (Exception e) {
|
||||
throw createTransformationException(e, format, /* isVideo= */ true, /* isDecoder= */ false);
|
||||
}
|
||||
return new Codec(adapter);
|
||||
}
|
||||
|
||||
private static final class MediaCodecFactory extends SynchronousMediaCodecAdapter.Factory {
|
||||
@Override
|
||||
protected MediaCodec createCodec(MediaCodecAdapter.Configuration configuration)
|
||||
throws IOException {
|
||||
String sampleMimeType =
|
||||
checkNotNull(configuration.mediaFormat.getString(MediaFormat.KEY_MIME));
|
||||
boolean isDecoder = (configuration.flags & MediaCodec.CONFIGURE_FLAG_ENCODE) == 0;
|
||||
return isDecoder
|
||||
? MediaCodec.createDecoderByType(checkNotNull(sampleMimeType))
|
||||
: MediaCodec.createEncoderByType(checkNotNull(sampleMimeType));
|
||||
}
|
||||
}
|
||||
|
||||
private static MediaCodecInfo createPlaceholderMediaCodecInfo() {
|
||||
return MediaCodecInfo.newInstance(
|
||||
/* name= */ "name-placeholder",
|
||||
/* mimeType= */ "mime-type-placeholder",
|
||||
/* codecMimeType= */ "mime-type-placeholder",
|
||||
/* capabilities= */ null,
|
||||
/* hardwareAccelerated= */ false,
|
||||
/* softwareOnly= */ false,
|
||||
/* vendor= */ false,
|
||||
/* forceDisableAdaptive= */ false,
|
||||
/* forceSecure= */ false);
|
||||
}
|
||||
|
||||
private static TransformationException createTransformationException(
|
||||
Exception cause, Format format, boolean isVideo, boolean isDecoder) {
|
||||
String componentName = (isVideo ? "Video" : "Audio") + (isDecoder ? "Decoder" : "Encoder");
|
||||
if (cause instanceof IOException) {
|
||||
return TransformationException.createForCodec(
|
||||
cause,
|
||||
componentName,
|
||||
format,
|
||||
isDecoder
|
||||
? TransformationException.ERROR_CODE_DECODER_INIT_FAILED
|
||||
: TransformationException.ERROR_CODE_ENCODER_INIT_FAILED);
|
||||
}
|
||||
if (cause instanceof IllegalArgumentException) {
|
||||
return TransformationException.createForCodec(
|
||||
cause,
|
||||
componentName,
|
||||
format,
|
||||
isDecoder
|
||||
? TransformationException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED
|
||||
: TransformationException.ERROR_CODE_ENCODING_FORMAT_UNSUPPORTED);
|
||||
}
|
||||
return TransformationException.createForUnexpected(cause);
|
||||
}
|
||||
}
|
||||
@ -1,471 +0,0 @@
|
||||
/*
|
||||
* Copyright 2021 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package androidx.media3.transformer;
|
||||
|
||||
import static androidx.media3.common.util.Assertions.checkArgument;
|
||||
import static androidx.media3.common.util.Assertions.checkNotNull;
|
||||
import static androidx.media3.common.util.Assertions.checkState;
|
||||
import static androidx.media3.common.util.Util.SDK_INT;
|
||||
|
||||
import android.annotation.SuppressLint;
|
||||
import android.media.MediaCodec;
|
||||
import android.media.MediaCodec.BufferInfo;
|
||||
import android.media.MediaCodecInfo.CodecCapabilities;
|
||||
import android.media.MediaFormat;
|
||||
import android.view.Surface;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.Format;
|
||||
import androidx.media3.common.MimeTypes;
|
||||
import androidx.media3.common.util.MediaFormatUtil;
|
||||
import androidx.media3.decoder.DecoderInputBuffer;
|
||||
import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter;
|
||||
import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter.Configuration;
|
||||
import androidx.media3.exoplayer.mediacodec.MediaCodecInfo;
|
||||
import androidx.media3.exoplayer.mediacodec.SynchronousMediaCodecAdapter;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.Map;
|
||||
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
/**
|
||||
* A wrapper around {@link MediaCodecAdapter}.
|
||||
*
|
||||
* <p>Provides a layer of abstraction for callers that need to interact with {@link MediaCodec}
|
||||
* through {@link MediaCodecAdapter}. This is done by simplifying the calls needed to queue and
|
||||
* dequeue buffers, removing the need to track buffer indices and codec events.
|
||||
*/
|
||||
/* package */ final class MediaCodecAdapterWrapper {
|
||||
|
||||
// MediaCodec decoders always output 16 bit PCM, unless configured to output PCM float.
|
||||
// https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers.
|
||||
private static final int MEDIA_CODEC_PCM_ENCODING = C.ENCODING_PCM_16BIT;
|
||||
|
||||
private final BufferInfo outputBufferInfo;
|
||||
private final MediaCodecAdapter codec;
|
||||
|
||||
private @MonotonicNonNull Format outputFormat;
|
||||
@Nullable private ByteBuffer outputBuffer;
|
||||
|
||||
private int inputBufferIndex;
|
||||
private int outputBufferIndex;
|
||||
private boolean inputStreamEnded;
|
||||
private boolean outputStreamEnded;
|
||||
|
||||
private static class Factory extends SynchronousMediaCodecAdapter.Factory {
|
||||
@Override
|
||||
protected MediaCodec createCodec(Configuration configuration) throws IOException {
|
||||
String sampleMimeType =
|
||||
checkNotNull(configuration.mediaFormat.getString(MediaFormat.KEY_MIME));
|
||||
boolean isDecoder = (configuration.flags & MediaCodec.CONFIGURE_FLAG_ENCODE) == 0;
|
||||
return isDecoder
|
||||
? MediaCodec.createDecoderByType(checkNotNull(sampleMimeType))
|
||||
: MediaCodec.createEncoderByType(checkNotNull(sampleMimeType));
|
||||
}
|
||||
}
|
||||
|
||||
private static MediaCodecInfo createPlaceholderMediaCodecInfo() {
|
||||
return MediaCodecInfo.newInstance(
|
||||
/* name= */ "name-placeholder",
|
||||
/* mimeType= */ "mime-type-placeholder",
|
||||
/* codecMimeType= */ "mime-type-placeholder",
|
||||
/* capabilities= */ null,
|
||||
/* hardwareAccelerated= */ false,
|
||||
/* softwareOnly= */ false,
|
||||
/* vendor= */ false,
|
||||
/* forceDisableAdaptive= */ false,
|
||||
/* forceSecure= */ false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link MediaCodecAdapterWrapper} for a configured and started {@link
|
||||
* MediaCodecAdapter} audio decoder.
|
||||
*
|
||||
* @param format The {@link Format} (of the input data) used to determine the underlying {@link
|
||||
* MediaCodec} and its configuration values.
|
||||
* @return A configured and started decoder wrapper.
|
||||
* @throws TransformationException If the underlying codec cannot be created.
|
||||
*/
|
||||
public static MediaCodecAdapterWrapper createForAudioDecoding(Format format)
|
||||
throws TransformationException {
|
||||
MediaFormat mediaFormat =
|
||||
MediaFormat.createAudioFormat(
|
||||
checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount);
|
||||
MediaFormatUtil.maybeSetInteger(
|
||||
mediaFormat, MediaFormat.KEY_MAX_INPUT_SIZE, format.maxInputSize);
|
||||
MediaFormatUtil.setCsdBuffers(mediaFormat, format.initializationData);
|
||||
|
||||
MediaCodecAdapter adapter;
|
||||
try {
|
||||
adapter =
|
||||
new Factory()
|
||||
.createAdapter(
|
||||
MediaCodecAdapter.Configuration.createForAudioDecoding(
|
||||
createPlaceholderMediaCodecInfo(), mediaFormat, format, /* crypto= */ null));
|
||||
} catch (Exception e) {
|
||||
throw createTransformationException(e, format, /* isVideo= */ false, /* isDecoder= */ true);
|
||||
}
|
||||
return new MediaCodecAdapterWrapper(adapter);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link MediaCodecAdapterWrapper} for a configured and started {@link
|
||||
* MediaCodecAdapter} video decoder.
|
||||
*
|
||||
* @param format The {@link Format} (of the input data) used to determine the underlying {@link
|
||||
* MediaCodec} and its configuration values.
|
||||
* @param surface The {@link Surface} to which the decoder output is rendered.
|
||||
* @return A configured and started decoder wrapper.
|
||||
* @throws TransformationException If the underlying codec cannot be created.
|
||||
*/
|
||||
@SuppressLint("InlinedApi")
|
||||
public static MediaCodecAdapterWrapper createForVideoDecoding(Format format, Surface surface)
|
||||
throws TransformationException {
|
||||
MediaFormat mediaFormat =
|
||||
MediaFormat.createVideoFormat(
|
||||
checkNotNull(format.sampleMimeType), format.width, format.height);
|
||||
MediaFormatUtil.maybeSetInteger(mediaFormat, MediaFormat.KEY_ROTATION, format.rotationDegrees);
|
||||
MediaFormatUtil.maybeSetInteger(
|
||||
mediaFormat, MediaFormat.KEY_MAX_INPUT_SIZE, format.maxInputSize);
|
||||
MediaFormatUtil.setCsdBuffers(mediaFormat, format.initializationData);
|
||||
if (SDK_INT >= 29) {
|
||||
// On API levels over 29, Transformer decodes as many frames as possible in one render
|
||||
// cycle. This key ensures no frame dropping when the decoder's output surface is full.
|
||||
mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0);
|
||||
}
|
||||
|
||||
MediaCodecAdapter adapter;
|
||||
try {
|
||||
adapter =
|
||||
new Factory()
|
||||
.createAdapter(
|
||||
MediaCodecAdapter.Configuration.createForVideoDecoding(
|
||||
createPlaceholderMediaCodecInfo(),
|
||||
mediaFormat,
|
||||
format,
|
||||
surface,
|
||||
/* crypto= */ null));
|
||||
} catch (Exception e) {
|
||||
throw createTransformationException(e, format, /* isVideo= */ true, /* isDecoder= */ true);
|
||||
}
|
||||
return new MediaCodecAdapterWrapper(adapter);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link MediaCodecAdapterWrapper} for a configured and started {@link
|
||||
* MediaCodecAdapter} audio encoder.
|
||||
*
|
||||
* @param format The {@link Format} (of the output data) used to determine the underlying {@link
|
||||
* MediaCodec} and its configuration values.
|
||||
* @return A configured and started encoder wrapper.
|
||||
* @throws TransformationException If the underlying codec cannot be created.
|
||||
*/
|
||||
public static MediaCodecAdapterWrapper createForAudioEncoding(Format format)
|
||||
throws TransformationException {
|
||||
MediaFormat mediaFormat =
|
||||
MediaFormat.createAudioFormat(
|
||||
checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.bitrate);
|
||||
|
||||
MediaCodecAdapter adapter;
|
||||
try {
|
||||
adapter =
|
||||
new Factory()
|
||||
.createAdapter(
|
||||
MediaCodecAdapter.Configuration.createForAudioEncoding(
|
||||
createPlaceholderMediaCodecInfo(), mediaFormat, format));
|
||||
} catch (Exception e) {
|
||||
throw createTransformationException(e, format, /* isVideo= */ false, /* isDecoder= */ false);
|
||||
}
|
||||
return new MediaCodecAdapterWrapper(adapter);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a {@link MediaCodecAdapterWrapper} for a configured and started {@link
|
||||
* MediaCodecAdapter} video encoder.
|
||||
*
|
||||
* @param format The {@link Format} (of the output data) used to determine the underlying {@link
|
||||
* MediaCodec} and its configuration values. {@link Format#sampleMimeType}, {@link
|
||||
* Format#width} and {@link Format#height} must be set to those of the desired output video
|
||||
* format. {@link Format#rotationDegrees} should be 0. The video should always be in landscape
|
||||
* orientation.
|
||||
* @param additionalEncoderConfig A map of {@link MediaFormat}'s integer settings, where the keys
|
||||
* are from {@code MediaFormat.KEY_*} constants. Its values will override those in {@code
|
||||
* format}.
|
||||
* @return A configured and started encoder wrapper.
|
||||
* @throws TransformationException If the underlying codec cannot be created.
|
||||
*/
|
||||
public static MediaCodecAdapterWrapper createForVideoEncoding(
|
||||
Format format, Map<String, Integer> additionalEncoderConfig) throws TransformationException {
|
||||
checkArgument(format.width != Format.NO_VALUE);
|
||||
checkArgument(format.height != Format.NO_VALUE);
|
||||
checkArgument(format.height < format.width);
|
||||
checkArgument(format.rotationDegrees == 0);
|
||||
|
||||
MediaFormat mediaFormat =
|
||||
MediaFormat.createVideoFormat(
|
||||
checkNotNull(format.sampleMimeType), format.width, format.height);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, CodecCapabilities.COLOR_FormatSurface);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
|
||||
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, 413_000);
|
||||
for (Map.Entry<String, Integer> encoderSetting : additionalEncoderConfig.entrySet()) {
|
||||
mediaFormat.setInteger(encoderSetting.getKey(), encoderSetting.getValue());
|
||||
}
|
||||
|
||||
MediaCodecAdapter adapter;
|
||||
try {
|
||||
adapter =
|
||||
new Factory()
|
||||
.createAdapter(
|
||||
MediaCodecAdapter.Configuration.createForVideoEncoding(
|
||||
createPlaceholderMediaCodecInfo(), mediaFormat, format));
|
||||
} catch (Exception e) {
|
||||
throw createTransformationException(e, format, /* isVideo= */ true, /* isDecoder= */ false);
|
||||
}
|
||||
return new MediaCodecAdapterWrapper(adapter);
|
||||
}
|
||||
|
||||
private MediaCodecAdapterWrapper(MediaCodecAdapter codec) {
|
||||
this.codec = codec;
|
||||
outputBufferInfo = new BufferInfo();
|
||||
inputBufferIndex = C.INDEX_UNSET;
|
||||
outputBufferIndex = C.INDEX_UNSET;
|
||||
}
|
||||
|
||||
/** Returns the input {@link Surface}, or null if the input is not a surface. */
|
||||
@Nullable
|
||||
public Surface getInputSurface() {
|
||||
return codec.getInputSurface();
|
||||
}
|
||||
|
||||
/**
|
||||
* Dequeues a writable input buffer, if available.
|
||||
*
|
||||
* @param inputBuffer The buffer where the dequeued buffer data is stored.
|
||||
* @return Whether an input buffer is ready to be used.
|
||||
*/
|
||||
@EnsuresNonNullIf(expression = "#1.data", result = true)
|
||||
public boolean maybeDequeueInputBuffer(DecoderInputBuffer inputBuffer) {
|
||||
if (inputStreamEnded) {
|
||||
return false;
|
||||
}
|
||||
if (inputBufferIndex < 0) {
|
||||
inputBufferIndex = codec.dequeueInputBufferIndex();
|
||||
if (inputBufferIndex < 0) {
|
||||
return false;
|
||||
}
|
||||
inputBuffer.data = codec.getInputBuffer(inputBufferIndex);
|
||||
inputBuffer.clear();
|
||||
}
|
||||
checkNotNull(inputBuffer.data);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Queues an input buffer to the decoder. No buffers may be queued after an {@link
|
||||
* DecoderInputBuffer#isEndOfStream() end of stream} buffer has been queued.
|
||||
*/
|
||||
public void queueInputBuffer(DecoderInputBuffer inputBuffer) {
|
||||
checkState(
|
||||
!inputStreamEnded, "Input buffer can not be queued after the input stream has ended.");
|
||||
|
||||
int offset = 0;
|
||||
int size = 0;
|
||||
if (inputBuffer.data != null && inputBuffer.data.hasRemaining()) {
|
||||
offset = inputBuffer.data.position();
|
||||
size = inputBuffer.data.remaining();
|
||||
}
|
||||
int flags = 0;
|
||||
if (inputBuffer.isEndOfStream()) {
|
||||
inputStreamEnded = true;
|
||||
flags = MediaCodec.BUFFER_FLAG_END_OF_STREAM;
|
||||
}
|
||||
codec.queueInputBuffer(inputBufferIndex, offset, size, inputBuffer.timeUs, flags);
|
||||
inputBufferIndex = C.INDEX_UNSET;
|
||||
inputBuffer.data = null;
|
||||
}
|
||||
|
||||
public void signalEndOfInputStream() {
|
||||
codec.signalEndOfInputStream();
|
||||
}
|
||||
|
||||
/** Returns the current output format, if available. */
|
||||
@Nullable
|
||||
public Format getOutputFormat() {
|
||||
// The format is updated when dequeueing a 'special' buffer index, so attempt to dequeue now.
|
||||
maybeDequeueOutputBuffer();
|
||||
return outputFormat;
|
||||
}
|
||||
|
||||
/** Returns the current output {@link ByteBuffer}, if available. */
|
||||
@Nullable
|
||||
public ByteBuffer getOutputBuffer() {
|
||||
return maybeDequeueAndSetOutputBuffer() ? outputBuffer : null;
|
||||
}
|
||||
|
||||
/** Returns the {@link BufferInfo} associated with the current output buffer, if available. */
|
||||
@Nullable
|
||||
public BufferInfo getOutputBufferInfo() {
|
||||
return maybeDequeueOutputBuffer() ? outputBufferInfo : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases the current output buffer.
|
||||
*
|
||||
* <p>This should be called after the buffer has been processed. The next output buffer will not
|
||||
* be available until the previous has been released.
|
||||
*/
|
||||
public void releaseOutputBuffer() {
|
||||
releaseOutputBuffer(/* render= */ false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases the current output buffer. If the {@link MediaCodec} was configured with an output
|
||||
* surface, setting {@code render} to {@code true} will first send the buffer to the output
|
||||
* surface. The surface will release the buffer back to the codec once it is no longer
|
||||
* used/displayed.
|
||||
*
|
||||
* <p>This should be called after the buffer has been processed. The next output buffer will not
|
||||
* be available until the previous has been released.
|
||||
*/
|
||||
public void releaseOutputBuffer(boolean render) {
|
||||
outputBuffer = null;
|
||||
codec.releaseOutputBuffer(outputBufferIndex, render);
|
||||
outputBufferIndex = C.INDEX_UNSET;
|
||||
}
|
||||
|
||||
/** Returns whether the codec output stream has ended, and no more data can be dequeued. */
|
||||
public boolean isEnded() {
|
||||
return outputStreamEnded && outputBufferIndex == C.INDEX_UNSET;
|
||||
}
|
||||
|
||||
/** Releases the underlying codec. */
|
||||
public void release() {
|
||||
outputBuffer = null;
|
||||
codec.release();
|
||||
}
|
||||
|
||||
/**
|
||||
* Tries obtaining an output buffer and sets {@link #outputBuffer} to the obtained output buffer.
|
||||
*
|
||||
* @return {@code true} if a buffer is successfully obtained, {@code false} otherwise.
|
||||
*/
|
||||
private boolean maybeDequeueAndSetOutputBuffer() {
|
||||
if (!maybeDequeueOutputBuffer()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
outputBuffer = checkNotNull(codec.getOutputBuffer(outputBufferIndex));
|
||||
outputBuffer.position(outputBufferInfo.offset);
|
||||
outputBuffer.limit(outputBufferInfo.offset + outputBufferInfo.size);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if there is already an output buffer pending. Otherwise attempts to dequeue an
|
||||
* output buffer and returns whether there is a new output buffer.
|
||||
*/
|
||||
private boolean maybeDequeueOutputBuffer() {
|
||||
if (outputBufferIndex >= 0) {
|
||||
return true;
|
||||
}
|
||||
if (outputStreamEnded) {
|
||||
return false;
|
||||
}
|
||||
|
||||
outputBufferIndex = codec.dequeueOutputBufferIndex(outputBufferInfo);
|
||||
if (outputBufferIndex < 0) {
|
||||
if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
|
||||
outputFormat = getFormat(codec.getOutputFormat());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if ((outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
|
||||
outputStreamEnded = true;
|
||||
if (outputBufferInfo.size == 0) {
|
||||
releaseOutputBuffer();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if ((outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
|
||||
// Encountered a CSD buffer, skip it.
|
||||
releaseOutputBuffer();
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static Format getFormat(MediaFormat mediaFormat) {
|
||||
ImmutableList.Builder<byte[]> csdBuffers = new ImmutableList.Builder<>();
|
||||
int csdIndex = 0;
|
||||
while (true) {
|
||||
@Nullable ByteBuffer csdByteBuffer = mediaFormat.getByteBuffer("csd-" + csdIndex);
|
||||
if (csdByteBuffer == null) {
|
||||
break;
|
||||
}
|
||||
byte[] csdBufferData = new byte[csdByteBuffer.remaining()];
|
||||
csdByteBuffer.get(csdBufferData);
|
||||
csdBuffers.add(csdBufferData);
|
||||
csdIndex++;
|
||||
}
|
||||
String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
|
||||
Format.Builder formatBuilder =
|
||||
new Format.Builder()
|
||||
.setSampleMimeType(mediaFormat.getString(MediaFormat.KEY_MIME))
|
||||
.setInitializationData(csdBuffers.build());
|
||||
if (MimeTypes.isVideo(mimeType)) {
|
||||
formatBuilder
|
||||
.setWidth(mediaFormat.getInteger(MediaFormat.KEY_WIDTH))
|
||||
.setHeight(mediaFormat.getInteger(MediaFormat.KEY_HEIGHT));
|
||||
} else if (MimeTypes.isAudio(mimeType)) {
|
||||
// TODO(internal b/178685617): Only set the PCM encoding for audio/raw, once we have a way to
|
||||
// simulate more realistic codec input/output formats in tests.
|
||||
formatBuilder
|
||||
.setChannelCount(mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT))
|
||||
.setSampleRate(mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE))
|
||||
.setPcmEncoding(MEDIA_CODEC_PCM_ENCODING);
|
||||
}
|
||||
return formatBuilder.build();
|
||||
}
|
||||
|
||||
private static TransformationException createTransformationException(
|
||||
Exception cause, Format format, boolean isVideo, boolean isDecoder) {
|
||||
String componentName = (isVideo ? "Video" : "Audio") + (isDecoder ? "Decoder" : "Encoder");
|
||||
if (cause instanceof IOException) {
|
||||
return TransformationException.createForCodec(
|
||||
cause,
|
||||
componentName,
|
||||
format,
|
||||
isDecoder
|
||||
? TransformationException.ERROR_CODE_DECODER_INIT_FAILED
|
||||
: TransformationException.ERROR_CODE_ENCODER_INIT_FAILED);
|
||||
}
|
||||
if (cause instanceof IllegalArgumentException) {
|
||||
return TransformationException.createForCodec(
|
||||
cause,
|
||||
componentName,
|
||||
format,
|
||||
isDecoder
|
||||
? TransformationException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED
|
||||
: TransformationException.ERROR_CODE_ENCODING_FORMAT_UNSUPPORTED);
|
||||
}
|
||||
return TransformationException.createForUnexpected(cause);
|
||||
}
|
||||
}
|
||||
@ -109,6 +109,7 @@ public final class Transformer {
|
||||
private DebugViewProvider debugViewProvider;
|
||||
private Looper looper;
|
||||
private Clock clock;
|
||||
private Codec.EncoderFactory encoderFactory;
|
||||
|
||||
/** @deprecated Use {@link #Builder(Context)} instead. */
|
||||
@Deprecated
|
||||
@ -120,6 +121,7 @@ public final class Transformer {
|
||||
listener = new Listener() {};
|
||||
looper = Util.getCurrentOrMainLooper();
|
||||
clock = Clock.DEFAULT;
|
||||
encoderFactory = Codec.EncoderFactory.DEFAULT;
|
||||
debugViewProvider = DebugViewProvider.NONE;
|
||||
}
|
||||
|
||||
@ -137,6 +139,7 @@ public final class Transformer {
|
||||
listener = new Listener() {};
|
||||
looper = Util.getCurrentOrMainLooper();
|
||||
clock = Clock.DEFAULT;
|
||||
encoderFactory = Codec.EncoderFactory.DEFAULT;
|
||||
debugViewProvider = DebugViewProvider.NONE;
|
||||
}
|
||||
|
||||
@ -155,6 +158,7 @@ public final class Transformer {
|
||||
this.videoMimeType = transformer.transformation.videoMimeType;
|
||||
this.listener = transformer.listener;
|
||||
this.looper = transformer.looper;
|
||||
this.encoderFactory = transformer.encoderFactory;
|
||||
this.debugViewProvider = transformer.debugViewProvider;
|
||||
this.clock = transformer.clock;
|
||||
}
|
||||
@ -362,6 +366,18 @@ public final class Transformer {
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the {@link Codec.EncoderFactory} that will be used by the transformer. The default value
|
||||
* is {@link Codec.EncoderFactory#DEFAULT}.
|
||||
*
|
||||
* @param encoderFactory The {@link Codec.EncoderFactory} instance.
|
||||
* @return This builder.
|
||||
*/
|
||||
public Builder setEncoderFactory(Codec.EncoderFactory encoderFactory) {
|
||||
this.encoderFactory = encoderFactory;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets a provider for views to show diagnostic information (if available) during
|
||||
* transformation. This is intended for debugging. The default value is {@link
|
||||
@ -450,6 +466,8 @@ public final class Transformer {
|
||||
listener,
|
||||
looper,
|
||||
clock,
|
||||
encoderFactory,
|
||||
Codec.DecoderFactory.DEFAULT,
|
||||
debugViewProvider);
|
||||
}
|
||||
|
||||
@ -538,6 +556,8 @@ public final class Transformer {
|
||||
private final Transformation transformation;
|
||||
private final Looper looper;
|
||||
private final Clock clock;
|
||||
private final Codec.EncoderFactory encoderFactory;
|
||||
private final Codec.DecoderFactory decoderFactory;
|
||||
private final Transformer.DebugViewProvider debugViewProvider;
|
||||
|
||||
private Transformer.Listener listener;
|
||||
@ -553,6 +573,8 @@ public final class Transformer {
|
||||
Transformer.Listener listener,
|
||||
Looper looper,
|
||||
Clock clock,
|
||||
Codec.EncoderFactory encoderFactory,
|
||||
Codec.DecoderFactory decoderFactory,
|
||||
Transformer.DebugViewProvider debugViewProvider) {
|
||||
checkState(
|
||||
!transformation.removeAudio || !transformation.removeVideo,
|
||||
@ -564,6 +586,8 @@ public final class Transformer {
|
||||
this.listener = listener;
|
||||
this.looper = looper;
|
||||
this.clock = clock;
|
||||
this.encoderFactory = encoderFactory;
|
||||
this.decoderFactory = decoderFactory;
|
||||
this.debugViewProvider = debugViewProvider;
|
||||
progressState = PROGRESS_STATE_NO_TRANSFORMATION;
|
||||
}
|
||||
@ -664,7 +688,12 @@ public final class Transformer {
|
||||
new ExoPlayer.Builder(
|
||||
context,
|
||||
new TransformerRenderersFactory(
|
||||
context, muxerWrapper, transformation, debugViewProvider))
|
||||
context,
|
||||
muxerWrapper,
|
||||
transformation,
|
||||
encoderFactory,
|
||||
decoderFactory,
|
||||
debugViewProvider))
|
||||
.setMediaSourceFactory(mediaSourceFactory)
|
||||
.setTrackSelector(trackSelector)
|
||||
.setLoadControl(loadControl)
|
||||
@ -753,16 +782,22 @@ public final class Transformer {
|
||||
private final MuxerWrapper muxerWrapper;
|
||||
private final TransformerMediaClock mediaClock;
|
||||
private final Transformation transformation;
|
||||
private final Codec.EncoderFactory encoderFactory;
|
||||
private final Codec.DecoderFactory decoderFactory;
|
||||
private final Transformer.DebugViewProvider debugViewProvider;
|
||||
|
||||
public TransformerRenderersFactory(
|
||||
Context context,
|
||||
MuxerWrapper muxerWrapper,
|
||||
Transformation transformation,
|
||||
Codec.EncoderFactory encoderFactory,
|
||||
Codec.DecoderFactory decoderFactory,
|
||||
Transformer.DebugViewProvider debugViewProvider) {
|
||||
this.context = context;
|
||||
this.muxerWrapper = muxerWrapper;
|
||||
this.transformation = transformation;
|
||||
this.encoderFactory = encoderFactory;
|
||||
this.decoderFactory = decoderFactory;
|
||||
this.debugViewProvider = debugViewProvider;
|
||||
mediaClock = new TransformerMediaClock();
|
||||
}
|
||||
@ -778,13 +813,21 @@ public final class Transformer {
|
||||
Renderer[] renderers = new Renderer[rendererCount];
|
||||
int index = 0;
|
||||
if (!transformation.removeAudio) {
|
||||
renderers[index] = new TransformerAudioRenderer(muxerWrapper, mediaClock, transformation);
|
||||
renderers[index] =
|
||||
new TransformerAudioRenderer(
|
||||
muxerWrapper, mediaClock, transformation, encoderFactory, decoderFactory);
|
||||
index++;
|
||||
}
|
||||
if (!transformation.removeVideo) {
|
||||
renderers[index] =
|
||||
new TransformerVideoRenderer(
|
||||
context, muxerWrapper, mediaClock, transformation, debugViewProvider);
|
||||
context,
|
||||
muxerWrapper,
|
||||
mediaClock,
|
||||
transformation,
|
||||
encoderFactory,
|
||||
decoderFactory,
|
||||
debugViewProvider);
|
||||
index++;
|
||||
}
|
||||
return renderers;
|
||||
|
||||
@ -32,11 +32,19 @@ import androidx.media3.extractor.metadata.mp4.SlowMotionData;
|
||||
|
||||
private static final String TAG = "TAudioRenderer";
|
||||
|
||||
private final Codec.EncoderFactory encoderFactory;
|
||||
private final Codec.DecoderFactory decoderFactory;
|
||||
private final DecoderInputBuffer decoderInputBuffer;
|
||||
|
||||
public TransformerAudioRenderer(
|
||||
MuxerWrapper muxerWrapper, TransformerMediaClock mediaClock, Transformation transformation) {
|
||||
MuxerWrapper muxerWrapper,
|
||||
TransformerMediaClock mediaClock,
|
||||
Transformation transformation,
|
||||
Codec.EncoderFactory encoderFactory,
|
||||
Codec.DecoderFactory decoderFactory) {
|
||||
super(C.TRACK_TYPE_AUDIO, muxerWrapper, mediaClock, transformation);
|
||||
this.encoderFactory = encoderFactory;
|
||||
this.decoderFactory = decoderFactory;
|
||||
decoderInputBuffer =
|
||||
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
|
||||
}
|
||||
@ -60,7 +68,8 @@ import androidx.media3.extractor.metadata.mp4.SlowMotionData;
|
||||
}
|
||||
Format inputFormat = checkNotNull(formatHolder.format);
|
||||
if (shouldTranscode(inputFormat)) {
|
||||
samplePipeline = new AudioSamplePipeline(inputFormat, transformation);
|
||||
samplePipeline =
|
||||
new AudioSamplePipeline(inputFormat, transformation, encoderFactory, decoderFactory);
|
||||
} else {
|
||||
samplePipeline = new PassthroughSamplePipeline(inputFormat);
|
||||
}
|
||||
|
||||
@ -34,6 +34,8 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
private static final String TAG = "TVideoRenderer";
|
||||
|
||||
private final Context context;
|
||||
private final Codec.EncoderFactory encoderFactory;
|
||||
private final Codec.DecoderFactory decoderFactory;
|
||||
private final Transformer.DebugViewProvider debugViewProvider;
|
||||
private final DecoderInputBuffer decoderInputBuffer;
|
||||
|
||||
@ -44,9 +46,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
MuxerWrapper muxerWrapper,
|
||||
TransformerMediaClock mediaClock,
|
||||
Transformation transformation,
|
||||
Codec.EncoderFactory encoderFactory,
|
||||
Codec.DecoderFactory decoderFactory,
|
||||
Transformer.DebugViewProvider debugViewProvider) {
|
||||
super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformation);
|
||||
this.context = context;
|
||||
this.encoderFactory = encoderFactory;
|
||||
this.decoderFactory = decoderFactory;
|
||||
this.debugViewProvider = debugViewProvider;
|
||||
decoderInputBuffer =
|
||||
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
|
||||
@ -72,7 +78,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
Format inputFormat = checkNotNull(formatHolder.format);
|
||||
if (shouldTranscode(inputFormat)) {
|
||||
samplePipeline =
|
||||
new VideoSamplePipeline(context, inputFormat, transformation, debugViewProvider);
|
||||
new VideoSamplePipeline(
|
||||
context,
|
||||
inputFormat,
|
||||
transformation,
|
||||
encoderFactory,
|
||||
decoderFactory,
|
||||
debugViewProvider);
|
||||
} else {
|
||||
samplePipeline = new PassthroughSamplePipeline(inputFormat);
|
||||
}
|
||||
|
||||
@ -27,7 +27,6 @@ import androidx.annotation.RequiresApi;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.Format;
|
||||
import androidx.media3.decoder.DecoderInputBuffer;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
/**
|
||||
@ -39,9 +38,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
|
||||
private final int outputRotationDegrees;
|
||||
private final DecoderInputBuffer decoderInputBuffer;
|
||||
private final MediaCodecAdapterWrapper decoder;
|
||||
private final Codec decoder;
|
||||
|
||||
private final MediaCodecAdapterWrapper encoder;
|
||||
private final Codec encoder;
|
||||
private final DecoderInputBuffer encoderOutputBuffer;
|
||||
|
||||
private @MonotonicNonNull FrameEditor frameEditor;
|
||||
@ -52,6 +51,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
Context context,
|
||||
Format inputFormat,
|
||||
Transformation transformation,
|
||||
Codec.EncoderFactory encoderFactory,
|
||||
Codec.DecoderFactory decoderFactory,
|
||||
Transformer.DebugViewProvider debugViewProvider)
|
||||
throws TransformationException {
|
||||
decoderInputBuffer =
|
||||
@ -85,7 +86,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
transformation.transformationMatrix.postRotate(outputRotationDegrees);
|
||||
|
||||
encoder =
|
||||
MediaCodecAdapterWrapper.createForVideoEncoding(
|
||||
encoderFactory.createForVideoEncoding(
|
||||
new Format.Builder()
|
||||
.setWidth(outputWidth)
|
||||
.setHeight(outputHeight)
|
||||
@ -94,8 +95,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
transformation.videoMimeType != null
|
||||
? transformation.videoMimeType
|
||||
: inputFormat.sampleMimeType)
|
||||
.build(),
|
||||
ImmutableMap.of());
|
||||
.build());
|
||||
if (inputFormat.height != outputHeight
|
||||
|| inputFormat.width != outputWidth
|
||||
|| !transformation.transformationMatrix.isIdentity()) {
|
||||
@ -109,7 +109,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
debugViewProvider);
|
||||
}
|
||||
decoder =
|
||||
MediaCodecAdapterWrapper.createForVideoDecoding(
|
||||
decoderFactory.createForVideoDecoding(
|
||||
inputFormat,
|
||||
frameEditor == null
|
||||
? checkNotNull(encoder.getInputSurface())
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user