loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make Codec an interface and introduce DefaultCodec.

Browse Source 
PiperOrigin-RevId: 427191610
This commit is contained in:
claincly 2022-02-08 15:56:26 +00:00 committed by Ian Baker
parent 9582a9f099
commit a41bdbad31
8 changed files with 486 additions and 454 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
libraries/transformer/src/main/java/androidx/media3/transformer/AudioTranscodingSamplePipeline.java
View File

@ -150,7 +150,7 @@ import org.checkerframework.dataflow.qual.Pure;
@Override
public void releaseOutputBuffer() throws TransformationException {
encoder.releaseOutputBuffer();
encoder.releaseOutputBuffer(/* render= */ false);
}
@Override
@ -188,7 +188,7 @@ import org.checkerframework.dataflow.qual.Pure;
feedEncoder(decoderOutputBuffer);
if (!decoderOutputBuffer.hasRemaining()) {
decoder.releaseOutputBuffer();
decoder.releaseOutputBuffer(/* render= */ false);
}
return true;
}
@ -243,7 +243,7 @@ import org.checkerframework.dataflow.qual.Pure;
speedChangingAudioProcessor.queueInput(decoderOutputBuffer);
if (!decoderOutputBuffer.hasRemaining()) {
decoder.releaseOutputBuffer();
decoder.releaseOutputBuffer(/* render= */ false);
}
return true;
}

376
libraries/transformer/src/main/java/androidx/media3/transformer/Codec.java
View File

@ -16,12 +16,7 @@
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import android.media.MediaCodec;
import android.media.MediaCodec.BufferInfo;
import android.media.MediaFormat;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
@ -29,24 +24,20 @@ import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.decoder.DecoderInputBuffer;
import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer;
import java.util.List;
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* A wrapper around {@link MediaCodec}.
* Provides a layer of abstraction for interacting with decoders and encoders.
*
* <p>Provides a layer of abstraction for callers that need to interact with {@link MediaCodec}.
* This is done by simplifying the calls needed to queue and dequeue buffers, removing the need to
* track buffer indices and codec events.
* <p>{@link DecoderInputBuffer DecoderInputBuffers} are used as both decoders' and encoders' input
* buffers.
*/
@UnstableApi
public final class Codec {
public interface Codec {
/** A factory for {@link Codec decoder} instances. */
public interface DecoderFactory {
interface DecoderFactory {
/** A default {@code DecoderFactory} implementation. */
DecoderFactory DEFAULT = new DefaultDecoderFactory();
@ -54,28 +45,28 @@ public final class Codec {
/**
* Returns a {@link Codec} for audio decoding.
*
* @param format The {@link Format} (of the input data) used to determine the underlying {@link
* MediaCodec} and its configuration values.
* @return A configured and started decoder wrapper.
* @throws TransformationException If no suitable codec can be created.
* @param format The {@link Format} (of the input data) used to determine the underlying decoder
* and its configuration values.
* @return A {@link Codec} for audio decoding.
* @throws TransformationException If no suitable {@link Codec} can be created.
*/
Codec createForAudioDecoding(Format format) throws TransformationException;
/**
* Returns a {@link Codec} for video decoding.
*
* @param format The {@link Format} (of the input data) used to determine the underlying {@link
* MediaCodec} and its configuration values.
* @param format The {@link Format} (of the input data) used to determine the underlying decoder
* and its configuration values.
* @param outputSurface The {@link Surface} to which the decoder output is rendered.
* @return A configured and started decoder wrapper.
* @throws TransformationException If no suitable codec can be created.
* @return A {@link Codec} for video decoding.
* @throws TransformationException If no suitable {@link Codec} can be created.
*/
Codec createForVideoDecoding(Format format, Surface outputSurface)
throws TransformationException;
}
/** A factory for {@link Codec encoder} instances. */
public interface EncoderFactory {
interface EncoderFactory {
/** A default {@code EncoderFactory} implementation. */
EncoderFactory DEFAULT = new DefaultEncoderFactory();
@ -87,12 +78,12 @@ public final class Codec {
* {@code allowedMimeTypes}. The {@link Format#sampleMimeType sample MIME type} given in {@code
* format} is not necessarily allowed.
*
* @param format The {@link Format} (of the output data) used to determine the underlying {@link
* MediaCodec} and its configuration values.
* @param format The {@link Format} (of the output data) used to determine the underlying
* encoder and its configuration values.
* @param allowedMimeTypes The non-empty list of allowed output sample {@link MimeTypes MIME
* types}.
* @return A configured and started encoder wrapper.
* @throws TransformationException If no suitable codec can be created.
* @return A {@link Codec} for audio encoding.
* @throws TransformationException If no suitable {@link Codec} can be created.
*/
Codec createForAudioEncoding(Format format, List<String> allowedMimeTypes)
throws TransformationException;
@ -104,334 +95,123 @@ public final class Codec {
* {@code allowedMimeTypes}. The {@link Format#sampleMimeType sample MIME type} given in {@code
* format} is not necessarily allowed.
*
* @param format The {@link Format} (of the output data) used to determine the underlying {@link
* MediaCodec} and its configuration values. {@link Format#sampleMimeType}, {@link
* Format#width} and {@link Format#height} must be set to those of the desired output video
* format. {@link Format#rotationDegrees} should be 0. The video should always be in
* landscape orientation.
* @param format The {@link Format} (of the output data) used to determine the underlying
* encoder and its configuration values. {@link Format#sampleMimeType}, {@link Format#width}
* and {@link Format#height} must be set to those of the desired output video format. {@link
* Format#rotationDegrees} should be 0. The video should always be in landscape orientation.
* @param allowedMimeTypes The non-empty list of allowed output sample {@link MimeTypes MIME
* types}.
* @return A configured and started encoder wrapper.
* @throws TransformationException If no suitable codec can be created.
* @return A {@link Codec} for video encoding.
* @throws TransformationException If no suitable {@link Codec} can be created.
*/
Codec createForVideoEncoding(Format format, List<String> allowedMimeTypes)
throws TransformationException;
}
// MediaCodec decoders always output 16 bit PCM, unless configured to output PCM float.
// https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers.
private static final int MEDIA_CODEC_PCM_ENCODING = C.ENCODING_PCM_16BIT;
private final BufferInfo outputBufferInfo;
private final MediaCodec mediaCodec;
private final Format configurationFormat;
@Nullable private final Surface inputSurface;
private @MonotonicNonNull Format outputFormat;
@Nullable private ByteBuffer outputBuffer;
private int inputBufferIndex;
private int outputBufferIndex;
private boolean inputStreamEnded;
private boolean outputStreamEnded;
/**
* Creates a {@code Codec} from a configured and started {@link MediaCodec}.
*
* @param mediaCodec The configured and started {@link MediaCodec}.
* @param configurationFormat See {@link #getConfigurationFormat()}.
* @param inputSurface The input {@link Surface} if the {@link MediaCodec} receives input from a
* surface.
*/
public Codec(MediaCodec mediaCodec, Format configurationFormat, @Nullable Surface inputSurface) {
this.mediaCodec = mediaCodec;
this.configurationFormat = configurationFormat;
this.inputSurface = inputSurface;
outputBufferInfo = new BufferInfo();
inputBufferIndex = C.INDEX_UNSET;
outputBufferIndex = C.INDEX_UNSET;
}
/**
* Returns the {@link Format} used for configuring the codec.
* Returns the {@link Format} used for configuring the {@code Codec}.
*
* <p>The configuration {@link Format} is the input {@link Format} used by the {@link
* DecoderFactory} or output {@link Format} used by the {@link EncoderFactory} for selecting and
* configuring the underlying {@link MediaCodec}.
* configuring the underlying decoder or encoder.
*/
public Format getConfigurationFormat() {
return configurationFormat;
}
Format getConfigurationFormat();
/** Returns the input {@link Surface}, or null if the input is not a surface. */
@Nullable
public Surface getInputSurface() {
return inputSurface;
}
/**
* Returns the input {@link Surface} of an underlying video encoder.
*
* <p>This method must only be called on video encoders because audio/video decoders and audio
* encoders don't use a {@link Surface} as input.
*/
Surface getInputSurface();
/**
* Dequeues a writable input buffer, if available.
*
* @param inputBuffer The buffer where the dequeued buffer data is stored.
* <p>This method must not be called from video encoders because they must use {@link Surface
* surfaces} as inputs.
*
* @param inputBuffer The buffer where the dequeued buffer data is stored, at {@link
* DecoderInputBuffer#data inputBuffer.data}.
* @return Whether an input buffer is ready to be used.
* @throws TransformationException If the underlying {@link MediaCodec} encounters a problem.
* @throws TransformationException If the underlying decoder or encoder encounters a problem.
*/
@EnsuresNonNullIf(expression = "#1.data", result = true)
public boolean maybeDequeueInputBuffer(DecoderInputBuffer inputBuffer)
throws TransformationException {
if (inputStreamEnded) {
return false;
}
if (inputBufferIndex < 0) {
try {
inputBufferIndex = mediaCodec.dequeueInputBuffer(/* timeoutUs= */ 0);
} catch (RuntimeException e) {
throw createTransformationException(e);
}
if (inputBufferIndex < 0) {
return false;
}
try {
inputBuffer.data = mediaCodec.getInputBuffer(inputBufferIndex);
} catch (RuntimeException e) {
throw createTransformationException(e);
}
inputBuffer.clear();
}
checkNotNull(inputBuffer.data);
return true;
}
boolean maybeDequeueInputBuffer(DecoderInputBuffer inputBuffer) throws TransformationException;
/**
* Queues an input buffer to the decoder. No buffers may be queued after an {@link
* Queues an input buffer to the {@code Codec}. No buffers may be queued after {@link
* DecoderInputBuffer#isEndOfStream() end of stream} buffer has been queued.
*
* <p>This method must not be called from video encoders because they must use {@link Surface
* surfaces} as inputs.
*
* @param inputBuffer The {@link DecoderInputBuffer input buffer}.
* @throws IllegalStateException If called again after an {@link
* DecoderInputBuffer#isEndOfStream() end of stream} buffer has been queued.
* @throws TransformationException If the underlying {@link MediaCodec} encounters a problem.
* @throws TransformationException If the underlying decoder or encoder encounters a problem.
*/
public void queueInputBuffer(DecoderInputBuffer inputBuffer) throws TransformationException {
checkState(
!inputStreamEnded, "Input buffer can not be queued after the input stream has ended.");
int offset = 0;
int size = 0;
if (inputBuffer.data != null && inputBuffer.data.hasRemaining()) {
offset = inputBuffer.data.position();
size = inputBuffer.data.remaining();
}
int flags = 0;
if (inputBuffer.isEndOfStream()) {
inputStreamEnded = true;
flags = MediaCodec.BUFFER_FLAG_END_OF_STREAM;
}
try {
mediaCodec.queueInputBuffer(inputBufferIndex, offset, size, inputBuffer.timeUs, flags);
} catch (RuntimeException e) {
throw createTransformationException(e);
}
inputBufferIndex = C.INDEX_UNSET;
inputBuffer.data = null;
}
void queueInputBuffer(DecoderInputBuffer inputBuffer) throws TransformationException;
/**
* Signals end-of-stream on input to a video encoder.
*
* <p>This method does not need to be called for audio/video decoders or audio encoders. For these
* the {@link MediaCodec#BUFFER_FLAG_END_OF_STREAM} flag should be set on the last input buffer
* {@link #queueInputBuffer(DecoderInputBuffer) queued}.
* <p>This method must only be called on video encoders because they must use a {@link Surface} as
* input. For audio/video decoders or audio encoders, the {@link C#BUFFER_FLAG_END_OF_STREAM} flag
* should be set on the last input buffer {@link #queueInputBuffer(DecoderInputBuffer) queued}.
*
* @throws IllegalStateException If the codec is not an encoder receiving input from a {@link
* Surface}.
* @throws TransformationException If the underlying {@link MediaCodec} encounters a problem.
* @throws TransformationException If the underlying video encoder encounters a problem.
*/
public void signalEndOfInputStream() throws TransformationException {
checkState(mediaCodec.getCodecInfo().isEncoder() && inputSurface != null);
try {
mediaCodec.signalEndOfInputStream();
} catch (RuntimeException e) {
throw createTransformationException(e);
}
}
void signalEndOfInputStream() throws TransformationException;
/**
* Returns the current output format, if available.
* Returns the current output format, or {@code null} if unavailable.
*
* @throws TransformationException If the underlying {@link MediaCodec} encounters a problem.
* @throws TransformationException If the underlying decoder or encoder encounters a problem.
*/
@Nullable
public Format getOutputFormat() throws TransformationException {
// The format is updated when dequeueing a 'special' buffer index, so attempt to dequeue now.
maybeDequeueOutputBuffer(/* setOutputBuffer= */ false);
return outputFormat;
}
Format getOutputFormat() throws TransformationException;
/**
* Returns the current output {@link ByteBuffer}, if available.
* Returns the current output {@link ByteBuffer}, or {@code null} if unavailable.
*
* @throws TransformationException If the underlying {@link MediaCodec} encounters a problem.
* <p>This method must not be called on video decoders because they must output to a {@link
* Surface}.
*
* @throws TransformationException If the underlying decoder or encoder encounters a problem.
*/
@Nullable
public ByteBuffer getOutputBuffer() throws TransformationException {
return maybeDequeueOutputBuffer(/* setOutputBuffer= */ true) ? outputBuffer : null;
}
ByteBuffer getOutputBuffer() throws TransformationException;
/**
* Returns the {@link BufferInfo} associated with the current output buffer, if available.
* Returns the {@link BufferInfo} associated with the current output buffer, or {@code null} if
* there is no output buffer available.
*
* @throws TransformationException If the underlying {@link MediaCodec} encounters a problem.
* <p>This method returns {@code null} if and only if {@link #getOutputBuffer()} returns null.
*
* @throws TransformationException If the underlying decoder or encoder encounters a problem.
*/
@Nullable
public BufferInfo getOutputBufferInfo() throws TransformationException {
return maybeDequeueOutputBuffer(/* setOutputBuffer= */ false) ? outputBufferInfo : null;
}
BufferInfo getOutputBufferInfo() throws TransformationException;
/**
* Releases the current output buffer.
*
* <p>This should be called after the buffer has been processed. The next output buffer will not
* be available until the previous has been released.
*
* @throws TransformationException If the underlying {@link MediaCodec} encounters a problem.
*/
public void releaseOutputBuffer() throws TransformationException {
releaseOutputBuffer(/* render= */ false);
}
/**
* Releases the current output buffer. If the {@link MediaCodec} was configured with an output
* surface, setting {@code render} to {@code true} will first send the buffer to the output
* surface. The surface will release the buffer back to the codec once it is no longer
* <p>Only set {@code render} to {@code true} when the {@code Codec} is a video decoder. Setting
* {@code render} to {@code true} will first render the buffer to the output surface. In this
* case, the surface will release the buffer back to the {@code Codec} once it is no longer
* used/displayed.
*
* <p>This should be called after the buffer has been processed. The next output buffer will not
* be available until the previous has been released.
* be available until the current output buffer has been released.
*
* @param render Whether the buffer needs to be sent to the output {@link Surface}.
* @throws TransformationException If the underlying {@link MediaCodec} encounters a problem.
* @param render Whether the buffer needs to be rendered to the output {@link Surface}.
* @throws TransformationException If the underlying decoder or encoder encounters a problem.
*/
public void releaseOutputBuffer(boolean render) throws TransformationException {
outputBuffer = null;
try {
mediaCodec.releaseOutputBuffer(outputBufferIndex, render);
} catch (RuntimeException e) {
throw createTransformationException(e);
}
outputBufferIndex = C.INDEX_UNSET;
}
/** Returns whether the codec output stream has ended, and no more data can be dequeued. */
public boolean isEnded() {
return outputStreamEnded && outputBufferIndex == C.INDEX_UNSET;
}
/** Releases the underlying codec. */
public void release() {
outputBuffer = null;
if (inputSurface != null) {
inputSurface.release();
}
mediaCodec.release();
}
void releaseOutputBuffer(boolean render) throws TransformationException;
/**
* Attempts to dequeue an output buffer if there is no output buffer pending. Does nothing
* otherwise.
*
* @param setOutputBuffer Whether to read the bytes of the dequeued output buffer and copy them
* into {@link #outputBuffer}.
* @return Whether there is an output buffer available.
* @throws TransformationException If the underlying {@link MediaCodec} encounters a problem.
* Returns whether the {@code Codec}'s output stream has ended, and no more data can be dequeued.
*/
private boolean maybeDequeueOutputBuffer(boolean setOutputBuffer) throws TransformationException {
if (outputBufferIndex >= 0) {
return true;
}
if (outputStreamEnded) {
return false;
}
boolean isEnded();
try {
outputBufferIndex = mediaCodec.dequeueOutputBuffer(outputBufferInfo, /* timeoutUs= */ 0);
} catch (RuntimeException e) {
throw createTransformationException(e);
}
if (outputBufferIndex < 0) {
if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
outputFormat = getFormat(mediaCodec.getOutputFormat());
}
return false;
}
if ((outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
outputStreamEnded = true;
if (outputBufferInfo.size == 0) {
releaseOutputBuffer();
return false;
}
}
if ((outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
// Encountered a CSD buffer, skip it.
releaseOutputBuffer();
return false;
}
if (setOutputBuffer) {
try {
outputBuffer = checkNotNull(mediaCodec.getOutputBuffer(outputBufferIndex));
} catch (RuntimeException e) {
throw createTransformationException(e);
}
outputBuffer.position(outputBufferInfo.offset);
outputBuffer.limit(outputBufferInfo.offset + outputBufferInfo.size);
}
return true;
}
private TransformationException createTransformationException(Exception cause) {
boolean isEncoder = mediaCodec.getCodecInfo().isEncoder();
boolean isVideo = MimeTypes.isVideo(configurationFormat.sampleMimeType);
String componentName = (isVideo ? "Video" : "Audio") + (isEncoder ? "Encoder" : "Decoder");
return TransformationException.createForCodec(
cause,
componentName,
configurationFormat,
mediaCodec.getName(),
isEncoder
? TransformationException.ERROR_CODE_ENCODING_FAILED
: TransformationException.ERROR_CODE_DECODING_FAILED);
}
private static Format getFormat(MediaFormat mediaFormat) {
ImmutableList.Builder<byte[]> csdBuffers = new ImmutableList.Builder<>();
int csdIndex = 0;
while (true) {
@Nullable ByteBuffer csdByteBuffer = mediaFormat.getByteBuffer("csd-" + csdIndex);
if (csdByteBuffer == null) {
break;
}
byte[] csdBufferData = new byte[csdByteBuffer.remaining()];
csdByteBuffer.get(csdBufferData);
csdBuffers.add(csdBufferData);
csdIndex++;
}
String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
Format.Builder formatBuilder =
new Format.Builder()
.setSampleMimeType(mediaFormat.getString(MediaFormat.KEY_MIME))
.setInitializationData(csdBuffers.build());
if (MimeTypes.isVideo(mimeType)) {
formatBuilder
.setWidth(mediaFormat.getInteger(MediaFormat.KEY_WIDTH))
.setHeight(mediaFormat.getInteger(MediaFormat.KEY_HEIGHT));
} else if (MimeTypes.isAudio(mimeType)) {
// TODO(internal b/178685617): Only set the PCM encoding for audio/raw, once we have a way to
// simulate more realistic codec input/output formats in tests.
formatBuilder
.setChannelCount(mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT))
.setSampleRate(mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE))
.setPcmEncoding(MEDIA_CODEC_PCM_ENCODING);
}
return formatBuilder.build();
}
/** Releases the {@code Codec}. */
void release();
}

119
libraries/transformer/src/main/java/androidx/media3/transformer/CodecFactoryUtil.java
View File

@ -1,119 +0,0 @@
/*
* Copyright 2021 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import android.media.MediaCodec;
import android.media.MediaFormat;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.media3.common.Format;
import androidx.media3.common.util.TraceUtil;
import java.io.IOException;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
/** Utility methods for {@link Codec}'s factory methods. */
/* package */ final class CodecFactoryUtil {
/** Creates a {@link Codec}. */
@RequiresNonNull("#1.sampleMimeType")
public static Codec createCodec(
Format format,
MediaFormat mediaFormat,
@Nullable String mediaCodecName,
boolean isVideo,
boolean isDecoder,
@Nullable Surface outputSurface)
throws TransformationException {
@Nullable MediaCodec mediaCodec = null;
@Nullable Surface inputSurface = null;
try {
mediaCodec =
mediaCodecName != null
? MediaCodec.createByCodecName(mediaCodecName)
: isDecoder
? MediaCodec.createDecoderByType(format.sampleMimeType)
: MediaCodec.createEncoderByType(format.sampleMimeType);
configureCodec(mediaCodec, mediaFormat, isDecoder, outputSurface);
if (isVideo && !isDecoder) {
inputSurface = mediaCodec.createInputSurface();
}
startCodec(mediaCodec);
} catch (Exception e) {
if (inputSurface != null) {
inputSurface.release();
}
if (mediaCodec != null) {
mediaCodecName = mediaCodec.getName();
mediaCodec.release();
}
throw createTransformationException(e, format, isVideo, isDecoder, mediaCodecName);
}
return new Codec(mediaCodec, format, inputSurface);
}
/** Creates a {@link TransformationException}. */
public static TransformationException createTransformationException(
Exception cause,
Format format,
boolean isVideo,
boolean isDecoder,
@Nullable String mediaCodecName) {
String componentName = (isVideo ? "Video" : "Audio") + (isDecoder ? "Decoder" : "Encoder");
if (cause instanceof IOException || cause instanceof MediaCodec.CodecException) {
return TransformationException.createForCodec(
cause,
componentName,
format,
mediaCodecName,
isDecoder
? TransformationException.ERROR_CODE_DECODER_INIT_FAILED
: TransformationException.ERROR_CODE_ENCODER_INIT_FAILED);
}
if (cause instanceof IllegalArgumentException) {
return TransformationException.createForCodec(
cause,
componentName,
format,
mediaCodecName,
isDecoder
? TransformationException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED
: TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
}
return TransformationException.createForUnexpected(cause);
}
private static void configureCodec(
MediaCodec codec,
MediaFormat mediaFormat,
boolean isDecoder,
@Nullable Surface outputSurface) {
TraceUtil.beginSection("configureCodec");
codec.configure(
mediaFormat,
outputSurface,
/* crypto= */ null,
isDecoder ? 0 : MediaCodec.CONFIGURE_FLAG_ENCODE);
TraceUtil.endSection();
}
private static void startCodec(MediaCodec codec) {
TraceUtil.beginSection("startCodec");
codec.start();
TraceUtil.endSection();
}
private CodecFactoryUtil() {}
}

383
libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodec.java Normal file
View File

@ -0,0 +1,383 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import android.media.MediaCodec;
import android.media.MediaCodec.BufferInfo;
import android.media.MediaFormat;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.TraceUtil;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.decoder.DecoderInputBuffer;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** A default {@link Codec} implementation that uses {@link MediaCodec}. */
@UnstableApi
public final class DefaultCodec implements Codec {
// MediaCodec decoders always output 16 bit PCM, unless configured to output PCM float.
// https://developer.android.com/reference/android/media/MediaCodec#raw-audio-buffers.
private static final int MEDIA_CODEC_PCM_ENCODING = C.ENCODING_PCM_16BIT;
private final BufferInfo outputBufferInfo;
private final Format configurationFormat;
private final MediaCodec mediaCodec;
@Nullable private final Surface inputSurface;
private @MonotonicNonNull Format outputFormat;
@Nullable private ByteBuffer outputBuffer;
private int inputBufferIndex;
private int outputBufferIndex;
private boolean inputStreamEnded;
private boolean outputStreamEnded;
/**
* Creates a {@code DefaultCodec}.
*
* @param configurationFormat The {@link Format} to configure the {@code DefaultCodec}. See {@link
* #getConfigurationFormat()}. The {@link Format#sampleMimeType sampleMimeType} must not be
* {@code null}.
* @param mediaFormat The {@link MediaFormat} to configure the underlying {@link MediaCodec}.
* @param mediaCodecName The name of a specific {@link MediaCodec} to instantiate. If {@code
* null}, {@code DefaultCodec} uses {@link Format#sampleMimeType
* configurationFormat.sampleMimeType} to create the underlying {@link MediaCodec codec}.
* @param isDecoder Whether the {@code DefaultCodec} is intended as a decoder.
* @param outputSurface The output {@link Surface} if the {@link MediaCodec} outputs to a surface.
*/
public DefaultCodec(
Format configurationFormat,
MediaFormat mediaFormat,
@Nullable String mediaCodecName,
boolean isDecoder,
@Nullable Surface outputSurface)
throws TransformationException {
this.configurationFormat = configurationFormat;
outputBufferInfo = new BufferInfo();
inputBufferIndex = C.INDEX_UNSET;
outputBufferIndex = C.INDEX_UNSET;
String sampleMimeType = checkNotNull(configurationFormat.sampleMimeType);
boolean isVideo = MimeTypes.isVideo(sampleMimeType);
@Nullable MediaCodec mediaCodec = null;
@Nullable Surface inputSurface = null;
try {
mediaCodec =
mediaCodecName != null
? MediaCodec.createByCodecName(mediaCodecName)
: isDecoder
? MediaCodec.createDecoderByType(sampleMimeType)
: MediaCodec.createEncoderByType(sampleMimeType);
configureCodec(mediaCodec, mediaFormat, isDecoder, outputSurface);
if (isVideo && !isDecoder) {
inputSurface = mediaCodec.createInputSurface();
}
startCodec(mediaCodec);
} catch (Exception e) {
if (inputSurface != null) {
inputSurface.release();
}
if (mediaCodec != null) {
mediaCodecName = mediaCodec.getName();
mediaCodec.release();
}
throw createInitializationTransformationException(
e, configurationFormat, isVideo, isDecoder, mediaCodecName);
}
this.mediaCodec = mediaCodec;
this.inputSurface = inputSurface;
}
@Override
public Format getConfigurationFormat() {
return configurationFormat;
}
@Override
public Surface getInputSurface() {
return checkStateNotNull(inputSurface);
}
@Override
@EnsuresNonNullIf(expression = "#1.data", result = true)
public boolean maybeDequeueInputBuffer(DecoderInputBuffer inputBuffer)
throws TransformationException {
if (inputStreamEnded) {
return false;
}
if (inputBufferIndex < 0) {
try {
inputBufferIndex = mediaCodec.dequeueInputBuffer(/* timeoutUs= */ 0);
} catch (RuntimeException e) {
throw createTransformationException(e);
}
if (inputBufferIndex < 0) {
return false;
}
try {
inputBuffer.data = mediaCodec.getInputBuffer(inputBufferIndex);
} catch (RuntimeException e) {
throw createTransformationException(e);
}
inputBuffer.clear();
}
checkNotNull(inputBuffer.data);
return true;
}
@Override
public void queueInputBuffer(DecoderInputBuffer inputBuffer) throws TransformationException {
checkState(
!inputStreamEnded, "Input buffer can not be queued after the input stream has ended.");
int offset = 0;
int size = 0;
if (inputBuffer.data != null && inputBuffer.data.hasRemaining()) {
offset = inputBuffer.data.position();
size = inputBuffer.data.remaining();
}
int flags = 0;
if (inputBuffer.isEndOfStream()) {
inputStreamEnded = true;
flags = MediaCodec.BUFFER_FLAG_END_OF_STREAM;
}
try {
mediaCodec.queueInputBuffer(inputBufferIndex, offset, size, inputBuffer.timeUs, flags);
} catch (RuntimeException e) {
throw createTransformationException(e);
}
inputBufferIndex = C.INDEX_UNSET;
inputBuffer.data = null;
}
@Override
public void signalEndOfInputStream() throws TransformationException {
try {
mediaCodec.signalEndOfInputStream();
} catch (RuntimeException e) {
throw createTransformationException(e);
}
}
@Override
@Nullable
public Format getOutputFormat() throws TransformationException {
// The format is updated when dequeueing a 'special' buffer index, so attempt to dequeue now.
maybeDequeueOutputBuffer(/* setOutputBuffer= */ false);
return outputFormat;
}
@Override
@Nullable
public ByteBuffer getOutputBuffer() throws TransformationException {
return maybeDequeueOutputBuffer(/* setOutputBuffer= */ true) ? outputBuffer : null;
}
@Override
@Nullable
public BufferInfo getOutputBufferInfo() throws TransformationException {
return maybeDequeueOutputBuffer(/* setOutputBuffer= */ false) ? outputBufferInfo : null;
}
@Override
public void releaseOutputBuffer(boolean render) throws TransformationException {
outputBuffer = null;
try {
mediaCodec.releaseOutputBuffer(outputBufferIndex, render);
} catch (RuntimeException e) {
throw createTransformationException(e);
}
outputBufferIndex = C.INDEX_UNSET;
}
@Override
public boolean isEnded() {
return outputStreamEnded && outputBufferIndex == C.INDEX_UNSET;
}
@Override
public void release() {
outputBuffer = null;
if (inputSurface != null) {
inputSurface.release();
}
mediaCodec.release();
}
/**
* Attempts to dequeue an output buffer if there is no output buffer pending. Does nothing
* otherwise.
*
* @param setOutputBuffer Whether to read the bytes of the dequeued output buffer and copy them
* into {@link #outputBuffer}.
* @return Whether there is an output buffer available.
* @throws TransformationException If the underlying {@link MediaCodec} encounters a problem.
*/
private boolean maybeDequeueOutputBuffer(boolean setOutputBuffer) throws TransformationException {
if (outputBufferIndex >= 0) {
return true;
}
if (outputStreamEnded) {
return false;
}
try {
outputBufferIndex = mediaCodec.dequeueOutputBuffer(outputBufferInfo, /* timeoutUs= */ 0);
} catch (RuntimeException e) {
throw createTransformationException(e);
}
if (outputBufferIndex < 0) {
if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
outputFormat = getFormat(mediaCodec.getOutputFormat());
}
return false;
}
if ((outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
outputStreamEnded = true;
if (outputBufferInfo.size == 0) {
releaseOutputBuffer(/* render= */ false);
return false;
}
}
if ((outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
// Encountered a CSD buffer, skip it.
releaseOutputBuffer(/* render= */ false);
return false;
}
if (setOutputBuffer) {
try {
outputBuffer = checkNotNull(mediaCodec.getOutputBuffer(outputBufferIndex));
} catch (RuntimeException e) {
throw createTransformationException(e);
}
outputBuffer.position(outputBufferInfo.offset);
outputBuffer.limit(outputBufferInfo.offset + outputBufferInfo.size);
}
return true;
}
private TransformationException createTransformationException(Exception cause) {
boolean isDecoder = !mediaCodec.getCodecInfo().isEncoder();
boolean isVideo = MimeTypes.isVideo(configurationFormat.sampleMimeType);
return TransformationException.createForCodec(
cause,
configurationFormat,
isVideo,
isDecoder,
mediaCodec.getName(),
isDecoder
? TransformationException.ERROR_CODE_DECODING_FAILED
: TransformationException.ERROR_CODE_ENCODING_FAILED);
}
private static TransformationException createInitializationTransformationException(
Exception cause,
Format format,
boolean isVideo,
boolean isDecoder,
@Nullable String mediaCodecName) {
if (cause instanceof IOException || cause instanceof MediaCodec.CodecException) {
return TransformationException.createForCodec(
cause,
format,
isVideo,
isDecoder,
mediaCodecName,
isDecoder
? TransformationException.ERROR_CODE_DECODER_INIT_FAILED
: TransformationException.ERROR_CODE_ENCODER_INIT_FAILED);
}
if (cause instanceof IllegalArgumentException) {
return TransformationException.createForCodec(
cause,
format,
isVideo,
isDecoder,
mediaCodecName,
isDecoder
? TransformationException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED
: TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
}
return TransformationException.createForUnexpected(cause);
}
private static Format getFormat(MediaFormat mediaFormat) {
ImmutableList.Builder<byte[]> csdBuffers = new ImmutableList.Builder<>();
int csdIndex = 0;
while (true) {
@Nullable ByteBuffer csdByteBuffer = mediaFormat.getByteBuffer("csd-" + csdIndex);
if (csdByteBuffer == null) {
break;
}
byte[] csdBufferData = new byte[csdByteBuffer.remaining()];
csdByteBuffer.get(csdBufferData);
csdBuffers.add(csdBufferData);
csdIndex++;
}
String mimeType = mediaFormat.getString(MediaFormat.KEY_MIME);
Format.Builder formatBuilder =
new Format.Builder()
.setSampleMimeType(mediaFormat.getString(MediaFormat.KEY_MIME))
.setInitializationData(csdBuffers.build());
if (MimeTypes.isVideo(mimeType)) {
formatBuilder
.setWidth(mediaFormat.getInteger(MediaFormat.KEY_WIDTH))
.setHeight(mediaFormat.getInteger(MediaFormat.KEY_HEIGHT));
} else if (MimeTypes.isAudio(mimeType)) {
// TODO(internal b/178685617): Only set the PCM encoding for audio/raw, once we have a way to
// simulate more realistic codec input/output formats in tests.
formatBuilder
.setChannelCount(mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT))
.setSampleRate(mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE))
.setPcmEncoding(MEDIA_CODEC_PCM_ENCODING);
}
return formatBuilder.build();
}
private static void configureCodec(
MediaCodec codec,
MediaFormat mediaFormat,
boolean isDecoder,
@Nullable Surface outputSurface) {
TraceUtil.beginSection("configureCodec");
codec.configure(
mediaFormat,
outputSurface,
/* crypto= */ null,
isDecoder ? 0 : MediaCodec.CONFIGURE_FLAG_ENCODE);
TraceUtil.endSection();
}
private static void startCodec(MediaCodec codec) {
TraceUtil.beginSection("startCodec");
codec.start();
TraceUtil.endSection();
}
}

13
libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java
View File

@ -18,7 +18,6 @@ package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Util.SDK_INT;
import static androidx.media3.transformer.CodecFactoryUtil.createCodec;
import android.media.MediaFormat;
import android.view.Surface;
@ -36,11 +35,10 @@ import androidx.media3.common.util.MediaFormatUtil;
mediaFormat, MediaFormat.KEY_MAX_INPUT_SIZE, format.maxInputSize);
MediaFormatUtil.setCsdBuffers(mediaFormat, format.initializationData);
return createCodec(
return new DefaultCodec(
format,
mediaFormat,
/* mediaCodecName= */ null,
/* isVideo= */ false,
/* isDecoder= */ true,
/* outputSurface= */ null);
}
@ -61,12 +59,7 @@ import androidx.media3.common.util.MediaFormatUtil;
mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0);
}
return createCodec(
format,
mediaFormat,
/* mediaCodecName= */ null,
/* isVideo= */ true,
/* isDecoder= */ true,
outputSurface);
return new DefaultCodec(
format, mediaFormat, /* mediaCodecName= */ null, /* isDecoder= */ true, outputSurface);
}
}

18
libraries/transformer/src/main/java/androidx/media3/transformer/DefaultEncoderFactory.java
View File

@ -21,8 +21,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.common.util.Util.SDK_INT;
import static androidx.media3.transformer.CodecFactoryUtil.createCodec;
import static androidx.media3.transformer.CodecFactoryUtil.createTransformationException;
import static java.lang.Math.abs;
import android.media.MediaCodecInfo;
@ -81,12 +79,13 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
// capabilities limitations.
format = format.buildUpon().setSampleMimeType(allowedMimeTypes.get(0)).build();
} else {
throw createTransformationException(
throw TransformationException.createForCodec(
new IllegalArgumentException("The requested output format is not supported."),
format,
/* isVideo= */ false,
/* isDecoder= */ false,
/* mediaCodecName= */ null);
/* mediaCodecName= */ null,
TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
}
}
MediaFormat mediaFormat =
@ -94,11 +93,10 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
checkNotNull(format.sampleMimeType), format.sampleRate, format.channelCount);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.bitrate);
return createCodec(
return new DefaultCodec(
format,
mediaFormat,
/* mediaCodecName= */ null,
/* isVideo= */ false,
/* isDecoder= */ false,
/* outputSurface= */ null);
}
@ -121,12 +119,13 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
findEncoderWithClosestFormatSupport(
format, videoEncoderSelector, allowedMimeTypes, disableFallback);
if (encoderAndClosestFormatSupport == null) {
throw createTransformationException(
throw TransformationException.createForCodec(
new IllegalArgumentException("The requested output format is not supported."),
format,
/* isVideo= */ true,
/* isDecoder= */ false,
/* mediaCodecName= */ null);
/* mediaCodecName= */ null,
TransformationException.ERROR_CODE_OUTPUT_FORMAT_UNSUPPORTED);
}
MediaCodecInfo encoderInfo = encoderAndClosestFormatSupport.first;
@ -198,11 +197,10 @@ public final class DefaultEncoderFactory implements Codec.EncoderFactory {
mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, DEFAULT_COLOR_FORMAT);
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, DEFAULT_I_FRAME_INTERVAL_SECS);
return createCodec(
return new DefaultCodec(
format,
mediaFormat,
encoderInfo.getName(),
/* isVideo= */ true,
/* isDecoder= */ false,
/* outputSurface= */ null);
}

17
libraries/transformer/src/main/java/androidx/media3/transformer/TransformationException.java
View File

@ -207,24 +207,23 @@ public final class TransformationException extends Exception {
* Creates an instance for a decoder or encoder related exception.
*
* @param cause The cause of the failure.
* @param componentName The name of the component used, e.g. 'VideoEncoder'.
* @param configurationFormat The {@link Format} used for configuring the decoder/encoder.
* @param format The {@link Format} used for configuring the decoder/encoder.
* @param isVideo Whether the decoder or encoder is configured for video.
* @param isDecoder Whether the exception is created for a decoder.
* @param mediaCodecName The name of the {@link MediaCodec} used, if known.
* @param errorCode See {@link #errorCode}.
* @return The created instance.
*/
public static TransformationException createForCodec(
Throwable cause,
String componentName,
Format configurationFormat,
Format format,
boolean isVideo,
boolean isDecoder,
@Nullable String mediaCodecName,
int errorCode) {
String componentName = (isVideo ? "Video" : "Audio") + (isDecoder ? "Decoder" : "Encoder");
return new TransformationException(
componentName
+ " error, format = "
+ configurationFormat
+ ", mediaCodecName="
+ mediaCodecName,
componentName + " error, format = " + format + ", mediaCodecName=" + mediaCodecName,
cause,
errorCode);
}

8
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -135,7 +135,7 @@ import org.checkerframework.dataflow.qual.Pure;
actualOutputFormat.height,
inputFormat.pixelWidthHeightRatio,
transformationMatrix,
/* outputSurface= */ checkNotNull(encoder.getInputSurface()),
/* outputSurface= */ encoder.getInputSurface(),
transformationRequest.enableHdrEditing,
debugViewProvider);
} else {
@ -145,9 +145,7 @@ import org.checkerframework.dataflow.qual.Pure;
decoder =
decoderFactory.createForVideoDecoding(
inputFormat,
frameEditor == null
? checkNotNull(encoder.getInputSurface())
: frameEditor.getInputSurface());
frameEditor == null ? encoder.getInputSurface() : frameEditor.getInputSurface());
}
@Override
@ -262,7 +260,7 @@ import org.checkerframework.dataflow.qual.Pure;
@Override
public void releaseOutputBuffer() throws TransformationException {
encoder.releaseOutputBuffer();
encoder.releaseOutputBuffer(/* render= */ false);
}
@Override