loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-05-08 16:10:38 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Deduce encoder video format from decoder format.

Browse Source 
When no encoder video MIME type is specified, the `TransformerTranscodingVideoRenderer` now uses the video MIME type of the input for the encoder format.
The input format is now read in a new method `ensureInputFormatRead` which is called before the other configuration methods. This removes the logic for reading the input format from `ensureDecoderConfigured`, because it is now needed for both encoder and decoder configuration but the encoder needs to be configured before GL and GL needs to be configured before the decoder, so the decoder can't read the format.
The width and height are now inferred from the input and the frame rate and bit rate are still hard-coded but set by the `MediaCodecAdapterWrapper` instead of `TranscodingTransformer`.

PiperOrigin-RevId: 405631263
This commit is contained in:
hschlueter 2021-10-26 14:20:24 +01:00 committed by bachinger
parent 68729ecd49
commit 649fe702f2
6 changed files with 65 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MediaCodecAdapterWrapper.java
View File

@ -19,7 +19,6 @@ package com.google.android.exoplayer2.transformer;
import static com.google.android.exoplayer2.util.Assertions.checkArgument;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import static com.google.android.exoplayer2.util.Assertions.checkState;
import static java.lang.Math.ceil;
import android.media.MediaCodec;
import android.media.MediaCodec.BufferInfo;
@ -202,9 +201,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* MediaCodecAdapter} video encoder.
*
* @param format The {@link Format} (of the output data) used to determine the underlying {@link
* MediaCodec} and its configuration values. {@link Format#width}, {@link Format#height},
* {@link Format#frameRate} and {@link Format#averageBitrate} must be set to those of the
* desired output video format.
* MediaCodec} and its configuration values. {@link Format#sampleMimeType}, {@link
* Format#width} and {@link Format#height} must be set to those of the desired output video
* format.
* @param additionalEncoderConfig A map of {@link MediaFormat}'s integer settings, where the keys
* are from {@code MediaFormat.KEY_*} constants. Its values will override those in {@code
* format}.
@ -215,8 +214,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
Format format, Map<String, Integer> additionalEncoderConfig) throws IOException {
checkArgument(format.width != Format.NO_VALUE);
checkArgument(format.height != Format.NO_VALUE);
checkArgument(format.frameRate != Format.NO_VALUE);
checkArgument(format.averageBitrate != Format.NO_VALUE);
@Nullable MediaCodecAdapter adapter = null;
try {
@ -224,9 +221,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
MediaFormat.createVideoFormat(
checkNotNull(format.sampleMimeType), format.width, format.height);
mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, CodecCapabilities.COLOR_FormatSurface);
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, (int) ceil(format.frameRate));
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30);
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, format.averageBitrate);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, 413_000);
for (Map.Entry<String, Integer> encoderSetting : additionalEncoderConfig.entrySet()) {
mediaFormat.setInteger(encoderSetting.getKey(), encoderSetting.getValue());

18
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TranscodingTransformer.java
View File

@ -38,7 +38,6 @@ import androidx.annotation.VisibleForTesting;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.DefaultLoadControl;
import com.google.android.exoplayer2.ExoPlayer;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.MediaItem;
import com.google.android.exoplayer2.PlaybackException;
import com.google.android.exoplayer2.Player;
@ -339,7 +338,12 @@ public final class TranscodingTransformer {
}
Transformation transformation =
new Transformation(
removeAudio, removeVideo, flattenForSlowMotion, outputMimeType, audioMimeType);
removeAudio,
removeVideo,
flattenForSlowMotion,
outputMimeType,
audioMimeType,
/* videoMimeType= */ null);
return new TranscodingTransformer(
context, mediaSourceFactory, muxerFactory, transformation, listener, looper, clock);
}
@ -639,17 +643,9 @@ public final class TranscodingTransformer {
index++;
}
if (!transformation.removeVideo) {
Format encoderOutputFormat =
new Format.Builder()
.setSampleMimeType(MimeTypes.VIDEO_H264)
.setWidth(480)
.setHeight(360)
.setAverageBitrate(413_000)
.setFrameRate(30)
.build();
renderers[index] =
new TransformerTranscodingVideoRenderer(
context, muxerWrapper, mediaClock, transformation, encoderOutputFormat);
context, muxerWrapper, mediaClock, transformation);
index++;
}
return renderers;

5
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformation.java
View File

@ -26,17 +26,20 @@ import androidx.annotation.Nullable;
public final boolean flattenForSlowMotion;
public final String outputMimeType;
@Nullable public final String audioMimeType;
@Nullable public final String videoMimeType;
public Transformation(
boolean removeAudio,
boolean removeVideo,
boolean flattenForSlowMotion,
String outputMimeType,
@Nullable String audioMimeType) {
@Nullable String audioMimeType,
@Nullable String videoMimeType) {
this.removeAudio = removeAudio;
this.removeVideo = removeVideo;
this.flattenForSlowMotion = flattenForSlowMotion;
this.outputMimeType = outputMimeType;
this.audioMimeType = audioMimeType;
this.videoMimeType = videoMimeType;
}
}

3
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java
View File

@ -304,7 +304,8 @@ public final class Transformer {
removeVideo,
flattenForSlowMotion,
outputMimeType,
/* audioMimeType= */ null);
/* audioMimeType= */ null,
/* videoMimeType= */ null);
return new Transformer(
context, mediaSourceFactory, muxerFactory, transformation, listener, looper, clock);
}

14
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java
View File

@ -51,7 +51,7 @@ import java.nio.ByteBuffer;
@Nullable private MediaCodecAdapterWrapper decoder;
@Nullable private MediaCodecAdapterWrapper encoder;
@Nullable private SpeedProvider speedProvider;
@Nullable private Format inputFormat;
@Nullable private Format decoderInputFormat;
@Nullable private AudioFormat encoderInputAudioFormat;
private ByteBuffer sonicOutputBuffer;
@ -100,7 +100,7 @@ import java.nio.ByteBuffer;
encoder = null;
}
speedProvider = null;
inputFormat = null;
decoderInputFormat = null;
encoderInputAudioFormat = null;
sonicOutputBuffer = AudioProcessor.EMPTY_BUFFER;
nextEncoderInputBufferTimeUs = 0;
@ -352,7 +352,7 @@ import java.nio.ByteBuffer;
}
String audioMimeType =
transformation.audioMimeType == null
? checkNotNull(inputFormat).sampleMimeType
? checkNotNull(decoderInputFormat).sampleMimeType
: transformation.audioMimeType;
try {
encoder =
@ -385,14 +385,14 @@ import java.nio.ByteBuffer;
if (result != C.RESULT_FORMAT_READ) {
return false;
}
inputFormat = checkNotNull(formatHolder.format);
decoderInputFormat = checkNotNull(formatHolder.format);
try {
decoder = MediaCodecAdapterWrapper.createForAudioDecoding(inputFormat);
decoder = MediaCodecAdapterWrapper.createForAudioDecoding(decoderInputFormat);
} catch (IOException e) {
// TODO (internal b/184262323): Assign an adequate error code.
throw createRendererException(e, PlaybackException.ERROR_CODE_UNSPECIFIED);
}
speedProvider = new SegmentSpeedProvider(inputFormat);
speedProvider = new SegmentSpeedProvider(decoderInputFormat);
currentSpeed = speedProvider.getSpeed(0);
return true;
}
@ -418,7 +418,7 @@ import java.nio.ByteBuffer;
cause,
TAG,
getIndex(),
inputFormat,
decoderInputFormat,
/* rendererFormatSupport= */ C.FORMAT_HANDLED,
/* isRecoverable= */ false,
errorCode);

67
library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerTranscodingVideoRenderer.java
View File

@ -42,6 +42,7 @@ import com.google.android.exoplayer2.util.GlUtil;
import com.google.common.collect.ImmutableMap;
import java.io.IOException;
import java.nio.ByteBuffer;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@RequiresApi(18)
/* package */ final class TransformerTranscodingVideoRenderer extends TransformerBaseRenderer {
@ -53,12 +54,12 @@ import java.nio.ByteBuffer;
private static final String TAG = "TransformerTranscodingVideoRenderer";
private final Context context;
/** The format the encoder is configured to output, may differ from the actual output format. */
private final Format encoderConfigurationOutputFormat;
private final DecoderInputBuffer decoderInputBuffer;
private final float[] decoderTextureTransformMatrix;
private @MonotonicNonNull Format decoderInputFormat;
@Nullable private EGLDisplay eglDisplay;
@Nullable private EGLContext eglContext;
@Nullable private EGLSurface eglSurface;
@ -81,11 +82,9 @@ import java.nio.ByteBuffer;
Context context,
MuxerWrapper muxerWrapper,
TransformerMediaClock mediaClock,
Transformation transformation,
Format encoderConfigurationOutputFormat) {
Transformation transformation) {
super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformation);
this.context = context;
this.encoderConfigurationOutputFormat = encoderConfigurationOutputFormat;
decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT);
decoderTextureTransformMatrix = new float[16];
decoderTextureId = GlUtil.TEXTURE_ID_UNSET;
@ -97,15 +96,13 @@ import java.nio.ByteBuffer;
}
@Override
protected void onStarted() throws ExoPlaybackException {
super.onStarted();
public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
if (!isRendererStarted || isEnded() || !ensureInputFormatRead()) {
return;
}
ensureEncoderConfigured();
ensureOpenGlConfigured();
}
@Override
public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
if (!isRendererStarted || isEnded() || !ensureDecoderConfigured()) {
if (!ensureDecoderConfigured()) {
return;
}
@ -153,6 +150,22 @@ import java.nio.ByteBuffer;
muxerWrapperTrackEnded = false;
}
private boolean ensureInputFormatRead() {
if (decoderInputFormat != null) {
return true;
}
FormatHolder formatHolder = getFormatHolder();
@SampleStream.ReadDataResult
int result =
readSource(
formatHolder, decoderInputBuffer, /* readFlags= */ SampleStream.FLAG_REQUIRE_FORMAT);
if (result != C.RESULT_FORMAT_READ) {
return false;
}
decoderInputFormat = checkNotNull(formatHolder.format);
return true;
}
private void ensureEncoderConfigured() throws ExoPlaybackException {
if (encoder != null) {
return;
@ -161,7 +174,15 @@ import java.nio.ByteBuffer;
try {
encoder =
MediaCodecAdapterWrapper.createForVideoEncoding(
encoderConfigurationOutputFormat, ImmutableMap.of());
new Format.Builder()
.setWidth(checkNotNull(decoderInputFormat).width)
.setHeight(decoderInputFormat.height)
.setSampleMimeType(
transformation.videoMimeType != null
? transformation.videoMimeType
: decoderInputFormat.sampleMimeType)
.build(),
ImmutableMap.of());
} catch (IOException e) {
throw createRendererException(
// TODO(claincly): should be "ENCODER_INIT_FAILED"
@ -190,8 +211,8 @@ import java.nio.ByteBuffer;
eglDisplay,
eglContext,
eglSurface,
encoderConfigurationOutputFormat.width,
encoderConfigurationOutputFormat.height);
checkNotNull(decoderInputFormat).width,
decoderInputFormat.height);
decoderTextureId = GlUtil.createExternalTexture();
String vertexShaderCode;
String fragmentShaderCode;
@ -248,26 +269,18 @@ import java.nio.ByteBuffer;
return true;
}
FormatHolder formatHolder = getFormatHolder();
@SampleStream.ReadDataResult
int result =
readSource(
formatHolder, decoderInputBuffer, /* readFlags= */ SampleStream.FLAG_REQUIRE_FORMAT);
if (result != C.RESULT_FORMAT_READ) {
return false;
}
Format inputFormat = checkNotNull(formatHolder.format);
checkState(decoderTextureId != GlUtil.TEXTURE_ID_UNSET);
decoderSurfaceTexture = new SurfaceTexture(decoderTextureId);
decoderSurfaceTexture.setOnFrameAvailableListener(
surfaceTexture -> isDecoderSurfacePopulated = true);
decoderSurface = new Surface(decoderSurfaceTexture);
try {
decoder = MediaCodecAdapterWrapper.createForVideoDecoding(inputFormat, decoderSurface);
decoder =
MediaCodecAdapterWrapper.createForVideoDecoding(
checkNotNull(decoderInputFormat), decoderSurface);
} catch (IOException e) {
throw createRendererException(
e, formatHolder.format, PlaybackException.ERROR_CODE_DECODER_INIT_FAILED);
e, decoderInputFormat, PlaybackException.ERROR_CODE_DECODER_INIT_FAILED);
}
return true;
}