loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Extend support for audio spatialization in MediaCodecAudioRenderer

Browse Source 
With this change, the MediaCodecAudioRenderer configures the MediaCodec
to not downmix audio only if spatialization can be applied. This way,
decoders who are downmixing by default are left doing so when
spatialization cannot be applied. The renderer re-initializes the codec
when spatialization properties change mid-playback.

PiperOrigin-RevId: 422822952
This commit is contained in:
christosts 2022-01-19 16:41:25 +00:00 committed by Ian Baker
parent c566ed91ad
commit 212e29865a
7 changed files with 227 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java
View File

@ -408,6 +408,13 @@ public interface AudioSink {
*/
void setAudioAttributes(AudioAttributes audioAttributes);
/**
* Returns the audio attributes used for audio playback, or {@code null} if the sink does not use
* audio attributes.
*/
@Nullable
AudioAttributes getAudioAttributes();
/** Sets the audio session id. */
void setAudioSessionId(int audioSessionId);

5
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java
View File

@ -1248,6 +1248,11 @@ public final class DefaultAudioSink implements AudioSink {
flush();
}
@Override
public AudioAttributes getAudioAttributes() {
return audioAttributes;
}
@Override
public void setAudioSessionId(int audioSessionId) {
if (this.audioSessionId != audioSessionId) {

6
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java
View File

@ -123,6 +123,12 @@ public class ForwardingAudioSink implements AudioSink {
sink.setAudioAttributes(audioAttributes);
}
@Override
@Nullable
public AudioAttributes getAudioAttributes() {
return sink.getAudioAttributes();
}
@Override
public void setAudioSessionId(int audioSessionId) {
sink.setAudioSessionId(audioSessionId);

191
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java
View File

@ -16,6 +16,7 @@
package androidx.media3.exoplayer.audio;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_MAX_INPUT_SIZE_EXCEEDED;
import static androidx.media3.exoplayer.DecoderReuseEvaluation.REUSE_RESULT_NO;
import static com.google.common.base.MoreObjects.firstNonNull;
@ -29,7 +30,9 @@ import android.media.MediaCrypto;
import android.media.MediaFormat;
import android.os.Handler;
import androidx.annotation.CallSuper;
import androidx.annotation.DoNotInline;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.media3.common.AudioAttributes;
import androidx.media3.common.AuxEffectInfo;
import androidx.media3.common.C;
@ -60,8 +63,10 @@ import androidx.media3.exoplayer.mediacodec.MediaCodecSelector;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil;
import androidx.media3.exoplayer.mediacodec.MediaCodecUtil.DecoderQueryException;
import com.google.common.collect.ImmutableList;
import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
import java.util.List;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/**
* Decodes and renders audio using {@link MediaCodec} and an {@link AudioSink}.
@ -98,6 +103,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
private final Context context;
private final EventDispatcher eventDispatcher;
private final AudioSink audioSink;
private final SpatializationHelper spatializationHelper;
private int codecMaxInputSize;
private boolean codecNeedsDiscardChannelsWorkaround;
@ -253,9 +259,11 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
mediaCodecSelector,
enableDecoderFallback,
/* assumedMinimumCodecOperatingRate= */ 44100);
this.context = context.getApplicationContext();
context = context.getApplicationContext();
this.context = context;
this.audioSink = audioSink;
eventDispatcher = new EventDispatcher(eventHandler, eventListener);
spatializationHelper = new SpatializationHelper(context, audioSink.getAudioAttributes());
audioSink.setListener(new AudioSinkListener());
}
@ -414,6 +422,11 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
return audioSink.supportsFormat(format);
}
@Override
protected boolean shouldReinitCodec() {
return spatializationHelper.shouldReinitCodec();
}
@Override
protected MediaCodecAdapter.Configuration getMediaCodecConfiguration(
MediaCodecInfo codecInfo,
@ -474,7 +487,11 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
@Override
protected void onCodecInitialized(
String name, long initializedTimestampMs, long initializationDurationMs) {
String name,
MediaCodecAdapter.Configuration configuration,
long initializedTimestampMs,
long initializationDurationMs) {
spatializationHelper.onCodecInitialized(configuration);
eventDispatcher.decoderInitialized(name, initializedTimestampMs, initializationDurationMs);
}
@ -565,6 +582,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
audioSink.disableTunneling();
}
audioSink.setPlayerId(getPlayerId());
spatializationHelper.enable();
}
@Override
@ -617,6 +635,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
audioSinkNeedsReset = false;
audioSink.reset();
}
spatializationHelper.reset();
}
}
@ -741,6 +760,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
case MSG_SET_AUDIO_ATTRIBUTES:
AudioAttributes audioAttributes = (AudioAttributes) message;
audioSink.setAudioAttributes(audioAttributes);
spatializationHelper.setAudioAttributes(audioSink.getAudioAttributes());
break;
case MSG_SET_AUX_EFFECT_INFO:
AuxEffectInfo auxEffectInfo = (AuxEffectInfo) message;
@ -852,14 +872,8 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
== AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY) {
mediaFormat.setInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_FLOAT);
}
spatializationHelper.configureForSpatialization(mediaFormat, format);
if (Util.SDK_INT >= 32) {
// Disable down-mixing in the decoder (for decoders that read the max-output-channel-count
// key).
// TODO[b/190759307]: Update key to use MediaFormat.KEY_MAX_OUTPUT_CHANNEL_COUNT once the
// compile SDK target is set to 32.
mediaFormat.setInteger("max-output-channel-count", 99);
}
return mediaFormat;
}
@ -943,4 +957,163 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
eventDispatcher.audioSinkError(audioSinkError);
}
}
/**
* A helper class that signals whether the codec needs to be re-initialized because spatialization
* properties changed.
*/
private static final class SpatializationHelper implements SpatializerDelegate.Listener {
// TODO[b/190759307] Remove and use MediaFormat.KEY_MAX_OUTPUT_CHANNEL_COUNT once the
// compile SDK target is set to 32.
private static final String KEY_MAX_OUTPUT_CHANNEL_COUNT = "max-output-channel-count";
private static final int SPATIALIZATION_CHANNEL_COUNT = 99;
@Nullable private final SpatializerDelegate spatializerDelegate;
private @MonotonicNonNull Handler handler;
@Nullable private AudioAttributes audioAttributes;
@Nullable private Format inputFormat;
private boolean codecConfiguredForSpatialization;
private boolean codecNeedsReinit;
private boolean listenerAdded;
/** Creates a new instance. */
public SpatializationHelper(Context context, @Nullable AudioAttributes audioAttributes) {
this.spatializerDelegate = maybeCreateSpatializer(context);
this.audioAttributes = audioAttributes;
}
/** Enables this helper. Call this method when the renderer is enabled. */
public void enable() {
maybeAddSpatalizationListener();
}
/** Resets the helper and releases any resources. Call this method when renderer is reset. */
public void reset() {
maybeRemoveSpatalizationListener();
}
/** Sets the audio attributes set by the player. */
public void setAudioAttributes(@Nullable AudioAttributes audioAttributes) {
if (Util.areEqual(this.audioAttributes, audioAttributes)) {
return;
}
this.audioAttributes = audioAttributes;
updateCodecNeedsReinit();
}
/**
* Sets keys for audio spatialization on the {@code mediaFormat} if the platform can apply
* spatialization to this {@code format}.
*/
public void configureForSpatialization(MediaFormat mediaFormat, Format format) {
if (canBeSpatialized(format)) {
mediaFormat.setInteger(KEY_MAX_OUTPUT_CHANNEL_COUNT, SPATIALIZATION_CHANNEL_COUNT);
}
}
/** Informs the helper that a codec was initialized. */
public void onCodecInitialized(MediaCodecAdapter.Configuration configuration) {
codecNeedsReinit = false;
inputFormat = configuration.format;
codecConfiguredForSpatialization =
configuration.mediaFormat.containsKey(KEY_MAX_OUTPUT_CHANNEL_COUNT)
&& configuration.mediaFormat.getInteger(KEY_MAX_OUTPUT_CHANNEL_COUNT)
== SPATIALIZATION_CHANNEL_COUNT;
}
/**
* Returns whether the codec should be re-initialized, caused by a change in the spatialization
* properties.
*/
public boolean shouldReinitCodec() {
return codecNeedsReinit;
}
// SpatializerDelegate.Listener
@Override
public void onSpatializerEnabledChanged(SpatializerDelegate spatializer, boolean enabled) {
updateCodecNeedsReinit();
}
@Override
public void onSpatializerAvailableChanged(SpatializerDelegate spatializer, boolean available) {
updateCodecNeedsReinit();
}
// Other internal methods
/** Returns whether this format can be spatialized by the platform. */
private boolean canBeSpatialized(@Nullable Format format) {
if (Util.SDK_INT < 32
|| format == null
|| audioAttributes == null
|| spatializerDelegate == null
|| spatializerDelegate.getImmersiveAudioLevel()
!= SpatializerDelegate.SPATIALIZER_IMMERSIVE_LEVEL_MULTICHANNEL
|| !spatializerDelegate.isAvailable()
|| !spatializerDelegate.isEnabled()) {
return false;
}
AudioFormat.Builder audioFormatBuilder =
new AudioFormat.Builder()
.setEncoding(AudioFormat.ENCODING_PCM_16BIT)
.setChannelMask(Util.getAudioTrackChannelConfig(format.channelCount));
if (format.sampleRate != Format.NO_VALUE) {
audioFormatBuilder.setSampleRate(format.sampleRate);
}
return spatializerDelegate.canBeSpatialized(
audioAttributes.getAudioAttributesV21(), audioFormatBuilder.build());
}
private void maybeAddSpatalizationListener() {
if (!listenerAdded && spatializerDelegate != null && Util.SDK_INT >= 32) {
if (handler == null) {
// Route callbacks to the playback thread.
handler = Util.createHandlerForCurrentLooper();
}
spatializerDelegate.addOnSpatializerStateChangedListener(handler::post, this);
listenerAdded = true;
}
}
private void maybeRemoveSpatalizationListener() {
if (listenerAdded && spatializerDelegate != null && Util.SDK_INT >= 32) {
spatializerDelegate.removeOnSpatializerStateChangedListener(this);
checkStateNotNull(handler).removeCallbacksAndMessages(null);
}
}
private void updateCodecNeedsReinit() {
codecNeedsReinit = codecConfiguredForSpatialization != canBeSpatialized(inputFormat);
}
@Nullable
private static SpatializerDelegate maybeCreateSpatializer(Context context) {
if (Util.SDK_INT >= 32) {
return Api32.createSpatializer(context);
}
return null;
}
}
@RequiresApi(32)
private static final class Api32 {
private Api32() {}
@DoNotInline
@Nullable
public static SpatializerDelegate createSpatializer(Context context) {
try {
return new SpatializerDelegate(context);
} catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException e) {
// Do nothing for these cases.
} catch (InvocationTargetException e) {
Log.w(TAG, "Failed to load Spatializer with reflection", e);
}
return null;
}
}
}

19
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java
View File

@ -566,6 +566,14 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
return true;
}
/**
* Returns whether the renderer needs to re-initialize the codec, possibly as a result of a change
* in device capabilities.
*/
protected boolean shouldReinitCodec() {
return false;
}
/**
* Returns whether the codec needs the renderer to propagate the end-of-stream signal directly,
* rather than by using an end-of-stream buffer queued to the codec.
@ -1120,7 +1128,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
decoderCounters.decoderInitCount++;
long elapsed = codecInitializedTimestamp - codecInitializingTimestamp;
onCodecInitialized(codecName, codecInitializedTimestamp, elapsed);
onCodecInitialized(codecName, configuration, codecInitializedTimestamp, elapsed);
}
private boolean shouldContinueRendering(long renderStartTimeMs) {
@ -1160,6 +1168,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
if (codec == null || codecDrainState == DRAIN_STATE_WAIT_END_OF_STREAM || inputStreamEnded) {
return false;
}
if (codecDrainState == DRAIN_STATE_NONE && shouldReinitCodec()) {
drainAndReinitializeCodec();
}
if (inputIndex < 0) {
inputIndex = codec.dequeueInputBufferIndex();
@ -1354,12 +1365,16 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
* <p>The default implementation is a no-op.
*
* @param name The name of the codec that was initialized.
* @param configuration The {@link MediaCodecAdapter.Configuration} used to configure the codec.
* @param initializedTimestampMs {@link SystemClock#elapsedRealtime()} when initialization
* finished.
* @param initializationDurationMs The time taken to initialize the codec in milliseconds.
*/
protected void onCodecInitialized(
String name, long initializedTimestampMs, long initializationDurationMs) {
String name,
MediaCodecAdapter.Configuration configuration,
long initializedTimestampMs,
long initializationDurationMs) {
// Do nothing.
}

5
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View File

@ -771,7 +771,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
@Override
protected void onCodecInitialized(
String name, long initializedTimestampMs, long initializationDurationMs) {
String name,
MediaCodecAdapter.Configuration configuration,
long initializedTimestampMs,
long initializationDurationMs) {
eventDispatcher.decoderInitialized(name, initializedTimestampMs, initializationDurationMs);
codecNeedsSetOutputSurfaceWorkaround = codecNeedsSetOutputSurfaceWorkaround(name);
codecHandlesHdr10PlusOutOfBandMetadata =

8
libraries/test_exoplayer_playback/src/androidTest/java/androidx/media3/test/exoplayer/playback/gts/DebugRenderersFactory.java
View File

@ -149,14 +149,18 @@ import java.util.ArrayList;
@Override
protected void onCodecInitialized(
String name, long initializedTimestampMs, long initializationDurationMs) {
String name,
MediaCodecAdapter.Configuration configuration,
long initializedTimestampMs,
long initializationDurationMs) {
// If the codec was initialized whilst the renderer is started, default behavior is to
// render the first frame (i.e. the keyframe before the current position), then drop frames up
// to the current playback position. For test runs that place a maximum limit on the number of
// dropped frames allowed, this is not desired behavior. Hence we skip (rather than drop)
// frames up to the current playback position [Internal: b/66494991].
skipToPositionBeforeRenderingFirstFrame = getState() == Renderer.STATE_STARTED;
super.onCodecInitialized(name, initializedTimestampMs, initializationDurationMs);
super.onCodecInitialized(
name, configuration, initializedTimestampMs, initializationDurationMs);
}
@Override