diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java index f114b20727..65d8a1b954 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioSink.java @@ -408,6 +408,13 @@ public interface AudioSink { */ void setAudioAttributes(AudioAttributes audioAttributes); + /** + * Returns the audio attributes used for audio playback, or {@code null} if the sink does not use + * audio attributes. + */ + @Nullable + AudioAttributes getAudioAttributes(); + /** Sets the audio session id. */ void setAudioSessionId(int audioSessionId); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java index 8c45a6347b..60e8fb04f9 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java @@ -1248,6 +1248,11 @@ public final class DefaultAudioSink implements AudioSink { flush(); } + @Override + public AudioAttributes getAudioAttributes() { + return audioAttributes; + } + @Override public void setAudioSessionId(int audioSessionId) { if (this.audioSessionId != audioSessionId) { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java index a9983b40dc..eda2b250de 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/ForwardingAudioSink.java @@ -123,6 +123,12 @@ public class ForwardingAudioSink implements AudioSink { sink.setAudioAttributes(audioAttributes); } + @Override + @Nullable + public AudioAttributes getAudioAttributes() { + return sink.getAudioAttributes(); + } + @Override public void setAudioSessionId(int audioSessionId) { sink.setAudioSessionId(audioSessionId); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java index 25aa466b76..172d0a3ca8 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java @@ -16,6 +16,7 @@ package androidx.media3.exoplayer.audio; import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkStateNotNull; import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_MAX_INPUT_SIZE_EXCEEDED; import static androidx.media3.exoplayer.DecoderReuseEvaluation.REUSE_RESULT_NO; import static com.google.common.base.MoreObjects.firstNonNull; @@ -29,7 +30,9 @@ import android.media.MediaCrypto; import android.media.MediaFormat; import android.os.Handler; import androidx.annotation.CallSuper; +import androidx.annotation.DoNotInline; import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; import androidx.media3.common.AudioAttributes; import androidx.media3.common.AuxEffectInfo; import androidx.media3.common.C; @@ -60,8 +63,10 @@ import androidx.media3.exoplayer.mediacodec.MediaCodecSelector; import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import androidx.media3.exoplayer.mediacodec.MediaCodecUtil.DecoderQueryException; import com.google.common.collect.ImmutableList; +import java.lang.reflect.InvocationTargetException; import java.nio.ByteBuffer; import java.util.List; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * Decodes and renders audio using {@link MediaCodec} and an {@link AudioSink}. @@ -98,6 +103,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media private final Context context; private final EventDispatcher eventDispatcher; private final AudioSink audioSink; + private final SpatializationHelper spatializationHelper; private int codecMaxInputSize; private boolean codecNeedsDiscardChannelsWorkaround; @@ -253,9 +259,11 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media mediaCodecSelector, enableDecoderFallback, /* assumedMinimumCodecOperatingRate= */ 44100); - this.context = context.getApplicationContext(); + context = context.getApplicationContext(); + this.context = context; this.audioSink = audioSink; eventDispatcher = new EventDispatcher(eventHandler, eventListener); + spatializationHelper = new SpatializationHelper(context, audioSink.getAudioAttributes()); audioSink.setListener(new AudioSinkListener()); } @@ -414,6 +422,11 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media return audioSink.supportsFormat(format); } + @Override + protected boolean shouldReinitCodec() { + return spatializationHelper.shouldReinitCodec(); + } + @Override protected MediaCodecAdapter.Configuration getMediaCodecConfiguration( MediaCodecInfo codecInfo, @@ -474,7 +487,11 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media @Override protected void onCodecInitialized( - String name, long initializedTimestampMs, long initializationDurationMs) { + String name, + MediaCodecAdapter.Configuration configuration, + long initializedTimestampMs, + long initializationDurationMs) { + spatializationHelper.onCodecInitialized(configuration); eventDispatcher.decoderInitialized(name, initializedTimestampMs, initializationDurationMs); } @@ -565,6 +582,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media audioSink.disableTunneling(); } audioSink.setPlayerId(getPlayerId()); + spatializationHelper.enable(); } @Override @@ -617,6 +635,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media audioSinkNeedsReset = false; audioSink.reset(); } + spatializationHelper.reset(); } } @@ -741,6 +760,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media case MSG_SET_AUDIO_ATTRIBUTES: AudioAttributes audioAttributes = (AudioAttributes) message; audioSink.setAudioAttributes(audioAttributes); + spatializationHelper.setAudioAttributes(audioSink.getAudioAttributes()); break; case MSG_SET_AUX_EFFECT_INFO: AuxEffectInfo auxEffectInfo = (AuxEffectInfo) message; @@ -852,14 +872,8 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media == AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY) { mediaFormat.setInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_FLOAT); } + spatializationHelper.configureForSpatialization(mediaFormat, format); - if (Util.SDK_INT >= 32) { - // Disable down-mixing in the decoder (for decoders that read the max-output-channel-count - // key). - // TODO[b/190759307]: Update key to use MediaFormat.KEY_MAX_OUTPUT_CHANNEL_COUNT once the - // compile SDK target is set to 32. - mediaFormat.setInteger("max-output-channel-count", 99); - } return mediaFormat; } @@ -943,4 +957,163 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media eventDispatcher.audioSinkError(audioSinkError); } } + + /** + * A helper class that signals whether the codec needs to be re-initialized because spatialization + * properties changed. + */ + private static final class SpatializationHelper implements SpatializerDelegate.Listener { + // TODO[b/190759307] Remove and use MediaFormat.KEY_MAX_OUTPUT_CHANNEL_COUNT once the + // compile SDK target is set to 32. + private static final String KEY_MAX_OUTPUT_CHANNEL_COUNT = "max-output-channel-count"; + private static final int SPATIALIZATION_CHANNEL_COUNT = 99; + + @Nullable private final SpatializerDelegate spatializerDelegate; + + private @MonotonicNonNull Handler handler; + @Nullable private AudioAttributes audioAttributes; + @Nullable private Format inputFormat; + private boolean codecConfiguredForSpatialization; + private boolean codecNeedsReinit; + private boolean listenerAdded; + + /** Creates a new instance. */ + public SpatializationHelper(Context context, @Nullable AudioAttributes audioAttributes) { + this.spatializerDelegate = maybeCreateSpatializer(context); + this.audioAttributes = audioAttributes; + } + + /** Enables this helper. Call this method when the renderer is enabled. */ + public void enable() { + maybeAddSpatalizationListener(); + } + + /** Resets the helper and releases any resources. Call this method when renderer is reset. */ + public void reset() { + maybeRemoveSpatalizationListener(); + } + + /** Sets the audio attributes set by the player. */ + public void setAudioAttributes(@Nullable AudioAttributes audioAttributes) { + if (Util.areEqual(this.audioAttributes, audioAttributes)) { + return; + } + + this.audioAttributes = audioAttributes; + updateCodecNeedsReinit(); + } + + /** + * Sets keys for audio spatialization on the {@code mediaFormat} if the platform can apply + * spatialization to this {@code format}. + */ + public void configureForSpatialization(MediaFormat mediaFormat, Format format) { + if (canBeSpatialized(format)) { + mediaFormat.setInteger(KEY_MAX_OUTPUT_CHANNEL_COUNT, SPATIALIZATION_CHANNEL_COUNT); + } + } + + /** Informs the helper that a codec was initialized. */ + public void onCodecInitialized(MediaCodecAdapter.Configuration configuration) { + codecNeedsReinit = false; + inputFormat = configuration.format; + codecConfiguredForSpatialization = + configuration.mediaFormat.containsKey(KEY_MAX_OUTPUT_CHANNEL_COUNT) + && configuration.mediaFormat.getInteger(KEY_MAX_OUTPUT_CHANNEL_COUNT) + == SPATIALIZATION_CHANNEL_COUNT; + } + + /** + * Returns whether the codec should be re-initialized, caused by a change in the spatialization + * properties. + */ + public boolean shouldReinitCodec() { + return codecNeedsReinit; + } + + // SpatializerDelegate.Listener + + @Override + public void onSpatializerEnabledChanged(SpatializerDelegate spatializer, boolean enabled) { + updateCodecNeedsReinit(); + } + + @Override + public void onSpatializerAvailableChanged(SpatializerDelegate spatializer, boolean available) { + updateCodecNeedsReinit(); + } + + // Other internal methods + + /** Returns whether this format can be spatialized by the platform. */ + private boolean canBeSpatialized(@Nullable Format format) { + if (Util.SDK_INT < 32 + || format == null + || audioAttributes == null + || spatializerDelegate == null + || spatializerDelegate.getImmersiveAudioLevel() + != SpatializerDelegate.SPATIALIZER_IMMERSIVE_LEVEL_MULTICHANNEL + || !spatializerDelegate.isAvailable() + || !spatializerDelegate.isEnabled()) { + return false; + } + AudioFormat.Builder audioFormatBuilder = + new AudioFormat.Builder() + .setEncoding(AudioFormat.ENCODING_PCM_16BIT) + .setChannelMask(Util.getAudioTrackChannelConfig(format.channelCount)); + if (format.sampleRate != Format.NO_VALUE) { + audioFormatBuilder.setSampleRate(format.sampleRate); + } + return spatializerDelegate.canBeSpatialized( + audioAttributes.getAudioAttributesV21(), audioFormatBuilder.build()); + } + + private void maybeAddSpatalizationListener() { + if (!listenerAdded && spatializerDelegate != null && Util.SDK_INT >= 32) { + if (handler == null) { + // Route callbacks to the playback thread. + handler = Util.createHandlerForCurrentLooper(); + } + spatializerDelegate.addOnSpatializerStateChangedListener(handler::post, this); + listenerAdded = true; + } + } + + private void maybeRemoveSpatalizationListener() { + if (listenerAdded && spatializerDelegate != null && Util.SDK_INT >= 32) { + spatializerDelegate.removeOnSpatializerStateChangedListener(this); + checkStateNotNull(handler).removeCallbacksAndMessages(null); + } + } + + private void updateCodecNeedsReinit() { + codecNeedsReinit = codecConfiguredForSpatialization != canBeSpatialized(inputFormat); + } + + @Nullable + private static SpatializerDelegate maybeCreateSpatializer(Context context) { + if (Util.SDK_INT >= 32) { + return Api32.createSpatializer(context); + } + return null; + } + } + + @RequiresApi(32) + private static final class Api32 { + private Api32() {} + + @DoNotInline + @Nullable + public static SpatializerDelegate createSpatializer(Context context) { + try { + return new SpatializerDelegate(context); + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException e) { + // Do nothing for these cases. + } catch (InvocationTargetException e) { + Log.w(TAG, "Failed to load Spatializer with reflection", e); + } + return null; + } + } } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index 939383ff17..6068e56264 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -566,6 +566,14 @@ public abstract class MediaCodecRenderer extends BaseRenderer { return true; } + /** + * Returns whether the renderer needs to re-initialize the codec, possibly as a result of a change + * in device capabilities. + */ + protected boolean shouldReinitCodec() { + return false; + } + /** * Returns whether the codec needs the renderer to propagate the end-of-stream signal directly, * rather than by using an end-of-stream buffer queued to the codec. @@ -1120,7 +1128,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { decoderCounters.decoderInitCount++; long elapsed = codecInitializedTimestamp - codecInitializingTimestamp; - onCodecInitialized(codecName, codecInitializedTimestamp, elapsed); + onCodecInitialized(codecName, configuration, codecInitializedTimestamp, elapsed); } private boolean shouldContinueRendering(long renderStartTimeMs) { @@ -1160,6 +1168,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer { if (codec == null || codecDrainState == DRAIN_STATE_WAIT_END_OF_STREAM || inputStreamEnded) { return false; } + if (codecDrainState == DRAIN_STATE_NONE && shouldReinitCodec()) { + drainAndReinitializeCodec(); + } if (inputIndex < 0) { inputIndex = codec.dequeueInputBufferIndex(); @@ -1354,12 +1365,16 @@ public abstract class MediaCodecRenderer extends BaseRenderer { *

The default implementation is a no-op. * * @param name The name of the codec that was initialized. + * @param configuration The {@link MediaCodecAdapter.Configuration} used to configure the codec. * @param initializedTimestampMs {@link SystemClock#elapsedRealtime()} when initialization * finished. * @param initializationDurationMs The time taken to initialize the codec in milliseconds. */ protected void onCodecInitialized( - String name, long initializedTimestampMs, long initializationDurationMs) { + String name, + MediaCodecAdapter.Configuration configuration, + long initializedTimestampMs, + long initializationDurationMs) { // Do nothing. } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index c8607616c7..d3aa4562ec 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -771,7 +771,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { @Override protected void onCodecInitialized( - String name, long initializedTimestampMs, long initializationDurationMs) { + String name, + MediaCodecAdapter.Configuration configuration, + long initializedTimestampMs, + long initializationDurationMs) { eventDispatcher.decoderInitialized(name, initializedTimestampMs, initializationDurationMs); codecNeedsSetOutputSurfaceWorkaround = codecNeedsSetOutputSurfaceWorkaround(name); codecHandlesHdr10PlusOutOfBandMetadata = diff --git a/libraries/test_exoplayer_playback/src/androidTest/java/androidx/media3/test/exoplayer/playback/gts/DebugRenderersFactory.java b/libraries/test_exoplayer_playback/src/androidTest/java/androidx/media3/test/exoplayer/playback/gts/DebugRenderersFactory.java index de6edcf479..c2f80194a6 100644 --- a/libraries/test_exoplayer_playback/src/androidTest/java/androidx/media3/test/exoplayer/playback/gts/DebugRenderersFactory.java +++ b/libraries/test_exoplayer_playback/src/androidTest/java/androidx/media3/test/exoplayer/playback/gts/DebugRenderersFactory.java @@ -149,14 +149,18 @@ import java.util.ArrayList; @Override protected void onCodecInitialized( - String name, long initializedTimestampMs, long initializationDurationMs) { + String name, + MediaCodecAdapter.Configuration configuration, + long initializedTimestampMs, + long initializationDurationMs) { // If the codec was initialized whilst the renderer is started, default behavior is to // render the first frame (i.e. the keyframe before the current position), then drop frames up // to the current playback position. For test runs that place a maximum limit on the number of // dropped frames allowed, this is not desired behavior. Hence we skip (rather than drop) // frames up to the current playback position [Internal: b/66494991]. skipToPositionBeforeRenderingFirstFrame = getState() == Renderer.STATE_STARTED; - super.onCodecInitialized(name, initializedTimestampMs, initializationDurationMs); + super.onCodecInitialized( + name, configuration, initializedTimestampMs, initializationDurationMs); } @Override