From 54f5e0729ed6515dfeff5e3fc87c6897e77e3bc1 Mon Sep 17 00:00:00 2001 From: tonihei Date: Thu, 18 Jul 2024 02:27:18 -0700 Subject: [PATCH] Support detached surface mode from API 35 This replaces the existing PlaceholderSurface mode with a more efficient solution that doesn't require a GL texture or a new thread. PiperOrigin-RevId: 653537596 --- RELEASENOTES.md | 4 +++ .../AsynchronousMediaCodecAdapter.java | 11 +++++++ .../mediacodec/MediaCodecAdapter.java | 8 +++++ .../exoplayer/mediacodec/MediaCodecInfo.java | 21 ++++++++++-- .../SynchronousMediaCodecAdapter.java | 16 ++++++++- .../video/MediaCodecVideoRenderer.java | 33 ++++++++++++++++--- .../mediacodec/MediaCodecInfoTest.java | 6 ++-- .../mediacodec/MediaCodecRendererTest.java | 5 +++ .../video/MediaCodecVideoRendererTest.java | 5 +++ .../test/utils/CapturingRenderersFactory.java | 6 ++++ 10 files changed, 106 insertions(+), 9 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index e5c1bc1d26..86baab181c 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -41,6 +41,10 @@ * Video: * `MediaCodecVideoRenderer` avoids decoding samples that are neither rendered nor used as reference by other samples. + * On API 35 and above, `MediaCodecAdapter` may now receive a `null` + `Surface` in `configure` and calls to a new method `detachOutputSurface` + to remove a previously set `Surface` if the codec supports this + (`MediaCodecInfo.detachedSurfaceSupported`). * Text: * TTML: Fix handling of percentage `tts:fontSize` values to ensure they are correctly inherited from parent nodes with percentage `tts:fontSize` diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java index ea1a31c013..84272048ee 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java @@ -116,6 +116,11 @@ import java.nio.ByteBuffer; codecAdapter = new AsynchronousMediaCodecAdapter(codec, callbackThreadSupplier.get(), bufferEnqueuer); TraceUtil.endSection(); + if (configuration.surface == null + && configuration.codecInfo.detachedSurfaceSupported + && Util.SDK_INT >= 35) { + flags |= MediaCodec.CONFIGURE_FLAG_DETACHED_SURFACE; + } codecAdapter.initialize( configuration.mediaFormat, configuration.surface, configuration.crypto, flags); return codecAdapter; @@ -295,6 +300,12 @@ import java.nio.ByteBuffer; codec.setOutputSurface(surface); } + @RequiresApi(35) + @Override + public void detachOutputSurface() { + codec.detachOutputSurface(); + } + @Override public void setParameters(Bundle params) { bufferEnqueuer.setParameters(params); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecAdapter.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecAdapter.java index 2184b28f0e..509f6905f3 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecAdapter.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecAdapter.java @@ -284,6 +284,14 @@ public interface MediaCodecAdapter { @RequiresApi(23) void setOutputSurface(Surface surface); + /** + * Detaches the current output surface. + * + * @see MediaCodec#detachOutputSurface() + */ + @RequiresApi(35) + void detachOutputSurface(); + /** * Communicate additional parameter changes to the {@link MediaCodec} instance. * diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfo.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfo.java index 7400ab745f..d9826a3b97 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfo.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfo.java @@ -138,6 +138,14 @@ public final class MediaCodecInfo { */ public final boolean vendor; + /** + * Whether the codec supports "detached" surface mode where it is able to decode without an + * attached surface. Only relevant for video codecs. + * + * @see android.media.MediaCodecInfo.CodecCapabilities#FEATURE_DetachedSurface + */ + public final boolean detachedSurfaceSupported; + private final boolean isVideo; /** @@ -179,7 +187,8 @@ public final class MediaCodecInfo { && isAdaptive(capabilities) && !needsDisableAdaptationWorkaround(name), /* tunneling= */ capabilities != null && isTunneling(capabilities), - /* secure= */ forceSecure || (capabilities != null && isSecure(capabilities))); + /* secure= */ forceSecure || (capabilities != null && isSecure(capabilities)), + isDetachedSurfaceSupported(capabilities)); } @VisibleForTesting @@ -193,7 +202,8 @@ public final class MediaCodecInfo { boolean vendor, boolean adaptive, boolean tunneling, - boolean secure) { + boolean secure, + boolean detachedSurfaceSupported) { this.name = Assertions.checkNotNull(name); this.mimeType = mimeType; this.codecMimeType = codecMimeType; @@ -204,6 +214,7 @@ public final class MediaCodecInfo { this.adaptive = adaptive; this.tunneling = tunneling; this.secure = secure; + this.detachedSurfaceSupported = detachedSurfaceSupported; isVideo = MimeTypes.isVideo(mimeType); } @@ -661,6 +672,12 @@ public final class MediaCodecInfo { return capabilities.isFeatureSupported(CodecCapabilities.FEATURE_SecurePlayback); } + private static boolean isDetachedSurfaceSupported(@Nullable CodecCapabilities capabilities) { + return Util.SDK_INT >= 35 + && capabilities != null + && capabilities.isFeatureSupported(CodecCapabilities.FEATURE_DetachedSurface); + } + private static boolean areSizeAndRateSupported( VideoCapabilities capabilities, int width, int height, double frameRate) { // Don't ever fail due to alignment. See: https://github.com/google/ExoPlayer/issues/6551. diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/SynchronousMediaCodecAdapter.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/SynchronousMediaCodecAdapter.java index 02c01e9935..88c80f12d6 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/SynchronousMediaCodecAdapter.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/SynchronousMediaCodecAdapter.java @@ -18,6 +18,7 @@ package androidx.media3.exoplayer.mediacodec; import static androidx.media3.common.util.Assertions.checkNotNull; +import android.annotation.SuppressLint; import android.media.MediaCodec; import android.media.MediaFormat; import android.os.Bundle; @@ -43,14 +44,21 @@ public final class SynchronousMediaCodecAdapter implements MediaCodecAdapter { /** A factory for {@link SynchronousMediaCodecAdapter} instances. */ public static class Factory implements MediaCodecAdapter.Factory { + @SuppressLint("WrongConstant") // Can't verify codec flag IntDef @Override public MediaCodecAdapter createAdapter(Configuration configuration) throws IOException { @Nullable MediaCodec codec = null; try { codec = createCodec(configuration); TraceUtil.beginSection("configureCodec"); + int flags = 0; + if (configuration.surface == null + && configuration.codecInfo.detachedSurfaceSupported + && Util.SDK_INT >= 35) { + flags |= MediaCodec.CONFIGURE_FLAG_DETACHED_SURFACE; + } codec.configure( - configuration.mediaFormat, configuration.surface, configuration.crypto, /* flags= */ 0); + configuration.mediaFormat, configuration.surface, configuration.crypto, flags); TraceUtil.endSection(); TraceUtil.beginSection("startCodec"); codec.start(); @@ -177,6 +185,12 @@ public final class SynchronousMediaCodecAdapter implements MediaCodecAdapter { codec.setOutputSurface(surface); } + @RequiresApi(35) + @Override + public void detachOutputSurface() { + codec.detachOutputSurface(); + } + @Override public void setParameters(Bundle params) { codec.setParameters(params); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 36aae0527e..f1ba1aab1d 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -948,7 +948,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer MediaCodecInfo codecInfo = checkNotNull(getCodecInfo()); boolean canUpdateSurface = hasSurfaceForCodec(codecInfo); if (Util.SDK_INT >= 23 && canUpdateSurface && !codecNeedsSetOutputSurfaceWorkaround) { - setOutputSurfaceV23(codec, getSurfaceForCodec(codecInfo)); + setOutputSurface(codec, getSurfaceForCodec(codecInfo)); } else { releaseCodec(); maybeInitCodecOrBypass(); @@ -1766,18 +1766,24 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer } private boolean hasSurfaceForCodec(MediaCodecInfo codecInfo) { - return displaySurface != null || shouldUsePlaceholderSurface(codecInfo); + return displaySurface != null + || shouldUseDetachedSurface(codecInfo) + || shouldUsePlaceholderSurface(codecInfo); } /** - * Returns surface to be set on the codec. Must only be called if {@link - * #hasSurfaceForCodec(MediaCodecInfo)} returns true. + * Returns surface to be set on the codec, or null to use detached surface mode. + * + *

Must only be called if {@link #hasSurfaceForCodec(MediaCodecInfo)} returns true. */ + @Nullable private Surface getSurfaceForCodec(MediaCodecInfo codecInfo) { if (videoSink != null) { return videoSink.getInputSurface(); } else if (displaySurface != null) { return displaySurface; + } else if (shouldUseDetachedSurface(codecInfo)) { + return null; } else { checkState(shouldUsePlaceholderSurface(codecInfo)); if (placeholderSurface != null && placeholderSurface.secure != codecInfo.secure) { @@ -1791,6 +1797,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer } } + protected boolean shouldUseDetachedSurface(MediaCodecInfo codecInfo) { + return Util.SDK_INT >= 35 && codecInfo.detachedSurfaceSupported; + } + private boolean shouldUsePlaceholderSurface(MediaCodecInfo codecInfo) { return Util.SDK_INT >= 23 && !tunneling @@ -1898,11 +1908,26 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer codec.setParameters(codecParameters); } + private void setOutputSurface(MediaCodecAdapter codec, @Nullable Surface surface) { + if (Util.SDK_INT >= 23 && surface != null) { + setOutputSurfaceV23(codec, surface); + } else if (Util.SDK_INT >= 35) { + detachOutputSurfaceV35(codec); + } else { + throw new IllegalStateException(); + } + } + @RequiresApi(23) protected void setOutputSurfaceV23(MediaCodecAdapter codec, Surface surface) { codec.setOutputSurface(surface); } + @RequiresApi(35) + protected void detachOutputSurfaceV35(MediaCodecAdapter codec) { + codec.detachOutputSurface(); + } + /** * Returns the framework {@link MediaFormat} that should be used to configure the decoder. * diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfoTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfoTest.java index c4e3f53565..5be9127024 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfoTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecInfoTest.java @@ -310,7 +310,8 @@ public final class MediaCodecInfoTest { /* vendor= */ true, adaptive, /* tunneling= */ false, - /* secure= */ false); + /* secure= */ false, + /* detachedSurfaceSupported= */ true); } private static MediaCodecInfo buildAacCodecInfo() { @@ -324,7 +325,8 @@ public final class MediaCodecInfoTest { /* vendor= */ false, /* adaptive= */ false, /* tunneling= */ false, - /* secure= */ false); + /* secure= */ false, + /* detachedSurfaceSupported= */ false); } private static ColorInfo buildHdrColorInfo(@C.ColorSpace int colorSpace) { diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java index dfb7030357..cec2df3a1d 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java @@ -733,6 +733,11 @@ public class MediaCodecRendererTest { throw exceptionSupplier.get(); } + @Override + public void detachOutputSurface() { + throw exceptionSupplier.get(); + } + @Override public void setParameters(Bundle params) { throw exceptionSupplier.get(); diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index 82cd1c9482..7f92d7efad 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -1978,6 +1978,11 @@ public class MediaCodecVideoRendererTest { adapter.setOutputSurface(surface); } + @Override + public void detachOutputSurface() { + adapter.detachOutputSurface(); + } + @Override public void setParameters(Bundle params) { adapter.setParameters(params); diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java index ce753e93d3..080f1f22cd 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/CapturingRenderersFactory.java @@ -373,6 +373,12 @@ public class CapturingRenderersFactory implements RenderersFactory, Dumper.Dumpa delegate.setOutputSurface(surface); } + @RequiresApi(35) + @Override + public void detachOutputSurface() { + delegate.detachOutputSurface(); + } + @Override public void setParameters(Bundle params) { delegate.setParameters(params);