From f0463bff8aa3b2693739da098ec38789b4d26f88 Mon Sep 17 00:00:00 2001 From: Daniele Sparano Date: Mon, 13 May 2024 14:54:56 +0100 Subject: [PATCH 1/4] =?UTF-8?q?=EF=BB=BFSet=20pixel=20aspect=20ratio=20fro?= =?UTF-8?q?m=20output=20media=20format=20if=20set?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../media3/exoplayer/video/MediaCodecVideoRenderer.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 1c4b727d69..867915ca03 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -1342,7 +1342,12 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer ? mediaFormat.getInteger(KEY_CROP_BOTTOM) - mediaFormat.getInteger(KEY_CROP_TOP) + 1 : mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); } - pixelWidthHeightRatio = format.pixelWidthHeightRatio; + boolean hasPixelAspectRatio = mediaFormat.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH) + && mediaFormat.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT); + pixelWidthHeightRatio = hasPixelAspectRatio ? + mediaFormat.getInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH) / + mediaFormat.getInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT) + : format.pixelWidthHeightRatio; // The decoder applies the rotation when rendering to the surface. For 90 and 270 degree // rotations, we need to flip the width, height and pixel aspect ratio to reflect the rotation // that was applied. From 7473156853fdc76dfc530a889d160486f5f3b781 Mon Sep 17 00:00:00 2001 From: Daniele Sparano Date: Wed, 28 Aug 2024 12:37:11 +0100 Subject: [PATCH 2/4] Fix output pixel aspect ratio and add test for it --- .../video/MediaCodecVideoRenderer.java | 4 +- .../video/MediaCodecVideoRendererTest.java | 91 ++++++++++++++++++- 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 867915ca03..247e54fdf8 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -1342,10 +1342,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer ? mediaFormat.getInteger(KEY_CROP_BOTTOM) - mediaFormat.getInteger(KEY_CROP_TOP) + 1 : mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); } - boolean hasPixelAspectRatio = mediaFormat.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH) + boolean hasPixelAspectRatio = (Util.SDK_INT >= 30) && mediaFormat != null && mediaFormat.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH) && mediaFormat.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT); pixelWidthHeightRatio = hasPixelAspectRatio ? - mediaFormat.getInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH) / + (float)mediaFormat.getInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH) / mediaFormat.getInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT) : format.pixelWidthHeightRatio; // The decoder applies the rotation when rendering to the surface. For 90 and 270 degree diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index 7f92d7efad..203dd3fcc3 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -50,6 +50,7 @@ import androidx.media3.common.MimeTypes; import androidx.media3.common.TrackGroup; import androidx.media3.common.VideoSize; import androidx.media3.common.util.Clock; +import androidx.media3.common.util.Util; import androidx.media3.decoder.CryptoInfo; import androidx.media3.exoplayer.DecoderCounters; import androidx.media3.exoplayer.ExoPlaybackException; @@ -100,6 +101,7 @@ import org.mockito.Mock; import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; import org.robolectric.Shadows; +import org.robolectric.annotation.Config; import org.robolectric.shadows.ShadowDisplay; import org.robolectric.shadows.ShadowLooper; import org.robolectric.shadows.ShadowSystemClock; @@ -747,6 +749,71 @@ public class MediaCodecVideoRendererTest { new VideoSize(VIDEO_H264.width, VIDEO_H264.height, VIDEO_H264.pixelWidthHeightRatio)); } + @Config(minSdk = 30) + @Test + public void render_withMediaCodecAlteringPixelAspectRatioWidthHeight_sendsVideoSizeChangeWithMediaFormatValues() throws Exception { + MediaCodecAdapter.Factory codecAdapterFactory = + configuration -> + new ForwardingSynchronousMediaCodecAdapterAlteringPixelAspectRatio( + new SynchronousMediaCodecAdapter.Factory().createAdapter(configuration)); + MediaCodecVideoRenderer mediaCodecVideoRendererWithCustomAdapter = + new MediaCodecVideoRenderer( + ApplicationProvider.getApplicationContext(), + codecAdapterFactory, + mediaCodecSelector, + /* allowedJoiningTimeMs= */ 0, + /* enableDecoderFallback= */ false, + /* eventHandler= */ new Handler(testMainLooper), + /* eventListener= */ eventListener, + /* maxDroppedFramesToNotify= */ 1) { + @Override + protected @Capabilities int supportsFormat( + MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + }; + mediaCodecVideoRendererWithCustomAdapter.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + surface = new Surface(new SurfaceTexture(/* texName= */ 0)); + mediaCodecVideoRendererWithCustomAdapter.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ VIDEO_H264, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRendererWithCustomAdapter.enable( + RendererConfiguration.DEFAULT, + new Format[] {VIDEO_H264}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + mediaCodecVideoRendererWithCustomAdapter.setCurrentStreamFinal(); + mediaCodecVideoRendererWithCustomAdapter.start(); + + int positionUs = 0; + do { + mediaCodecVideoRendererWithCustomAdapter.render(positionUs, SystemClock.elapsedRealtime() * 1000); + positionUs += 10; + } while (!mediaCodecVideoRendererWithCustomAdapter.isEnded()); + shadowOf(testMainLooper).idle(); + + verify(eventListener) + .onVideoSizeChanged( + new VideoSize( + VIDEO_H264.width, + VIDEO_H264.height, + VIDEO_H264.pixelWidthHeightRatio / 2)); + } + @Test public void render_withMultipleQueued_sendsVideoSizeChangedWithCorrectPixelAspectRatioWhenMultipleQueued() @@ -1901,9 +1968,31 @@ public class MediaCodecVideoRendererTest { } } + private static final class ForwardingSynchronousMediaCodecAdapterAlteringPixelAspectRatio + extends ForwardingSynchronousMediaCodecAdapter { + + ForwardingSynchronousMediaCodecAdapterAlteringPixelAspectRatio(MediaCodecAdapter adapter) { + super(adapter); + } + + @Override + public MediaFormat getOutputFormat() { + MediaFormat mediaFormat = adapter.getOutputFormat(); + if (Util.SDK_INT >= 30) { + int pixelAspectRatioHeight = 1 << 30; + int pixelAspectRatioWidth = (int) (0.5f * pixelAspectRatioHeight); + mediaFormat.setInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH, + pixelAspectRatioWidth); + mediaFormat.setInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT, + pixelAspectRatioHeight); + } + return mediaFormat; + } + } + private abstract static class ForwardingSynchronousMediaCodecAdapter implements MediaCodecAdapter { - private final MediaCodecAdapter adapter; + protected final MediaCodecAdapter adapter; ForwardingSynchronousMediaCodecAdapter(MediaCodecAdapter adapter) { this.adapter = adapter; From 0ac26c4004219fa888bdbf481eba5de5e7195703 Mon Sep 17 00:00:00 2001 From: microkatz Date: Wed, 28 Aug 2024 12:41:44 +0000 Subject: [PATCH 3/4] Format with google-java-format --- .../video/MediaCodecVideoRenderer.java | 16 ++++-- .../video/MediaCodecVideoRendererTest.java | 56 ++++++++----------- 2 files changed, 34 insertions(+), 38 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 247e54fdf8..bf74a4b8ed 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -1342,12 +1342,16 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer ? mediaFormat.getInteger(KEY_CROP_BOTTOM) - mediaFormat.getInteger(KEY_CROP_TOP) + 1 : mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); } - boolean hasPixelAspectRatio = (Util.SDK_INT >= 30) && mediaFormat != null && mediaFormat.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH) - && mediaFormat.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT); - pixelWidthHeightRatio = hasPixelAspectRatio ? - (float)mediaFormat.getInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH) / - mediaFormat.getInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT) - : format.pixelWidthHeightRatio; + boolean hasPixelAspectRatio = + (Util.SDK_INT >= 30) + && mediaFormat != null + && mediaFormat.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH) + && mediaFormat.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT); + pixelWidthHeightRatio = + hasPixelAspectRatio + ? (float) mediaFormat.getInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH) + / mediaFormat.getInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT) + : format.pixelWidthHeightRatio; // The decoder applies the rotation when rendering to the surface. For 90 and 270 degree // rotations, we need to flip the width, height and pixel aspect ratio to reflect the rotation // that was applied. diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index 203dd3fcc3..fa084d8e10 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -751,11 +751,27 @@ public class MediaCodecVideoRendererTest { @Config(minSdk = 30) @Test - public void render_withMediaCodecAlteringPixelAspectRatioWidthHeight_sendsVideoSizeChangeWithMediaFormatValues() throws Exception { + public void + render_withMediaCodecModifyingPixelAspectRatioWidthHeight_sendsVideoSizeChangeWithMediaFormatValues() + throws Exception { MediaCodecAdapter.Factory codecAdapterFactory = configuration -> - new ForwardingSynchronousMediaCodecAdapterAlteringPixelAspectRatio( - new SynchronousMediaCodecAdapter.Factory().createAdapter(configuration)); + new ForwardingSynchronousMediaCodecAdapter( + new SynchronousMediaCodecAdapter.Factory().createAdapter(configuration)) { + @Override + public MediaFormat getOutputFormat() { + MediaFormat mediaFormat = adapter.getOutputFormat(); + if (Util.SDK_INT >= 30) { + int pixelAspectRatioHeight = 1 << 30; // Max integer power of 2. + int pixelAspectRatioWidth = (int) (0.5f * pixelAspectRatioHeight); + mediaFormat.setInteger( + MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH, pixelAspectRatioWidth); + mediaFormat.setInteger( + MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT, pixelAspectRatioHeight); + } + return mediaFormat; + } + }; MediaCodecVideoRenderer mediaCodecVideoRendererWithCustomAdapter = new MediaCodecVideoRenderer( ApplicationProvider.getApplicationContext(), @@ -773,9 +789,8 @@ public class MediaCodecVideoRendererTest { } }; mediaCodecVideoRendererWithCustomAdapter.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); - surface = new Surface(new SurfaceTexture(/* texName= */ 0)); - mediaCodecVideoRendererWithCustomAdapter.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); - + mediaCodecVideoRendererWithCustomAdapter.handleMessage( + Renderer.MSG_SET_VIDEO_OUTPUT, new Surface(new SurfaceTexture(/* texName= */ 0))); FakeSampleStream fakeSampleStream = new FakeSampleStream( new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), @@ -801,7 +816,8 @@ public class MediaCodecVideoRendererTest { int positionUs = 0; do { - mediaCodecVideoRendererWithCustomAdapter.render(positionUs, SystemClock.elapsedRealtime() * 1000); + mediaCodecVideoRendererWithCustomAdapter.render( + positionUs, SystemClock.elapsedRealtime() * 1000); positionUs += 10; } while (!mediaCodecVideoRendererWithCustomAdapter.isEnded()); shadowOf(testMainLooper).idle(); @@ -809,9 +825,7 @@ public class MediaCodecVideoRendererTest { verify(eventListener) .onVideoSizeChanged( new VideoSize( - VIDEO_H264.width, - VIDEO_H264.height, - VIDEO_H264.pixelWidthHeightRatio / 2)); + VIDEO_H264.width, VIDEO_H264.height, VIDEO_H264.pixelWidthHeightRatio / 2)); } @Test @@ -1968,28 +1982,6 @@ public class MediaCodecVideoRendererTest { } } - private static final class ForwardingSynchronousMediaCodecAdapterAlteringPixelAspectRatio - extends ForwardingSynchronousMediaCodecAdapter { - - ForwardingSynchronousMediaCodecAdapterAlteringPixelAspectRatio(MediaCodecAdapter adapter) { - super(adapter); - } - - @Override - public MediaFormat getOutputFormat() { - MediaFormat mediaFormat = adapter.getOutputFormat(); - if (Util.SDK_INT >= 30) { - int pixelAspectRatioHeight = 1 << 30; - int pixelAspectRatioWidth = (int) (0.5f * pixelAspectRatioHeight); - mediaFormat.setInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH, - pixelAspectRatioWidth); - mediaFormat.setInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT, - pixelAspectRatioHeight); - } - return mediaFormat; - } - } - private abstract static class ForwardingSynchronousMediaCodecAdapter implements MediaCodecAdapter { protected final MediaCodecAdapter adapter; From f1654732a658364f8c7aa0bf5e1b3ce449474cd9 Mon Sep 17 00:00:00 2001 From: microkatz Date: Wed, 28 Aug 2024 12:42:57 +0000 Subject: [PATCH 4/4] Added release note for change --- RELEASENOTES.md | 3 +++ .../video/MediaCodecVideoRenderer.java | 21 ++++++++++--------- .../video/MediaCodecVideoRendererTest.java | 14 ++++++------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 582527a796..c81a93d3f2 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -66,6 +66,9 @@ `Surface` in `configure` and calls to a new method `detachOutputSurface` to remove a previously set `Surface` if the codec supports this (`MediaCodecInfo.detachedSurfaceSupported`). + * Use `MediaCodecAdapter` supplied pixel aspect ratio values if provided + when processing `onOutputFormatChanged` + ([#1371](https://github.com/androidx/media/pull/1371)). * Text: * Metadata: * Image: diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index bf74a4b8ed..a7c31e671e 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -1342,16 +1342,17 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer ? mediaFormat.getInteger(KEY_CROP_BOTTOM) - mediaFormat.getInteger(KEY_CROP_TOP) + 1 : mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); } - boolean hasPixelAspectRatio = - (Util.SDK_INT >= 30) - && mediaFormat != null - && mediaFormat.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH) - && mediaFormat.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT); - pixelWidthHeightRatio = - hasPixelAspectRatio - ? (float) mediaFormat.getInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH) - / mediaFormat.getInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT) - : format.pixelWidthHeightRatio; + + pixelWidthHeightRatio = format.pixelWidthHeightRatio; + if (Util.SDK_INT >= 30 + && mediaFormat != null + && mediaFormat.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH) + && mediaFormat.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT)) { + pixelWidthHeightRatio = + (float) mediaFormat.getInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH) + / mediaFormat.getInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT); + } + // The decoder applies the rotation when rendering to the surface. For 90 and 270 degree // rotations, we need to flip the width, height and pixel aspect ratio to reflect the rotation // that was applied. diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index fa084d8e10..1954a14697 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -116,6 +116,7 @@ public class MediaCodecVideoRendererTest { .setSampleMimeType(MimeTypes.VIDEO_H264) .setWidth(1920) .setHeight(1080) + .setPixelWidthHeightRatio(1.0f) .build(); private static final TrackGroup TRACK_GROUP_H264 = new TrackGroup(VIDEO_H264); @@ -762,12 +763,9 @@ public class MediaCodecVideoRendererTest { public MediaFormat getOutputFormat() { MediaFormat mediaFormat = adapter.getOutputFormat(); if (Util.SDK_INT >= 30) { - int pixelAspectRatioHeight = 1 << 30; // Max integer power of 2. - int pixelAspectRatioWidth = (int) (0.5f * pixelAspectRatioHeight); - mediaFormat.setInteger( - MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH, pixelAspectRatioWidth); - mediaFormat.setInteger( - MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT, pixelAspectRatioHeight); + // Change to 9:16 Ratio + mediaFormat.setInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH, 9); + mediaFormat.setInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT, 16); } return mediaFormat; } @@ -825,7 +823,9 @@ public class MediaCodecVideoRendererTest { verify(eventListener) .onVideoSizeChanged( new VideoSize( - VIDEO_H264.width, VIDEO_H264.height, VIDEO_H264.pixelWidthHeightRatio / 2)); + VIDEO_H264.width, + VIDEO_H264.height, + /* pixelWidthHeightRatio= */ VIDEO_H264.pixelWidthHeightRatio * (9.0f / 16.0f))); } @Test