From 3ebf94cd453842f40bc112ff13ea3f9ff871d183 Mon Sep 17 00:00:00 2001 From: olly Date: Wed, 24 Mar 2021 19:56:10 +0000 Subject: [PATCH] DataSources: Remove position-out-of-range workarounds PiperOrigin-RevId: 364871094 --- RELEASENOTES.md | 6 + .../PixelCopySurfaceCapturerV24Test.java | 428 ++++++++++++++++++ .../VideoRendererOutputCapturerTest.java | 301 ++++++++++++ .../upstream/cache/CacheDataSource.java | 16 - .../upstream/cache/CacheWriter.java | 12 - .../PixelCopySurfaceCapturerV24.java | 108 +++++ .../SingleFrameMediaCodecVideoRenderer.java | 107 +++++ .../surfacecapturer/SurfaceCapturer.java | 84 ++++ .../VideoRendererOutputCapturer.java | 260 +++++++++++ .../video/surfacecapturer/package-info.java | 19 + 10 files changed, 1313 insertions(+), 28 deletions(-) create mode 100644 library/core/src/androidTest/java/com/google/android/exoplayer2/video/surfacecapturer/PixelCopySurfaceCapturerV24Test.java create mode 100644 library/core/src/androidTest/java/com/google/android/exoplayer2/video/surfacecapturer/VideoRendererOutputCapturerTest.java create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/PixelCopySurfaceCapturerV24.java create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/SingleFrameMediaCodecVideoRenderer.java create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/SurfaceCapturer.java create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/VideoRendererOutputCapturer.java create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/package-info.java diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 33321f3dfb..be87a12a6b 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -31,6 +31,12 @@ * Analytics: * Add `onAudioCodecError` and `onVideoCodecError` to `AnalyticsListener`. * Downloads and caching: + * Fix `CacheWriter` to correctly handle cases where the request `DataSpec` + extends beyond the end of the underlying resource. Caching will now + succeed in this case, with data up to the end of the resource being + cached. This behaviour is enabled by default, and so the + `allowShortContent` parameter has been removed + ([#7326](https://github.com/google/ExoPlayer/issues/7326)). * Fix `CacheWriter` to correctly handle `DataSource.close` failures, for which it cannot be assumed that data was successfully written to the cache. diff --git a/library/core/src/androidTest/java/com/google/android/exoplayer2/video/surfacecapturer/PixelCopySurfaceCapturerV24Test.java b/library/core/src/androidTest/java/com/google/android/exoplayer2/video/surfacecapturer/PixelCopySurfaceCapturerV24Test.java new file mode 100644 index 0000000000..1724e94920 --- /dev/null +++ b/library/core/src/androidTest/java/com/google/android/exoplayer2/video/surfacecapturer/PixelCopySurfaceCapturerV24Test.java @@ -0,0 +1,428 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.video.surfacecapturer; + +import static com.google.android.exoplayer2.testutil.TestUtil.getBitmap; +import static com.google.common.truth.Truth.assertThat; + +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.Paint; +import android.os.Handler; +import android.os.Looper; +import android.view.Surface; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.filters.SdkSuppress; +import androidx.test.platform.app.InstrumentationRegistry; +import com.google.android.exoplayer2.testutil.DummyMainThread; +import com.google.android.exoplayer2.testutil.TestUtil; +import com.google.android.exoplayer2.util.ConditionVariable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit test for {@link PixelCopySurfaceCapturerV24}. */ +@RunWith(AndroidJUnit4.class) +@SdkSuppress(minSdkVersion = 24) +public final class PixelCopySurfaceCapturerV24Test { + + private PixelCopySurfaceCapturerV24 pixelCopySurfaceCapturer; + + private DummyMainThread testThread; + private List resultBitmaps; + private List resultExceptions; + private ConditionVariable callbackCalledCondition; + + private final SurfaceCapturer.Callback defaultCallback = + new SurfaceCapturer.Callback() { + @Override + public void onSurfaceCaptured(Bitmap bitmap) { + resultBitmaps.add(bitmap); + callbackCalledCondition.open(); + } + + @Override + public void onSurfaceCaptureError(Exception e) { + resultExceptions.add(e); + callbackCalledCondition.open(); + } + }; + + @Before + public void setUp() { + resultBitmaps = new ArrayList<>(); + resultExceptions = new ArrayList<>(); + testThread = new DummyMainThread(); + callbackCalledCondition = new ConditionVariable(); + } + + @After + public void tearDown() { + testThread.runOnMainThread( + () -> { + if (pixelCopySurfaceCapturer != null) { + pixelCopySurfaceCapturer.release(); + } + }); + testThread.release(); + } + + @Test + public void getSurface_notNull() { + int outputWidth = 80; + int outputHeight = 60; + testThread.runOnMainThread( + () -> { + pixelCopySurfaceCapturer = + new PixelCopySurfaceCapturerV24( + defaultCallback, outputWidth, outputHeight, new Handler(Looper.myLooper())); + Surface surface = pixelCopySurfaceCapturer.getSurface(); + assertThat(surface).isNotNull(); + }); + } + + @Test + public void captureSurface_bmpFile_originalSize() throws IOException, InterruptedException { + int outputWidth = 80; + int outputHeight = 60; + Bitmap originalBitmap = + getBitmap( + InstrumentationRegistry.getInstrumentation().getTargetContext(), + "media/bitmap/image_80_60.bmp"); + + testThread.runOnMainThread( + () -> { + pixelCopySurfaceCapturer = + new PixelCopySurfaceCapturerV24( + defaultCallback, outputWidth, outputHeight, new Handler(Looper.myLooper())); + drawBitmapOnSurface(originalBitmap); + }); + callbackCalledCondition.block(); + assertThat(resultExceptions).isEmpty(); + assertThat(resultBitmaps).hasSize(1); + assertBitmapsAreSimilar(originalBitmap, resultBitmaps.get(0)); + } + + @Test + public void captureSurface_bmpFile_largerSize_sameRatio() + throws IOException, InterruptedException { + int outputWidth = 160; + int outputHeight = 120; + Bitmap originalBitmap = + getBitmap( + InstrumentationRegistry.getInstrumentation().getTargetContext(), + "media/bitmap/image_80_60.bmp"); + Bitmap expectedBitmap = + Bitmap.createScaledBitmap(originalBitmap, outputWidth, outputHeight, /* filter= */ true); + testThread.runOnMainThread( + () -> { + pixelCopySurfaceCapturer = + new PixelCopySurfaceCapturerV24( + defaultCallback, outputWidth, outputHeight, new Handler(Looper.myLooper())); + drawBitmapOnSurface(originalBitmap); + }); + + callbackCalledCondition.block(); + assertThat(resultExceptions).isEmpty(); + assertThat(resultBitmaps).hasSize(1); + assertBitmapsAreSimilar(expectedBitmap, resultBitmaps.get(0)); + } + + @Test + public void captureSurface_bmpFile_largerSize_notSameRatio() + throws IOException, InterruptedException { + int outputWidth = 89; + int outputHeight = 67; + Bitmap originalBitmap = + getBitmap( + InstrumentationRegistry.getInstrumentation().getTargetContext(), + "media/bitmap/image_80_60.bmp"); + Bitmap expectedBitmap = + Bitmap.createScaledBitmap(originalBitmap, outputWidth, outputHeight, /* filter= */ true); + testThread.runOnMainThread( + () -> { + pixelCopySurfaceCapturer = + new PixelCopySurfaceCapturerV24( + defaultCallback, outputWidth, outputHeight, new Handler(Looper.myLooper())); + drawBitmapOnSurface(originalBitmap); + }); + + callbackCalledCondition.block(); + assertThat(resultExceptions).isEmpty(); + assertThat(resultBitmaps).hasSize(1); + assertBitmapsAreSimilar(expectedBitmap, resultBitmaps.get(0)); + } + + @Test + public void captureSurface_bmpFile_smallerSize_sameRatio() + throws IOException, InterruptedException { + int outputWidth = 40; + int outputHeight = 30; + Bitmap originalBitmap = + getBitmap( + InstrumentationRegistry.getInstrumentation().getTargetContext(), + "media/bitmap/image_80_60.bmp"); + Bitmap expectedBitmap = + Bitmap.createScaledBitmap(originalBitmap, outputWidth, outputHeight, /* filter= */ true); + testThread.runOnMainThread( + () -> { + pixelCopySurfaceCapturer = + new PixelCopySurfaceCapturerV24( + defaultCallback, outputWidth, outputHeight, new Handler(Looper.myLooper())); + drawBitmapOnSurface(originalBitmap); + }); + + callbackCalledCondition.block(); + assertThat(resultExceptions).isEmpty(); + assertThat(resultBitmaps).hasSize(1); + assertBitmapsAreSimilar(expectedBitmap, resultBitmaps.get(0)); + } + + @Test + public void captureSurface_bmpFile_smallerSize_notSameRatio() + throws IOException, InterruptedException { + int outputWidth = 32; + int outputHeight = 12; + Bitmap originalBitmap = + getBitmap( + InstrumentationRegistry.getInstrumentation().getTargetContext(), + "media/bitmap/image_80_60.bmp"); + Bitmap expectedBitmap = + Bitmap.createScaledBitmap(originalBitmap, outputWidth, outputHeight, /* filter= */ true); + testThread.runOnMainThread( + () -> { + pixelCopySurfaceCapturer = + new PixelCopySurfaceCapturerV24( + defaultCallback, outputWidth, outputHeight, new Handler(Looper.myLooper())); + drawBitmapOnSurface(originalBitmap); + }); + + callbackCalledCondition.block(); + assertThat(resultExceptions).isEmpty(); + assertThat(resultBitmaps).hasSize(1); + assertBitmapsAreSimilar(expectedBitmap, resultBitmaps.get(0)); + } + + @Test + public void captureSurface_pngFile_originalSize() throws IOException, InterruptedException { + int outputWidth = 256; + int outputHeight = 256; + Bitmap originalBitmap = + getBitmap( + InstrumentationRegistry.getInstrumentation().getTargetContext(), + "media/bitmap/image_256_256.png"); + + testThread.runOnMainThread( + () -> { + pixelCopySurfaceCapturer = + new PixelCopySurfaceCapturerV24( + defaultCallback, outputWidth, outputHeight, new Handler(Looper.myLooper())); + drawBitmapOnSurface(originalBitmap); + }); + callbackCalledCondition.block(); + assertThat(resultExceptions).isEmpty(); + assertThat(resultBitmaps).hasSize(1); + assertBitmapsAreSimilar(originalBitmap, resultBitmaps.get(0)); + } + + @Test + public void captureSurface_pngFile_largerSize_sameRatio() + throws IOException, InterruptedException { + int outputWidth = 512; + int outputHeight = 512; + Bitmap originalBitmap = + getBitmap( + InstrumentationRegistry.getInstrumentation().getTargetContext(), + "media/bitmap/image_256_256.png"); + Bitmap expectedBitmap = + Bitmap.createScaledBitmap(originalBitmap, outputWidth, outputHeight, /* filter= */ true); + testThread.runOnMainThread( + () -> { + pixelCopySurfaceCapturer = + new PixelCopySurfaceCapturerV24( + defaultCallback, outputWidth, outputHeight, new Handler(Looper.myLooper())); + drawBitmapOnSurface(originalBitmap); + }); + + callbackCalledCondition.block(); + assertThat(resultExceptions).isEmpty(); + assertThat(resultBitmaps).hasSize(1); + assertBitmapsAreSimilar(expectedBitmap, resultBitmaps.get(0)); + } + + @Test + public void captureSurface_pngFile_largerSize_notSameRatio() + throws IOException, InterruptedException { + int outputWidth = 567; + int outputHeight = 890; + Bitmap originalBitmap = + getBitmap( + InstrumentationRegistry.getInstrumentation().getTargetContext(), + "media/bitmap/image_256_256.png"); + Bitmap expectedBitmap = + Bitmap.createScaledBitmap(originalBitmap, outputWidth, outputHeight, /* filter= */ true); + testThread.runOnMainThread( + () -> { + pixelCopySurfaceCapturer = + new PixelCopySurfaceCapturerV24( + defaultCallback, outputWidth, outputHeight, new Handler(Looper.myLooper())); + drawBitmapOnSurface(originalBitmap); + }); + + callbackCalledCondition.block(); + assertThat(resultExceptions).isEmpty(); + assertThat(resultBitmaps).hasSize(1); + assertBitmapsAreSimilar(expectedBitmap, resultBitmaps.get(0)); + } + + @Test + public void captureSurface_pngFile_smallerSize_sameRatio() + throws IOException, InterruptedException { + int outputWidth = 128; + int outputHeight = 128; + Bitmap originalBitmap = + getBitmap( + InstrumentationRegistry.getInstrumentation().getTargetContext(), + "media/bitmap/image_256_256.png"); + Bitmap expectedBitmap = + Bitmap.createScaledBitmap(originalBitmap, outputWidth, outputHeight, /* filter= */ true); + testThread.runOnMainThread( + () -> { + pixelCopySurfaceCapturer = + new PixelCopySurfaceCapturerV24( + defaultCallback, outputWidth, outputHeight, new Handler(Looper.myLooper())); + drawBitmapOnSurface(originalBitmap); + }); + + callbackCalledCondition.block(); + assertThat(resultExceptions).isEmpty(); + assertThat(resultBitmaps).hasSize(1); + assertBitmapsAreSimilar(expectedBitmap, resultBitmaps.get(0)); + } + + @Test + public void captureSurface_pngFile_smallerSize_notSameRatio() + throws IOException, InterruptedException { + int outputWidth = 210; + int outputHeight = 123; + Bitmap originalBitmap = + getBitmap( + InstrumentationRegistry.getInstrumentation().getTargetContext(), + "media/bitmap/image_256_256.png"); + Bitmap expectedBitmap = + Bitmap.createScaledBitmap(originalBitmap, outputWidth, outputHeight, /* filter= */ true); + testThread.runOnMainThread( + () -> { + pixelCopySurfaceCapturer = + new PixelCopySurfaceCapturerV24( + defaultCallback, outputWidth, outputHeight, new Handler(Looper.myLooper())); + drawBitmapOnSurface(originalBitmap); + }); + + callbackCalledCondition.block(); + assertThat(resultExceptions).isEmpty(); + assertThat(resultBitmaps).hasSize(1); + assertBitmapsAreSimilar(expectedBitmap, resultBitmaps.get(0)); + } + + @Test + public void captureSurface_multipleTimes() throws IOException, InterruptedException { + int outputWidth = 500; + int outputHeight = 400; + Bitmap originalBitmap1 = + getBitmap( + InstrumentationRegistry.getInstrumentation().getTargetContext(), + "media/bitmap/image_80_60.bmp"); + Bitmap originalBitmap2 = + getBitmap( + InstrumentationRegistry.getInstrumentation().getTargetContext(), + "media/bitmap/image_256_256.png"); + Bitmap expectedBitmap1 = + Bitmap.createScaledBitmap(originalBitmap1, outputWidth, outputHeight, /* filter= */ true); + Bitmap expectedBitmap2 = + Bitmap.createScaledBitmap(originalBitmap2, outputWidth, outputHeight, /* filter= */ true); + + testThread.runOnMainThread( + () -> { + pixelCopySurfaceCapturer = + new PixelCopySurfaceCapturerV24( + defaultCallback, outputWidth, outputHeight, new Handler(Looper.myLooper())); + drawBitmapOnSurface(originalBitmap1); + }); + // Wait for the first bitmap to finish draw on the pixelCopySurfaceCapturer's surface. + callbackCalledCondition.block(); + callbackCalledCondition.close(); + testThread.runOnMainThread(() -> drawBitmapOnSurface(originalBitmap2)); + callbackCalledCondition.block(); + + assertThat(resultExceptions).isEmpty(); + assertThat(resultBitmaps).hasSize(2); + assertBitmapsAreSimilar(expectedBitmap1, resultBitmaps.get(0)); + assertBitmapsAreSimilar(expectedBitmap2, resultBitmaps.get(1)); + } + + @Test + public void getOutputWidth() { + int outputWidth = 500; + int outputHeight = 400; + testThread.runOnMainThread( + () -> + pixelCopySurfaceCapturer = + new PixelCopySurfaceCapturerV24( + defaultCallback, outputWidth, outputHeight, new Handler(Looper.myLooper()))); + assertThat(pixelCopySurfaceCapturer.getOutputWidth()).isEqualTo(outputWidth); + } + + @Test + public void getOutputHeight() { + int outputWidth = 500; + int outputHeight = 400; + testThread.runOnMainThread( + () -> + pixelCopySurfaceCapturer = + new PixelCopySurfaceCapturerV24( + defaultCallback, outputWidth, outputHeight, new Handler(Looper.myLooper()))); + assertThat(pixelCopySurfaceCapturer.getOutputHeight()).isEqualTo(outputHeight); + } + + // Internal methods + + private void drawBitmapOnSurface(Bitmap bitmap) { + pixelCopySurfaceCapturer.setDefaultSurfaceTextureBufferSize( + bitmap.getWidth(), bitmap.getHeight()); + Surface surface = pixelCopySurfaceCapturer.getSurface(); + Canvas canvas = surface.lockCanvas(/* inOutDirty= */ null); + canvas.drawBitmap(bitmap, /* left= */ 0, /* top= */ 0, new Paint()); + surface.unlockCanvasAndPost(canvas); + } + + /** + * Asserts whether actual bitmap is very similar to the expected bitmap. + * + *

This is defined as their PSNR value is greater than or equal to 30. + * + * @param expectedBitmap The expected bitmap. + * @param actualBitmap The actual bitmap. + */ + private static void assertBitmapsAreSimilar(Bitmap expectedBitmap, Bitmap actualBitmap) { + // TODO: Default PSNR threshold of 35 is quite low. Try to increase this without breaking tests. + TestUtil.assertBitmapsAreSimilar(expectedBitmap, actualBitmap, /* psnrThresholdDb= */ 35); + } +} diff --git a/library/core/src/androidTest/java/com/google/android/exoplayer2/video/surfacecapturer/VideoRendererOutputCapturerTest.java b/library/core/src/androidTest/java/com/google/android/exoplayer2/video/surfacecapturer/VideoRendererOutputCapturerTest.java new file mode 100644 index 0000000000..3d5f136422 --- /dev/null +++ b/library/core/src/androidTest/java/com/google/android/exoplayer2/video/surfacecapturer/VideoRendererOutputCapturerTest.java @@ -0,0 +1,301 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.video.surfacecapturer; + +import static com.google.android.exoplayer2.testutil.TestUtil.assertBitmapsAreSimilar; +import static com.google.android.exoplayer2.testutil.TestUtil.getBitmap; +import static com.google.common.truth.Truth.assertThat; + +import android.content.Context; +import android.graphics.Bitmap; +import android.os.Handler; +import android.os.Looper; +import android.util.Pair; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.filters.SdkSuppress; +import androidx.test.platform.app.InstrumentationRegistry; +import com.google.android.exoplayer2.ExoPlayer; +import com.google.android.exoplayer2.MediaItem; +import com.google.android.exoplayer2.mediacodec.MediaCodecSelector; +import com.google.android.exoplayer2.source.MediaSource; +import com.google.android.exoplayer2.source.ProgressiveMediaSource; +import com.google.android.exoplayer2.testutil.DummyMainThread; +import com.google.android.exoplayer2.upstream.DefaultDataSourceFactory; +import com.google.android.exoplayer2.util.ConditionVariable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit test for {@link VideoRendererOutputCapturer}. */ +@RunWith(AndroidJUnit4.class) +@SdkSuppress(minSdkVersion = 24) +public final class VideoRendererOutputCapturerTest { + + private static final String TEST_VIDEO_URI = "asset:///media/mp4/testvid_1022ms.mp4"; + private static final List FRAMES_TO_CAPTURE = + Collections.unmodifiableList(Arrays.asList(0, 14, 15, 16, 29)); + private static final List CAPTURE_FRAMES_TIME_MS = + Collections.unmodifiableList(Arrays.asList(0, 467, 501, 534, 969)); + + // TODO: PSNR threshold of 20 is really low. This is partly due to a bug with Texture rendering. + // To be updated when the internal bug has been resolved. See [Internal: b/80516628]. + private static final double PSNR_THRESHOLD = 20; + + private DummyMainThread testThread; + + private List> resultOutputSizes; + private List resultBitmaps; + private AtomicReference testException; + private ExoPlayer exoPlayer; + private VideoRendererOutputCapturer videoRendererOutputCapturer; + private SingleFrameMediaCodecVideoRenderer mediaCodecVideoRenderer; + private TestRunner testRunner; + + @Before + public void setUp() { + testThread = new DummyMainThread(); + resultOutputSizes = new ArrayList<>(); + resultBitmaps = new ArrayList<>(); + testException = new AtomicReference<>(); + testRunner = new TestRunner(); + testThread.runOnMainThread( + () -> { + Context context = ApplicationProvider.getApplicationContext(); + mediaCodecVideoRenderer = + new SingleFrameMediaCodecVideoRenderer(context, MediaCodecSelector.DEFAULT); + exoPlayer = new ExoPlayer.Builder(context, mediaCodecVideoRenderer).build(); + exoPlayer.setMediaSource(getMediaSource(context, TEST_VIDEO_URI)); + exoPlayer.prepare(); + + videoRendererOutputCapturer = + new VideoRendererOutputCapturer( + testRunner, new Handler(Looper.myLooper()), mediaCodecVideoRenderer, exoPlayer); + }); + } + + @After + public void tearDown() { + testThread.runOnMainThread( + () -> { + if (exoPlayer != null) { + exoPlayer.release(); + } + if (videoRendererOutputCapturer != null) { + videoRendererOutputCapturer.release(); + } + }); + testThread.release(); + } + + @Test + public void setOutputSize() throws InterruptedException { + testThread.runOnMainThread( + () -> { + videoRendererOutputCapturer = + new VideoRendererOutputCapturer( + testRunner, new Handler(Looper.myLooper()), mediaCodecVideoRenderer, exoPlayer); + videoRendererOutputCapturer.setOutputSize(800, 600); + }); + + testRunner.outputSetCondition.block(); + assertNoExceptionOccurred(); + assertThat(resultOutputSizes).containsExactly(new Pair<>(800, 600)); + } + + @Test + public void getFrame_getAllFramesCorrectly_originalSize() throws Exception { + int outputWidth = 480; + int outputHeight = 360; + startFramesCaptureProcess(outputWidth, outputHeight, CAPTURE_FRAMES_TIME_MS); + + assertNoExceptionOccurred(); + assertExtractedFramesMatchExpectation( + FRAMES_TO_CAPTURE, outputWidth, outputHeight, resultBitmaps); + } + + @Test + public void getFrame_getAllFramesCorrectly_largerSize_SameRatio() throws Exception { + int outputWidth = 720; + int outputHeight = 540; + startFramesCaptureProcess(outputWidth, outputHeight, CAPTURE_FRAMES_TIME_MS); + + assertNoExceptionOccurred(); + assertExtractedFramesMatchExpectation( + FRAMES_TO_CAPTURE, outputWidth, outputHeight, resultBitmaps); + } + + @Test + public void getFrame_getAllFramesCorrectly_largerSize_NotSameRatio() throws Exception { + int outputWidth = 987; + int outputHeight = 654; + startFramesCaptureProcess(outputWidth, outputHeight, CAPTURE_FRAMES_TIME_MS); + + assertNoExceptionOccurred(); + assertExtractedFramesMatchExpectation( + FRAMES_TO_CAPTURE, outputWidth, outputHeight, resultBitmaps); + } + + @Test + public void getFrame_getAllFramesCorrectly_smallerSize_SameRatio() throws Exception { + int outputWidth = 320; + int outputHeight = 240; + startFramesCaptureProcess(outputWidth, outputHeight, CAPTURE_FRAMES_TIME_MS); + + assertNoExceptionOccurred(); + assertExtractedFramesMatchExpectation( + FRAMES_TO_CAPTURE, outputWidth, outputHeight, resultBitmaps); + } + + @Test + public void getFrame_getAllFramesCorrectly_smallerSize_NotSameRatio() throws Exception { + int outputWidth = 432; + int outputHeight = 321; + startFramesCaptureProcess(outputWidth, outputHeight, CAPTURE_FRAMES_TIME_MS); + + assertNoExceptionOccurred(); + assertExtractedFramesMatchExpectation( + FRAMES_TO_CAPTURE, outputWidth, outputHeight, resultBitmaps); + } + + @Ignore // [Internal ref: b/111542655] + @Test + public void getFrame_getAllFramesCorrectly_setSurfaceMultipleTimes() throws Exception { + int firstOutputWidth = 480; + int firstOutputHeight = 360; + + startFramesCaptureProcess(firstOutputWidth, firstOutputHeight, CAPTURE_FRAMES_TIME_MS); + int secondOutputWidth = 432; + int secondOutputHeight = 321; + + startFramesCaptureProcess(secondOutputWidth, secondOutputHeight, CAPTURE_FRAMES_TIME_MS); + + List firstHalfResult = + new ArrayList<>(resultBitmaps.subList(0, FRAMES_TO_CAPTURE.size())); + List secondHalfResult = + new ArrayList<>( + resultBitmaps.subList(FRAMES_TO_CAPTURE.size(), 2 * FRAMES_TO_CAPTURE.size())); + assertThat(resultBitmaps).hasSize(FRAMES_TO_CAPTURE.size() * 2); + + assertNoExceptionOccurred(); + assertExtractedFramesMatchExpectation( + FRAMES_TO_CAPTURE, firstOutputWidth, firstOutputHeight, firstHalfResult); + assertExtractedFramesMatchExpectation( + FRAMES_TO_CAPTURE, secondOutputWidth, secondOutputHeight, secondHalfResult); + } + + private void startFramesCaptureProcess( + int outputWidth, int outputHeight, List listFrameToCaptureMs) + throws InterruptedException { + + testRunner.captureFinishedCondition.close(); + testThread.runOnMainThread( + () -> { + videoRendererOutputCapturer.setOutputSize(outputWidth, outputHeight); + testRunner.setListFramesToCapture(listFrameToCaptureMs); + testRunner.startCapturingProcess(); + }); + testRunner.captureFinishedCondition.block(); + } + + private MediaSource getMediaSource(Context context, String testVideoUri) { + return new ProgressiveMediaSource.Factory(new DefaultDataSourceFactory(context)) + .createMediaSource(MediaItem.fromUri(testVideoUri)); + } + + private void assertNoExceptionOccurred() { + if (testException.get() != null) { + throw new AssertionError("Unexpected exception", testException.get()); + } + } + + private void assertExtractedFramesMatchExpectation( + List framesToExtract, int outputWidth, int outputHeight, List resultBitmaps) + throws IOException { + assertThat(resultBitmaps).hasSize(framesToExtract.size()); + for (int i = 0; i < framesToExtract.size(); i++) { + int frameIndex = framesToExtract.get(i); + String expectedBitmapFileName = + String.format("media/mp4/testvid_1022ms_%03d.png", frameIndex); + Bitmap referenceFrame = + getBitmap( + InstrumentationRegistry.getInstrumentation().getTargetContext(), + expectedBitmapFileName); + Bitmap expectedBitmap = + Bitmap.createScaledBitmap(referenceFrame, outputWidth, outputHeight, /* filter= */ true); + assertBitmapsAreSimilar(expectedBitmap, resultBitmaps.get(i), PSNR_THRESHOLD); + } + } + + /** A {@link VideoRendererOutputCapturer.Callback} implementation that facilities testing. */ + private final class TestRunner implements VideoRendererOutputCapturer.Callback { + + private final ConditionVariable outputSetCondition; + private final ConditionVariable captureFinishedCondition; + private final List captureFrameTimeMs; + private final AtomicInteger currentFrameIndex; + + TestRunner() { + captureFrameTimeMs = new ArrayList<>(); + currentFrameIndex = new AtomicInteger(); + outputSetCondition = new ConditionVariable(); + captureFinishedCondition = new ConditionVariable(); + } + + public void setListFramesToCapture(List listFrameToCaptureMs) { + captureFrameTimeMs.clear(); + captureFrameTimeMs.addAll(listFrameToCaptureMs); + } + + public void startCapturingProcess() { + currentFrameIndex.set(0); + exoPlayer.seekTo(captureFrameTimeMs.get(currentFrameIndex.get())); + } + + @Override + public void onOutputSizeSet(int width, int height) { + resultOutputSizes.add(new Pair<>(width, height)); + outputSetCondition.open(); + } + + @Override + public void onSurfaceCaptured(Bitmap bitmap) { + resultBitmaps.add(bitmap); + int frameIndex = currentFrameIndex.incrementAndGet(); + if (frameIndex == captureFrameTimeMs.size()) { + captureFinishedCondition.open(); + } else { + exoPlayer.seekTo(captureFrameTimeMs.get(frameIndex)); + } + } + + @Override + public void onSurfaceCaptureError(Exception exception) { + testException.set(exception); + // By default, if there is any thrown exception, we will finish the test. + captureFinishedCondition.open(); + } + } +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheDataSource.java b/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheDataSource.java index 2c1b52ad3b..efccb0cf78 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheDataSource.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheDataSource.java @@ -630,22 +630,6 @@ public final class CacheDataSource implements DataSource { return read(buffer, offset, readLength); } return bytesRead; - } catch (IOException e) { - // TODO: This is not correct, because position-out-of-range exceptions should only be thrown - // if the requested position is more than one byte beyond the end of the resource. Conversely, - // this code is assuming that a position-out-of-range exception indicates the requested - // position is exactly one byte beyond the end of the resource, which is not a case for which - // this type of exception should be thrown. This exception handling may be required for - // interop with current HttpDataSource implementations that do (incorrectly) throw a - // position-out-of-range exception at this position. It should be removed when the - // HttpDataSource implementations are fixed. - if (currentDataSpec.length == C.LENGTH_UNSET - && DataSourceException.isCausedByPositionOutOfRange(e)) { - setNoBytesRemainingAndMaybeStoreLength(castNonNull(requestDataSpec.key)); - return C.RESULT_END_OF_INPUT; - } - handleBeforeThrow(e); - throw e; } catch (Throwable e) { handleBeforeThrow(e); throw e; diff --git a/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheWriter.java b/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheWriter.java index a68f0764a3..7e9b3794b9 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheWriter.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/upstream/cache/CacheWriter.java @@ -18,7 +18,6 @@ package com.google.android.exoplayer2.upstream.cache; import androidx.annotation.Nullable; import androidx.annotation.WorkerThread; import com.google.android.exoplayer2.C; -import com.google.android.exoplayer2.upstream.DataSourceException; import com.google.android.exoplayer2.upstream.DataSpec; import com.google.android.exoplayer2.util.PriorityTaskManager; import com.google.android.exoplayer2.util.PriorityTaskManager.PriorityTooLowException; @@ -160,17 +159,6 @@ public final class CacheWriter { isDataSourceOpen = true; } catch (IOException e) { Util.closeQuietly(dataSource); - // TODO: This exception handling may be required for interop with current HttpDataSource - // implementations that (incorrectly) throw a position-out-of-range when opened exactly one - // byte beyond the end of the resource. It should be removed when the HttpDataSource - // implementations are fixed. - if (isLastBlock && DataSourceException.isCausedByPositionOutOfRange(e)) { - // The length of the request exceeds the length of the content. If we allow shorter - // content and are reading the last block, fall through and try again with an unbounded - // request to read up to the end of the content. - } else { - throw e; - } } } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/PixelCopySurfaceCapturerV24.java b/library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/PixelCopySurfaceCapturerV24.java new file mode 100644 index 0000000000..00719619ad --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/PixelCopySurfaceCapturerV24.java @@ -0,0 +1,108 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.video.surfacecapturer; + +import android.graphics.Bitmap; +import android.graphics.SurfaceTexture; +import android.os.Handler; +import android.view.PixelCopy; +import android.view.Surface; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import com.google.android.exoplayer2.util.EGLSurfaceTexture; + +/** + * A {@link SurfaceCapturer} implementation that uses {@link PixelCopy} APIs to perform image copy + * from a {@link SurfaceTexture} into a {@link Bitmap}. + */ +@RequiresApi(24) +/* package */ final class PixelCopySurfaceCapturerV24 extends SurfaceCapturer + implements EGLSurfaceTexture.TextureImageListener, PixelCopy.OnPixelCopyFinishedListener { + + /** Exception to be thrown if there is some problem capturing images from the surface. */ + public static final class SurfaceCapturerException extends Exception { + + /** + * One of the {@link PixelCopy} {@code ERROR_*} values return from the {@link + * PixelCopy#request(Surface, Bitmap, PixelCopy.OnPixelCopyFinishedListener, Handler)} + */ + public final int errorCode; + + /** + * Constructs a new instance. + * + * @param message The error message. + * @param errorCode The error code. + */ + public SurfaceCapturerException(String message, int errorCode) { + super(message); + this.errorCode = errorCode; + } + } + + private final EGLSurfaceTexture eglSurfaceTexture; + private final Handler handler; + private final Surface decoderSurface; + + @Nullable private Bitmap bitmap; + + @SuppressWarnings("nullness") + /* package */ PixelCopySurfaceCapturerV24( + Callback callback, int outputWidth, int outputHeight, Handler imageRenderingHandler) { + super(callback, outputWidth, outputHeight); + this.handler = imageRenderingHandler; + eglSurfaceTexture = new EGLSurfaceTexture(imageRenderingHandler, /* callback= */ this); + eglSurfaceTexture.init(EGLSurfaceTexture.SECURE_MODE_NONE); + decoderSurface = new Surface(eglSurfaceTexture.getSurfaceTexture()); + } + + @Override + public Surface getSurface() { + return decoderSurface; + } + + @Override + public void release() { + eglSurfaceTexture.release(); + decoderSurface.release(); + } + + /** @see SurfaceTexture#setDefaultBufferSize(int, int) */ + public void setDefaultSurfaceTextureBufferSize(int width, int height) { + eglSurfaceTexture.getSurfaceTexture().setDefaultBufferSize(width, height); + } + + // TextureImageListener + + @Override + public void onFrameAvailable() { + bitmap = Bitmap.createBitmap(getOutputWidth(), getOutputHeight(), Bitmap.Config.ARGB_8888); + PixelCopy.request(decoderSurface, bitmap, this, handler); + } + + // OnPixelCopyFinishedListener + + @Override + public void onPixelCopyFinished(int copyResult) { + Callback callback = getCallback(); + if (copyResult == PixelCopy.SUCCESS && bitmap != null) { + callback.onSurfaceCaptured(bitmap); + } else { + callback.onSurfaceCaptureError( + new SurfaceCapturerException("Couldn't copy image from surface", copyResult)); + } + } +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/SingleFrameMediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/SingleFrameMediaCodecVideoRenderer.java new file mode 100644 index 0000000000..941d952e72 --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/SingleFrameMediaCodecVideoRenderer.java @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.video.surfacecapturer; + +import android.content.Context; +import android.media.MediaCodec; +import android.view.Surface; +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.ExoPlaybackException; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.mediacodec.MediaCodecAdapter; +import com.google.android.exoplayer2.mediacodec.MediaCodecSelector; +import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.Util; +import com.google.android.exoplayer2.video.MediaCodecVideoRenderer; +import java.nio.ByteBuffer; + +/** + * Decodes and renders video using {@link MediaCodec}. + * + *

This video renderer will only render the first frame after position reset (seeking), or after + * being re-enabled. + */ +public class SingleFrameMediaCodecVideoRenderer extends MediaCodecVideoRenderer { + + private static final String TAG = "SingleFrameMediaCodecVideoRenderer"; + private boolean hasRenderedFirstFrame; + @Nullable private Surface surface; + + public SingleFrameMediaCodecVideoRenderer( + Context context, MediaCodecSelector mediaCodecSelector) { + super(context, mediaCodecSelector); + } + + @Override + public String getName() { + return TAG; + } + + @Override + public void handleMessage(int messageType, @Nullable Object message) throws ExoPlaybackException { + if (messageType == MSG_SET_SURFACE) { + this.surface = (Surface) message; + } + super.handleMessage(messageType, message); + } + + @Override + protected void onEnabled(boolean joining, boolean mayRenderStartOfStream) + throws ExoPlaybackException { + hasRenderedFirstFrame = false; + super.onEnabled(joining, mayRenderStartOfStream); + } + + @Override + protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException { + hasRenderedFirstFrame = false; + super.onPositionReset(positionUs, joining); + } + + @Override + protected boolean processOutputBuffer( + long positionUs, + long elapsedRealtimeUs, + @Nullable MediaCodecAdapter codec, + @Nullable ByteBuffer buffer, + int bufferIndex, + int bufferFlags, + int sampleCount, + long bufferPresentationTimeUs, + boolean isDecodeOnlyBuffer, + boolean isLastBuffer, + Format format) + throws ExoPlaybackException { + Assertions.checkNotNull(codec); // Can not render video without codec + + long presentationTimeUs = bufferPresentationTimeUs - getOutputStreamOffsetUs(); + if (isDecodeOnlyBuffer && !isLastBuffer) { + skipOutputBuffer(codec, bufferIndex, presentationTimeUs); + return true; + } + if (surface == null || hasRenderedFirstFrame) { + return false; + } + + hasRenderedFirstFrame = true; + if (Util.SDK_INT >= 21) { + renderOutputBufferV21(codec, bufferIndex, presentationTimeUs, System.nanoTime()); + } else { + renderOutputBuffer(codec, bufferIndex, presentationTimeUs); + } + return true; + } +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/SurfaceCapturer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/SurfaceCapturer.java new file mode 100644 index 0000000000..788b8ac875 --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/SurfaceCapturer.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.video.surfacecapturer; + +import android.graphics.Bitmap; +import android.view.Surface; + +/** + * A surface capturer, which captures image drawn into its surface as bitmaps. + * + *

It constructs a {@link Surface}, which can be used as the output surface for an image producer + * to draw images to. As images are being drawn into this surface, this capturer will capture these + * images, and return them via {@link Callback}. The output images will have a fixed frame size of + * (width, height), and any image drawn into the surface will be stretched to fit this frame size. + */ +public abstract class SurfaceCapturer { + + /** The callback to be notified of the image capturing result. */ + public interface Callback { + + /** + * Called when the surface capturer has been able to capture its surface into a {@link Bitmap}. + * This will happen whenever the producer updates the image on the wrapped surface. + */ + void onSurfaceCaptured(Bitmap bitmap); + + /** Called when the surface capturer couldn't capture its surface due to an error. */ + void onSurfaceCaptureError(Exception e); + } + + /** The callback to be notified of the image capturing result. */ + private final Callback callback; + /** The width of the output images. */ + private final int outputWidth; + /** The height of the output images. */ + private final int outputHeight; + + /** + * Constructs a new instance. + * + * @param callback See {@link #callback}. + * @param outputWidth See {@link #outputWidth}. + * @param outputHeight See {@link #outputHeight}. + */ + protected SurfaceCapturer(Callback callback, int outputWidth, int outputHeight) { + this.callback = callback; + this.outputWidth = outputWidth; + this.outputHeight = outputHeight; + } + + /** Returns the callback to be notified of the image capturing result. */ + protected Callback getCallback() { + return callback; + } + + /** Returns the width of the output images. */ + public int getOutputWidth() { + return outputWidth; + } + + /** Returns the height of the output images. */ + public int getOutputHeight() { + return outputHeight; + } + + /** Returns a {@link Surface} that image producers (camera, video codec etc...) can draw to. */ + public abstract Surface getSurface(); + + /** Releases all kept resources. This instance cannot be used after this call. */ + public abstract void release(); +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/VideoRendererOutputCapturer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/VideoRendererOutputCapturer.java new file mode 100644 index 0000000000..c6e37db62f --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/VideoRendererOutputCapturer.java @@ -0,0 +1,260 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.video.surfacecapturer; + +import android.graphics.Bitmap; +import android.os.Handler; +import android.os.HandlerThread; +import android.os.Message; +import android.view.Surface; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import com.google.android.exoplayer2.ExoPlayer; +import com.google.android.exoplayer2.Renderer; +import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.Util; + +/** + * A capturer that can capture the output of a video {@link SingleFrameMediaCodecVideoRenderer} into + * bitmaps. + * + *

Start by setting the output size via {@link #setOutputSize(int, int)}. The capturer will + * create a surface and set this up as the output for the video renderer. + * + *

Once the surface setup is done, the capturer will call {@link Callback#onOutputSizeSet(int, + * int)}. After this call, the capturer will capture all images rendered by the {@link Renderer}, + * and deliver the captured bitmaps via {@link Callback#onSurfaceCaptured(Bitmap)}, or failure via + * {@link Callback#onSurfaceCaptureError(Exception)}. You can change the output image size at any + * time by calling {@link #setOutputSize(int, int)}. + * + *

When this capturer is no longer needed, you need to call {@link #release()} to release all + * resources it is holding. After this call returns, no callback will be called anymore. + */ +public final class VideoRendererOutputCapturer implements Handler.Callback { + + /** The callback to be notified of the video image capturing result. */ + public interface Callback extends SurfaceCapturer.Callback { + /** Called when output surface has been set properly. */ + void onOutputSizeSet(int width, int height); + } + + private static final int MSG_SET_OUTPUT = 1; + private static final int MSG_RELEASE = 2; + + private final HandlerThread handlerThread; + private final Handler handler; + private final ExoPlayer exoPlayer; + private final EventDispatcher eventDispatcher; + private final Renderer renderer; + + @Nullable private SurfaceCapturer surfaceCapturer; + + private volatile boolean released; + + /** + * Constructs a new instance. + * + * @param callback The callback to be notified of image capturing result. + * @param callbackHandler The {@link Handler} that the callback will be called on. + * @param videoRenderer A {@link SingleFrameMediaCodecVideoRenderer} that will be used to render + * video frames, which this capturer will capture. + * @param exoPlayer The {@link ExoPlayer} instance that is using the video renderer. + */ + public VideoRendererOutputCapturer( + Callback callback, + Handler callbackHandler, + SingleFrameMediaCodecVideoRenderer videoRenderer, + ExoPlayer exoPlayer) { + this.renderer = Assertions.checkNotNull(videoRenderer); + this.exoPlayer = Assertions.checkNotNull(exoPlayer); + this.eventDispatcher = new EventDispatcher(callbackHandler, callback); + + // Use a separate thread to handle all operations in this class, because bitmap copying may take + // time and should not be handled on callbackHandler (which maybe run on main thread). + handlerThread = new HandlerThread("ExoPlayer:VideoRendererOutputCapturer"); + handlerThread.start(); + handler = Util.createHandler(handlerThread.getLooper(), /* callback= */ this); + } + + /** + * Sets the size of the video renderer surface's with and height. + * + *

This call is performed asynchronously. Only after the {@code callback} receives a call to + * {@link Callback#onOutputSizeSet(int, int)}, the output frames will conform to the new size. + * Output frames before the callback will still conform to previous size. + * + * @param width The target width of the output frame. + * @param height The target height of the output frame. + */ + public void setOutputSize(int width, int height) { + handler.obtainMessage(MSG_SET_OUTPUT, width, height).sendToTarget(); + } + + /** Releases all kept resources. This instance cannot be used after this call. */ + public synchronized void release() { + if (released) { + return; + } + + // Some long running or waiting operations may run on the handler thread, so we try to + // interrupt the thread to end these operations quickly. + handlerThread.interrupt(); + handler.removeCallbacksAndMessages(null); + handler.sendEmptyMessage(MSG_RELEASE); + boolean wasInterrupted = false; + while (!released) { + try { + wait(); + } catch (InterruptedException e) { + wasInterrupted = true; + } + } + if (wasInterrupted) { + // Restore the interrupted status. + Thread.currentThread().interrupt(); + } + } + + // Handler.Callback + + @Override + public boolean handleMessage(Message message) { + switch (message.what) { + case MSG_SET_OUTPUT: + handleSetOutput(/* width= */ message.arg1, /* height= */ message.arg2); + return true; + case MSG_RELEASE: + handleRelease(); + return true; + default: + return false; + } + } + + // Internal methods + + private void handleSetOutput(int width, int height) { + if (surfaceCapturer == null + || surfaceCapturer.getOutputWidth() != width + || surfaceCapturer.getOutputHeight() != height) { + updateSurfaceCapturer(width, height); + } + eventDispatcher.onOutputSizeSet(width, height); + } + + private void updateSurfaceCapturer(int width, int height) { + SurfaceCapturer oldSurfaceCapturer = surfaceCapturer; + if (oldSurfaceCapturer != null) { + blockingSetRendererSurface(/* surface= */ null); + oldSurfaceCapturer.release(); + } + surfaceCapturer = createSurfaceCapturer(width, height); + blockingSetRendererSurface(surfaceCapturer.getSurface()); + } + + private SurfaceCapturer createSurfaceCapturer(int width, int height) { + if (Util.SDK_INT >= 24) { + return createSurfaceCapturerV24(width, height); + } else { + // TODO: Use different SurfaceCapturer based on API level, flags etc... + throw new UnsupportedOperationException( + "Creating Surface Capturer is not supported for API < 24 yet"); + } + } + + @RequiresApi(24) + private SurfaceCapturer createSurfaceCapturerV24(int width, int height) { + return new PixelCopySurfaceCapturerV24(eventDispatcher, width, height, handler); + } + + private void blockingSetRendererSurface(@Nullable Surface surface) { + try { + exoPlayer + .createMessage(renderer) + .setType(Renderer.MSG_SET_SURFACE) + .setPayload(surface) + .send() + .blockUntilDelivered(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private void handleRelease() { + eventDispatcher.release(); + handler.removeCallbacksAndMessages(null); + if (surfaceCapturer != null) { + surfaceCapturer.release(); + } + handlerThread.quit(); + synchronized (this) { + released = true; + notifyAll(); + } + } + + /** Dispatches {@link Callback} events using a callback handler. */ + private static final class EventDispatcher implements Callback { + + private final Handler callbackHandler; + private final Callback callback; + + private volatile boolean released; + + private EventDispatcher(Handler callbackHandler, Callback callback) { + this.callbackHandler = callbackHandler; + this.callback = callback; + } + + @Override + public void onOutputSizeSet(int width, int height) { + callbackHandler.post( + () -> { + if (released) { + return; + } + callback.onOutputSizeSet(width, height); + }); + } + + @Override + public void onSurfaceCaptured(Bitmap bitmap) { + callbackHandler.post( + () -> { + if (released) { + return; + } + callback.onSurfaceCaptured(bitmap); + }); + } + + @Override + public void onSurfaceCaptureError(Exception exception) { + callbackHandler.post( + () -> { + if (released) { + return; + } + callback.onSurfaceCaptureError(exception); + }); + } + + /** Releases this event dispatcher. No event will be dispatched after this call. */ + public void release() { + released = true; + } + } +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/package-info.java b/library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/package-info.java new file mode 100644 index 0000000000..c9375dc27c --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/surfacecapturer/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@NonNullApi +package com.google.android.exoplayer2.video.surfacecapturer; + +import com.google.android.exoplayer2.util.NonNullApi;