diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index 6fa803e65e..6c0cc836fa 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -79,10 +79,10 @@ public interface VideoFrameProcessor { * @param inputColorInfo The {@link ColorInfo} for input frames. * @param outputColorInfo The {@link ColorInfo} for output frames. * @param inputType The {@link InputType}. - * @param releaseFramesAutomatically If {@code true}, the instance will render output frames to + * @param renderFramesAutomatically If {@code true}, the instance will render output frames to * the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface} automatically as * {@link VideoFrameProcessor} is done processing them. If {@code false}, the {@link - * VideoFrameProcessor} will block until {@link #releaseOutputFrame(long)} is called, to + * VideoFrameProcessor} will block until {@link #renderOutputFrame(long)} is called, to * render or drop the frame. * @param executor The {@link Executor} on which the {@code listener} is invoked. * @param listener A {@link Listener}. @@ -97,7 +97,7 @@ public interface VideoFrameProcessor { ColorInfo inputColorInfo, ColorInfo outputColorInfo, @InputType int inputType, - boolean releaseFramesAutomatically, + boolean renderFramesAutomatically, Executor executor, Listener listener) throws VideoFrameProcessingException; @@ -128,7 +128,7 @@ public interface VideoFrameProcessor { * * @param presentationTimeUs The presentation time of the frame, in microseconds. */ - void onOutputFrameAvailable(long presentationTimeUs); + void onOutputFrameAvailableForRendering(long presentationTimeUs); /** * Called when an exception occurs during asynchronous video frame processing. @@ -143,12 +143,12 @@ public interface VideoFrameProcessor { } /** - * Indicates the frame should be released immediately after {@link #releaseOutputFrame(long)} is + * Indicates the frame should be rendered immediately after {@link #renderOutputFrame(long)} is * invoked. */ - long RELEASE_OUTPUT_FRAME_IMMEDIATELY = -1; + long RENDER_OUTPUT_FRAME_IMMEDIATELY = -1; - /** Indicates the frame should be dropped after {@link #releaseOutputFrame(long)} is invoked. */ + /** Indicates the frame should be dropped after {@link #renderOutputFrame(long)} is invoked. */ long DROP_OUTPUT_FRAME = -2; /** @@ -226,7 +226,7 @@ public interface VideoFrameProcessor { int getPendingInputFrameCount(); /** - * Sets the output surface and supporting information. When output frames are released and not + * Sets the output surface and supporting information. When output frames are rendered and not * dropped, they will be rendered to this output {@link SurfaceInfo}. * *

The new output {@link SurfaceInfo} is applied from the next output frame rendered onwards. @@ -244,24 +244,25 @@ public interface VideoFrameProcessor { void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo); /** - * Releases the oldest unreleased output frame that has become {@linkplain - * Listener#onOutputFrameAvailable(long) available} at the given {@code releaseTimeNs}. + * Renders the oldest unrendered output frame that has become {@linkplain + * Listener#onOutputFrameAvailableForRendering(long) available for rendering} at the given {@code + * renderTimeNs}. * *

This will either render the output frame to the {@linkplain #setOutputSurfaceInfo output - * surface}, or drop the frame, per {@code releaseTimeNs}. + * surface}, or drop the frame, per {@code renderTimeNs}. * - *

This method must only be called if {@code releaseFramesAutomatically} was set to {@code + *

This method must only be called if {@code renderFramesAutomatically} was set to {@code * false} using the {@link Factory} and should be called exactly once for each frame that becomes - * {@linkplain Listener#onOutputFrameAvailable(long) available}. + * {@linkplain Listener#onOutputFrameAvailableForRendering(long) available for rendering}. * - *

The {@code releaseTimeNs} may be passed to {@link EGLExt#eglPresentationTimeANDROID} + *

The {@code renderTimeNs} may be passed to {@link EGLExt#eglPresentationTimeANDROID} * depending on the implementation. * - * @param releaseTimeNs The release time to use for the frame, in nanoseconds. The release time - * can be before of after the current system time. Use {@link #DROP_OUTPUT_FRAME} to drop the - * frame, or {@link #RELEASE_OUTPUT_FRAME_IMMEDIATELY} to release the frame immediately. + * @param renderTimeNs The render time to use for the frame, in nanoseconds. The render time can + * be before or after the current system time. Use {@link #DROP_OUTPUT_FRAME} to drop the + * frame, or {@link #RENDER_OUTPUT_FRAME_IMMEDIATELY} to render the frame immediately. */ - void releaseOutputFrame(long releaseTimeNs); + void renderOutputFrame(long renderTimeNs); /** * Informs the {@code VideoFrameProcessor} that no further input frames should be accepted. diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java index 146341aaf2..be659202aa 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorImageFrameOutputTest.java @@ -114,7 +114,7 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest { Queue actualPresentationTimesUs = new ConcurrentLinkedQueue<>(); videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId) - .setOnOutputFrameAvailableListener(actualPresentationTimesUs::add) + .setOnOutputFrameAvailableForRenderingListener(actualPresentationTimesUs::add) .build(); long offsetUs = 1_000_000L; @@ -137,7 +137,7 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest { Queue actualPresentationTimesUs = new ConcurrentLinkedQueue<>(); videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId) - .setOnOutputFrameAvailableListener(actualPresentationTimesUs::add) + .setOnOutputFrameAvailableForRenderingListener(actualPresentationTimesUs::add) .build(); long offsetUs1 = 1_000_000L; @@ -172,7 +172,7 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest { Queue actualPresentationTimesUs = new ConcurrentLinkedQueue<>(); videoFrameProcessorTestRunner = getDefaultFrameProcessorTestRunnerBuilder(testId) - .setOnOutputFrameAvailableListener(actualPresentationTimesUs::add) + .setOnOutputFrameAvailableForRenderingListener(actualPresentationTimesUs::add) .build(); videoFrameProcessorTestRunner.queueInputBitmap( @@ -197,7 +197,7 @@ public class DefaultVideoFrameProcessorImageFrameOutputTest { .setVideoFrameProcessorFactory(new DefaultVideoFrameProcessor.Factory.Builder().build()) .setInputType(INPUT_TYPE_BITMAP) .setInputColorInfo(ColorInfo.SRGB_BT709_FULL) - .setOnOutputFrameAvailableListener( + .setOnOutputFrameAvailableForRenderingListener( unused -> checkNotNull(framesProduced).incrementAndGet()); } } diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java similarity index 73% rename from libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java rename to libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java index 2cacecc40b..b77466a130 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameReleaseTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorVideoFrameRenderingTest.java @@ -54,23 +54,23 @@ import org.junit.After; import org.junit.Test; import org.junit.runner.RunWith; -/** Tests for frame release in {@link DefaultVideoFrameProcessor}. */ +/** Tests for frame rendering in {@link DefaultVideoFrameProcessor}. */ @RunWith(AndroidJUnit4.class) -public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { +public final class DefaultVideoFrameProcessorVideoFrameRenderingTest { private static final int WIDTH = 200; private static final int HEIGHT = 100; /** - * Time to wait between releasing frames to avoid frame drops between GL and the {@link + * Time to wait between rendering frames to avoid frame drops between GL and the {@link * ImageReader}. */ - private static final long PER_FRAME_RELEASE_WAIT_TIME_MS = 1000L; - /** Maximum time to wait for each released frame to be notified. */ + private static final long PER_FRAME_RENDERING_WAIT_TIME_MS = 1000L; + /** Maximum time to wait for each rendered frame to be notified. */ private static final long PER_FRAME_TIMEOUT_MS = 5000L; private static final long MICROS_TO_NANOS = 1000L; - private final LinkedBlockingQueue outputReleaseTimesNs = new LinkedBlockingQueue<>(); + private final LinkedBlockingQueue outputRenderTimesNs = new LinkedBlockingQueue<>(); private @MonotonicNonNull DefaultVideoFrameProcessor defaultVideoFrameProcessor; @@ -82,22 +82,22 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { } @Test - public void automaticFrameRelease_withOneFrame_reusesInputTimestamp() throws Exception { + public void automaticFrameRendering_withOneFrame_reusesInputTimestamp() throws Exception { long originalPresentationTimeUs = 1234; AtomicLong actualPresentationTimeUs = new AtomicLong(); processFramesToEndOfStream( /* inputPresentationTimesUs= */ new long[] {originalPresentationTimeUs}, /* onFrameAvailableListener= */ actualPresentationTimeUs::set, - /* releaseFramesAutomatically= */ true); + /* renderFramesAutomatically= */ true); assertThat(actualPresentationTimeUs.get()).isEqualTo(originalPresentationTimeUs); - ImmutableList actualReleaseTimesNs = - waitForFrameReleaseAndGetReleaseTimesNs(/* expectedFrameCount= */ 1); - assertThat(actualReleaseTimesNs).containsExactly(MICROS_TO_NANOS * originalPresentationTimeUs); + ImmutableList actualRenderTimesNs = + waitForFrameRenderingAndGetRenderTimesNs(/* expectedFrameCount= */ 1); + assertThat(actualRenderTimesNs).containsExactly(MICROS_TO_NANOS * originalPresentationTimeUs); } @Test - public void automaticFrameRelease_withThreeFrames_reusesInputTimestamps() throws Exception { + public void automaticFrameRendering_withThreeFrames_reusesInputTimestamps() throws Exception { long[] originalPresentationTimesUs = new long[] {1234, 3456, 4567}; ArrayList actualPresentationTimesUs = new ArrayList<>(); processFramesToEndOfStream( @@ -108,12 +108,12 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { // TODO(b/264252759): Investigate output frames being dropped and remove sleep. // Frames can be dropped silently between EGL and the ImageReader. Sleep after each call // to swap buffers, to avoid this behavior. - Thread.sleep(PER_FRAME_RELEASE_WAIT_TIME_MS); + Thread.sleep(PER_FRAME_RENDERING_WAIT_TIME_MS); } catch (InterruptedException e) { throw new IllegalStateException(e); } }, - /* releaseFramesAutomatically= */ true); + /* renderFramesAutomatically= */ true); assertThat(actualPresentationTimesUs) .containsExactly( @@ -121,9 +121,9 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { originalPresentationTimesUs[1], originalPresentationTimesUs[2]) .inOrder(); - ImmutableList actualReleaseTimesNs = - waitForFrameReleaseAndGetReleaseTimesNs(/* expectedFrameCount= */ 3); - assertThat(actualReleaseTimesNs) + ImmutableList actualRenderTimesNs = + waitForFrameRenderingAndGetRenderTimesNs(/* expectedFrameCount= */ 3); + assertThat(actualRenderTimesNs) .containsExactly( MICROS_TO_NANOS * originalPresentationTimesUs[0], MICROS_TO_NANOS * originalPresentationTimesUs[1], @@ -132,67 +132,66 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { } @Test - public void controlledFrameRelease_withOneFrame_usesGivenTimestamp() throws Exception { + public void controlledFrameRendering_withOneFrame_usesGivenTimestamp() throws Exception { long originalPresentationTimeUs = 1234; - long releaseTimesNs = System.nanoTime() + 345678; + long renderTimesNs = System.nanoTime() + 345678; AtomicLong actualPresentationTimeUs = new AtomicLong(); processFramesToEndOfStream( /* inputPresentationTimesUs= */ new long[] {originalPresentationTimeUs}, /* onFrameAvailableListener= */ presentationTimeUs -> { actualPresentationTimeUs.set(presentationTimeUs); - checkNotNull(defaultVideoFrameProcessor).releaseOutputFrame(releaseTimesNs); + checkNotNull(defaultVideoFrameProcessor).renderOutputFrame(renderTimesNs); }, - /* releaseFramesAutomatically= */ false); + /* renderFramesAutomatically= */ false); - ImmutableList actualReleaseTimesNs = - waitForFrameReleaseAndGetReleaseTimesNs(/* expectedFrameCount= */ 1); - assertThat(actualReleaseTimesNs).containsExactly(releaseTimesNs); + ImmutableList actualRenderTimesNs = + waitForFrameRenderingAndGetRenderTimesNs(/* expectedFrameCount= */ 1); + assertThat(actualRenderTimesNs).containsExactly(renderTimesNs); } @Test - public void controlledFrameRelease_withOneFrameRequestImmediateRelease_releasesFrame() + public void controlledFrameRendering_withOneFrameRequestImmediateRender_rendersframe() throws Exception { long originalPresentationTimeUs = 1234; - long releaseTimesNs = VideoFrameProcessor.RELEASE_OUTPUT_FRAME_IMMEDIATELY; + long renderTimesNs = VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY; AtomicLong actualPresentationTimeUs = new AtomicLong(); processFramesToEndOfStream( /* inputPresentationTimesUs= */ new long[] {originalPresentationTimeUs}, /* onFrameAvailableListener= */ presentationTimeUs -> { actualPresentationTimeUs.set(presentationTimeUs); - checkNotNull(defaultVideoFrameProcessor).releaseOutputFrame(releaseTimesNs); + checkNotNull(defaultVideoFrameProcessor).renderOutputFrame(renderTimesNs); }, - /* releaseFramesAutomatically= */ false); + /* renderFramesAutomatically= */ false); assertThat(actualPresentationTimeUs.get()).isEqualTo(originalPresentationTimeUs); - // The actual release time is determined by the VideoFrameProcessor when releasing the frame. - ImmutableList actualReleaseTimesNs = - waitForFrameReleaseAndGetReleaseTimesNs(/* expectedFrameCount= */ 1); - assertThat(actualReleaseTimesNs).hasSize(1); + // The actual render time is determined by the VideoFrameProcessor when rendering the frame. + ImmutableList actualRenderTimesNs = + waitForFrameRenderingAndGetRenderTimesNs(/* expectedFrameCount= */ 1); + assertThat(actualRenderTimesNs).hasSize(1); } @Test - public void controlledFrameRelease_withLateFrame_releasesFrame() throws Exception { + public void controlledFrameRendering_withLateFrame_rendersframe() throws Exception { long originalPresentationTimeUs = 1234; - long releaseTimeBeforeCurrentTimeNs = System.nanoTime() - 345678; + long renderTimeBeforeCurrentTimeNs = System.nanoTime() - 345678; AtomicLong actualPresentationTimeUs = new AtomicLong(); processFramesToEndOfStream( /* inputPresentationTimesUs= */ new long[] {originalPresentationTimeUs}, /* onFrameAvailableListener= */ presentationTimeUs -> { actualPresentationTimeUs.set(presentationTimeUs); - checkNotNull(defaultVideoFrameProcessor) - .releaseOutputFrame(releaseTimeBeforeCurrentTimeNs); + checkNotNull(defaultVideoFrameProcessor).renderOutputFrame(renderTimeBeforeCurrentTimeNs); }, - /* releaseFramesAutomatically= */ false); + /* renderFramesAutomatically= */ false); - ImmutableList actualReleaseTimesNs = - waitForFrameReleaseAndGetReleaseTimesNs(/* expectedFrameCount= */ 1); - assertThat(actualReleaseTimesNs).hasSize(1); - // The actual release time is determined by the VideoFrameProcessor when releasing the frame. - assertThat(actualReleaseTimesNs.get(0)).isAtLeast(releaseTimeBeforeCurrentTimeNs); + ImmutableList actualRenderTimesNs = + waitForFrameRenderingAndGetRenderTimesNs(/* expectedFrameCount= */ 1); + assertThat(actualRenderTimesNs).hasSize(1); + // The actual render time is determined by the VideoFrameProcessor when rendering the frame. + assertThat(actualRenderTimesNs.get(0)).isAtLeast(renderTimeBeforeCurrentTimeNs); } @Test - public void controlledFrameRelease_requestsFrameDropping_dropsFrame() throws Exception { + public void controlledFrameRendering_requestsFrameDropping_dropsFrame() throws Exception { long originalPresentationTimeUs = 1234; AtomicLong actualPresentationTimeUs = new AtomicLong(); processFramesToEndOfStream( @@ -200,19 +199,19 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { /* onFrameAvailableListener= */ presentationTimeNs -> { actualPresentationTimeUs.set(presentationTimeNs); checkNotNull(defaultVideoFrameProcessor) - .releaseOutputFrame(VideoFrameProcessor.DROP_OUTPUT_FRAME); + .renderOutputFrame(VideoFrameProcessor.DROP_OUTPUT_FRAME); }, - /* releaseFramesAutomatically= */ false); + /* renderFramesAutomatically= */ false); - waitForFrameReleaseAndGetReleaseTimesNs(/* expectedFrameCount= */ 0); + waitForFrameRenderingAndGetRenderTimesNs(/* expectedFrameCount= */ 0); } @Test - public void controlledFrameRelease_withThreeIndividualFrames_usesGivenTimestamps() + public void controlledFrameRendering_withThreeIndividualFrames_usesGivenTimestamps() throws Exception { long[] originalPresentationTimesUs = new long[] {1234, 3456, 4567}; long offsetNs = System.nanoTime(); - long[] releaseTimesNs = new long[] {offsetNs + 123456, offsetNs + 234567, offsetNs + 345678}; + long[] renderTimesNs = new long[] {offsetNs + 123456, offsetNs + 234567, offsetNs + 345678}; ArrayList actualPresentationTimesUs = new ArrayList<>(); AtomicInteger frameIndex = new AtomicInteger(); processFramesToEndOfStream( @@ -220,17 +219,17 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { /* onFrameAvailableListener= */ presentationTimeUs -> { actualPresentationTimesUs.add(presentationTimeUs); checkNotNull(defaultVideoFrameProcessor) - .releaseOutputFrame(releaseTimesNs[frameIndex.getAndIncrement()]); + .renderOutputFrame(renderTimesNs[frameIndex.getAndIncrement()]); try { // TODO(b/264252759): Investigate output frames being dropped and remove sleep. // Frames can be dropped silently between EGL and the ImageReader. Sleep after each call // to swap buffers, to avoid this behavior. - Thread.sleep(PER_FRAME_RELEASE_WAIT_TIME_MS); + Thread.sleep(PER_FRAME_RENDERING_WAIT_TIME_MS); } catch (InterruptedException e) { throw new IllegalStateException(e); } }, - /* releaseFramesAutomatically= */ false); + /* renderFramesAutomatically= */ false); assertThat(actualPresentationTimesUs) .containsExactly( @@ -240,31 +239,32 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { .inOrder(); int actualFrameCount = frameIndex.get(); assertThat(actualFrameCount).isEqualTo(originalPresentationTimesUs.length); - long[] actualReleaseTimesNs = - Longs.toArray(waitForFrameReleaseAndGetReleaseTimesNs(actualFrameCount)); - assertThat(actualReleaseTimesNs).isEqualTo(releaseTimesNs); + long[] actualRenderTimesNs = + Longs.toArray(waitForFrameRenderingAndGetRenderTimesNs(actualFrameCount)); + assertThat(actualRenderTimesNs).isEqualTo(renderTimesNs); } @Test - public void controlledFrameRelease_withThreeFramesAtOnce_usesGivenTimestamps() throws Exception { + public void controlledFrameRendering_withThreeFramesAtOnce_usesGivenTimestamps() + throws Exception { long[] originalPresentationTimesUs = new long[] {1234, 3456, 4567}; long offsetNs = System.nanoTime(); - long[] releaseTimesNs = new long[] {offsetNs + 123456, offsetNs + 234567, offsetNs + 345678}; + long[] renderTimesNs = new long[] {offsetNs + 123456, offsetNs + 234567, offsetNs + 345678}; ArrayList actualPresentationTimesUs = new ArrayList<>(); processFramesToEndOfStream( /* inputPresentationTimesUs= */ originalPresentationTimesUs, /* onFrameAvailableListener= */ actualPresentationTimesUs::add, - /* releaseFramesAutomatically= */ false); + /* renderFramesAutomatically= */ false); // TODO(b/264252759): Investigate output frames being dropped and remove sleep. // Frames can be dropped silently between EGL and the ImageReader. Sleep after each call // to swap buffers, to avoid this behavior. - defaultVideoFrameProcessor.releaseOutputFrame(releaseTimesNs[0]); - Thread.sleep(PER_FRAME_RELEASE_WAIT_TIME_MS); - defaultVideoFrameProcessor.releaseOutputFrame(releaseTimesNs[1]); - Thread.sleep(PER_FRAME_RELEASE_WAIT_TIME_MS); - defaultVideoFrameProcessor.releaseOutputFrame(releaseTimesNs[2]); - Thread.sleep(PER_FRAME_RELEASE_WAIT_TIME_MS); + defaultVideoFrameProcessor.renderOutputFrame(renderTimesNs[0]); + Thread.sleep(PER_FRAME_RENDERING_WAIT_TIME_MS); + defaultVideoFrameProcessor.renderOutputFrame(renderTimesNs[1]); + Thread.sleep(PER_FRAME_RENDERING_WAIT_TIME_MS); + defaultVideoFrameProcessor.renderOutputFrame(renderTimesNs[2]); + Thread.sleep(PER_FRAME_RENDERING_WAIT_TIME_MS); assertThat(actualPresentationTimesUs) .containsExactly( @@ -272,20 +272,20 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { originalPresentationTimesUs[1], originalPresentationTimesUs[2]) .inOrder(); - long[] actualReleaseTimesNs = - Longs.toArray(waitForFrameReleaseAndGetReleaseTimesNs(/* expectedFrameCount= */ 3)); - assertThat(actualReleaseTimesNs).isEqualTo(releaseTimesNs); + long[] actualRenderTimesNs = + Longs.toArray(waitForFrameRenderingAndGetRenderTimesNs(/* expectedFrameCount= */ 3)); + assertThat(actualRenderTimesNs).isEqualTo(renderTimesNs); } - private interface OnOutputFrameAvailableListener { - void onFrameAvailable(long presentationTimeUs); + private interface OnOutputFrameAvailableForRenderingListener { + void onFrameAvailableForRendering(long presentationTimeUs); } @EnsuresNonNull("defaultVideoFrameProcessor") private void processFramesToEndOfStream( long[] inputPresentationTimesUs, - OnOutputFrameAvailableListener onFrameAvailableListener, - boolean releaseFramesAutomatically) + OnOutputFrameAvailableForRenderingListener onFrameAvailableListener, + boolean renderFramesAutomatically) throws Exception { AtomicReference<@NullableType VideoFrameProcessingException> videoFrameProcessingExceptionReference = new AtomicReference<>(); @@ -302,7 +302,7 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { /* inputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, /* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, INPUT_TYPE_SURFACE, - releaseFramesAutomatically, + renderFramesAutomatically, MoreExecutors.directExecutor(), new VideoFrameProcessor.Listener() { @Override @@ -319,15 +319,15 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { outputImageReader.setOnImageAvailableListener( imageReader -> { try (Image image = imageReader.acquireNextImage()) { - outputReleaseTimesNs.add(image.getTimestamp()); + outputRenderTimesNs.add(image.getTimestamp()); } }, Util.createHandlerForCurrentOrMainLooper()); } @Override - public void onOutputFrameAvailable(long presentationTimeUs) { - onFrameAvailableListener.onFrameAvailable(presentationTimeUs); + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + onFrameAvailableListener.onFrameAvailableForRendering(presentationTimeUs); } @Override @@ -364,15 +364,15 @@ public final class DefaultVideoFrameProcessorVideoFrameReleaseTest { } } - private ImmutableList waitForFrameReleaseAndGetReleaseTimesNs(int expectedFrameCount) + private ImmutableList waitForFrameRenderingAndGetRenderTimesNs(int expectedFrameCount) throws Exception { ImmutableList.Builder listBuilder = new ImmutableList.Builder<>(); for (int i = 0; i < expectedFrameCount; i++) { - listBuilder.add(checkNotNull(outputReleaseTimesNs.poll(PER_FRAME_TIMEOUT_MS, MILLISECONDS))); + listBuilder.add(checkNotNull(outputRenderTimesNs.poll(PER_FRAME_TIMEOUT_MS, MILLISECONDS))); } // This is a best-effort check because there's no guarantee that frames aren't added to the - // release times after this method has been called. - assertThat(outputReleaseTimesNs).isEmpty(); + // render times after this method has been called. + assertThat(outputRenderTimesNs).isEmpty(); return listBuilder.build(); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index ec1cf7b0a0..32ad6073b5 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -188,7 +188,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ColorInfo inputColorInfo, ColorInfo outputColorInfo, @InputType int inputType, - boolean releaseFramesAutomatically, + boolean renderFramesAutomatically, Executor listenerExecutor, Listener listener) throws VideoFrameProcessingException { @@ -227,7 +227,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { outputColorInfo, enableColorTransfers, inputType, - releaseFramesAutomatically, + renderFramesAutomatically, singleThreadExecutorService, listenerExecutor, listener, @@ -253,7 +253,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { private final EGLContext eglContext; private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor; private final InputHandler inputHandler; - private final boolean releaseFramesAutomatically; + private final boolean renderFramesAutomatically; private final FinalShaderProgramWrapper finalShaderProgramWrapper; private final ImmutableList allShaderPrograms; // A queue of input streams that have not been fully processed identified by their input types. @@ -271,13 +271,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { @InputType int inputType, VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor, ImmutableList shaderPrograms, - boolean releaseFramesAutomatically) + boolean renderFramesAutomatically) throws VideoFrameProcessingException { this.eglDisplay = eglDisplay; this.eglContext = eglContext; this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor; - this.releaseFramesAutomatically = releaseFramesAutomatically; + this.renderFramesAutomatically = renderFramesAutomatically; this.unprocessedInputStreams = new ConcurrentLinkedQueue<>(); checkState(!shaderPrograms.isEmpty()); @@ -411,12 +411,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { } @Override - public void releaseOutputFrame(long releaseTimeNs) { + public void renderOutputFrame(long renderTimeNs) { checkState( - !releaseFramesAutomatically, - "Calling this method is not allowed when releaseFramesAutomatically is enabled"); + !renderFramesAutomatically, + "Calling this method is not allowed when renderFramesAutomatically is enabled"); videoFrameProcessingTaskExecutor.submitWithHighPriority( - () -> finalShaderProgramWrapper.releaseOutputFrame(releaseTimeNs)); + () -> finalShaderProgramWrapper.renderOutputFrame(renderTimeNs)); } @Override @@ -494,7 +494,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ColorInfo outputColorInfo, boolean enableColorTransfers, @InputType int inputType, - boolean releaseFramesAutomatically, + boolean renderFramesAutomatically, ExecutorService singleThreadExecutorService, Executor executor, Listener listener, @@ -514,9 +514,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { glObjectsProvider.createEglContext(eglDisplay, openGlVersion, configAttributes); glObjectsProvider.createFocusedPlaceholderEglSurface(eglContext, eglDisplay, configAttributes); - // Not releaseFramesAutomatically means outputting to a display surface. HDR display surfaces + // Not renderFramesAutomatically means outputting to a display surface. HDR display surfaces // require the BT2020 PQ GL extension. - if (!releaseFramesAutomatically && ColorInfo.isTransferHdr(outputColorInfo)) { + if (!renderFramesAutomatically && ColorInfo.isTransferHdr(outputColorInfo)) { // Display hardware supports PQ only. checkArgument(outputColorInfo.colorTransfer == C.COLOR_TRANSFER_ST2084); if (Util.SDK_INT < 33 || !GlUtil.isBt2020PqExtensionSupported()) { @@ -538,7 +538,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { outputColorInfo, enableColorTransfers, inputType, - releaseFramesAutomatically, + renderFramesAutomatically, executor, listener, glObjectsProvider, @@ -555,7 +555,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { inputType, videoFrameProcessingTaskExecutor, shaderPrograms, - releaseFramesAutomatically); + renderFramesAutomatically); } /** @@ -579,7 +579,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { ColorInfo outputColorInfo, boolean enableColorTransfers, @InputType int inputType, - boolean releaseFramesAutomatically, + boolean renderFramesAutomatically, Executor executor, Listener listener, GlObjectsProvider glObjectsProvider, @@ -666,7 +666,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { enableColorTransfers, sampleFromInputTexture, inputType, - releaseFramesAutomatically, + renderFramesAutomatically, executor, listener, glObjectsProvider, diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java index 71070c3806..6185f37103 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java @@ -87,7 +87,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final ColorInfo inputColorInfo; private final ColorInfo outputColorInfo; private final boolean enableColorTransfers; - private final boolean releaseFramesAutomatically; + private final boolean renderFramesAutomatically; private final Executor videoFrameProcessorListenerExecutor; private final VideoFrameProcessor.Listener videoFrameProcessorListener; private final float[] textureTransformMatrix; @@ -129,7 +129,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; boolean enableColorTransfers, boolean sampleFromInputTexture, @VideoFrameProcessor.InputType int inputType, - boolean releaseFramesAutomatically, + boolean renderFramesAutomatically, Executor videoFrameProcessorListenerExecutor, VideoFrameProcessor.Listener videoFrameProcessorListener, GlObjectsProvider glObjectsProvider, @@ -145,7 +145,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.inputColorInfo = inputColorInfo; this.outputColorInfo = outputColorInfo; this.enableColorTransfers = enableColorTransfers; - this.releaseFramesAutomatically = releaseFramesAutomatically; + this.renderFramesAutomatically = renderFramesAutomatically; this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor; this.videoFrameProcessorListener = videoFrameProcessorListener; this.glObjectsProvider = glObjectsProvider; @@ -203,9 +203,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) { frameProcessingStarted = true; videoFrameProcessorListenerExecutor.execute( - () -> videoFrameProcessorListener.onOutputFrameAvailable(presentationTimeUs)); - if (releaseFramesAutomatically) { - renderFrame(inputTexture, presentationTimeUs, /* releaseTimeNs= */ presentationTimeUs * 1000); + () -> videoFrameProcessorListener.onOutputFrameAvailableForRendering(presentationTimeUs)); + if (renderFramesAutomatically) { + renderFrame(inputTexture, presentationTimeUs, /* renderTimeNs= */ presentationTimeUs * 1000); } else { availableFrames.add(Pair.create(inputTexture, presentationTimeUs)); } @@ -218,20 +218,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throw new UnsupportedOperationException(); } - public void releaseOutputFrame(long releaseTimeNs) { + public void renderOutputFrame(long renderTimeNs) { frameProcessingStarted = true; - checkState(!releaseFramesAutomatically); + checkState(!renderFramesAutomatically); Pair oldestAvailableFrame = availableFrames.remove(); renderFrame( /* inputTexture= */ oldestAvailableFrame.first, /* presentationTimeUs= */ oldestAvailableFrame.second, - releaseTimeNs); + renderTimeNs); } @Override public void flush() { frameProcessingStarted = true; - // Drops all frames that aren't released yet. + // Drops all frames that aren't rendered yet. availableFrames.clear(); if (defaultShaderProgram != null) { defaultShaderProgram.flush(); @@ -302,15 +302,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } private synchronized void renderFrame( - GlTextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) { + GlTextureInfo inputTexture, long presentationTimeUs, long renderTimeNs) { try { - if (releaseTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME + if (renderTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME || !ensureConfigured(inputTexture.width, inputTexture.height)) { inputListener.onInputFrameProcessed(inputTexture); return; // Drop frames when requested, or there is no output surface. } if (outputSurfaceInfo != null) { - renderFrameToOutputSurface(inputTexture, presentationTimeUs, releaseTimeNs); + renderFrameToOutputSurface(inputTexture, presentationTimeUs, renderTimeNs); } if (textureOutputListener != null) { renderFrameToOutputTexture(inputTexture, presentationTimeUs); @@ -329,7 +329,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } private synchronized void renderFrameToOutputSurface( - GlTextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs) + GlTextureInfo inputTexture, long presentationTimeUs, long renderTimeNs) throws VideoFrameProcessingException, GlUtil.GlException { EGLSurface outputEglSurface = checkNotNull(this.outputEglSurface); SurfaceInfo outputSurfaceInfo = checkNotNull(this.outputSurfaceInfo); @@ -347,9 +347,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; EGLExt.eglPresentationTimeANDROID( eglDisplay, outputEglSurface, - releaseTimeNs == VideoFrameProcessor.RELEASE_OUTPUT_FRAME_IMMEDIATELY + renderTimeNs == VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY ? System.nanoTime() - : releaseTimeNs); + : renderTimeNs); EGL14.eglSwapBuffers(eglDisplay, outputEglSurface); } @@ -427,8 +427,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; eglDisplay, outputSurfaceInfo.surface, outputColorInfo.colorTransfer, - // Frames are only released automatically when outputting to an encoder. - /* isEncoderInputSurface= */ releaseFramesAutomatically); + // Frames are only rendered automatically when outputting to an encoder. + /* isEncoderInputSurface= */ renderFramesAutomatically); } @Nullable diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 4c43ad3f44..61cea535f2 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -2027,7 +2027,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { inputAndOutputColorInfos.first, inputAndOutputColorInfos.second, INPUT_TYPE_SURFACE, - /* releaseFramesAutomatically= */ false, + /* renderFramesAutomatically= */ false, /* executor= */ handler::post, new VideoFrameProcessor.Listener() { @Override @@ -2048,7 +2048,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } @Override - public void onOutputFrameAvailable(long presentationTimeUs) { + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { if (registeredLastFrame) { checkState(lastCodecBufferPresentationTimestampUs != C.TIME_UNSET); } @@ -2254,7 +2254,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { boolean shouldReleaseFrameImmediately = renderer.shouldForceRender(positionUs, earlyUs); if (shouldReleaseFrameImmediately) { releaseProcessedFrameInternal( - VideoFrameProcessor.RELEASE_OUTPUT_FRAME_IMMEDIATELY, isLastFrame); + VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY, isLastFrame); break; } else if (!isStarted || positionUs == renderer.initialPositionUs) { return; @@ -2313,8 +2313,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } private void releaseProcessedFrameInternal(long releaseTimeNs, boolean isLastFrame) { + // VideoFrameProcessor renders to its output surface using + // VideoFrameProcessor.renderOutputFrame, to release the MediaCodecVideoRenderer frame. checkStateNotNull(videoFrameProcessor); - videoFrameProcessor.releaseOutputFrame(releaseTimeNs); + videoFrameProcessor.renderOutputFrame(releaseTimeNs); processedFramesTimestampsUs.remove(); renderer.lastRenderRealtimeUs = SystemClock.elapsedRealtime() * 1000; if (releaseTimeNs != VideoFrameProcessor.DROP_OUTPUT_FRAME) { diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java index a7b6c7d796..146caad124 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/VideoFrameProcessorTestRunner.java @@ -68,7 +68,7 @@ public final class VideoFrameProcessorTestRunner { private @MonotonicNonNull ColorInfo inputColorInfo; private @MonotonicNonNull ColorInfo outputColorInfo; private @VideoFrameProcessor.InputType int inputType; - private OnOutputFrameAvailableListener onOutputFrameAvailableListener; + private OnOutputFrameAvailableForRenderingListener onOutputFrameAvailableListener; /** Creates a new instance with default values. */ public Builder() { @@ -202,13 +202,14 @@ public final class VideoFrameProcessorTestRunner { } /** - * Sets the method to be called in {@link VideoFrameProcessor.Listener#onOutputFrameAvailable}. + * Sets the method to be called in {@link + * VideoFrameProcessor.Listener#onOutputFrameAvailableForRendering}. * *

The default value is a no-op. */ @CanIgnoreReturnValue - public Builder setOnOutputFrameAvailableListener( - OnOutputFrameAvailableListener onOutputFrameAvailableListener) { + public Builder setOnOutputFrameAvailableForRenderingListener( + OnOutputFrameAvailableForRenderingListener onOutputFrameAvailableListener) { this.onOutputFrameAvailableListener = onOutputFrameAvailableListener; return this; } @@ -260,7 +261,7 @@ public final class VideoFrameProcessorTestRunner { ColorInfo inputColorInfo, ColorInfo outputColorInfo, @VideoFrameProcessor.InputType int inputType, - OnOutputFrameAvailableListener onOutputFrameAvailableListener) + OnOutputFrameAvailableForRenderingListener onOutputFrameAvailableForRenderingListener) throws VideoFrameProcessingException { this.testId = testId; this.bitmapReader = bitmapReader; @@ -277,7 +278,7 @@ public final class VideoFrameProcessorTestRunner { inputColorInfo, outputColorInfo, inputType, - /* releaseFramesAutomatically= */ true, + /* renderFramesAutomatically= */ true, MoreExecutors.directExecutor(), new VideoFrameProcessor.Listener() { @Override @@ -296,9 +297,10 @@ public final class VideoFrameProcessorTestRunner { } @Override - public void onOutputFrameAvailable(long presentationTimeUs) { - // Do nothing as frames are released automatically. - onOutputFrameAvailableListener.onFrameAvailable(presentationTimeUs); + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + // Do nothing as frames are rendered automatically. + onOutputFrameAvailableForRenderingListener.onFrameAvailableForRendering( + presentationTimeUs); } @Override @@ -378,8 +380,8 @@ public final class VideoFrameProcessorTestRunner { } } - public interface OnOutputFrameAvailableListener { - void onFrameAvailable(long presentationTimeUs); + public interface OnOutputFrameAvailableForRenderingListener { + void onFrameAvailableForRendering(long presentationTimeUs); } /** Reads a {@link Bitmap} from {@link VideoFrameProcessor} output. */ diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java index da307a304a..5188d25b52 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java @@ -159,7 +159,7 @@ import org.checkerframework.dataflow.qual.Pure; videoFrameProcessorInputColor, videoFrameProcessorOutputColor, inputType, - /* releaseFramesAutomatically= */ true, + /* renderFramesAutomatically= */ true, MoreExecutors.directExecutor(), new VideoFrameProcessor.Listener() { private long lastProcessedFramePresentationTimeUs; @@ -175,8 +175,8 @@ import org.checkerframework.dataflow.qual.Pure; } @Override - public void onOutputFrameAvailable(long presentationTimeUs) { - // Frames are released automatically. + public void onOutputFrameAvailableForRendering(long presentationTimeUs) { + // Frames are rendered automatically. if (presentationTimeUs == 0) { encoderExpectsTimestampZero = true; }