diff --git a/libraries/common/src/main/java/androidx/media3/common/FrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/FrameProcessor.java index c52bbb922e..9ce300eabd 100644 --- a/libraries/common/src/main/java/androidx/media3/common/FrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/FrameProcessor.java @@ -47,6 +47,10 @@ public interface FrameProcessor { * @param effects The {@link Effect} instances to apply to each frame. * @param debugViewProvider A {@link DebugViewProvider}. * @param colorInfo The {@link ColorInfo} for input and output frames. + * @param releaseFramesAutomatically If {@code true}, the {@link FrameProcessor} will release + * output frames to the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface} + * automatically as they become available. If {@code false}, the {@link FrameProcessor} will + * wait to release each frame until {@link #releaseOutputFrame(long)} is called. * @return A new instance. * @throws FrameProcessingException If a problem occurs while creating the {@link * FrameProcessor}. @@ -56,7 +60,8 @@ public interface FrameProcessor { Listener listener, List effects, DebugViewProvider debugViewProvider, - ColorInfo colorInfo) + ColorInfo colorInfo, + boolean releaseFramesAutomatically) throws FrameProcessingException; } @@ -75,6 +80,13 @@ public interface FrameProcessor { */ void onOutputSizeChanged(int width, int height); + /** + * Called when an output frame with the given {@code presentationTimeNs} becomes available. + * + * @param presentationTimeNs The presentation time of the frame, in nanoseconds. + */ + void onOutputFrameAvailable(long presentationTimeNs); + /** * Called when an exception occurs during asynchronous frame processing. * @@ -137,6 +149,20 @@ public interface FrameProcessor { */ void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo); + /** + * Releases the oldest unreleased output frame that has become {@linkplain + * Listener#onOutputFrameAvailable(long) available} at the given {@code releaseTimeNs}. + * + *

This method must only be called if {@code releaseFramesAutomatically} was set to {@code + * false} using the {@link Factory} and should be called exactly once for each frame that becomes + * {@linkplain Listener#onOutputFrameAvailable(long) available}. + * + * @param releaseTimeNs The release time to use for the frame, in nanoseconds. Use {@link + * C#TIME_UNSET} to drop the frame. If {@code releaseTimeNs} is after {@link + * System#nanoTime()} at the time of the release, the frame is also dropped. + */ + void releaseOutputFrame(long releaseTimeNs); + /** * Informs the {@code FrameProcessor} that no further input frames should be accepted. * diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorFrameReleaseTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorFrameReleaseTest.java new file mode 100644 index 0000000000..8d4eab2f2a --- /dev/null +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorFrameReleaseTest.java @@ -0,0 +1,361 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.common.truth.Truth.assertThat; + +import android.graphics.PixelFormat; +import android.media.Image; +import android.media.ImageReader; +import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.DebugViewProvider; +import androidx.media3.common.FrameInfo; +import androidx.media3.common.FrameProcessingException; +import androidx.media3.common.FrameProcessor; +import androidx.media3.common.SurfaceInfo; +import androidx.media3.common.util.GlUtil; +import androidx.media3.common.util.Util; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Tests for frame release in {@link GlEffectsFrameProcessor}. */ +@RunWith(AndroidJUnit4.class) +public final class GlEffectsFrameProcessorFrameReleaseTest { + + private static final int WIDTH = 200; + private static final int HEIGHT = 100; + private static final long FRAME_PROCESSING_WAIT_MS = 5000L; + private static final long MILLIS_TO_NANOS = 1_000_000L; + private static final long MICROS_TO_NANOS = 1000L; + + private final AtomicReference frameProcessingException = + new AtomicReference<>(); + private final Queue outputReleaseTimesNs = new ConcurrentLinkedQueue<>(); + + private @MonotonicNonNull GlEffectsFrameProcessor glEffectsFrameProcessor; + private volatile @MonotonicNonNull Runnable produceBlankFramesTask; + + @After + public void release() { + if (glEffectsFrameProcessor != null) { + glEffectsFrameProcessor.release(); + } + } + + @Test + public void automaticFrameRelease_withOneFrame_reusesInputTimestamp() throws Exception { + long originalPresentationTimeUs = 1234; + AtomicLong actualPresentationTimeNs = new AtomicLong(); + setupGlEffectsFrameProcessorWithBlankFrameProducer( + /* inputPresentationTimesUs= */ new long[] {originalPresentationTimeUs}, + /* onFrameAvailableListener= */ actualPresentationTimeNs::set, + /* releaseFramesAutomatically= */ true); + + checkNotNull(produceBlankFramesTask).run(); + Thread.sleep(FRAME_PROCESSING_WAIT_MS); + + assertThat(frameProcessingException.get()).isNull(); + assertThat(actualPresentationTimeNs.get()) + .isEqualTo(MICROS_TO_NANOS * originalPresentationTimeUs); + assertThat(outputReleaseTimesNs).containsExactly(MICROS_TO_NANOS * originalPresentationTimeUs); + } + + @Test + public void automaticFrameRelease_withThreeFrames_reusesInputTimestamps() throws Exception { + long[] originalPresentationTimesUs = new long[] {1234, 3456, 4567}; + ArrayList actualPresentationTimesNs = new ArrayList<>(); + setupGlEffectsFrameProcessorWithBlankFrameProducer( + originalPresentationTimesUs, + /* onFrameAvailableListener= */ actualPresentationTimesNs::add, + /* releaseFramesAutomatically= */ true); + + checkNotNull(produceBlankFramesTask).run(); + Thread.sleep(FRAME_PROCESSING_WAIT_MS); + + assertThat(frameProcessingException.get()).isNull(); + assertThat(actualPresentationTimesNs) + .containsExactly( + MICROS_TO_NANOS * originalPresentationTimesUs[0], + MICROS_TO_NANOS * originalPresentationTimesUs[1], + MICROS_TO_NANOS * originalPresentationTimesUs[2]) + .inOrder(); + assertThat(outputReleaseTimesNs).containsExactlyElementsIn(actualPresentationTimesNs).inOrder(); + } + + @Test + public void controlledFrameRelease_withOneFrame_usesGivenTimestamp() throws Exception { + long originalPresentationTimeUs = 1234; + long releaseTimesNs = System.nanoTime() + MILLIS_TO_NANOS * FRAME_PROCESSING_WAIT_MS + 345678; + AtomicLong actualPresentationTimeNs = new AtomicLong(); + setupGlEffectsFrameProcessorWithBlankFrameProducer( + /* inputPresentationTimesUs= */ new long[] {originalPresentationTimeUs}, + /* onFrameAvailableListener= */ presentationTimeNs -> { + actualPresentationTimeNs.set(presentationTimeNs); + checkNotNull(glEffectsFrameProcessor).releaseOutputFrame(releaseTimesNs); + }, + /* releaseFramesAutomatically= */ false); + + checkNotNull(produceBlankFramesTask).run(); + Thread.sleep(FRAME_PROCESSING_WAIT_MS); + + assertThat(frameProcessingException.get()).isNull(); + assertThat(actualPresentationTimeNs.get()) + .isEqualTo(MICROS_TO_NANOS * originalPresentationTimeUs); + assertThat(outputReleaseTimesNs).containsExactly(releaseTimesNs); + } + + @Test + public void controlledFrameRelease_withLateFrame_dropsFrame() throws Exception { + long originalPresentationTimeUs = 1234; + long releaseTimeBeforeCurrentTimeNs = System.nanoTime() - 345678; + AtomicLong actualPresentationTimeNs = new AtomicLong(); + setupGlEffectsFrameProcessorWithBlankFrameProducer( + /* inputPresentationTimesUs= */ new long[] {originalPresentationTimeUs}, + /* onFrameAvailableListener= */ presentationTimeNs -> { + actualPresentationTimeNs.set(presentationTimeNs); + checkNotNull(glEffectsFrameProcessor).releaseOutputFrame(releaseTimeBeforeCurrentTimeNs); + }, + /* releaseFramesAutomatically= */ false); + + checkNotNull(produceBlankFramesTask).run(); + Thread.sleep(FRAME_PROCESSING_WAIT_MS); + + assertThat(frameProcessingException.get()).isNull(); + assertThat(actualPresentationTimeNs.get()) + .isEqualTo(MICROS_TO_NANOS * originalPresentationTimeUs); + assertThat(outputReleaseTimesNs).isEmpty(); + } + + @Test + public void controlledFrameRelease_withUnsetReleaseTime_dropsFrame() throws Exception { + long originalPresentationTimeUs = 1234; + AtomicLong actualPresentationTimeNs = new AtomicLong(); + setupGlEffectsFrameProcessorWithBlankFrameProducer( + /* inputPresentationTimesUs= */ new long[] {originalPresentationTimeUs}, + /* onFrameAvailableListener= */ presentationTimeNs -> { + actualPresentationTimeNs.set(presentationTimeNs); + checkNotNull(glEffectsFrameProcessor) + .releaseOutputFrame(/* releaseTimeNs= */ C.TIME_UNSET); + }, + /* releaseFramesAutomatically= */ false); + + checkNotNull(produceBlankFramesTask).run(); + Thread.sleep(FRAME_PROCESSING_WAIT_MS); + + assertThat(frameProcessingException.get()).isNull(); + assertThat(actualPresentationTimeNs.get()) + .isEqualTo(MICROS_TO_NANOS * originalPresentationTimeUs); + assertThat(outputReleaseTimesNs).isEmpty(); + } + + @Test + public void controlledFrameRelease_withThreeIndividualFrames_usesGivenTimestamps() + throws Exception { + long[] originalPresentationTimesUs = new long[] {1234, 3456, 4567}; + long offsetNs = System.nanoTime() + MILLIS_TO_NANOS * FRAME_PROCESSING_WAIT_MS; + long[] releaseTimesNs = new long[] {offsetNs + 123456, offsetNs + 234567, offsetNs + 345678}; + ArrayList actualPresentationTimesNs = new ArrayList<>(); + AtomicInteger frameIndex = new AtomicInteger(); + setupGlEffectsFrameProcessorWithBlankFrameProducer( + /* inputPresentationTimesUs= */ originalPresentationTimesUs, + /* onFrameAvailableListener= */ presentationTimeNs -> { + actualPresentationTimesNs.add(presentationTimeNs); + checkNotNull(glEffectsFrameProcessor) + .releaseOutputFrame(releaseTimesNs[frameIndex.getAndIncrement()]); + }, + /* releaseFramesAutomatically= */ false); + + checkNotNull(produceBlankFramesTask).run(); + Thread.sleep(FRAME_PROCESSING_WAIT_MS); + + assertThat(frameProcessingException.get()).isNull(); + assertThat(actualPresentationTimesNs) + .containsExactly( + MICROS_TO_NANOS * originalPresentationTimesUs[0], + MICROS_TO_NANOS * originalPresentationTimesUs[1], + MICROS_TO_NANOS * originalPresentationTimesUs[2]) + .inOrder(); + assertThat(frameIndex.get()).isEqualTo(originalPresentationTimesUs.length); + assertThat(outputReleaseTimesNs) + .containsExactly(releaseTimesNs[0], releaseTimesNs[1], releaseTimesNs[2]) + .inOrder(); + } + + @Test + public void controlledFrameRelease_withThreeFramesAtOnce_usesGivenTimestamps() throws Exception { + long[] originalPresentationTimesUs = new long[] {1234, 3456, 4567}; + long offsetNs = System.nanoTime() + MILLIS_TO_NANOS * 2 * FRAME_PROCESSING_WAIT_MS; + long[] releaseTimesNs = new long[] {offsetNs + 123456, offsetNs + 234567, offsetNs + 345678}; + ArrayList actualPresentationTimesNs = new ArrayList<>(); + setupGlEffectsFrameProcessorWithBlankFrameProducer( + /* inputPresentationTimesUs= */ originalPresentationTimesUs, + /* onFrameAvailableListener= */ actualPresentationTimesNs::add, + /* releaseFramesAutomatically= */ false); + + checkNotNull(produceBlankFramesTask).run(); + Thread.sleep(FRAME_PROCESSING_WAIT_MS); + glEffectsFrameProcessor.releaseOutputFrame(releaseTimesNs[0]); + glEffectsFrameProcessor.releaseOutputFrame(releaseTimesNs[1]); + glEffectsFrameProcessor.releaseOutputFrame(releaseTimesNs[2]); + Thread.sleep(FRAME_PROCESSING_WAIT_MS); + + assertThat(frameProcessingException.get()).isNull(); + assertThat(actualPresentationTimesNs) + .containsExactly( + MICROS_TO_NANOS * originalPresentationTimesUs[0], + MICROS_TO_NANOS * originalPresentationTimesUs[1], + MICROS_TO_NANOS * originalPresentationTimesUs[2]) + .inOrder(); + assertThat(outputReleaseTimesNs) + .containsExactly(releaseTimesNs[0], releaseTimesNs[1], releaseTimesNs[2]) + .inOrder(); + } + + private interface OnFrameAvailableListener { + void onFrameAvailable(long presentationTimeNs); + } + + @EnsuresNonNull("glEffectsFrameProcessor") + private void setupGlEffectsFrameProcessorWithBlankFrameProducer( + long[] inputPresentationTimesUs, + OnFrameAvailableListener onFrameAvailableListener, + boolean releaseFramesAutomatically) + throws Exception { + glEffectsFrameProcessor = + checkNotNull( + new GlEffectsFrameProcessor.Factory() + .create( + getApplicationContext(), + new FrameProcessor.Listener() { + @Override + public void onOutputSizeChanged(int width, int height) { + ImageReader outputImageReader = + ImageReader.newInstance( + width, + height, + PixelFormat.RGBA_8888, + /* maxImages= */ inputPresentationTimesUs.length); + checkNotNull(glEffectsFrameProcessor) + .setOutputSurfaceInfo( + new SurfaceInfo(outputImageReader.getSurface(), width, height)); + outputImageReader.setOnImageAvailableListener( + imageReader -> { + try (Image image = imageReader.acquireNextImage()) { + outputReleaseTimesNs.add(image.getTimestamp()); + } + }, + Util.createHandlerForCurrentOrMainLooper()); + } + + @Override + public void onOutputFrameAvailable(long presentationTimeNs) { + onFrameAvailableListener.onFrameAvailable(presentationTimeNs); + } + + @Override + public void onFrameProcessingError(FrameProcessingException exception) { + frameProcessingException.set(exception); + } + + @Override + public void onFrameProcessingEnded() {} + }, + ImmutableList.of( + (GlEffect) + (context, useHdr) -> + new BlankFrameProducer(inputPresentationTimesUs, useHdr)), + DebugViewProvider.NONE, + ColorInfo.SDR_BT709_LIMITED, + releaseFramesAutomatically)); + + glEffectsFrameProcessor.setInputFrameInfo( + new FrameInfo(WIDTH, HEIGHT, /* pixelWidthHeightRatio= */ 1, /* streamOffsetUs= */ 0)); + // A frame needs to be registered despite not queuing any external input to ensure that the + // frame processor knows about the stream offset. + glEffectsFrameProcessor.registerInputFrame(); + } + + /** Produces blank frames with the given timestamps. */ + private final class BlankFrameProducer implements GlTextureProcessor { + + private final TextureInfo blankTexture; + private final long[] presentationTimesUs; + + public BlankFrameProducer(long[] presentationTimesUs, boolean useHdr) + throws FrameProcessingException { + this.presentationTimesUs = presentationTimesUs; + try { + int texId = GlUtil.createTexture(WIDTH, HEIGHT, useHdr); + int fboId = GlUtil.createFboForTexture(texId); + blankTexture = new TextureInfo(texId, fboId, WIDTH, HEIGHT); + GlUtil.focusFramebufferUsingCurrentContext(fboId, WIDTH, HEIGHT); + GlUtil.clearOutputFrame(); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e); + } + } + + @Override + public void setInputListener(InputListener inputListener) {} + + @Override + public void setOutputListener(OutputListener outputListener) { + produceBlankFramesTask = + () -> { + for (long presentationTimeUs : presentationTimesUs) { + outputListener.onOutputFrameAvailable(blankTexture, presentationTimeUs); + } + }; + } + + @Override + public void setErrorListener(ErrorListener errorListener) {} + + @Override + public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { + // No input is queued in these tests. The BlankFrameProducer is used to produce frames. + throw new UnsupportedOperationException(); + } + + @Override + public void releaseOutputFrame(TextureInfo outputTexture) {} + + @Override + public void signalEndOfCurrentInputStream() { + // The tests don't end the input stream. + throw new UnsupportedOperationException(); + } + + @Override + public void release() { + // Do nothing as destroying the OpenGL context destroys the texture. + } + } +} diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java index 0945738b8e..06d3139ef2 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java @@ -476,6 +476,11 @@ public final class GlEffectsFrameProcessorPixelTest { new SurfaceInfo(outputImageReader.getSurface(), width, height)); } + @Override + public void onOutputFrameAvailable(long presentationTimeNs) { + // Do nothing as frames are released automatically. + } + @Override public void onFrameProcessingError(FrameProcessingException exception) { frameProcessingException.set(exception); @@ -488,7 +493,8 @@ public final class GlEffectsFrameProcessorPixelTest { }, effects, DebugViewProvider.NONE, - ColorInfo.SDR_BT709_LIMITED)); + ColorInfo.SDR_BT709_LIMITED, + /* releaseFramesAutomatically= */ true)); glEffectsFrameProcessor.setInputFrameInfo( new FrameInfo(inputWidth, inputHeight, pixelWidthHeightRatio, /* streamOffsetUs= */ 0)); glEffectsFrameProcessor.registerInputFrame(); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java index e0fd531987..747e9f04bc 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java @@ -16,6 +16,7 @@ package androidx.media3.effect; import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Assertions.checkStateNotNull; import android.content.Context; import android.opengl.EGL14; @@ -71,8 +72,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final FrameProcessor.Listener frameProcessorListener; private final boolean sampleFromExternalTexture; private final ColorInfo colorInfo; + private final boolean releaseFramesAutomatically; private final float[] textureTransformMatrix; private final Queue streamOffsetUsQueue; + private final Queue> availableFrames; private int inputWidth; private int inputHeight; @@ -100,7 +103,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; FrameProcessor.Listener frameProcessorListener, DebugViewProvider debugViewProvider, boolean sampleFromExternalTexture, - ColorInfo colorInfo) { + ColorInfo colorInfo, + boolean releaseFramesAutomatically) { this.context = context; this.matrixTransformations = matrixTransformations; this.eglDisplay = eglDisplay; @@ -109,11 +113,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.frameProcessorListener = frameProcessorListener; this.sampleFromExternalTexture = sampleFromExternalTexture; this.colorInfo = colorInfo; + this.releaseFramesAutomatically = releaseFramesAutomatically; textureTransformMatrix = new float[16]; Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0); streamOffsetUsQueue = new ConcurrentLinkedQueue<>(); inputListener = new InputListener() {}; + availableFrames = new ConcurrentLinkedQueue<>(); } @Override @@ -136,54 +142,149 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @Override public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { - checkState(!streamOffsetUsQueue.isEmpty(), "No input stream specified."); + long streamOffsetUs = + checkStateNotNull(streamOffsetUsQueue.peek(), "No input stream specified."); + long presentationTimeNs = (presentationTimeUs + streamOffsetUs) * 1000; + frameProcessorListener.onOutputFrameAvailable(presentationTimeNs); + if (releaseFramesAutomatically) { + renderFrameToSurfaces( + inputTexture, + presentationTimeUs, + /* releaseTimeNs= */ presentationTimeNs, + /* dropLateFrame= */ false); + } else { + availableFrames.add(Pair.create(inputTexture, presentationTimeUs)); + } + inputListener.onReadyToAcceptInputFrame(); + } - try { - synchronized (this) { - if (!ensureConfigured(inputTexture.width, inputTexture.height)) { - inputListener.onInputFrameProcessed(inputTexture); - return; // Drop frames when there is no output surface. - } + @Override + public void releaseOutputFrame(TextureInfo outputTexture) { + // The final texture processor writes to a surface so there is no texture to release. + throw new UnsupportedOperationException(); + } - EGLSurface outputEglSurface = this.outputEglSurface; - SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo; - MatrixTransformationProcessor matrixTransformationProcessor = - this.matrixTransformationProcessor; + @WorkerThread + public void releaseOutputFrame(long releaseTimeNs) { + checkState(!releaseFramesAutomatically); - GlUtil.focusEglSurface( - eglDisplay, - eglContext, - outputEglSurface, - outputSurfaceInfo.width, - outputSurfaceInfo.height); - GlUtil.clearOutputFrame(); - matrixTransformationProcessor.drawFrame(inputTexture.texId, presentationTimeUs); - EGLExt.eglPresentationTimeANDROID( - eglDisplay, - outputEglSurface, - /* presentationTimeNs= */ (presentationTimeUs + streamOffsetUsQueue.element()) * 1000); - EGL14.eglSwapBuffers(eglDisplay, outputEglSurface); + Pair oldestAvailableFrame = availableFrames.remove(); + renderFrameToSurfaces( + /* inputTexture= */ oldestAvailableFrame.first, + /* presentationTimeUs= */ oldestAvailableFrame.second, + releaseTimeNs, + /* dropLateFrame= */ true); + } + + @Override + public void signalEndOfCurrentInputStream() { + checkState(!streamOffsetUsQueue.isEmpty(), "No input stream to end."); + + streamOffsetUsQueue.remove(); + if (streamOffsetUsQueue.isEmpty()) { + frameProcessorListener.onFrameProcessingEnded(); + } + } + + @Override + @WorkerThread + public void release() throws FrameProcessingException { + if (matrixTransformationProcessor != null) { + matrixTransformationProcessor.release(); + } + } + + @Override + public void setTextureTransformMatrix(float[] textureTransformMatrix) { + System.arraycopy( + /* src= */ textureTransformMatrix, + /* srcPos= */ 0, + /* dest= */ this.textureTransformMatrix, + /* destPost= */ 0, + /* length= */ textureTransformMatrix.length); + + if (matrixTransformationProcessor != null) { + matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix); + } + } + + /** + * Signals that there will be another input stream after all previously appended input streams + * have {@linkplain #signalEndOfCurrentInputStream() ended}. + * + *

This method does not need to be called on the GL thread, but the caller must ensure that + * stream offsets are appended in the correct order. + * + * @param streamOffsetUs The presentation timestamp offset, in microseconds. + */ + public void appendStream(long streamOffsetUs) { + streamOffsetUsQueue.add(streamOffsetUs); + } + + /** + * Sets the output {@link SurfaceInfo}. + * + * @see FrameProcessor#setOutputSurfaceInfo(SurfaceInfo) + */ + public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { + if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) { + if (outputSurfaceInfo != null + && this.outputSurfaceInfo != null + && !this.outputSurfaceInfo.surface.equals(outputSurfaceInfo.surface)) { + this.outputEglSurface = null; } + outputSizeOrRotationChanged = + this.outputSurfaceInfo == null + || outputSurfaceInfo == null + || this.outputSurfaceInfo.width != outputSurfaceInfo.width + || this.outputSurfaceInfo.height != outputSurfaceInfo.height + || this.outputSurfaceInfo.orientationDegrees != outputSurfaceInfo.orientationDegrees; + this.outputSurfaceInfo = outputSurfaceInfo; + } + } + + private void renderFrameToSurfaces( + TextureInfo inputTexture, + long presentationTimeUs, + long releaseTimeNs, + boolean dropLateFrame) { + try { + maybeRenderFrameToOutputSurface( + inputTexture, presentationTimeUs, releaseTimeNs, dropLateFrame); } catch (FrameProcessingException | GlUtil.GlException e) { frameProcessorListener.onFrameProcessingError( FrameProcessingException.from(e, presentationTimeUs)); } - - if (debugSurfaceViewWrapper != null && matrixTransformationProcessor != null) { - MatrixTransformationProcessor matrixTransformationProcessor = - this.matrixTransformationProcessor; - try { - debugSurfaceViewWrapper.maybeRenderToSurfaceView( - () -> { - GlUtil.clearOutputFrame(); - matrixTransformationProcessor.drawFrame(inputTexture.texId, presentationTimeUs); - }); - } catch (FrameProcessingException | GlUtil.GlException e) { - Log.d(TAG, "Error rendering to debug preview", e); - } - } + maybeRenderFrameToDebugSurface(inputTexture, presentationTimeUs); inputListener.onInputFrameProcessed(inputTexture); - inputListener.onReadyToAcceptInputFrame(); + } + + private synchronized void maybeRenderFrameToOutputSurface( + TextureInfo inputTexture, long presentationTimeUs, long releaseTimeNs, boolean dropLateFrame) + throws FrameProcessingException, GlUtil.GlException { + if (!ensureConfigured(inputTexture.width, inputTexture.height)) { + return; // Drop frames when there is no output surface. + } + + EGLSurface outputEglSurface = this.outputEglSurface; + SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo; + MatrixTransformationProcessor matrixTransformationProcessor = + this.matrixTransformationProcessor; + + GlUtil.focusEglSurface( + eglDisplay, + eglContext, + outputEglSurface, + outputSurfaceInfo.width, + outputSurfaceInfo.height); + GlUtil.clearOutputFrame(); + matrixTransformationProcessor.drawFrame(inputTexture.texId, presentationTimeUs); + + if (dropLateFrame && System.nanoTime() > releaseTimeNs) { + return; + } + EGLExt.eglPresentationTimeANDROID(eglDisplay, outputEglSurface, releaseTimeNs); + EGL14.eglSwapBuffers(eglDisplay, outputEglSurface); } @EnsuresNonNullIf( @@ -282,76 +383,21 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return matrixTransformationProcessor; } - @Override - public void releaseOutputFrame(TextureInfo outputTexture) { - // The final texture processor writes to a surface so there is no texture to release. - throw new UnsupportedOperationException(); - } - - @Override - public void signalEndOfCurrentInputStream() { - checkState(!streamOffsetUsQueue.isEmpty(), "No input stream to end."); - - streamOffsetUsQueue.remove(); - if (streamOffsetUsQueue.isEmpty()) { - frameProcessorListener.onFrameProcessingEnded(); + private void maybeRenderFrameToDebugSurface(TextureInfo inputTexture, long presentationTimeUs) { + if (debugSurfaceViewWrapper == null || matrixTransformationProcessor == null) { + return; } - } - @Override - @WorkerThread - public void release() throws FrameProcessingException { - if (matrixTransformationProcessor != null) { - matrixTransformationProcessor.release(); - } - } - - @Override - public void setTextureTransformMatrix(float[] textureTransformMatrix) { - System.arraycopy( - /* src= */ textureTransformMatrix, - /* srcPos= */ 0, - /* dest= */ this.textureTransformMatrix, - /* destPost= */ 0, - /* length= */ textureTransformMatrix.length); - - if (matrixTransformationProcessor != null) { - matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix); - } - } - - /** - * Signals that there will be another input stream after all previously appended input streams - * have {@linkplain #signalEndOfCurrentInputStream() ended}. - * - *

This method does not need to be called on the GL thread, but the caller must ensure that - * stream offsets are appended in the correct order. - * - * @param streamOffsetUs The presentation timestamp offset, in microseconds. - */ - public void appendStream(long streamOffsetUs) { - streamOffsetUsQueue.add(streamOffsetUs); - } - - /** - * Sets the output {@link SurfaceInfo}. - * - * @see FrameProcessor#setOutputSurfaceInfo(SurfaceInfo) - */ - public synchronized void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { - if (!Util.areEqual(this.outputSurfaceInfo, outputSurfaceInfo)) { - if (outputSurfaceInfo != null - && this.outputSurfaceInfo != null - && !this.outputSurfaceInfo.surface.equals(outputSurfaceInfo.surface)) { - this.outputEglSurface = null; - } - outputSizeOrRotationChanged = - this.outputSurfaceInfo == null - || outputSurfaceInfo == null - || this.outputSurfaceInfo.width != outputSurfaceInfo.width - || this.outputSurfaceInfo.height != outputSurfaceInfo.height - || this.outputSurfaceInfo.orientationDegrees != outputSurfaceInfo.orientationDegrees; - this.outputSurfaceInfo = outputSurfaceInfo; + MatrixTransformationProcessor matrixTransformationProcessor = + this.matrixTransformationProcessor; + try { + debugSurfaceViewWrapper.maybeRenderToSurfaceView( + () -> { + GlUtil.clearOutputFrame(); + matrixTransformationProcessor.drawFrame(inputTexture.texId, presentationTimeUs); + }); + } catch (FrameProcessingException | GlUtil.GlException e) { + Log.d(TAG, "Error rendering to debug preview", e); } } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java index f0656749cc..324ff05c4e 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java @@ -67,7 +67,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { FrameProcessor.Listener listener, List effects, DebugViewProvider debugViewProvider, - ColorInfo colorInfo) + ColorInfo colorInfo, + boolean releaseFramesAutomatically) throws FrameProcessingException { ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME); @@ -81,6 +82,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { effects, debugViewProvider, colorInfo, + releaseFramesAutomatically, singleThreadExecutorService)); try { @@ -111,6 +113,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { List effects, DebugViewProvider debugViewProvider, ColorInfo colorInfo, + boolean releaseFramesAutomatically, ExecutorService singleThreadExecutorService) throws GlUtil.GlException, FrameProcessingException { checkState(Thread.currentThread().getName().equals(THREAD_NAME)); @@ -135,13 +138,24 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { ImmutableList textureProcessors = getGlTextureProcessorsForGlEffects( - context, effects, eglDisplay, eglContext, listener, debugViewProvider, colorInfo); + context, + effects, + eglDisplay, + eglContext, + listener, + debugViewProvider, + colorInfo, + releaseFramesAutomatically); FrameProcessingTaskExecutor frameProcessingTaskExecutor = new FrameProcessingTaskExecutor(singleThreadExecutorService, listener); chainTextureProcessorsWithListeners(textureProcessors, frameProcessingTaskExecutor, listener); return new GlEffectsFrameProcessor( - eglDisplay, eglContext, frameProcessingTaskExecutor, textureProcessors); + eglDisplay, + eglContext, + frameProcessingTaskExecutor, + textureProcessors, + releaseFramesAutomatically); } /** @@ -163,7 +177,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { EGLContext eglContext, FrameProcessor.Listener listener, DebugViewProvider debugViewProvider, - ColorInfo colorInfo) + ColorInfo colorInfo, + boolean releaseFramesAutomatically) throws FrameProcessingException { ImmutableList.Builder textureProcessorListBuilder = new ImmutableList.Builder<>(); @@ -244,7 +259,8 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { listener, debugViewProvider, sampleFromExternalTexture, - colorInfo)); + colorInfo, + releaseFramesAutomatically)); return textureProcessorListBuilder.build(); } @@ -278,6 +294,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { private final FrameProcessingTaskExecutor frameProcessingTaskExecutor; private final ExternalTextureManager inputExternalTextureManager; private final Surface inputSurface; + private final boolean releaseFramesAutomatically; private final FinalMatrixTransformationProcessorWrapper finalTextureProcessorWrapper; private final ImmutableList allTextureProcessors; @@ -293,12 +310,14 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { EGLDisplay eglDisplay, EGLContext eglContext, FrameProcessingTaskExecutor frameProcessingTaskExecutor, - ImmutableList textureProcessors) + ImmutableList textureProcessors, + boolean releaseFramesAutomatically) throws FrameProcessingException { this.eglDisplay = eglDisplay; this.eglContext = eglContext; this.frameProcessingTaskExecutor = frameProcessingTaskExecutor; + this.releaseFramesAutomatically = releaseFramesAutomatically; checkState(!textureProcessors.isEmpty()); checkState(textureProcessors.get(0) instanceof ExternalTextureProcessor); @@ -349,6 +368,15 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { finalTextureProcessorWrapper.setOutputSurfaceInfo(outputSurfaceInfo); } + @Override + public void releaseOutputFrame(long releaseTimeNs) { + checkState( + !releaseFramesAutomatically, + "Calling this method is not allowed when releaseFramesAutomatically is enabled"); + frameProcessingTaskExecutor.submitWithHighPriority( + () -> finalTextureProcessorWrapper.releaseOutputFrame(releaseTimeNs)); + } + @Override public void signalEndOfInput() { checkState(!inputStreamEnded); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index cbc13d2d43..991784ffa7 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -125,6 +125,11 @@ import org.checkerframework.dataflow.qual.Pure; } } + @Override + public void onOutputFrameAvailable(long presentationTimeNs) { + // Do nothing as frames are released automatically. + } + @Override public void onFrameProcessingError(FrameProcessingException exception) { asyncErrorListener.onTransformationException( @@ -147,7 +152,8 @@ import org.checkerframework.dataflow.qual.Pure; // This implies that the OpenGL EXT_YUV_target extension is supported and hence the // default FrameProcessor, GlEffectsFrameProcessor, also supports HDR. Otherwise, tone // mapping is applied, which ensures the decoder outputs SDR output for an HDR input. - encoderWrapper.getSupportedInputColor()); + encoderWrapper.getSupportedInputColor(), + /* releaseFramesAutomatically= */ true); } catch (FrameProcessingException e) { throw TransformationException.createForFrameProcessingException( e, TransformationException.ERROR_CODE_FRAME_PROCESSING_FAILED);